From 0e2c37e8fa4ec6a3e6c26a00ecbe3eb5ba60856d Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Fri, 10 Jul 2026 03:26:18 +0200 Subject: [PATCH] Validate Leo production apply receipts --- .../current-truth-index.md | 5 +- ...dler-direct-claim-suite-score-current.json | 2 +- ...andler-direct-claim-suite-score-current.md | 2 +- ...-leo-authorized-apply-runbook-current.json | 10 +- ...ng-leo-authorized-apply-runbook-current.md | 7 +- .../working-leo-current-state-20260709.md | 2 + .../working-leo-execution-plan-current.md | 3 +- ...tion-apply-receipt-validation-current.json | 27 ++ ...uction-apply-receipt-validation-current.md | 22 ++ ...ld_working_leo_authorized_apply_runbook.py | 11 + ...validate_working_leo_production_receipt.py | 271 ++++++++++++++++++ ...ld_working_leo_authorized_apply_runbook.py | 1 + ...validate_working_leo_production_receipt.py | 127 ++++++++ 13 files changed, 484 insertions(+), 6 deletions(-) create mode 100644 docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.json create mode 100644 docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md create mode 100644 scripts/validate_working_leo_production_receipt.py create mode 100644 tests/test_validate_working_leo_production_receipt.py diff --git a/docs/reports/leo-working-state-20260709/current-truth-index.md b/docs/reports/leo-working-state-20260709/current-truth-index.md index 57d4c95..936c092 100644 --- a/docs/reports/leo-working-state-20260709/current-truth-index.md +++ b/docs/reports/leo-working-state-20260709/current-truth-index.md @@ -76,6 +76,8 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c - Working Leo authorized-apply readiness verifier JSON: `docs/reports/leo-working-state-20260709/working-leo-apply-readiness-current.json` - Working Leo authorized apply operator runbook, not executed: `docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md` - Working Leo authorized apply operator runbook JSON: `docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json` +- Working Leo production apply receipt validation, currently missing receipt: `docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md` +- Working Leo production apply receipt validation JSON: `docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.json` - Working Leo cleanup readback: `docs/reports/leo-working-state-20260709/working-leo-cleanup-readback-current.md` - Working Leo cleanup readback JSON: `docs/reports/leo-working-state-20260709/working-leo-cleanup-readback-current.json` - Helmer live rollback log: `docs/reports/leo-working-state-20260709/helmer-7powers-live-preflight-rollback-current.log` @@ -121,7 +123,8 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c - The governance/concept-map schema companion write is clone-proven and reversible: after the mapped base packet exists, it creates minimal concept-map and claim-to-governance link tables, then inserts `1` concept map, `1` concept link, and `2` governance links in a disposable clone; production stayed unchanged. - The full integrated packet set is now clone-proven in production dependency order: mapped base, cross-surface anchor, Rio strategy context, governance/concept schema, and Helmer 7 Powers applied together in `teleo_integrated_rehearsal_20260710`, then rolled back in reverse order; production stayed unchanged and the disposable clone/temp files were removed. - The authorized-apply readiness verifier now passes: packet files exist and hash, apply/rollback order matches the manifest, expected counts match, clone apply/rollback markers are ordered, clone cleanup is proven, service stayed stable, and `production_apply_executed=false`. -- The authorized apply operator runbook is generated and ready-not-executed: it gives preflight/apply/postflight/rollback command templates in manifest order, requires `TELEO_DATABASE_URL`, and now also includes a post-apply regression command bundle plus required production-apply receipt fields. It still does not authorize or execute production SQL. +- The authorized apply operator runbook is generated and ready-not-executed: it gives preflight/apply/postflight/rollback command templates in manifest order, requires `TELEO_DATABASE_URL`, and now also includes a post-apply regression command bundle plus required production-apply receipt fields and an offline receipt validator. It still does not authorize or execute production SQL. +- The production-apply receipt validator currently reports `missing_receipt`, which is correct because no production apply has been authorized or executed. - DB-first identity rendering is now documented from the July 9 PDFs plus VPS readback: canonical identity rows feed rendered `SOUL.md`, direct SOUL edits are not canonical DB updates, and no active renderer/timer/hook is proven in the current VPS readback. - The decision-matrix approval schema is still a target design on the current VPS: `kb_stage.matrix_voters`, `kb_stage.proposal_votes`, and `kb_stage.proposal_decisions` were not present in live schema readback. - Pending/approved document proposals are real `kb_stage.kb_proposals` rows with `source_ref`s, but inspected `source_ref`s did not directly match `public.sources`; they still need review/apply mapping into canonical sources/evidence/edges. diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.json b/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.json index 5e7a8c7..9b249be 100644 --- a/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.json +++ b/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.json @@ -21,7 +21,7 @@ "runtime": "disposable_clone_or_sandbox_first" } ], - "generated_at_utc": "2026-07-10T01:20:35.275732+00:00", + "generated_at_utc": "2026-07-10T01:25:53.065461+00:00", "include_cory_style_scenarios": false, "include_direct_claim_followups": true, "mode": "retained_evidence_score", diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.md b/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.md index 1270f68..b2a2e42 100644 --- a/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.md +++ b/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.md @@ -1,6 +1,6 @@ # Working Leo Open-Ended Benchmark Score -Generated UTC: `2026-07-10T01:20:35.275732+00:00` +Generated UTC: `2026-07-10T01:25:53.065461+00:00` Mode: `retained_evidence_score` Source: `docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-current.json` Coverage: `partial` diff --git a/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json b/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json index 8854969..0b8b473 100644 --- a/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json +++ b/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json @@ -26,7 +26,7 @@ "rio_strategy_nodes": 2, "sources": 28 }, - "generated_at_utc": "2026-07-10T01:16:00.171472+00:00", + "generated_at_utc": "2026-07-10T01:25:52.995962+00:00", "mode": "operator_runbook_generation_only", "operator_environment": { "operator_commands_run_sql_if_executed": true, @@ -318,6 +318,14 @@ "id": "gateway_service_readback", "mutates_production_db": false, "posts_to_telegram": false + }, + { + "command_template": ".venv/bin/python scripts/validate_working_leo_production_receipt.py --receipt docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json", + "description": "Validate the retained production-apply receipt against the authorized runbook requirements.", + "expected_artifact": "docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md", + "id": "production_receipt_validator", + "mutates_production_db": false, + "posts_to_telegram": false } ], "post_apply_required_actions": [ diff --git a/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md b/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md index c0354f0..cf3703a 100644 --- a/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md +++ b/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md @@ -1,6 +1,6 @@ # Working Leo Authorized Apply Runbook -Generated UTC: `2026-07-10T01:16:00.171472+00:00` +Generated UTC: `2026-07-10T01:25:52.995962+00:00` Mode: `operator_runbook_generation_only` Status: `ready_not_executed` Ready for authorized operator: `True` @@ -119,6 +119,11 @@ Do not run apply commands unless the operator explicitly authorizes production D - artifact: `retained service readback with ActiveState=active and NRestarts reviewed` - mutates production DB: `False` - posts to Telegram: `False` +- `production_receipt_validator`: Validate the retained production-apply receipt against the authorized runbook requirements. + - command: `.venv/bin/python scripts/validate_working_leo_production_receipt.py --receipt docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json` + - artifact: `docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md` + - mutates production DB: `False` + - posts to Telegram: `False` ## Production Apply Receipt Required Fields diff --git a/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md b/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md index fc806dc..ba734dd 100644 --- a/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md +++ b/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md @@ -138,6 +138,7 @@ A working Leo, for Cory/m3taversal's current expectation, is a Telegram-facing a - command templates cover preflight, apply, postflight, and rollback in manifest dependency order - post-apply regression bundle now requires the non-posting direct-claim handler suite, strict scorer, focused local regression tests, and gateway service readback - production apply receipt schema now requires explicit authorization record, operator identity, git SHA, preflight/apply/postflight output paths, expected-vs-actual counts, regression paths, service readback, cleanup readback, rollback packet paths, and residual risks + - offline receipt validator is installed and currently reports `missing_receipt`, as expected before any authorized production apply - generator does not connect to Postgres or run SQL; production apply still requires explicit authorization - VPS/Hermes/DB identity and document-linking map is now documented: - the supplied July 9 master/build and database-reference PDFs confirm the intended architecture is DB-first identity: canonical Postgres identity graph rows render to `SOUL.md`, and evidence depth remains on demand through `teleo-kb` @@ -184,6 +185,7 @@ A working Leo, for Cory/m3taversal's current expectation, is a Telegram-facing a - Integrated clone proof log: `outputs/rich-proposal-creation-plan-20260709/integrated-20260710/working-leo-integrated-clone-rehearsal-current.log` - Authorized-apply readiness verifier: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-apply-readiness-current.md` - Authorized apply operator runbook: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md` +- Production apply receipt validation: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md` - Cleanup readback: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-cleanup-readback-current.md` - VPS/Hermes/DB mapping: `outputs/vps-hermes-db-mapping-20260709.md` - Identity render / decision matrix / document artifact image: `outputs/leo-db-state-22-identity-render-matrix-artifacts.svg` diff --git a/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md b/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md index 17d2e03..a0a4e0b 100644 --- a/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md +++ b/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md @@ -96,7 +96,7 @@ Claim ceiling: VPS Telegram memory/KB audit, full live-safe open-ended Telegram ## WL-EXEC-06B - Integrated production-order packet rehearsal - Status: `done_integrated_clone_ready_not_executed` -- Result: Applied the full prepared packet set in one disposable clone in dependency order: mapped base, cross-surface anchor, Rio strategy context, governance/concept schema, and Helmer 7 Powers. Expected integrated footprint was `12` claims, `28` sources, `18` claim edges, `34` claim evidence rows, `1` reasoning tool, `2` Rio strategy nodes, `5` Rio strategy anchors, `1` concept map, `1` concept link, `2` governance links, one Helmer ledger row marked applied, and one cross-surface anchor update. Reverse rollback restored all generated rows/schema to zero/absent, restored the cross-surface anchor to the old claim, restored the Helmer ledger to `approved`, dropped the disposable DB, removed temp files, and left production unchanged. The apply-readiness verifier now passes over the retained manifest/files/logs with `ready_for_authorized_production_apply_packet=true`, `production_apply_executed=false`, and `production_apply_authorization_present=false`. The authorized apply runbook now emits preflight/apply/postflight/rollback command templates in manifest order, a post-apply regression command bundle, and a required production-apply receipt schema with `status=ready_not_executed`. +- Result: Applied the full prepared packet set in one disposable clone in dependency order: mapped base, cross-surface anchor, Rio strategy context, governance/concept schema, and Helmer 7 Powers. Expected integrated footprint was `12` claims, `28` sources, `18` claim edges, `34` claim evidence rows, `1` reasoning tool, `2` Rio strategy nodes, `5` Rio strategy anchors, `1` concept map, `1` concept link, `2` governance links, one Helmer ledger row marked applied, and one cross-surface anchor update. Reverse rollback restored all generated rows/schema to zero/absent, restored the cross-surface anchor to the old claim, restored the Helmer ledger to `approved`, dropped the disposable DB, removed temp files, and left production unchanged. The apply-readiness verifier now passes over the retained manifest/files/logs with `ready_for_authorized_production_apply_packet=true`, `production_apply_executed=false`, and `production_apply_authorization_present=false`. The authorized apply runbook now emits preflight/apply/postflight/rollback command templates in manifest order, a post-apply regression command bundle, a required production-apply receipt schema, and an offline receipt validator with `status=ready_not_executed`. - Next action: If production apply is explicitly authorized, use the generated runbook order, run every postflight SQL file, run the post-apply regression bundle, retain the production receipt, and then rerun any user-visible Telegram canary needed for the demo/meeting tier. - Not done condition: Production canonical rows/schema remain unapplied until explicit production apply authorization. - Evidence: @@ -104,6 +104,7 @@ Claim ceiling: VPS Telegram memory/KB audit, full live-safe open-ended Telegram - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/rich-proposal-creation-plan-20260709/integrated-20260710/working-leo-integrated-clone-rehearsal-current.log` - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-apply-readiness-current.md` - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md` + - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md` ## WL-EXEC-07 - Leo-in-the-loop staging from Telegram - Status: `done_for_guarded_kb_stage_canary` diff --git a/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.json b/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.json new file mode 100644 index 0000000..84c2707 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.json @@ -0,0 +1,27 @@ +{ + "claim_ceiling": "No production apply receipt was validated.", + "errors": [ + "receipt file does not exist: docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json" + ], + "generated_at_utc": "2026-07-10T01:25:57.290551+00:00", + "missing_fields": [ + "explicit_authorization_record", + "operator_identity", + "git_commit_sha", + "preflight_output_paths", + "apply_output_paths", + "postflight_output_paths", + "expected_after_apply_counts", + "actual_after_apply_counts", + "telegram_or_handler_regression_paths", + "direct_claim_score_path", + "service_readback", + "cleanup_readback", + "rollback_packet_paths", + "known_residual_risks" + ], + "receipt_path": "docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json", + "runbook_path": "docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json", + "status": "missing_receipt", + "valid": false +} diff --git a/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md b/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md new file mode 100644 index 0000000..38e46b7 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md @@ -0,0 +1,22 @@ +# Working Leo Production Apply Receipt Validation + +Generated UTC: `2026-07-10T01:25:57.290551+00:00` +Status: `missing_receipt` +Valid: `False` +Receipt: `docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json` +Runbook: `docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json` + +## Checks + + +## Errors + +- receipt file does not exist: docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json + +## Warnings + +- none + +## Claim Ceiling + +No production apply receipt was validated. diff --git a/scripts/build_working_leo_authorized_apply_runbook.py b/scripts/build_working_leo_authorized_apply_runbook.py index 90ac5e3..9cc6e26 100644 --- a/scripts/build_working_leo_authorized_apply_runbook.py +++ b/scripts/build_working_leo_authorized_apply_runbook.py @@ -103,6 +103,17 @@ POST_APPLY_REGRESSION_COMMANDS: list[dict[str, Any]] = [ "mutates_production_db": False, "posts_to_telegram": False, }, + { + "id": "production_receipt_validator", + "description": "Validate the retained production-apply receipt against the authorized runbook requirements.", + "command_template": ( + ".venv/bin/python scripts/validate_working_leo_production_receipt.py " + "--receipt docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json" + ), + "expected_artifact": str(REPO_REPORT_DIR / "working-leo-production-apply-receipt-validation-current.md"), + "mutates_production_db": False, + "posts_to_telegram": False, + }, ] PRODUCTION_APPLY_RECEIPT_REQUIRED_FIELDS: list[str] = [ diff --git a/scripts/validate_working_leo_production_receipt.py b/scripts/validate_working_leo_production_receipt.py new file mode 100644 index 0000000..3f05b91 --- /dev/null +++ b/scripts/validate_working_leo_production_receipt.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Validate a Working Leo production-apply receipt. + +This validator is intentionally offline. It reads the retained authorized apply +runbook plus a receipt JSON file and checks that the receipt proves the +post-apply state expected by the runbook. It never opens a database connection, +never runs SQL, and never posts to Telegram. +""" + +from __future__ import annotations + +import argparse +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +REPORT_DIR = Path("docs/reports/leo-working-state-20260709") +DEFAULT_RUNBOOK = REPORT_DIR / "working-leo-authorized-apply-runbook-current.json" +DEFAULT_RECEIPT = REPORT_DIR / "working-leo-production-apply-receipt-current.json" +DEFAULT_OUT = REPORT_DIR / "working-leo-production-apply-receipt-validation-current.json" +DEFAULT_MARKDOWN_OUT = REPORT_DIR / "working-leo-production-apply-receipt-validation-current.md" + +EMPTY_PLACEHOLDERS = {"", "todo", "tbd", "n/a", "none", "null", "placeholder"} + + +def _load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _is_nonempty(value: Any) -> bool: + if value is None: + return False + if isinstance(value, str): + return value.strip().lower() not in EMPTY_PLACEHOLDERS + if isinstance(value, dict | list): + return bool(value) + return True + + +def _as_list(value: Any) -> list[Any]: + if isinstance(value, list): + return value + if value is None: + return [] + return [value] + + +def _resolve(path: str, *, repo_root: Path) -> Path: + candidate = Path(path) + return candidate if candidate.is_absolute() else repo_root / candidate + + +def _paths_exist(receipt: dict[str, Any], field: str, *, repo_root: Path) -> tuple[bool, list[str]]: + values = _as_list(receipt.get(field)) + missing = [str(value) for value in values if not _resolve(str(value), repo_root=repo_root).exists()] + return bool(values) and not missing, missing + + +def _counts_match(expected: dict[str, Any], actual: Any) -> bool: + if not isinstance(actual, dict): + return False + try: + normalized_actual = {str(key): int(value) for key, value in actual.items()} + normalized_expected = {str(key): int(value) for key, value in expected.items()} + except (TypeError, ValueError): + return False + return normalized_actual == normalized_expected + + +def _direct_claim_score_passes(path_value: Any, *, repo_root: Path) -> tuple[bool, str]: + values = _as_list(path_value) + if not values: + return False, "missing direct_claim_score_path" + path = _resolve(str(values[0]), repo_root=repo_root) + if not path.exists(): + return False, f"missing direct claim score artifact: {path}" + text = path.read_text(encoding="utf-8") + if path.suffix == ".json": + data = json.loads(text) + score = data.get("score", {}) + if score.get("pass") is True and score.get("passes") == score.get("expected_prompt_count") == 6: + return True, "json score pass" + return False, "json score is not 6/6 pass" + if "Overall pass for scored coverage: `True`" in text and "Prompts scored: `6/6`" in text: + return True, "markdown score pass" + return False, "score artifact does not show 6/6 pass" + + +def _service_readback_ok(value: Any) -> bool: + if not isinstance(value, dict): + return False + return ( + value.get("ActiveState") == "active" + and value.get("SubState") == "running" + and _is_nonempty(value.get("MainPID")) + and _is_nonempty(value.get("NRestarts")) + ) + + +def _cleanup_readback_ok(value: Any) -> bool: + if not isinstance(value, dict): + return False + checks = value.get("checks") if isinstance(value.get("checks"), dict) else value + leftovers = checks.get("leftover_rehearsal_clone_or_staging_databases", []) + return checks.get("cleanup_ok") is True and leftovers == [] + + +def validate_receipt(receipt_path: Path, runbook_path: Path = DEFAULT_RUNBOOK, *, repo_root: Path = Path(".")) -> dict[str, Any]: + repo_root = repo_root.resolve() + runbook = _load_json(runbook_path) + required_fields = list(runbook["production_apply_receipt_required_fields"]) + expected_counts = dict(runbook["expected_after_apply_counts"]) + + if not receipt_path.exists(): + return { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "status": "missing_receipt", + "valid": False, + "receipt_path": str(receipt_path), + "runbook_path": str(runbook_path), + "missing_fields": required_fields, + "errors": [f"receipt file does not exist: {receipt_path}"], + "claim_ceiling": "No production apply receipt was validated.", + } + + receipt = _load_json(receipt_path) + missing_fields = [field for field in required_fields if field not in receipt] + errors: list[str] = [] + warnings: list[str] = [] + checks: dict[str, bool] = {} + + checks["all_required_fields_present"] = not missing_fields + if missing_fields: + errors.append(f"missing required fields: {', '.join(missing_fields)}") + + for field in ("explicit_authorization_record", "operator_identity", "git_commit_sha"): + checks[f"{field}_present"] = _is_nonempty(receipt.get(field)) + if not checks[f"{field}_present"]: + errors.append(f"{field} is missing or placeholder") + + git_sha = str(receipt.get("git_commit_sha", "")) + checks["git_commit_sha_shape"] = bool(re.fullmatch(r"[0-9a-f]{7,40}", git_sha)) + if not checks["git_commit_sha_shape"]: + errors.append("git_commit_sha must be a 7-40 char lowercase hex SHA") + + authorization = receipt.get("explicit_authorization_record") + checks["authorization_record_not_false"] = not ( + isinstance(authorization, dict) and authorization.get("authorized") is False + ) + if not checks["authorization_record_not_false"]: + errors.append("explicit_authorization_record explicitly says authorized=false") + + for field in ( + "preflight_output_paths", + "apply_output_paths", + "postflight_output_paths", + "telegram_or_handler_regression_paths", + "rollback_packet_paths", + ): + ok, missing = _paths_exist(receipt, field, repo_root=repo_root) + checks[f"{field}_exist"] = ok + if missing: + errors.append(f"{field} contains missing path(s): {', '.join(missing)}") + elif not _as_list(receipt.get(field)): + errors.append(f"{field} must be a non-empty path list") + + checks["expected_counts_match_runbook"] = _counts_match(expected_counts, receipt.get("expected_after_apply_counts")) + if not checks["expected_counts_match_runbook"]: + errors.append("expected_after_apply_counts does not match the authorized runbook") + + checks["actual_counts_match_expected"] = _counts_match(expected_counts, receipt.get("actual_after_apply_counts")) + if not checks["actual_counts_match_expected"]: + errors.append("actual_after_apply_counts does not match the authorized runbook expected counts") + + score_ok, score_note = _direct_claim_score_passes(receipt.get("direct_claim_score_path"), repo_root=repo_root) + checks["direct_claim_score_passes"] = score_ok + if not score_ok: + errors.append(score_note) + + checks["service_readback_ok"] = _service_readback_ok(receipt.get("service_readback")) + if not checks["service_readback_ok"]: + errors.append("service_readback must show active/running service with MainPID and NRestarts") + + checks["cleanup_readback_ok"] = _cleanup_readback_ok(receipt.get("cleanup_readback")) + if not checks["cleanup_readback_ok"]: + errors.append("cleanup_readback must show cleanup_ok=true and no rehearsal/clone/staging DB leftovers") + + checks["known_residual_risks_list"] = isinstance(receipt.get("known_residual_risks"), list) + if not checks["known_residual_risks_list"]: + errors.append("known_residual_risks must be a list") + elif not receipt["known_residual_risks"]: + warnings.append("known_residual_risks is empty; verify that this is intentional") + + valid = all(checks.values()) and not errors + return { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "status": "valid" if valid else "invalid", + "valid": valid, + "receipt_path": str(receipt_path), + "runbook_path": str(runbook_path), + "checks": checks, + "missing_fields": missing_fields, + "errors": errors, + "warnings": warnings, + "claim_ceiling": ( + "This validates a retained production-apply receipt against the authorized runbook. It does not run SQL, " + "does not mutate production, and does not create authorization." + ), + } + + +def write_markdown(path: Path, result: dict[str, Any]) -> None: + lines = [ + "# Working Leo Production Apply Receipt Validation", + "", + f"Generated UTC: `{result['generated_at_utc']}`", + f"Status: `{result['status']}`", + f"Valid: `{result['valid']}`", + f"Receipt: `{result['receipt_path']}`", + f"Runbook: `{result['runbook_path']}`", + "", + "## Checks", + "", + ] + for key, value in sorted(result.get("checks", {}).items()): + lines.append(f"- `{key}`: `{value}`") + lines.extend(["", "## Errors", ""]) + errors = result.get("errors", []) + lines.extend(f"- {item}" for item in errors) if errors else lines.append("- none") + lines.extend(["", "## Warnings", ""]) + warnings = result.get("warnings", []) + lines.extend(f"- {item}" for item in warnings) if warnings else lines.append("- none") + lines.extend(["", "## Claim Ceiling", "", result["claim_ceiling"], ""]) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines), encoding="utf-8") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--receipt", type=Path, default=DEFAULT_RECEIPT) + parser.add_argument("--runbook", type=Path, default=DEFAULT_RUNBOOK) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + result = validate_receipt(args.receipt, args.runbook) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_markdown(args.markdown_out, result) + print( + json.dumps( + { + "out": str(args.out), + "markdown_out": str(args.markdown_out), + "status": result["status"], + "valid": result["valid"], + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if result["valid"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_build_working_leo_authorized_apply_runbook.py b/tests/test_build_working_leo_authorized_apply_runbook.py index 848f32e..40bffe9 100644 --- a/tests/test_build_working_leo_authorized_apply_runbook.py +++ b/tests/test_build_working_leo_authorized_apply_runbook.py @@ -38,6 +38,7 @@ def test_build_runbook_is_ready_not_executed(): "direct_claim_strict_score", "focused_local_regression_tests", "gateway_service_readback", + "production_receipt_validator", } assert all(item["mutates_production_db"] is False for item in report["post_apply_regression_commands"]) assert all(item["posts_to_telegram"] is False for item in report["post_apply_regression_commands"]) diff --git a/tests/test_validate_working_leo_production_receipt.py b/tests/test_validate_working_leo_production_receipt.py new file mode 100644 index 0000000..557dd9a --- /dev/null +++ b/tests/test_validate_working_leo_production_receipt.py @@ -0,0 +1,127 @@ +"""Tests for scripts/validate_working_leo_production_receipt.py.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import build_working_leo_authorized_apply_runbook as runbook_builder # noqa: E402 +import validate_working_leo_production_receipt as validator # noqa: E402 + + +def write_json(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def make_artifact(root: Path, relative: str, text: str = "ok\n") -> str: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return relative + + +def make_runbook(tmp_path: Path) -> Path: + runbook = runbook_builder.build_runbook() + path = tmp_path / "runbook.json" + write_json(path, runbook) + return path + + +def make_valid_receipt(tmp_path: Path, runbook_path: Path) -> Path: + runbook = json.loads(runbook_path.read_text(encoding="utf-8")) + score_path = make_artifact( + tmp_path, + "reports/direct-score.md", + "Overall pass for scored coverage: `True`\nPrompts scored: `6/6`\n", + ) + receipt = { + "actual_after_apply_counts": runbook["expected_after_apply_counts"], + "apply_output_paths": [make_artifact(tmp_path, "reports/apply.log")], + "cleanup_readback": { + "checks": { + "cleanup_ok": True, + "leftover_rehearsal_clone_or_staging_databases": [], + } + }, + "direct_claim_score_path": score_path, + "explicit_authorization_record": { + "authorized": True, + "authorized_by": "operator", + "scope": "integrated Working Leo packet set", + }, + "expected_after_apply_counts": runbook["expected_after_apply_counts"], + "git_commit_sha": "3351d2d", + "known_residual_risks": ["GCP parity remains separate"], + "operator_identity": "operator@example.com", + "postflight_output_paths": [make_artifact(tmp_path, "reports/postflight.log")], + "preflight_output_paths": [make_artifact(tmp_path, "reports/preflight.log")], + "rollback_packet_paths": [make_artifact(tmp_path, "reports/rollback.sql")], + "service_readback": { + "ActiveState": "active", + "MainPID": "1908033", + "NRestarts": "0", + "SubState": "running", + }, + "telegram_or_handler_regression_paths": [make_artifact(tmp_path, "reports/handler.json", "{}\n")], + } + path = tmp_path / "receipt.json" + write_json(path, receipt) + return path + + +def test_valid_receipt_passes(tmp_path: Path): + runbook_path = make_runbook(tmp_path) + receipt_path = make_valid_receipt(tmp_path, runbook_path) + + result = validator.validate_receipt(receipt_path, runbook_path, repo_root=tmp_path) + + assert result["valid"] is True + assert result["status"] == "valid" + assert result["errors"] == [] + assert all(result["checks"].values()) + + +def test_missing_receipt_fails(tmp_path: Path): + runbook_path = make_runbook(tmp_path) + + result = validator.validate_receipt(tmp_path / "missing.json", runbook_path, repo_root=tmp_path) + + assert result["valid"] is False + assert result["status"] == "missing_receipt" + assert "receipt file does not exist" in result["errors"][0] + + +def test_receipt_rejects_missing_paths_and_bad_counts(tmp_path: Path): + runbook_path = make_runbook(tmp_path) + receipt_path = make_valid_receipt(tmp_path, runbook_path) + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + receipt["actual_after_apply_counts"] = {"claims": 1} + receipt["apply_output_paths"] = ["reports/missing.log"] + write_json(receipt_path, receipt) + + result = validator.validate_receipt(receipt_path, runbook_path, repo_root=tmp_path) + + assert result["valid"] is False + assert result["checks"]["actual_counts_match_expected"] is False + assert result["checks"]["apply_output_paths_exist"] is False + assert any("actual_after_apply_counts" in error for error in result["errors"]) + assert any("apply_output_paths" in error for error in result["errors"]) + + +def test_receipt_rejects_failed_direct_claim_score(tmp_path: Path): + runbook_path = make_runbook(tmp_path) + receipt_path = make_valid_receipt(tmp_path, runbook_path) + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + bad_score = make_artifact(tmp_path, "reports/direct-score.md", "Prompts scored: `3/6`\n") + receipt["direct_claim_score_path"] = bad_score + write_json(receipt_path, receipt) + + result = validator.validate_receipt(receipt_path, runbook_path, repo_root=tmp_path) + + assert result["valid"] is False + assert result["checks"]["direct_claim_score_passes"] is False