Validate Leo production apply receipts
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
This commit is contained in:
twentyOne2x
parent
3351d2d886
commit
0e2c37e8fa
13 changed files with 484 additions and 6 deletions
|
|
@ -76,6 +76,8 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c
|
|||
- Working Leo authorized-apply readiness verifier JSON: `docs/reports/leo-working-state-20260709/working-leo-apply-readiness-current.json`
|
||||
- Working Leo authorized apply operator runbook, not executed: `docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md`
|
||||
- Working Leo authorized apply operator runbook JSON: `docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json`
|
||||
- Working Leo production apply receipt validation, currently missing receipt: `docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md`
|
||||
- Working Leo production apply receipt validation JSON: `docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.json`
|
||||
- Working Leo cleanup readback: `docs/reports/leo-working-state-20260709/working-leo-cleanup-readback-current.md`
|
||||
- Working Leo cleanup readback JSON: `docs/reports/leo-working-state-20260709/working-leo-cleanup-readback-current.json`
|
||||
- Helmer live rollback log: `docs/reports/leo-working-state-20260709/helmer-7powers-live-preflight-rollback-current.log`
|
||||
|
|
@ -121,7 +123,8 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c
|
|||
- The governance/concept-map schema companion write is clone-proven and reversible: after the mapped base packet exists, it creates minimal concept-map and claim-to-governance link tables, then inserts `1` concept map, `1` concept link, and `2` governance links in a disposable clone; production stayed unchanged.
|
||||
- The full integrated packet set is now clone-proven in production dependency order: mapped base, cross-surface anchor, Rio strategy context, governance/concept schema, and Helmer 7 Powers applied together in `teleo_integrated_rehearsal_20260710`, then rolled back in reverse order; production stayed unchanged and the disposable clone/temp files were removed.
|
||||
- The authorized-apply readiness verifier now passes: packet files exist and hash, apply/rollback order matches the manifest, expected counts match, clone apply/rollback markers are ordered, clone cleanup is proven, service stayed stable, and `production_apply_executed=false`.
|
||||
- The authorized apply operator runbook is generated and ready-not-executed: it gives preflight/apply/postflight/rollback command templates in manifest order, requires `TELEO_DATABASE_URL`, and now also includes a post-apply regression command bundle plus required production-apply receipt fields. It still does not authorize or execute production SQL.
|
||||
- The authorized apply operator runbook is generated and ready-not-executed: it gives preflight/apply/postflight/rollback command templates in manifest order, requires `TELEO_DATABASE_URL`, and now also includes a post-apply regression command bundle plus required production-apply receipt fields and an offline receipt validator. It still does not authorize or execute production SQL.
|
||||
- The production-apply receipt validator currently reports `missing_receipt`, which is correct because no production apply has been authorized or executed.
|
||||
- DB-first identity rendering is now documented from the July 9 PDFs plus VPS readback: canonical identity rows feed rendered `SOUL.md`, direct SOUL edits are not canonical DB updates, and no active renderer/timer/hook is proven in the current VPS readback.
|
||||
- The decision-matrix approval schema is still a target design on the current VPS: `kb_stage.matrix_voters`, `kb_stage.proposal_votes`, and `kb_stage.proposal_decisions` were not present in live schema readback.
|
||||
- Pending/approved document proposals are real `kb_stage.kb_proposals` rows with `source_ref`s, but inspected `source_ref`s did not directly match `public.sources`; they still need review/apply mapping into canonical sources/evidence/edges.
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
"runtime": "disposable_clone_or_sandbox_first"
|
||||
}
|
||||
],
|
||||
"generated_at_utc": "2026-07-10T01:20:35.275732+00:00",
|
||||
"generated_at_utc": "2026-07-10T01:25:53.065461+00:00",
|
||||
"include_cory_style_scenarios": false,
|
||||
"include_direct_claim_followups": true,
|
||||
"mode": "retained_evidence_score",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Working Leo Open-Ended Benchmark Score
|
||||
|
||||
Generated UTC: `2026-07-10T01:20:35.275732+00:00`
|
||||
Generated UTC: `2026-07-10T01:25:53.065461+00:00`
|
||||
Mode: `retained_evidence_score`
|
||||
Source: `docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-current.json`
|
||||
Coverage: `partial`
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@
|
|||
"rio_strategy_nodes": 2,
|
||||
"sources": 28
|
||||
},
|
||||
"generated_at_utc": "2026-07-10T01:16:00.171472+00:00",
|
||||
"generated_at_utc": "2026-07-10T01:25:52.995962+00:00",
|
||||
"mode": "operator_runbook_generation_only",
|
||||
"operator_environment": {
|
||||
"operator_commands_run_sql_if_executed": true,
|
||||
|
|
@ -318,6 +318,14 @@
|
|||
"id": "gateway_service_readback",
|
||||
"mutates_production_db": false,
|
||||
"posts_to_telegram": false
|
||||
},
|
||||
{
|
||||
"command_template": ".venv/bin/python scripts/validate_working_leo_production_receipt.py --receipt docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json",
|
||||
"description": "Validate the retained production-apply receipt against the authorized runbook requirements.",
|
||||
"expected_artifact": "docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md",
|
||||
"id": "production_receipt_validator",
|
||||
"mutates_production_db": false,
|
||||
"posts_to_telegram": false
|
||||
}
|
||||
],
|
||||
"post_apply_required_actions": [
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Working Leo Authorized Apply Runbook
|
||||
|
||||
Generated UTC: `2026-07-10T01:16:00.171472+00:00`
|
||||
Generated UTC: `2026-07-10T01:25:52.995962+00:00`
|
||||
Mode: `operator_runbook_generation_only`
|
||||
Status: `ready_not_executed`
|
||||
Ready for authorized operator: `True`
|
||||
|
|
@ -119,6 +119,11 @@ Do not run apply commands unless the operator explicitly authorizes production D
|
|||
- artifact: `retained service readback with ActiveState=active and NRestarts reviewed`
|
||||
- mutates production DB: `False`
|
||||
- posts to Telegram: `False`
|
||||
- `production_receipt_validator`: Validate the retained production-apply receipt against the authorized runbook requirements.
|
||||
- command: `.venv/bin/python scripts/validate_working_leo_production_receipt.py --receipt docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json`
|
||||
- artifact: `docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md`
|
||||
- mutates production DB: `False`
|
||||
- posts to Telegram: `False`
|
||||
|
||||
## Production Apply Receipt Required Fields
|
||||
|
||||
|
|
|
|||
|
|
@ -138,6 +138,7 @@ A working Leo, for Cory/m3taversal's current expectation, is a Telegram-facing a
|
|||
- command templates cover preflight, apply, postflight, and rollback in manifest dependency order
|
||||
- post-apply regression bundle now requires the non-posting direct-claim handler suite, strict scorer, focused local regression tests, and gateway service readback
|
||||
- production apply receipt schema now requires explicit authorization record, operator identity, git SHA, preflight/apply/postflight output paths, expected-vs-actual counts, regression paths, service readback, cleanup readback, rollback packet paths, and residual risks
|
||||
- offline receipt validator is installed and currently reports `missing_receipt`, as expected before any authorized production apply
|
||||
- generator does not connect to Postgres or run SQL; production apply still requires explicit authorization
|
||||
- VPS/Hermes/DB identity and document-linking map is now documented:
|
||||
- the supplied July 9 master/build and database-reference PDFs confirm the intended architecture is DB-first identity: canonical Postgres identity graph rows render to `SOUL.md`, and evidence depth remains on demand through `teleo-kb`
|
||||
|
|
@ -184,6 +185,7 @@ A working Leo, for Cory/m3taversal's current expectation, is a Telegram-facing a
|
|||
- Integrated clone proof log: `outputs/rich-proposal-creation-plan-20260709/integrated-20260710/working-leo-integrated-clone-rehearsal-current.log`
|
||||
- Authorized-apply readiness verifier: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-apply-readiness-current.md`
|
||||
- Authorized apply operator runbook: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md`
|
||||
- Production apply receipt validation: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md`
|
||||
- Cleanup readback: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-cleanup-readback-current.md`
|
||||
- VPS/Hermes/DB mapping: `outputs/vps-hermes-db-mapping-20260709.md`
|
||||
- Identity render / decision matrix / document artifact image: `outputs/leo-db-state-22-identity-render-matrix-artifacts.svg`
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ Claim ceiling: VPS Telegram memory/KB audit, full live-safe open-ended Telegram
|
|||
|
||||
## WL-EXEC-06B - Integrated production-order packet rehearsal
|
||||
- Status: `done_integrated_clone_ready_not_executed`
|
||||
- Result: Applied the full prepared packet set in one disposable clone in dependency order: mapped base, cross-surface anchor, Rio strategy context, governance/concept schema, and Helmer 7 Powers. Expected integrated footprint was `12` claims, `28` sources, `18` claim edges, `34` claim evidence rows, `1` reasoning tool, `2` Rio strategy nodes, `5` Rio strategy anchors, `1` concept map, `1` concept link, `2` governance links, one Helmer ledger row marked applied, and one cross-surface anchor update. Reverse rollback restored all generated rows/schema to zero/absent, restored the cross-surface anchor to the old claim, restored the Helmer ledger to `approved`, dropped the disposable DB, removed temp files, and left production unchanged. The apply-readiness verifier now passes over the retained manifest/files/logs with `ready_for_authorized_production_apply_packet=true`, `production_apply_executed=false`, and `production_apply_authorization_present=false`. The authorized apply runbook now emits preflight/apply/postflight/rollback command templates in manifest order, a post-apply regression command bundle, and a required production-apply receipt schema with `status=ready_not_executed`.
|
||||
- Result: Applied the full prepared packet set in one disposable clone in dependency order: mapped base, cross-surface anchor, Rio strategy context, governance/concept schema, and Helmer 7 Powers. Expected integrated footprint was `12` claims, `28` sources, `18` claim edges, `34` claim evidence rows, `1` reasoning tool, `2` Rio strategy nodes, `5` Rio strategy anchors, `1` concept map, `1` concept link, `2` governance links, one Helmer ledger row marked applied, and one cross-surface anchor update. Reverse rollback restored all generated rows/schema to zero/absent, restored the cross-surface anchor to the old claim, restored the Helmer ledger to `approved`, dropped the disposable DB, removed temp files, and left production unchanged. The apply-readiness verifier now passes over the retained manifest/files/logs with `ready_for_authorized_production_apply_packet=true`, `production_apply_executed=false`, and `production_apply_authorization_present=false`. The authorized apply runbook now emits preflight/apply/postflight/rollback command templates in manifest order, a post-apply regression command bundle, a required production-apply receipt schema, and an offline receipt validator with `status=ready_not_executed`.
|
||||
- Next action: If production apply is explicitly authorized, use the generated runbook order, run every postflight SQL file, run the post-apply regression bundle, retain the production receipt, and then rerun any user-visible Telegram canary needed for the demo/meeting tier.
|
||||
- Not done condition: Production canonical rows/schema remain unapplied until explicit production apply authorization.
|
||||
- Evidence:
|
||||
|
|
@ -104,6 +104,7 @@ Claim ceiling: VPS Telegram memory/KB audit, full live-safe open-ended Telegram
|
|||
- `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/rich-proposal-creation-plan-20260709/integrated-20260710/working-leo-integrated-clone-rehearsal-current.log`
|
||||
- `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-apply-readiness-current.md`
|
||||
- `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.md`
|
||||
- `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-validation-current.md`
|
||||
|
||||
## WL-EXEC-07 - Leo-in-the-loop staging from Telegram
|
||||
- Status: `done_for_guarded_kb_stage_canary`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"claim_ceiling": "No production apply receipt was validated.",
|
||||
"errors": [
|
||||
"receipt file does not exist: docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json"
|
||||
],
|
||||
"generated_at_utc": "2026-07-10T01:25:57.290551+00:00",
|
||||
"missing_fields": [
|
||||
"explicit_authorization_record",
|
||||
"operator_identity",
|
||||
"git_commit_sha",
|
||||
"preflight_output_paths",
|
||||
"apply_output_paths",
|
||||
"postflight_output_paths",
|
||||
"expected_after_apply_counts",
|
||||
"actual_after_apply_counts",
|
||||
"telegram_or_handler_regression_paths",
|
||||
"direct_claim_score_path",
|
||||
"service_readback",
|
||||
"cleanup_readback",
|
||||
"rollback_packet_paths",
|
||||
"known_residual_risks"
|
||||
],
|
||||
"receipt_path": "docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json",
|
||||
"runbook_path": "docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json",
|
||||
"status": "missing_receipt",
|
||||
"valid": false
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# Working Leo Production Apply Receipt Validation
|
||||
|
||||
Generated UTC: `2026-07-10T01:25:57.290551+00:00`
|
||||
Status: `missing_receipt`
|
||||
Valid: `False`
|
||||
Receipt: `docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json`
|
||||
Runbook: `docs/reports/leo-working-state-20260709/working-leo-authorized-apply-runbook-current.json`
|
||||
|
||||
## Checks
|
||||
|
||||
|
||||
## Errors
|
||||
|
||||
- receipt file does not exist: docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json
|
||||
|
||||
## Warnings
|
||||
|
||||
- none
|
||||
|
||||
## Claim Ceiling
|
||||
|
||||
No production apply receipt was validated.
|
||||
|
|
@ -103,6 +103,17 @@ POST_APPLY_REGRESSION_COMMANDS: list[dict[str, Any]] = [
|
|||
"mutates_production_db": False,
|
||||
"posts_to_telegram": False,
|
||||
},
|
||||
{
|
||||
"id": "production_receipt_validator",
|
||||
"description": "Validate the retained production-apply receipt against the authorized runbook requirements.",
|
||||
"command_template": (
|
||||
".venv/bin/python scripts/validate_working_leo_production_receipt.py "
|
||||
"--receipt docs/reports/leo-working-state-20260709/working-leo-production-apply-receipt-current.json"
|
||||
),
|
||||
"expected_artifact": str(REPO_REPORT_DIR / "working-leo-production-apply-receipt-validation-current.md"),
|
||||
"mutates_production_db": False,
|
||||
"posts_to_telegram": False,
|
||||
},
|
||||
]
|
||||
|
||||
PRODUCTION_APPLY_RECEIPT_REQUIRED_FIELDS: list[str] = [
|
||||
|
|
|
|||
271
scripts/validate_working_leo_production_receipt.py
Normal file
271
scripts/validate_working_leo_production_receipt.py
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Validate a Working Leo production-apply receipt.
|
||||
|
||||
This validator is intentionally offline. It reads the retained authorized apply
|
||||
runbook plus a receipt JSON file and checks that the receipt proves the
|
||||
post-apply state expected by the runbook. It never opens a database connection,
|
||||
never runs SQL, and never posts to Telegram.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPORT_DIR = Path("docs/reports/leo-working-state-20260709")
|
||||
DEFAULT_RUNBOOK = REPORT_DIR / "working-leo-authorized-apply-runbook-current.json"
|
||||
DEFAULT_RECEIPT = REPORT_DIR / "working-leo-production-apply-receipt-current.json"
|
||||
DEFAULT_OUT = REPORT_DIR / "working-leo-production-apply-receipt-validation-current.json"
|
||||
DEFAULT_MARKDOWN_OUT = REPORT_DIR / "working-leo-production-apply-receipt-validation-current.md"
|
||||
|
||||
EMPTY_PLACEHOLDERS = {"", "todo", "tbd", "n/a", "none", "null", "placeholder"}
|
||||
|
||||
|
||||
def _load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _is_nonempty(value: Any) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() not in EMPTY_PLACEHOLDERS
|
||||
if isinstance(value, dict | list):
|
||||
return bool(value)
|
||||
return True
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list[Any]:
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if value is None:
|
||||
return []
|
||||
return [value]
|
||||
|
||||
|
||||
def _resolve(path: str, *, repo_root: Path) -> Path:
|
||||
candidate = Path(path)
|
||||
return candidate if candidate.is_absolute() else repo_root / candidate
|
||||
|
||||
|
||||
def _paths_exist(receipt: dict[str, Any], field: str, *, repo_root: Path) -> tuple[bool, list[str]]:
|
||||
values = _as_list(receipt.get(field))
|
||||
missing = [str(value) for value in values if not _resolve(str(value), repo_root=repo_root).exists()]
|
||||
return bool(values) and not missing, missing
|
||||
|
||||
|
||||
def _counts_match(expected: dict[str, Any], actual: Any) -> bool:
|
||||
if not isinstance(actual, dict):
|
||||
return False
|
||||
try:
|
||||
normalized_actual = {str(key): int(value) for key, value in actual.items()}
|
||||
normalized_expected = {str(key): int(value) for key, value in expected.items()}
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return normalized_actual == normalized_expected
|
||||
|
||||
|
||||
def _direct_claim_score_passes(path_value: Any, *, repo_root: Path) -> tuple[bool, str]:
|
||||
values = _as_list(path_value)
|
||||
if not values:
|
||||
return False, "missing direct_claim_score_path"
|
||||
path = _resolve(str(values[0]), repo_root=repo_root)
|
||||
if not path.exists():
|
||||
return False, f"missing direct claim score artifact: {path}"
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if path.suffix == ".json":
|
||||
data = json.loads(text)
|
||||
score = data.get("score", {})
|
||||
if score.get("pass") is True and score.get("passes") == score.get("expected_prompt_count") == 6:
|
||||
return True, "json score pass"
|
||||
return False, "json score is not 6/6 pass"
|
||||
if "Overall pass for scored coverage: `True`" in text and "Prompts scored: `6/6`" in text:
|
||||
return True, "markdown score pass"
|
||||
return False, "score artifact does not show 6/6 pass"
|
||||
|
||||
|
||||
def _service_readback_ok(value: Any) -> bool:
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
return (
|
||||
value.get("ActiveState") == "active"
|
||||
and value.get("SubState") == "running"
|
||||
and _is_nonempty(value.get("MainPID"))
|
||||
and _is_nonempty(value.get("NRestarts"))
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_readback_ok(value: Any) -> bool:
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
checks = value.get("checks") if isinstance(value.get("checks"), dict) else value
|
||||
leftovers = checks.get("leftover_rehearsal_clone_or_staging_databases", [])
|
||||
return checks.get("cleanup_ok") is True and leftovers == []
|
||||
|
||||
|
||||
def validate_receipt(receipt_path: Path, runbook_path: Path = DEFAULT_RUNBOOK, *, repo_root: Path = Path(".")) -> dict[str, Any]:
|
||||
repo_root = repo_root.resolve()
|
||||
runbook = _load_json(runbook_path)
|
||||
required_fields = list(runbook["production_apply_receipt_required_fields"])
|
||||
expected_counts = dict(runbook["expected_after_apply_counts"])
|
||||
|
||||
if not receipt_path.exists():
|
||||
return {
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"status": "missing_receipt",
|
||||
"valid": False,
|
||||
"receipt_path": str(receipt_path),
|
||||
"runbook_path": str(runbook_path),
|
||||
"missing_fields": required_fields,
|
||||
"errors": [f"receipt file does not exist: {receipt_path}"],
|
||||
"claim_ceiling": "No production apply receipt was validated.",
|
||||
}
|
||||
|
||||
receipt = _load_json(receipt_path)
|
||||
missing_fields = [field for field in required_fields if field not in receipt]
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
checks: dict[str, bool] = {}
|
||||
|
||||
checks["all_required_fields_present"] = not missing_fields
|
||||
if missing_fields:
|
||||
errors.append(f"missing required fields: {', '.join(missing_fields)}")
|
||||
|
||||
for field in ("explicit_authorization_record", "operator_identity", "git_commit_sha"):
|
||||
checks[f"{field}_present"] = _is_nonempty(receipt.get(field))
|
||||
if not checks[f"{field}_present"]:
|
||||
errors.append(f"{field} is missing or placeholder")
|
||||
|
||||
git_sha = str(receipt.get("git_commit_sha", ""))
|
||||
checks["git_commit_sha_shape"] = bool(re.fullmatch(r"[0-9a-f]{7,40}", git_sha))
|
||||
if not checks["git_commit_sha_shape"]:
|
||||
errors.append("git_commit_sha must be a 7-40 char lowercase hex SHA")
|
||||
|
||||
authorization = receipt.get("explicit_authorization_record")
|
||||
checks["authorization_record_not_false"] = not (
|
||||
isinstance(authorization, dict) and authorization.get("authorized") is False
|
||||
)
|
||||
if not checks["authorization_record_not_false"]:
|
||||
errors.append("explicit_authorization_record explicitly says authorized=false")
|
||||
|
||||
for field in (
|
||||
"preflight_output_paths",
|
||||
"apply_output_paths",
|
||||
"postflight_output_paths",
|
||||
"telegram_or_handler_regression_paths",
|
||||
"rollback_packet_paths",
|
||||
):
|
||||
ok, missing = _paths_exist(receipt, field, repo_root=repo_root)
|
||||
checks[f"{field}_exist"] = ok
|
||||
if missing:
|
||||
errors.append(f"{field} contains missing path(s): {', '.join(missing)}")
|
||||
elif not _as_list(receipt.get(field)):
|
||||
errors.append(f"{field} must be a non-empty path list")
|
||||
|
||||
checks["expected_counts_match_runbook"] = _counts_match(expected_counts, receipt.get("expected_after_apply_counts"))
|
||||
if not checks["expected_counts_match_runbook"]:
|
||||
errors.append("expected_after_apply_counts does not match the authorized runbook")
|
||||
|
||||
checks["actual_counts_match_expected"] = _counts_match(expected_counts, receipt.get("actual_after_apply_counts"))
|
||||
if not checks["actual_counts_match_expected"]:
|
||||
errors.append("actual_after_apply_counts does not match the authorized runbook expected counts")
|
||||
|
||||
score_ok, score_note = _direct_claim_score_passes(receipt.get("direct_claim_score_path"), repo_root=repo_root)
|
||||
checks["direct_claim_score_passes"] = score_ok
|
||||
if not score_ok:
|
||||
errors.append(score_note)
|
||||
|
||||
checks["service_readback_ok"] = _service_readback_ok(receipt.get("service_readback"))
|
||||
if not checks["service_readback_ok"]:
|
||||
errors.append("service_readback must show active/running service with MainPID and NRestarts")
|
||||
|
||||
checks["cleanup_readback_ok"] = _cleanup_readback_ok(receipt.get("cleanup_readback"))
|
||||
if not checks["cleanup_readback_ok"]:
|
||||
errors.append("cleanup_readback must show cleanup_ok=true and no rehearsal/clone/staging DB leftovers")
|
||||
|
||||
checks["known_residual_risks_list"] = isinstance(receipt.get("known_residual_risks"), list)
|
||||
if not checks["known_residual_risks_list"]:
|
||||
errors.append("known_residual_risks must be a list")
|
||||
elif not receipt["known_residual_risks"]:
|
||||
warnings.append("known_residual_risks is empty; verify that this is intentional")
|
||||
|
||||
valid = all(checks.values()) and not errors
|
||||
return {
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"status": "valid" if valid else "invalid",
|
||||
"valid": valid,
|
||||
"receipt_path": str(receipt_path),
|
||||
"runbook_path": str(runbook_path),
|
||||
"checks": checks,
|
||||
"missing_fields": missing_fields,
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
"claim_ceiling": (
|
||||
"This validates a retained production-apply receipt against the authorized runbook. It does not run SQL, "
|
||||
"does not mutate production, and does not create authorization."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def write_markdown(path: Path, result: dict[str, Any]) -> None:
|
||||
lines = [
|
||||
"# Working Leo Production Apply Receipt Validation",
|
||||
"",
|
||||
f"Generated UTC: `{result['generated_at_utc']}`",
|
||||
f"Status: `{result['status']}`",
|
||||
f"Valid: `{result['valid']}`",
|
||||
f"Receipt: `{result['receipt_path']}`",
|
||||
f"Runbook: `{result['runbook_path']}`",
|
||||
"",
|
||||
"## Checks",
|
||||
"",
|
||||
]
|
||||
for key, value in sorted(result.get("checks", {}).items()):
|
||||
lines.append(f"- `{key}`: `{value}`")
|
||||
lines.extend(["", "## Errors", ""])
|
||||
errors = result.get("errors", [])
|
||||
lines.extend(f"- {item}" for item in errors) if errors else lines.append("- none")
|
||||
lines.extend(["", "## Warnings", ""])
|
||||
warnings = result.get("warnings", [])
|
||||
lines.extend(f"- {item}" for item in warnings) if warnings else lines.append("- none")
|
||||
lines.extend(["", "## Claim Ceiling", "", result["claim_ceiling"], ""])
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--receipt", type=Path, default=DEFAULT_RECEIPT)
|
||||
parser.add_argument("--runbook", type=Path, default=DEFAULT_RUNBOOK)
|
||||
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
||||
parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
result = validate_receipt(args.receipt, args.runbook)
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
write_markdown(args.markdown_out, result)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"out": str(args.out),
|
||||
"markdown_out": str(args.markdown_out),
|
||||
"status": result["status"],
|
||||
"valid": result["valid"],
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0 if result["valid"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -38,6 +38,7 @@ def test_build_runbook_is_ready_not_executed():
|
|||
"direct_claim_strict_score",
|
||||
"focused_local_regression_tests",
|
||||
"gateway_service_readback",
|
||||
"production_receipt_validator",
|
||||
}
|
||||
assert all(item["mutates_production_db"] is False for item in report["post_apply_regression_commands"])
|
||||
assert all(item["posts_to_telegram"] is False for item in report["post_apply_regression_commands"])
|
||||
|
|
|
|||
127
tests/test_validate_working_leo_production_receipt.py
Normal file
127
tests/test_validate_working_leo_production_receipt.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
"""Tests for scripts/validate_working_leo_production_receipt.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
import build_working_leo_authorized_apply_runbook as runbook_builder # noqa: E402
|
||||
import validate_working_leo_production_receipt as validator # noqa: E402
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def make_artifact(root: Path, relative: str, text: str = "ok\n") -> str:
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return relative
|
||||
|
||||
|
||||
def make_runbook(tmp_path: Path) -> Path:
|
||||
runbook = runbook_builder.build_runbook()
|
||||
path = tmp_path / "runbook.json"
|
||||
write_json(path, runbook)
|
||||
return path
|
||||
|
||||
|
||||
def make_valid_receipt(tmp_path: Path, runbook_path: Path) -> Path:
|
||||
runbook = json.loads(runbook_path.read_text(encoding="utf-8"))
|
||||
score_path = make_artifact(
|
||||
tmp_path,
|
||||
"reports/direct-score.md",
|
||||
"Overall pass for scored coverage: `True`\nPrompts scored: `6/6`\n",
|
||||
)
|
||||
receipt = {
|
||||
"actual_after_apply_counts": runbook["expected_after_apply_counts"],
|
||||
"apply_output_paths": [make_artifact(tmp_path, "reports/apply.log")],
|
||||
"cleanup_readback": {
|
||||
"checks": {
|
||||
"cleanup_ok": True,
|
||||
"leftover_rehearsal_clone_or_staging_databases": [],
|
||||
}
|
||||
},
|
||||
"direct_claim_score_path": score_path,
|
||||
"explicit_authorization_record": {
|
||||
"authorized": True,
|
||||
"authorized_by": "operator",
|
||||
"scope": "integrated Working Leo packet set",
|
||||
},
|
||||
"expected_after_apply_counts": runbook["expected_after_apply_counts"],
|
||||
"git_commit_sha": "3351d2d",
|
||||
"known_residual_risks": ["GCP parity remains separate"],
|
||||
"operator_identity": "operator@example.com",
|
||||
"postflight_output_paths": [make_artifact(tmp_path, "reports/postflight.log")],
|
||||
"preflight_output_paths": [make_artifact(tmp_path, "reports/preflight.log")],
|
||||
"rollback_packet_paths": [make_artifact(tmp_path, "reports/rollback.sql")],
|
||||
"service_readback": {
|
||||
"ActiveState": "active",
|
||||
"MainPID": "1908033",
|
||||
"NRestarts": "0",
|
||||
"SubState": "running",
|
||||
},
|
||||
"telegram_or_handler_regression_paths": [make_artifact(tmp_path, "reports/handler.json", "{}\n")],
|
||||
}
|
||||
path = tmp_path / "receipt.json"
|
||||
write_json(path, receipt)
|
||||
return path
|
||||
|
||||
|
||||
def test_valid_receipt_passes(tmp_path: Path):
|
||||
runbook_path = make_runbook(tmp_path)
|
||||
receipt_path = make_valid_receipt(tmp_path, runbook_path)
|
||||
|
||||
result = validator.validate_receipt(receipt_path, runbook_path, repo_root=tmp_path)
|
||||
|
||||
assert result["valid"] is True
|
||||
assert result["status"] == "valid"
|
||||
assert result["errors"] == []
|
||||
assert all(result["checks"].values())
|
||||
|
||||
|
||||
def test_missing_receipt_fails(tmp_path: Path):
|
||||
runbook_path = make_runbook(tmp_path)
|
||||
|
||||
result = validator.validate_receipt(tmp_path / "missing.json", runbook_path, repo_root=tmp_path)
|
||||
|
||||
assert result["valid"] is False
|
||||
assert result["status"] == "missing_receipt"
|
||||
assert "receipt file does not exist" in result["errors"][0]
|
||||
|
||||
|
||||
def test_receipt_rejects_missing_paths_and_bad_counts(tmp_path: Path):
|
||||
runbook_path = make_runbook(tmp_path)
|
||||
receipt_path = make_valid_receipt(tmp_path, runbook_path)
|
||||
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
||||
receipt["actual_after_apply_counts"] = {"claims": 1}
|
||||
receipt["apply_output_paths"] = ["reports/missing.log"]
|
||||
write_json(receipt_path, receipt)
|
||||
|
||||
result = validator.validate_receipt(receipt_path, runbook_path, repo_root=tmp_path)
|
||||
|
||||
assert result["valid"] is False
|
||||
assert result["checks"]["actual_counts_match_expected"] is False
|
||||
assert result["checks"]["apply_output_paths_exist"] is False
|
||||
assert any("actual_after_apply_counts" in error for error in result["errors"])
|
||||
assert any("apply_output_paths" in error for error in result["errors"])
|
||||
|
||||
|
||||
def test_receipt_rejects_failed_direct_claim_score(tmp_path: Path):
|
||||
runbook_path = make_runbook(tmp_path)
|
||||
receipt_path = make_valid_receipt(tmp_path, runbook_path)
|
||||
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
||||
bad_score = make_artifact(tmp_path, "reports/direct-score.md", "Prompts scored: `3/6`\n")
|
||||
receipt["direct_claim_score_path"] = bad_score
|
||||
write_json(receipt_path, receipt)
|
||||
|
||||
result = validator.validate_receipt(receipt_path, runbook_path, repo_root=tmp_path)
|
||||
|
||||
assert result["valid"] is False
|
||||
assert result["checks"]["direct_claim_score_passes"] is False
|
||||
Loading…
Reference in a new issue