#!/usr/bin/env python3 """Exercise guarded approve_claim review/apply in a disposable database clone. The configured source database is read only. The harness copies its schema into a disposable clone, applies the current role/function prerequisites there, stages pending proposals, approves them through ``kb_review``, and applies them through ``kb_apply``. It compares exact persisted rows, probes forbidden direct mutations, rejects SQL built against approval metadata that changes before execution, and always drops the clone with cleanup/readback evidence. """ from __future__ import annotations import argparse import hashlib import json import re import subprocess import sys import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any HERE = Path(__file__).resolve().parent REPO_ROOT = HERE.parent sys.path.insert(0, str(HERE)) import apply_proposal as ap # noqa: E402 SAFE_DB_RE = re.compile(r"^[a-z][a-z0-9_]{1,62}$") REVIEWER_HANDLE = "m3ta" REVIEW_NOTE = "Reviewed exact strict payload inside the disposable clone." DEFAULT_REVIEW_SECRETS_FILE = ( "/home/teleo/.hermes/profiles/leoclean/secrets/kb-review.env" ) SECURITY_DEFINER_DEPENDENCIES = { "approve_strict_proposal": "kb_review", "assert_approved_proposal": "kb_apply", "finish_approved_proposal": "kb_apply", } SECURITY_DEFINER_ARGUMENT_TYPES = { "approve_strict_proposal": "uuid, text, jsonb, text, text", "assert_approved_proposal": "uuid, text, jsonb, text, uuid, timestamp with time zone, text", "finish_approved_proposal": "uuid, text, jsonb, text, uuid, timestamp with time zone, text, text", } def _run( command: list[str], *, input_text: str | None = None, check: bool = True, env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: result = subprocess.run( command, input=input_text, text=True, capture_output=True, check=False, env=env, ) if check and result.returncode != 0: raise RuntimeError( f"command failed ({result.returncode}): {' '.join(command[:4])}\n" f"stdout={result.stdout.strip()}\nstderr={result.stderr.strip()}" ) return result def _command_readback(result: subprocess.CompletedProcess[str]) -> dict[str, Any]: return { "returncode": result.returncode, "stdout": result.stdout.strip(), "stderr": result.stderr.strip(), } def _repo_source_record(path: Path) -> dict[str, Any]: resolved = path.resolve(strict=True) try: relative = resolved.relative_to(REPO_ROOT.resolve()) except ValueError as exc: raise ValueError(f"source file is outside the repository: {resolved}") from exc if not resolved.is_file(): raise ValueError(f"source path is not a file: {relative.as_posix()}") content = resolved.read_bytes() return { "repo_relative_path": relative.as_posix(), "sha256": hashlib.sha256(content).hexdigest(), "size_bytes": len(content), } def _source_file_manifest(args: argparse.Namespace) -> list[dict[str, Any]]: paths = { Path(__file__).resolve(), (HERE / "run_approve_claim_isolated_container_canary.sh").resolve(), Path(args.approve_script).resolve(), Path(args.apply_script).resolve(), Path(args.prereqs).resolve(), } return sorted( (_repo_source_record(path) for path in paths), key=lambda row: row["repo_relative_path"], ) def _psql(container: str, database: str, sql: str, *, tuples: bool = True) -> str: command = [ "docker", "exec", "-i", container, "psql", "-U", "postgres", "-d", database, "-v", "ON_ERROR_STOP=1", ] if tuples: command.extend(["-At", "-q"]) return _run(command, input_text=sql).stdout.strip() def _role_psql( container: str, database: str, sql: str, *, role: str, password: str, ) -> subprocess.CompletedProcess[str]: command = [ "docker", "exec", "-e", "PGPASSWORD", "-i", container, "psql", "-U", role, "-h", "127.0.0.1", "-d", database, "-v", "ON_ERROR_STOP=1", "-At", "-q", ] return _run( command, input_text=sql, check=False, env={"PGPASSWORD": password, "PATH": "/usr/bin:/bin:/usr/local/bin"}, ) def _schema_copy(container: str, source_db: str, clone_db: str) -> None: dump_command = [ "docker", "exec", container, "pg_dump", "-U", "postgres", "-d", source_db, "--schema-only", "--no-owner", "--no-privileges", ] restore_command = [ "docker", "exec", "-i", container, "psql", "-U", "postgres", "-d", clone_db, "-v", "ON_ERROR_STOP=1", "-q", ] dump = subprocess.Popen( dump_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) assert dump.stdout is not None assert dump.stderr is not None restore = subprocess.run( restore_command, stdin=dump.stdout, text=True, capture_output=True, check=False, ) dump.stdout.close() dump_stderr = dump.stderr.read() dump_returncode = dump.wait() if dump_returncode != 0 or restore.returncode != 0: raise RuntimeError( "schema copy failed: " f"pg_dump={dump_returncode} {dump_stderr.strip()} " f"psql={restore.returncode} {restore.stderr.strip()}" ) def _service_state(service: str) -> dict[str, Any]: result = _run( [ "systemctl", "show", service, "--no-pager", "-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts", "-p", "ExecMainStartTimestamp", ], check=False, ) state: dict[str, Any] = {"returncode": result.returncode} for line in result.stdout.splitlines(): key, separator, value = line.partition("=") if separator: state[key] = value return state def _base_counts(container: str, database: str) -> dict[str, int]: raw = _psql( container, database, """begin transaction read only; select json_build_object( 'public.claims', (select count(*) from public.claims), 'public.sources', (select count(*) from public.sources), 'public.claim_evidence', (select count(*) from public.claim_evidence), 'public.claim_edges', (select count(*) from public.claim_edges), 'public.reasoning_tools', (select count(*) from public.reasoning_tools), 'kb_stage.kb_proposals', (select count(*) from kb_stage.kb_proposals) )::text; rollback; """, ) return {key: int(value) for key, value in json.loads(raw).items()} def _cluster_role_readback(container: str, database: str) -> list[dict[str, Any]]: raw = _psql( container, database, """begin transaction read only; select coalesce(json_agg(json_build_object( 'rolname', rolname, 'rolsuper', rolsuper, 'rolinherit', rolinherit, 'rolcreaterole', rolcreaterole, 'rolcreatedb', rolcreatedb, 'rolcanlogin', rolcanlogin, 'rolreplication', rolreplication, 'rolconnlimit', rolconnlimit, 'rolvaliduntil', rolvaliduntil::text, 'rolbypassrls', rolbypassrls ) order by rolname), '[]'::json)::text from pg_catalog.pg_roles where rolname in ('kb_apply', 'kb_gate_owner', 'kb_review'); rollback; """, ) return json.loads(raw) def _cluster_roles_are_preprovisioned(rows: list[dict[str, Any]]) -> bool: by_name = {row["rolname"]: row for row in rows} if set(by_name) != {"kb_apply", "kb_gate_owner", "kb_review"}: return False for name, row in by_name.items(): if any( row.get(key) is True for key in ( "rolsuper", "rolcreaterole", "rolcreatedb", "rolreplication", "rolbypassrls", ) ): return False if row.get("rolinherit") is not False: return False if row.get("rolcanlogin") != (name != "kb_gate_owner"): return False return True def _clone_prerequisites_sql(prereqs: str) -> tuple[str, dict[str, Any]]: """Remove cluster-global role DDL while preserving clone-local objects/ACLs.""" start_marker = "do $roles$" end_marker = ( "alter role kb_apply login noinherit nosuperuser nocreatedb nocreaterole " "noreplication nobypassrls;" ) start = prereqs.lower().find(start_marker) end_start = prereqs.lower().find(end_marker, start) if start < 0 or end_start < 0: raise RuntimeError( "prerequisites role bootstrap block changed; refusing unsafe clone execution" ) end = end_start + len(end_marker) removed = prereqs[start:end] sanitized = ( prereqs[:start] + "-- Cluster-global role DDL omitted by disposable clone canary.\n" + prereqs[end:] ) if re.search(r"\b(?:create|alter|drop)\s+role\b", sanitized, re.IGNORECASE): raise RuntimeError( "prerequisites still contain cluster-global role DDL after sanitization" ) return sanitized, { "source_sha256": hashlib.sha256(prereqs.encode()).hexdigest(), "clone_sql_sha256": hashlib.sha256(sanitized.encode()).hexdigest(), "cluster_role_ddl_removed": True, "removed_create_role_statements": len( re.findall(r"\bcreate\s+role\b", removed, re.IGNORECASE) ), "removed_alter_role_statements": len( re.findall(r"\balter\s+role\b", removed, re.IGNORECASE) ), } def build_fixture(namespace: str) -> dict[str, Any]: def stable(label: str) -> str: return str( uuid.uuid5( uuid.NAMESPACE_URL, f"working-leo:approve-claim-canary:{namespace}:{label}", ) ) claim_a = stable("claim-a") claim_b = stable("claim-b") source_a = stable("source-a") source_b = stable("source-b") return { "proposal_id": stable("proposal"), "conflict_proposal_id": stable("conflict-proposal"), "conflict_claim_id": stable("conflict-claim"), "conflict_source_id": stable("conflict-source"), "permission_claim_id": stable("permission-claim"), "permission_source_id": stable("permission-source"), "apply_payload": { "contract_version": 2, "agent_id": None, "claims": [ { "id": claim_a, "type": "structural", "text": "An approved strict proposal can create exact canonical rows atomically.", "status": "open", "confidence": 0.9, "tags": ["working-leo", "clone-canary"], "created_by": None, "superseded_by": None, }, { "id": claim_b, "type": "concept", "text": "Row-level proof is the authority for canonical KB state.", "status": "open", "confidence": 0.85, "tags": ["working-leo", "row-proof"], "created_by": None, "superseded_by": None, }, ], "sources": [ { "id": source_a, "source_type": "observation", "url": None, "storage_path": None, "excerpt": "Disposable clone lifecycle canary source A.", "hash": f"approve-claim-canary-{namespace}-a", "created_by": None, }, { "id": source_b, "source_type": "observation", "url": None, "storage_path": None, "excerpt": "Disposable clone lifecycle canary source B.", "hash": f"approve-claim-canary-{namespace}-b", "created_by": None, }, ], "evidence": [ { "claim_id": claim_a, "source_id": source_a, "role": "grounds", "weight": 0.9, "created_by": None, }, { "claim_id": claim_b, "source_id": source_b, "role": "illustrates", "weight": 0.8, "created_by": None, }, ], "edges": [ { "from_claim": claim_a, "to_claim": claim_b, "edge_type": "supports", "weight": 0.75, "created_by": None, } ], "reasoning_tools": [ { "id": stable("reasoning-tool"), "agent_id": None, "name": "Canonical row proof canary", "description": "Verify approved proposal to canonical graph lifecycle in a disposable clone.", "category": "verification", } ], }, } def load_normalized_fixture(path: Path, namespace: str) -> dict[str, Any]: data = json.loads(path.read_text(encoding="utf-8")) previews = data.get("normalization_previews") if isinstance(data, dict) else None if not isinstance(previews, list) or len(previews) != 1: raise ValueError( "normalization JSON must contain exactly one normalization_previews row" ) preview = previews[0] children = preview.get("strict_child_proposals") if ( preview.get("normalization_state") != "strict_children_ready" or preview.get("blocked_count") != 0 or not isinstance(children, list) or len(children) != 1 or children[0].get("proposal_type") != "approve_claim" ): raise ValueError( "normalization JSON is not one unblocked strict approve_claim child" ) apply_payload = (children[0].get("payload") or {}).get("apply_payload") if not isinstance(apply_payload, dict): raise ValueError("strict approve_claim child has no apply_payload") source_proposal_id = str(preview.get("proposal_id") or "") def stable(label: str) -> str: return str( uuid.uuid5( uuid.NAMESPACE_URL, f"working-leo:approve-claim-normalized-canary:{source_proposal_id}:{namespace}:{label}", ) ) return { "source_proposal_id": source_proposal_id, "proposal_id": stable("strict-child-proposal"), "conflict_proposal_id": stable("conflict-proposal"), "conflict_claim_id": stable("conflict-claim"), "conflict_source_id": stable("conflict-source"), "permission_claim_id": stable("permission-claim"), "permission_source_id": stable("permission-source"), "apply_payload": apply_payload, } def _external_claim_ids(payload: dict[str, Any]) -> list[str]: new_ids = {str(row["id"]) for row in payload.get("claims") or []} references = set() for row in payload.get("edges") or []: references.update([str(row["from_claim"]), str(row["to_claim"])]) for row in payload.get("evidence") or []: references.add(str(row["claim_id"])) return sorted(references - new_ids) def _payload_agent_ids(payload: dict[str, Any]) -> list[str]: values: set[str] = set() if payload.get("agent_id"): values.add(str(payload["agent_id"])) for collection, field in ( ("claims", "created_by"), ("sources", "created_by"), ("evidence", "created_by"), ("edges", "created_by"), ("reasoning_tools", "agent_id"), ): for row in payload.get(collection) or []: if row.get(field): values.add(str(row[field])) return sorted(values) def _seed_payload_agents( container: str, source_db: str, clone_db: str, payload: dict[str, Any] ) -> dict[str, Any]: required_ids = _payload_agent_ids(payload) if not required_ids: return { "required_ids": [], "source_rows": [], "clone_rows": [], "exact_clone_copy": True, } id_sql = ", ".join( f"{ap.sql_literal(value)}::uuid" for value in required_ids ) source_raw = _psql( container, source_db, f"""begin transaction read only; select coalesce(json_agg(json_build_object( 'id', id::text, 'handle', handle, 'kind', kind::text ) order by id), '[]'::json)::text from public.agents where id in ({id_sql}); rollback; """, ) source_rows = json.loads(source_raw) found = {row["id"] for row in source_rows} missing = sorted(set(required_ids) - found) if missing: raise RuntimeError(f"strict bundle references missing source agents: {missing}") statements = [ f"""insert into public.agents (id, handle, kind) values ({ap.sql_literal(row['id'])}::uuid, {ap.sql_literal(row['handle'])}, {ap.sql_literal(row['kind'])}) on conflict (id) do nothing;""" for row in source_rows ] _psql(container, clone_db, "\n".join(statements), tuples=False) clone_raw = _psql( container, clone_db, f"""select coalesce(json_agg(json_build_object( 'id', id::text, 'handle', handle, 'kind', kind::text ) order by id), '[]'::json)::text from public.agents where id in ({id_sql});""", ) clone_rows = json.loads(clone_raw) return { "required_ids": required_ids, "source_rows": source_rows, "clone_rows": clone_rows, "exact_clone_copy": clone_rows == source_rows, } def _seed_reviewer_agent( container: str, source_db: str, clone_db: str, reviewer_handle: str, ) -> dict[str, Any]: raw = _psql( container, source_db, f"""begin transaction read only; select coalesce((select json_build_object( 'id', id::text, 'handle', handle, 'kind', kind::text ) from public.agents where handle = {ap.sql_literal(reviewer_handle)}), 'null'::json)::text; rollback; """, ) source_row = json.loads(raw) if not source_row: raise RuntimeError( f"reviewer principal dependency is missing from source agents: {reviewer_handle}" ) _psql( container, clone_db, f"""insert into public.agents (id, handle, kind) values ({ap.sql_literal(source_row['id'])}::uuid, {ap.sql_literal(source_row['handle'])}, {ap.sql_literal(source_row['kind'])});""", tuples=False, ) clone_raw = _psql( container, clone_db, f"""select json_build_object( 'id', id::text, 'handle', handle, 'kind', kind::text )::text from public.agents where handle = {ap.sql_literal(reviewer_handle)};""", ) clone_row = json.loads(clone_raw) return { "reviewer_handle": reviewer_handle, "source_row": source_row, "clone_row": clone_row, "exact_clone_copy": clone_row == source_row, } def _seed_external_claims( container: str, source_db: str, clone_db: str, payload: dict[str, Any] ) -> dict[str, Any]: external_ids = _external_claim_ids(payload) if not external_ids: return { "required_ids": [], "source_rows": [], "clone_rows": [], "exact_clone_copy": True, } id_sql = ", ".join( f"{ap.sql_literal(value)}::uuid" for value in external_ids ) raw = _psql( container, source_db, f"""begin transaction read only; select coalesce(json_agg(json_build_object( 'id', id::text, 'type', type::text, 'text', text, 'status', status::text, 'confidence', confidence, 'tags', tags, 'created_by', created_by::text, 'superseded_by', superseded_by::text ) order by id), '[]'::json)::text from public.claims where id in ({id_sql}); rollback; """, ) rows = json.loads(raw) found = {row["id"] for row in rows} missing = sorted(set(external_ids) - found) if missing: raise RuntimeError( f"normalized bundle references missing source claims: {missing}" ) statements = [] for row in rows: tags = row.get("tags") or [] tags_sql = ( "'{}'::text[]" if not tags else "array[" + ", ".join(ap.sql_literal(tag) for tag in tags) + "]::text[]" ) statements.append( f"""insert into public.claims (id, type, text, status, confidence, tags, created_by, superseded_by) values ({ap.sql_literal(row['id'])}::uuid, {ap.sql_literal(row['type'])}, {ap.sql_literal(row['text'])}, {ap.sql_literal(row['status'])}, {ap.sql_literal(row.get('confidence'))}, {tags_sql}, {ap.sql_literal(row.get('created_by'))}::uuid, {ap.sql_literal(row.get('superseded_by'))}::uuid);""" ) _psql(container, clone_db, "\n".join(statements), tuples=False) clone_raw = _psql( container, clone_db, f"""select coalesce(json_agg(json_build_object( 'id', id::text, 'type', type::text, 'text', text, 'status', status::text, 'confidence', confidence, 'tags', tags, 'created_by', created_by::text, 'superseded_by', superseded_by::text ) order by id), '[]'::json)::text from public.claims where id in ({id_sql});""", ) clone_rows = json.loads(clone_raw) return { "required_ids": external_ids, "source_rows": rows, "clone_rows": clone_rows, "exact_clone_copy": clone_rows == rows, } def _id_predicate(column: str, values: list[str]) -> str: if not values: return "false" return f"{column} in (" + ", ".join( f"{ap.sql_literal(value)}::uuid" for value in values ) + ")" def _key_predicate(rows: list[dict[str, Any]], fields: list[tuple[str, str]]) -> str: if not rows: return "false" clauses = [] for row in rows: parts = [] for column, cast in fields: value = row[column] if cast == "uuid": parts.append(f"{column} = {ap.sql_literal(value)}::uuid") else: parts.append(f"{column}::text = {ap.sql_literal(value)}") clauses.append("(" + " and ".join(parts) + ")") return "(" + " or ".join(clauses) + ")" def _expected_bundle_rows(payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: agent_id = payload.get("agent_id") def inherited(row: dict[str, Any], key: str) -> Any: return row.get(key, agent_id) claims = [ { "id": str(row["id"]), "type": row["type"], "text": row["text"], "status": row.get("status", "open"), "confidence": row.get("confidence"), "tags": row.get("tags") or [], "created_by": inherited(row, "created_by"), "superseded_by": row.get("superseded_by"), } for row in payload.get("claims") or [] ] sources = [ { "id": str(row["id"]), "source_type": row["source_type"], "url": row.get("url"), "storage_path": row.get("storage_path"), "excerpt": row.get("excerpt"), "hash": row["hash"], "created_by": inherited(row, "created_by"), } for row in payload.get("sources") or [] ] evidence = [ { "claim_id": str(row["claim_id"]), "source_id": str(row["source_id"]), "role": row.get("role", "grounds"), "weight": row.get("weight"), "created_by": inherited(row, "created_by"), } for row in payload.get("evidence") or [] ] edges = [ { "from_claim": str(row["from_claim"]), "to_claim": str(row["to_claim"]), "edge_type": row["edge_type"], "weight": row.get("weight"), "created_by": inherited(row, "created_by"), } for row in payload.get("edges") or [] ] tools = [ { "id": str(row["id"]), "agent_id": inherited(row, "agent_id"), "name": row["name"], "description": row["description"], "category": row.get("category"), } for row in payload.get("reasoning_tools") or [] ] return { "claims": sorted(claims, key=lambda row: row["id"]), "sources": sorted(sources, key=lambda row: row["id"]), "claim_evidence": sorted( evidence, key=lambda row: (row["claim_id"], row["source_id"], row["role"]), ), "claim_edges": sorted( edges, key=lambda row: ( row["from_claim"], row["to_claim"], row["edge_type"], ), ), "reasoning_tools": sorted(tools, key=lambda row: row["id"]), } def _expected_table_deltas( expected_rows: dict[str, list[dict[str, Any]]], ) -> dict[str, int]: return { "public.claims": len(expected_rows["claims"]), "public.sources": len(expected_rows["sources"]), "public.claim_evidence": len(expected_rows["claim_evidence"]), "public.claim_edges": len(expected_rows["claim_edges"]), "public.reasoning_tools": len(expected_rows["reasoning_tools"]), "kb_stage.kb_proposals": 0, } def _table_deltas(before: dict[str, int], after: dict[str, int]) -> dict[str, int]: if set(before) != set(after): raise ValueError( "cannot compare table counts with different keys: " f"before={sorted(before)} after={sorted(after)}" ) return {key: after[key] - before[key] for key in sorted(before)} def _exact_bundle_readback( container: str, database: str, payload: dict[str, Any] ) -> dict[str, list[dict[str, Any]]]: expected = _expected_bundle_rows(payload) claim_predicate = _id_predicate( "id", [row["id"] for row in expected["claims"]] ) source_predicate = _id_predicate( "id", [row["id"] for row in expected["sources"]] ) evidence_predicate = _key_predicate( expected["claim_evidence"], [("claim_id", "uuid"), ("source_id", "uuid"), ("role", "text")], ) edge_predicate = _key_predicate( expected["claim_edges"], [ ("from_claim", "uuid"), ("to_claim", "uuid"), ("edge_type", "text"), ], ) tool_predicate = _id_predicate( "id", [row["id"] for row in expected["reasoning_tools"]] ) raw = _psql( container, database, f"""begin transaction read only; select json_build_object( 'claims', coalesce((select json_agg(row_to_json(q) order by q.id) from ( select id::text as id, type::text as type, text, status::text as status, confidence, tags, created_by::text as created_by, superseded_by::text as superseded_by from public.claims where {claim_predicate} ) q), '[]'::json), 'sources', coalesce((select json_agg(row_to_json(q) order by q.id) from ( select id::text as id, source_type::text as source_type, url, storage_path, excerpt, hash, created_by::text as created_by from public.sources where {source_predicate} ) q), '[]'::json), 'claim_evidence', coalesce((select json_agg(row_to_json(q) order by q.claim_id, q.source_id, q.role) from ( select claim_id::text as claim_id, source_id::text as source_id, role::text as role, weight, created_by::text as created_by from public.claim_evidence where {evidence_predicate} ) q), '[]'::json), 'claim_edges', coalesce((select json_agg(row_to_json(q) order by q.from_claim, q.to_claim, q.edge_type) from ( select from_claim::text as from_claim, to_claim::text as to_claim, edge_type::text as edge_type, weight, created_by::text as created_by from public.claim_edges where {edge_predicate} ) q), '[]'::json), 'reasoning_tools', coalesce((select json_agg(row_to_json(q) order by q.id) from ( select id::text as id, agent_id::text as agent_id, name, description, category from public.reasoning_tools where {tool_predicate} ) q), '[]'::json) )::text; rollback; """, ) return json.loads(raw) def _proposal_readback( container: str, database: str, proposal_id: str ) -> dict[str, Any] | None: raw = _psql( container, database, f"""begin transaction read only; select coalesce((select json_build_object( 'id', id::text, 'proposal_type', proposal_type, 'status', status, 'channel', channel, 'source_ref', source_ref, 'rationale', rationale, 'payload', payload, 'reviewed_by_handle', reviewed_by_handle, 'reviewed_by_agent_id', reviewed_by_agent_id::text, 'reviewed_at', reviewed_at::text, 'review_note', review_note, 'applied_by_handle', applied_by_handle, 'applied_by_agent_id', applied_by_agent_id::text, 'applied_at', applied_at::text ) from kb_stage.kb_proposals where id = {ap.sql_literal(proposal_id)}::uuid), 'null'::json)::text; rollback; """, ) return json.loads(raw) def _approval_snapshot_readback( container: str, database: str, proposal_id: str ) -> dict[str, Any] | None: raw = _psql( container, database, f"""begin transaction read only; select coalesce((select json_build_object( 'proposal_id', proposal_id::text, 'proposal_type', proposal_type, 'payload', payload, 'reviewed_by_handle', reviewed_by_handle, 'reviewed_by_agent_id', reviewed_by_agent_id::text, 'reviewed_by_db_role', reviewed_by_db_role::text, 'reviewed_at', reviewed_at::text, 'review_note', review_note ) from kb_stage.kb_proposal_approvals where proposal_id = {ap.sql_literal(proposal_id)}::uuid), 'null'::json)::text; rollback; """, ) return json.loads(raw) def _stage_proposal( container: str, database: str, proposal_id: str, payload: dict[str, Any], *, source_ref: str, ) -> None: sql = f"""insert into kb_stage.kb_proposals (id, proposal_type, status, channel, source_ref, rationale, payload) values ({ap.sql_literal(proposal_id)}::uuid, 'approve_claim', 'pending_review', 'clone_canary', {ap.sql_literal(source_ref)}, 'Exercise strict canonical bundle review and apply lifecycle.', {ap.sql_literal(json.dumps({'apply_payload': payload}, sort_keys=True))}::jsonb); """ _psql(container, database, sql) def _approve( args: argparse.Namespace, proposal_id: str, database: str, *, dry_run: bool = False, ) -> subprocess.CompletedProcess[str]: command = [ sys.executable, str(args.approve_script), proposal_id, "--reviewed-by", REVIEWER_HANDLE, "--review-note", REVIEW_NOTE, "--secrets-file", args.review_secrets_file, "--container", args.container, "--db", database, "--host", "127.0.0.1", "--role", args.review_role, ] if dry_run: command.append("--dry-run") return _run(command, check=False) def _apply( args: argparse.Namespace, proposal_id: str, database: str, *, dry_run: bool = False, ) -> subprocess.CompletedProcess[str]: command = [ sys.executable, str(args.apply_script), proposal_id, "--applied-by", "kb-apply", "--secrets-file", args.secrets_file, "--container", args.container, "--db", database, "--host", "127.0.0.1", "--role", args.apply_role, ] if dry_run: command.append("--dry-run") return _run(command, check=False) def _security_definer_readback(container: str, database: str) -> dict[str, Any]: expected_values = ",\n".join( f"({ap.sql_literal(name)}, {ap.sql_literal(role)}, " f"{ap.sql_literal(SECURITY_DEFINER_ARGUMENT_TYPES[name])})" for name, role in SECURITY_DEFINER_DEPENDENCIES.items() ) raw = _psql( container, database, f"""begin transaction read only; with expected(function_name, expected_role, expected_argument_types) as ( values {expected_values} ), function_rows as ( select e.function_name, e.expected_role, e.expected_argument_types, p.oid, p.prosecdef, p.proacl, p.proowner, pg_get_userbyid(p.proowner) as owner, oidvectortypes(p.proargtypes) as argument_types, pg_get_function_identity_arguments(p.oid) as identity_arguments from expected e left join lateral ( select candidate.* from pg_proc candidate join pg_namespace n on n.oid = candidate.pronamespace where n.nspname = 'kb_stage' and candidate.proname = e.function_name and oidvectortypes(candidate.proargtypes) = e.expected_argument_types ) p on true ) select json_build_object( 'functions', (select json_agg(json_build_object( 'function', 'kb_stage.' || function_name, 'argument_types', argument_types, 'identity_arguments', identity_arguments, 'present', oid is not null, 'security_definer', prosecdef, 'owner', owner, 'expected_role', expected_role, 'expected_role_execute', case when oid is null or to_regrole(expected_role) is null then null else has_function_privilege(to_regrole(expected_role), oid, 'EXECUTE') end, 'other_runtime_role_execute', case when oid is null then null when expected_role = 'kb_apply' and to_regrole('kb_review') is not null then has_function_privilege(to_regrole('kb_review'), oid, 'EXECUTE') when expected_role = 'kb_review' and to_regrole('kb_apply') is not null then has_function_privilege(to_regrole('kb_apply'), oid, 'EXECUTE') else null end, 'public_execute', case when oid is null then null else exists ( select 1 from aclexplode(coalesce(proacl, acldefault('f', proowner))) acl where acl.grantee = 0 and acl.privilege_type = 'EXECUTE' ) end ) order by function_name) from function_rows), 'unexpected_overloads', ( select coalesce(json_agg(json_build_object( 'function', 'kb_stage.' || candidate.proname, 'argument_types', oidvectortypes(candidate.proargtypes), 'owner', pg_get_userbyid(candidate.proowner) ) order by candidate.proname, oidvectortypes(candidate.proargtypes)), '[]'::json) from pg_proc candidate join pg_namespace n on n.oid = candidate.pronamespace where n.nspname = 'kb_stage' and candidate.proname in ( 'approve_strict_proposal', 'assert_approved_proposal', 'finish_approved_proposal' ) and not exists ( select 1 from expected e where e.function_name = candidate.proname and e.expected_argument_types = oidvectortypes(candidate.proargtypes) ) ), 'protected_role_memberships', ( select coalesce(json_agg(json_build_object( 'member', member_role.rolname, 'granted_role', granted_role.rolname ) order by member_role.rolname, granted_role.rolname), '[]'::json) from pg_auth_members membership join pg_roles member_role on member_role.oid = membership.member join pg_roles granted_role on granted_role.oid = membership.roleid where member_role.rolname in ('kb_gate_owner', 'kb_review', 'kb_apply') or granted_role.rolname in ('kb_gate_owner', 'kb_review', 'kb_apply') ), 'login_role_owned_objects', ( select coalesce(json_agg(owned_object order by owned_object), '[]'::json) from ( select n.nspname || '.' || c.relname as owned_object from pg_class c join pg_namespace n on n.oid = c.relnamespace join pg_roles owner_role on owner_role.oid = c.relowner where owner_role.rolname in ('kb_review', 'kb_apply') and n.nspname in ('public', 'kb_stage') union all select n.nspname || '.' || p.proname || '(' || oidvectortypes(p.proargtypes) || ')' as owned_object from pg_proc p join pg_namespace n on n.oid = p.pronamespace join pg_roles owner_role on owner_role.oid = p.proowner where owner_role.rolname in ('kb_review', 'kb_apply') and n.nspname in ('public', 'kb_stage') ) owned ), 'role_privileges', json_build_object( 'kb_review_proposal_select', case when to_regrole('kb_review') is null then null else has_table_privilege(to_regrole('kb_review'), 'kb_stage.kb_proposals', 'SELECT') end, 'kb_review_proposal_update', case when to_regrole('kb_review') is null then null else has_table_privilege(to_regrole('kb_review'), 'kb_stage.kb_proposals', 'UPDATE') end, 'kb_apply_proposal_select', case when to_regrole('kb_apply') is null then null else has_table_privilege(to_regrole('kb_apply'), 'kb_stage.kb_proposals', 'SELECT') end, 'kb_apply_proposal_update', case when to_regrole('kb_apply') is null then null else has_table_privilege(to_regrole('kb_apply'), 'kb_stage.kb_proposals', 'UPDATE') end, 'kb_apply_claim_evidence_insert', case when to_regrole('kb_apply') is null then null else has_table_privilege(to_regrole('kb_apply'), 'public.claim_evidence', 'INSERT') end, 'kb_apply_claim_evidence_update', case when to_regrole('kb_apply') is null then null else has_table_privilege(to_regrole('kb_apply'), 'public.claim_evidence', 'UPDATE') end, 'kb_review_approval_snapshot_select', case when to_regrole('kb_review') is null then null else has_table_privilege(to_regrole('kb_review'), 'kb_stage.kb_proposal_approvals', 'SELECT') end, 'kb_review_approval_snapshot_update', case when to_regrole('kb_review') is null then null else has_table_privilege(to_regrole('kb_review'), 'kb_stage.kb_proposal_approvals', 'UPDATE') end, 'kb_apply_approval_snapshot_select', case when to_regrole('kb_apply') is null then null else has_table_privilege(to_regrole('kb_apply'), 'kb_stage.kb_proposal_approvals', 'SELECT') end, 'kb_apply_approval_snapshot_update', case when to_regrole('kb_apply') is null then null else has_table_privilege(to_regrole('kb_apply'), 'kb_stage.kb_proposal_approvals', 'UPDATE') end ) )::text; rollback; """, ) return json.loads(raw) def _security_dependencies_ready(readback: dict[str, Any]) -> bool: functions = readback.get("functions") or [] functions_ready = len(functions) == len(SECURITY_DEFINER_DEPENDENCIES) and all( row.get("present") is True and row.get("argument_types") == SECURITY_DEFINER_ARGUMENT_TYPES[row["function"].removeprefix("kb_stage.")] and row.get("security_definer") is True and row.get("expected_role_execute") is True and row.get("other_runtime_role_execute") is False and row.get("public_execute") is False and row.get("owner") == "kb_gate_owner" for row in functions ) privileges = readback.get("role_privileges") or {} catalog_ready = ( readback.get("unexpected_overloads") == [] and readback.get("protected_role_memberships") == [] and readback.get("login_role_owned_objects") == [] ) return functions_ready and catalog_ready and privileges == { "kb_review_proposal_select": True, "kb_review_proposal_update": False, "kb_apply_proposal_select": True, "kb_apply_proposal_update": False, "kb_apply_claim_evidence_insert": True, "kb_apply_claim_evidence_update": False, "kb_review_approval_snapshot_select": False, "kb_review_approval_snapshot_update": False, "kb_apply_approval_snapshot_select": False, "kb_apply_approval_snapshot_update": False, } def _approval_transition_valid( pending: dict[str, Any] | None, approved: dict[str, Any] | None ) -> bool: if not pending or not approved: return False immutable = [ "id", "proposal_type", "channel", "source_ref", "rationale", "payload", ] return ( pending.get("status") == "pending_review" and all(pending.get(key) == approved.get(key) for key in immutable) and all( pending.get(key) is None for key in ( "reviewed_by_handle", "reviewed_by_agent_id", "reviewed_at", "review_note", "applied_by_handle", "applied_by_agent_id", "applied_at", ) ) and approved.get("status") == "approved" and approved.get("reviewed_by_handle") == REVIEWER_HANDLE and bool(approved.get("reviewed_at")) and approved.get("review_note") == REVIEW_NOTE and approved.get("applied_by_handle") is None and approved.get("applied_by_agent_id") is None and approved.get("applied_at") is None ) def _apply_transition_valid( approved: dict[str, Any] | None, applied: dict[str, Any] | None ) -> bool: if not approved or not applied: return False preserved = [ "id", "proposal_type", "channel", "source_ref", "rationale", "payload", "reviewed_by_handle", "reviewed_by_agent_id", "reviewed_at", "review_note", ] return ( approved.get("status") == "approved" and all(approved.get(key) == applied.get(key) for key in preserved) and applied.get("status") == "applied" and applied.get("applied_by_handle") == "kb-apply" and bool(applied.get("applied_by_agent_id")) and bool(applied.get("applied_at")) ) def _approval_snapshot_matches( proposal: dict[str, Any] | None, snapshot: dict[str, Any] | None, ) -> bool: if not proposal or not snapshot: return False return snapshot == { "proposal_id": proposal["id"], "proposal_type": proposal["proposal_type"], "payload": proposal["payload"], "reviewed_by_handle": proposal["reviewed_by_handle"], "reviewed_by_agent_id": proposal["reviewed_by_agent_id"], "reviewed_by_db_role": "kb_review", "reviewed_at": proposal["reviewed_at"], "review_note": proposal["review_note"], } def _seed_evidence_permission_probe( container: str, database: str, fixture: dict[str, Any] ) -> None: source_hash = f"approve-claim-permission-probe-{fixture['permission_source_id']}" _psql( container, database, f"""insert into public.claims (id, type, text, status, confidence, tags, created_by, superseded_by) values ({ap.sql_literal(fixture['permission_claim_id'])}::uuid, 'structural', 'Existing claim_evidence permission probe.', 'open', 0.42, array['clone-canary-permission-probe']::text[], null, null); insert into public.sources (id, source_type, url, storage_path, excerpt, hash, created_by) values ({ap.sql_literal(fixture['permission_source_id'])}::uuid, 'observation', null, null, 'Existing claim_evidence permission probe source.', {ap.sql_literal(source_hash)}, null); insert into public.claim_evidence (claim_id, source_id, role, weight, created_by) values ({ap.sql_literal(fixture['permission_claim_id'])}::uuid, {ap.sql_literal(fixture['permission_source_id'])}::uuid, 'grounds', 0.42, null); """, tuples=False, ) def _evidence_permission_probe_readback( container: str, database: str, fixture: dict[str, Any] ) -> dict[str, Any] | None: raw = _psql( container, database, f"""begin transaction read only; select coalesce((select json_build_object( 'claim_id', claim_id::text, 'source_id', source_id::text, 'role', role::text, 'weight', weight, 'created_by', created_by::text ) from public.claim_evidence where claim_id = {ap.sql_literal(fixture['permission_claim_id'])}::uuid and source_id = {ap.sql_literal(fixture['permission_source_id'])}::uuid and role = 'grounds'), 'null'::json)::text; rollback; """, ) return json.loads(raw) def _permission_probe( container: str, database: str, sql: str, *, role: str, password: str, ) -> dict[str, Any]: result = _role_psql( container, database, f"begin;\n{sql}\nrollback;\n", role=role, password=password, ) readback = _command_readback(result) readback["refused"] = result.returncode != 0 return readback def _empty_bundle_readback(payload: dict[str, Any]) -> dict[str, Any]: return {key: [] for key in _expected_bundle_rows(payload)} def _conflict_payload(fixture: dict[str, Any]) -> dict[str, Any]: return { "contract_version": 2, "agent_id": None, "claims": [ { "id": fixture["conflict_claim_id"], "type": "structural", "text": "This row must roll back with the conflicting source.", "status": "open", "confidence": 0.5, "tags": ["conflict-canary"], "created_by": None, "superseded_by": None, } ], "sources": [ { "id": fixture["conflict_source_id"], "source_type": "observation", "url": None, "storage_path": None, "excerpt": "Conflicting source id.", "hash": ( f"approve-claim-permission-probe-" f"{fixture['permission_source_id']}" ), "created_by": None, } ], "evidence": [], "edges": [], "reasoning_tools": [], } def run_canary(args: argparse.Namespace) -> dict[str, Any]: if args.review_role != "kb_review" or args.apply_role != "kb_apply": raise ValueError( "the canary is pinned to the intended kb_review/kb_apply authority split" ) suffix = uuid.uuid4().hex[:10] clone_db = f"{args.database_prefix}_{suffix}" if not SAFE_DB_RE.match(clone_db): raise ValueError(f"unsafe generated database name: {clone_db}") fixture = ( load_normalized_fixture(args.normalization_json, suffix) if args.normalization_json else build_fixture(suffix) ) payload = fixture["apply_payload"] expected_rows = _expected_bundle_rows(payload) expected_table_deltas = _expected_table_deltas(expected_rows) conflict_payload = _conflict_payload(fixture) source_files_before = _source_file_manifest(args) result: dict[str, Any] = { "schema": "working-leo.approve-claim-clone-canary.v3", "generated_at_utc": datetime.now(timezone.utc).isoformat(), "required_tier": "T2_runtime", "source_database": args.source_db, "clone_database": clone_db, "source_database_mutated": False, "telegram_messages_sent": False, "service_restarted": False, "clone_created": False, "clone_dropped": False, "fixture_ids": { key: value for key, value in fixture.items() if key != "apply_payload" }, "dependencies": { "approve_script": Path(args.approve_script) .resolve() .relative_to(REPO_ROOT.resolve()) .as_posix(), "apply_script": Path(args.apply_script) .resolve() .relative_to(REPO_ROOT.resolve()) .as_posix(), "prerequisites_sql": Path(args.prereqs) .resolve() .relative_to(REPO_ROOT.resolve()) .as_posix(), "review_role": args.review_role, "apply_role": args.apply_role, "gate_owner_role": "kb_gate_owner", "reviewer_handle": REVIEWER_HANDLE, "immutable_approval_table": "kb_stage.kb_proposal_approvals", "security_definer_functions": [ f"kb_stage.{name}" for name in SECURITY_DEFINER_DEPENDENCIES ], "security_definer_argument_types": { f"kb_stage.{name}": argument_types for name, argument_types in SECURITY_DEFINER_ARGUMENT_TYPES.items() }, "credential_files": { "review": { "runtime_location": "ephemeral_or_operator_managed", "present_at_start": Path(args.review_secrets_file).is_file(), }, "apply": { "runtime_location": "ephemeral_or_operator_managed", "present_at_start": Path(args.secrets_file).is_file(), }, }, }, "payload_projection": { "payload_sha256": hashlib.sha256( json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() ).hexdigest(), "expected_rows": expected_rows, "expected_table_deltas": expected_table_deltas, }, "source_binding": { "files": source_files_before, "unchanged_during_run": False, }, } try: result["service_before"] = _service_state(args.service) result["source_counts_before"] = _base_counts( args.container, args.source_db ) result["source_cluster_roles_before"] = _cluster_role_readback( args.container, args.source_db ) if not _cluster_roles_are_preprovisioned( result["source_cluster_roles_before"] ): raise RuntimeError( "refusing clone prerequisites until kb_apply, kb_review, and " "NOLOGIN kb_gate_owner are pre-provisioned without elevated attributes" ) missing_credentials = [ name for name, readback in result["dependencies"]["credential_files"].items() if not readback["present_at_start"] ] if missing_credentials: raise RuntimeError( "required separated credential file(s) are absent: " + ", ".join(missing_credentials) ) result["source_target_rows_before"] = _exact_bundle_readback( args.container, args.source_db, payload ) prereqs = Path(args.prereqs).read_text(encoding="utf-8") clone_prereqs, prereq_sanitization = _clone_prerequisites_sql(prereqs) result["clone_prerequisite_sanitization"] = prereq_sanitization _psql(args.container, "postgres", f"create database {clone_db};", tuples=False) result["clone_created"] = True _schema_copy(args.container, args.source_db, clone_db) result["reviewer_agent_seed"] = _seed_reviewer_agent( args.container, args.source_db, clone_db, REVIEWER_HANDLE, ) _psql(args.container, clone_db, clone_prereqs, tuples=False) result["security_definer_readback"] = _security_definer_readback( args.container, clone_db ) result["payload_agent_seed"] = _seed_payload_agents( args.container, args.source_db, clone_db, payload ) result["external_claim_seed"] = _seed_external_claims( args.container, args.source_db, clone_db, payload ) _seed_evidence_permission_probe(args.container, clone_db, fixture) evidence_before = _evidence_permission_probe_readback( args.container, clone_db, fixture ) apply_password = ap.load_password(args.secrets_file) evidence_update = _permission_probe( args.container, clone_db, f"""update public.claim_evidence set weight = 0.99 where claim_id = {ap.sql_literal(fixture['permission_claim_id'])}::uuid and source_id = {ap.sql_literal(fixture['permission_source_id'])}::uuid and role = 'grounds';""", role=args.apply_role, password=apply_password, ) evidence_after = _evidence_permission_probe_readback( args.container, clone_db, fixture ) result["kb_apply_claim_evidence_update_probe"] = { "before": evidence_before, "attempt": evidence_update, "after": evidence_after, "exact_row_unchanged": evidence_before == evidence_after, } _stage_proposal( args.container, clone_db, fixture["proposal_id"], payload, source_ref="working-leo:approve-claim-clone-canary", ) pending_proposal = _proposal_readback( args.container, clone_db, fixture["proposal_id"] ) pending_approval_snapshot = _approval_snapshot_readback( args.container, clone_db, fixture["proposal_id"] ) review_dry_run = _approve( args, fixture["proposal_id"], clone_db, dry_run=True ) review_apply = _approve(args, fixture["proposal_id"], clone_db) approved_proposal = _proposal_readback( args.container, clone_db, fixture["proposal_id"] ) approved_snapshot = _approval_snapshot_readback( args.container, clone_db, fixture["proposal_id"] ) result["review_transition"] = { "pending_readback": pending_proposal, "pending_immutable_approval": pending_approval_snapshot, "review_dry_run": { "returncode": review_dry_run.returncode, "uses_approve_strict_proposal": ( "kb_stage.approve_strict_proposal" in review_dry_run.stdout ), "stderr": review_dry_run.stderr.strip(), }, "review_apply": _command_readback(review_apply), "approved_readback": approved_proposal, "immutable_approval_readback": approved_snapshot, "immutable_approval_matches_ledger": _approval_snapshot_matches( approved_proposal, approved_snapshot ), "exact_transition": _approval_transition_valid( pending_proposal, approved_proposal ), } approval_update = _permission_probe( args.container, clone_db, f"""update kb_stage.kb_proposals set status = 'pending_review', review_note = 'kb_apply direct mutation probe' where id = {ap.sql_literal(fixture['proposal_id'])}::uuid;""", role=args.apply_role, password=apply_password, ) payload_update = _permission_probe( args.container, clone_db, f"""update kb_stage.kb_proposals set payload = payload || '{{"kb_apply_direct_mutation_probe": true}}'::jsonb where id = {ap.sql_literal(fixture['proposal_id'])}::uuid;""", role=args.apply_role, password=apply_password, ) immutable_approval_update = _permission_probe( args.container, clone_db, f"""update kb_stage.kb_proposal_approvals set payload = payload || '{{"kb_apply_direct_mutation_probe": true}}'::jsonb where proposal_id = {ap.sql_literal(fixture['proposal_id'])}::uuid;""", role=args.apply_role, password=apply_password, ) proposal_after_permission_probes = _proposal_readback( args.container, clone_db, fixture["proposal_id"] ) snapshot_after_permission_probes = _approval_snapshot_readback( args.container, clone_db, fixture["proposal_id"] ) result["kb_apply_proposal_mutation_probes"] = { "approval_fields": approval_update, "payload": payload_update, "immutable_approval_payload": immutable_approval_update, "before": approved_proposal, "after": proposal_after_permission_probes, "immutable_approval_before": approved_snapshot, "immutable_approval_after": snapshot_after_permission_probes, "exact_row_unchanged": ( approved_proposal == proposal_after_permission_probes ), "immutable_approval_exactly_unchanged": ( approved_snapshot == snapshot_after_permission_probes ), } built_apply = _apply(args, fixture["proposal_id"], clone_db, dry_run=True) built_sql = built_apply.stdout if built_apply.returncode != 0 or not built_sql.strip(): raise RuntimeError( "failed to build apply SQL before mutation: " f"{built_apply.stderr.strip()}" ) _psql( args.container, clone_db, f"""update kb_stage.kb_proposals set payload = payload || '{{"mutation_after_build": true}}'::jsonb, updated_at = now() where id = {ap.sql_literal(fixture['proposal_id'])}::uuid;""", tuples=False, ) proposal_after_mutation = _proposal_readback( args.container, clone_db, fixture["proposal_id"] ) snapshot_after_ledger_mutation = _approval_snapshot_readback( args.container, clone_db, fixture["proposal_id"] ) stale_execute = _role_psql( args.container, clone_db, built_sql, role=args.apply_role, password=apply_password, ) rows_after_stale_execute = _exact_bundle_readback( args.container, clone_db, payload ) proposal_after_stale_execute = _proposal_readback( args.container, clone_db, fixture["proposal_id"] ) snapshot_after_stale_execute = _approval_snapshot_readback( args.container, clone_db, fixture["proposal_id"] ) assert approved_proposal is not None _psql( args.container, clone_db, f"""update kb_stage.kb_proposals set payload = {ap.sql_literal(json.dumps(approved_proposal['payload'], sort_keys=True))}::jsonb, updated_at = now() where id = {ap.sql_literal(fixture['proposal_id'])}::uuid;""", tuples=False, ) restored_proposal = _proposal_readback( args.container, clone_db, fixture["proposal_id"] ) restored_snapshot = _approval_snapshot_readback( args.container, clone_db, fixture["proposal_id"] ) result["mutation_between_build_and_execute"] = { "build_returncode": built_apply.returncode, "built_sql_sha256": hashlib.sha256(built_sql.encode()).hexdigest(), "built_sql_uses_assert_approved_proposal": ( "kb_stage.assert_approved_proposal" in built_sql ), "built_sql_uses_finish_approved_proposal": ( "kb_stage.finish_approved_proposal" in built_sql ), "approved_before_mutation": approved_proposal, "immutable_approval_before_mutation": approved_snapshot, "mutated_readback": proposal_after_mutation, "immutable_approval_after_ledger_mutation": ( snapshot_after_ledger_mutation ), "stale_execute": _command_readback(stale_execute), "rows_after_stale_execute": rows_after_stale_execute, "proposal_after_stale_execute": proposal_after_stale_execute, "immutable_approval_after_stale_execute": ( snapshot_after_stale_execute ), "restored_readback": restored_proposal, "restored_immutable_approval": restored_snapshot, "stale_apply_refused": stale_execute.returncode != 0, "canonical_rows_unchanged": ( rows_after_stale_execute == _empty_bundle_readback(payload) ), "approval_restored_exactly": restored_proposal == approved_proposal, "immutable_approval_never_changed": all( snapshot == approved_snapshot for snapshot in ( snapshot_after_ledger_mutation, snapshot_after_stale_execute, restored_snapshot, ) ), } result["clone_counts_before_apply"] = _base_counts( args.container, clone_db ) first_apply = _apply(args, fixture["proposal_id"], clone_db) result["first_apply"] = _command_readback(first_apply) result["row_readback"] = _exact_bundle_readback( args.container, clone_db, payload ) result["clone_counts_after_apply"] = _base_counts( args.container, clone_db ) result["clone_apply_table_deltas"] = _table_deltas( result["clone_counts_before_apply"], result["clone_counts_after_apply"], ) applied_proposal = _proposal_readback( args.container, clone_db, fixture["proposal_id"] ) applied_snapshot = _approval_snapshot_readback( args.container, clone_db, fixture["proposal_id"] ) result["apply_transition"] = { "approved_readback": approved_proposal, "applied_readback": applied_proposal, "immutable_approval_readback": applied_snapshot, "immutable_approval_matches_applied_ledger": ( _approval_snapshot_matches(applied_proposal, applied_snapshot) ), "exact_transition": _apply_transition_valid( approved_proposal, applied_proposal ), } replay = _apply(args, fixture["proposal_id"], clone_db) result["idempotent_replay"] = { "returncode": replay.returncode, "already_applied_refusal": ( "already applied" in (replay.stdout + replay.stderr).lower() ), "stderr": replay.stderr.strip(), } _stage_proposal( args.container, clone_db, fixture["conflict_proposal_id"], conflict_payload, source_ref="working-leo:approve-claim-conflict-canary", ) conflict_pending = _proposal_readback( args.container, clone_db, fixture["conflict_proposal_id"] ) conflict_pending_snapshot = _approval_snapshot_readback( args.container, clone_db, fixture["conflict_proposal_id"] ) conflict_review = _approve(args, fixture["conflict_proposal_id"], clone_db) conflict_approved = _proposal_readback( args.container, clone_db, fixture["conflict_proposal_id"] ) conflict_approved_snapshot = _approval_snapshot_readback( args.container, clone_db, fixture["conflict_proposal_id"] ) conflict = _apply(args, fixture["conflict_proposal_id"], clone_db) conflict_after = _proposal_readback( args.container, clone_db, fixture["conflict_proposal_id"] ) conflict_snapshot_after = _approval_snapshot_readback( args.container, clone_db, fixture["conflict_proposal_id"] ) conflict_rows = _exact_bundle_readback( args.container, clone_db, conflict_payload ) result["conflict_review_transition"] = { "pending_readback": conflict_pending, "pending_immutable_approval": conflict_pending_snapshot, "review_apply": _command_readback(conflict_review), "approved_readback": conflict_approved, "immutable_approval_readback": conflict_approved_snapshot, "immutable_approval_matches_ledger": _approval_snapshot_matches( conflict_approved, conflict_approved_snapshot ), "exact_transition": _approval_transition_valid( conflict_pending, conflict_approved ), } result["conflict_apply"] = { "returncode": conflict.returncode, "source_hash_collision_refusal": ( "source hash collision" in (conflict.stdout + conflict.stderr).lower() ), "stderr": conflict.stderr.strip(), } result["conflict_readback"] = { "proposal": conflict_after, "immutable_approval": conflict_snapshot_after, "canonical_rows": conflict_rows, "proposal_approval_unchanged": conflict_after == conflict_approved, "immutable_approval_unchanged": ( conflict_snapshot_after == conflict_approved_snapshot ), "canonical_rows_absent": ( conflict_rows == _empty_bundle_readback(conflict_payload) ), } except Exception as exc: result["error"] = f"{type(exc).__name__}: {exc}" finally: cleanup_readback: dict[str, Any] = { "drop_attempted": bool(result["clone_created"]) } if result["clone_created"]: cleanup_sql = f"""select pg_terminate_backend(pid) from pg_stat_activity where datname = {ap.sql_literal(clone_db)} and pid <> pg_backend_pid(); drop database if exists {clone_db}; """ cleanup = _run( [ "docker", "exec", "-i", args.container, "psql", "-U", "postgres", "-d", "postgres", "-v", "ON_ERROR_STOP=1", ], input_text=cleanup_sql, check=False, ) result["clone_dropped"] = cleanup.returncode == 0 result["cleanup_returncode"] = cleanup.returncode cleanup_readback.update(_command_readback(cleanup)) result["leftover_clone_count"] = int( _psql( args.container, "postgres", f"select count(*) from pg_database where datname = {ap.sql_literal(clone_db)};", ) or "0" ) cleanup_readback["leftover_clone_database_count"] = result[ "leftover_clone_count" ] result["cleanup_readback"] = cleanup_readback result["source_counts_after"] = _base_counts( args.container, args.source_db ) if "source_counts_before" in result: result["source_table_deltas"] = _table_deltas( result["source_counts_before"], result["source_counts_after"], ) result["source_cluster_roles_after"] = _cluster_role_readback( args.container, args.source_db ) result["source_target_rows_after"] = _exact_bundle_readback( args.container, args.source_db, payload ) result["service_after"] = _service_state(args.service) try: source_files_after = _source_file_manifest(args) result["source_binding"]["post_run_files"] = source_files_after result["source_binding"]["unchanged_during_run"] = ( source_files_after == source_files_before ) except Exception as exc: result["source_binding"]["post_run_error"] = ( f"{type(exc).__name__}: {exc}" ) result["checks"] = { "security_definer_dependencies_ready": _security_dependencies_ready( result.get("security_definer_readback") or {} ), "clone_prerequisites_omit_cluster_role_ddl": ( result.get("clone_prerequisite_sanitization", {}).get( "cluster_role_ddl_removed" ) is True ), "reviewer_agent_seeded_exactly": ( result.get("reviewer_agent_seed", {}).get("exact_clone_copy") is True ), "payload_agents_seeded_exactly": ( result.get("payload_agent_seed", {}).get("exact_clone_copy") is True ), "kb_review_approval_succeeded": ( result.get("review_transition", {}) .get("review_apply", {}) .get("returncode") == 0 ), "kb_review_used_security_definer": ( result.get("review_transition", {}) .get("review_dry_run", {}) .get("uses_approve_strict_proposal") is True ), "review_transition_exact": ( result.get("review_transition", {}).get("exact_transition") is True and result.get("review_transition", {}).get( "immutable_approval_matches_ledger" ) is True ), "kb_apply_cannot_mutate_proposal_approval": ( result.get("kb_apply_proposal_mutation_probes", {}) .get("approval_fields", {}) .get("refused") is True ), "kb_apply_cannot_mutate_proposal_payload": ( result.get("kb_apply_proposal_mutation_probes", {}) .get("payload", {}) .get("refused") is True ), "kb_apply_cannot_mutate_immutable_approval": ( result.get("kb_apply_proposal_mutation_probes", {}) .get("immutable_approval_payload", {}) .get("refused") is True ), "proposal_exact_after_permission_probes": ( result.get("kb_apply_proposal_mutation_probes", {}).get( "exact_row_unchanged" ) is True and result.get("kb_apply_proposal_mutation_probes", {}).get( "immutable_approval_exactly_unchanged" ) is True ), "kb_apply_cannot_update_existing_claim_evidence": ( result.get("kb_apply_claim_evidence_update_probe", {}) .get("attempt", {}) .get("refused") is True ), "claim_evidence_exact_after_permission_probe": ( result.get("kb_apply_claim_evidence_update_probe", {}).get( "exact_row_unchanged" ) is True ), "mutation_between_build_and_execute_refused": all( result.get("mutation_between_build_and_execute", {}).get(key) is True for key in ( "built_sql_uses_assert_approved_proposal", "built_sql_uses_finish_approved_proposal", "stale_apply_refused", "canonical_rows_unchanged", "approval_restored_exactly", "immutable_approval_never_changed", ) ), "first_apply_succeeded": result.get("first_apply", {}).get("returncode") == 0, "payload_projection_exact": result.get("row_readback") == expected_rows, "clone_apply_table_deltas_exact": ( result.get("clone_apply_table_deltas") == expected_table_deltas ), "apply_transition_exact": ( result.get("apply_transition", {}).get("exact_transition") is True and result.get("apply_transition", {}).get( "immutable_approval_matches_applied_ledger" ) is True ), "idempotent_replay_refused": ( result.get("idempotent_replay", {}).get("returncode", 0) != 0 and result.get("idempotent_replay", {}).get( "already_applied_refusal" ) is True ), "conflict_review_separate_and_exact": ( result.get("conflict_review_transition", {}).get("exact_transition") is True and result.get("conflict_review_transition", {}).get( "immutable_approval_matches_ledger" ) is True ), "source_hash_conflict_refused": ( result.get("conflict_apply", {}).get("returncode", 0) != 0 and result.get("conflict_apply", {}).get( "source_hash_collision_refusal" ) is True ), "conflict_transaction_rolled_back_exactly": ( result.get("conflict_readback", {}).get( "proposal_approval_unchanged" ) is True and result.get("conflict_readback", {}).get( "immutable_approval_unchanged" ) is True and result.get("conflict_readback", {}).get("canonical_rows_absent") is True ), "source_target_rows_unchanged": ( result.get("source_target_rows_before") == result.get("source_target_rows_after") ), "source_cluster_roles_preexisting": ( _cluster_roles_are_preprovisioned( result.get("source_cluster_roles_before") or [] ) ), "source_cluster_roles_unchanged": ( result.get("source_cluster_roles_before") == result.get("source_cluster_roles_after") ), "source_table_deltas_exactly_zero": ( bool(result.get("source_table_deltas")) and all(delta == 0 for delta in result["source_table_deltas"].values()) ), "source_files_unchanged_during_run": ( result.get("source_binding", {}).get("unchanged_during_run") is True ), "gateway_process_unchanged": ( result.get("service_before", {}).get("MainPID") == result.get("service_after", {}).get("MainPID") and result.get("service_before", {}).get("NRestarts") == result.get("service_after", {}).get("NRestarts") ), "clone_cleanup_complete": ( result.get("clone_dropped") is True and result.get("leftover_clone_count") == 0 ), } result["source_counts_observed_unchanged"] = ( result.get("source_counts_before") == result.get("source_counts_after") ) result["status"] = ( "pass" if all(result["checks"].values()) and "error" not in result else "fail" ) result["source_database_mutated"] = not all( result["checks"][key] for key in ( "source_target_rows_unchanged", "source_cluster_roles_unchanged", "source_table_deltas_exactly_zero", ) ) result["service_restarted"] = not result["checks"][ "gateway_process_unchanged" ] result["post_run_cleanup"] = { "leftover_clone_database_count": result.get("leftover_clone_count"), "gateway_active_state": result.get("service_after", {}).get("ActiveState"), "gateway_sub_state": result.get("service_after", {}).get("SubState"), "gateway_main_pid": result.get("service_after", {}).get("MainPID"), "gateway_restart_count": result.get("service_after", {}).get("NRestarts"), } return result def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--container", default=ap.DEFAULT_CONTAINER) parser.add_argument("--source-db", default=ap.DEFAULT_DB) parser.add_argument("--database-prefix", default="teleo_approve_claim_clone") parser.add_argument("--service", default="leoclean-gateway.service") parser.add_argument( "--approve-script", type=Path, default=HERE / "approve_proposal.py" ) parser.add_argument( "--apply-script", type=Path, default=HERE / "apply_proposal.py" ) parser.add_argument( "--prereqs", type=Path, default=HERE / "kb_apply_prereqs.sql" ) parser.add_argument("--review-secrets-file", default=DEFAULT_REVIEW_SECRETS_FILE) parser.add_argument("--secrets-file", default=ap.DEFAULT_SECRETS_FILE) parser.add_argument("--review-role", default="kb_review") parser.add_argument("--apply-role", default=ap.DEFAULT_ROLE) parser.add_argument( "--normalization-json", type=Path, help="optional one-row normalization output containing one strict approve_claim child", ) parser.add_argument("--output", type=Path) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = parse_args(sys.argv[1:] if argv is None else argv) result = run_canary(args) rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(rendered, encoding="utf-8") print(rendered, end="") return 0 if result["status"] == "pass" else 1 if __name__ == "__main__": raise SystemExit(main())