diff --git a/.agents/skills/teleo-corpus-knowledge-rebuild/SKILL.md b/.agents/skills/teleo-corpus-knowledge-rebuild/SKILL.md new file mode 100644 index 0000000..df4091b --- /dev/null +++ b/.agents/skills/teleo-corpus-knowledge-rebuild/SKILL.md @@ -0,0 +1,153 @@ +--- +name: teleo-corpus-knowledge-rebuild +description: Use when locally snapshotting an extracted UTF-8 Forgejo or text corpus into a deterministic Teleo source library, or sharding its verified extraction-work index for parallel processing before any claim generation, review, admission, or database write. +--- + +# Teleo Corpus Knowledge Rebuild + +## Job + +Turn one local UTF-8 source directory into a deterministic, self-contained +source snapshot bundle. This is the source-library phase only: it preserves +bytes, paths, hashes, and syntactic section locators without interpreting any +section as a claim. It does not perform semantic claim extraction. + +After that inventory verifies, optionally divide its exact work rows into +deterministic local shards so independent extractors can process a large corpus +in parallel. Sharding schedules work; it does not interpret source content. + +## Operator Path + +From the repository root, build into a path that does not exist: + +```bash +python3 ops/run_local_corpus_knowledge_rebuild.py \ + --source-root tests/fixtures/teleo_corpus_rebuild \ + --output /private/tmp/teleo-corpus-rebuild-bundle \ + --mode source-inventory-only +``` + +Then run the read-only verifier: + +```bash +python3 ops/run_local_corpus_knowledge_rebuild.py \ + --verify-bundle /private/tmp/teleo-corpus-rebuild-bundle +``` + +For parallel processing, choose a shard count from 1 through 1024 that does not +exceed the verified work-item count. Build into another path that does not +exist: + +```bash +python3 ops/shard_corpus_extraction_work.py \ + --inventory-bundle /private/tmp/teleo-corpus-rebuild-bundle \ + --output-dir /private/tmp/teleo-corpus-extraction-shards \ + --shard-count 64 +``` + +Then verify the shards against the same inventory: + +```bash +python3 ops/shard_corpus_extraction_work.py \ + --inventory-bundle /private/tmp/teleo-corpus-rebuild-bundle \ + --verify-bundle /private/tmp/teleo-corpus-extraction-shards +``` + +Inspect these stable entry points: + +```text +source-manifest.json +extraction-work-index.json +receipt.json +snapshots/sha256//.utf8 +shard-manifest.json +shards/shard--of-.json +``` + +## Required Readback + +Require all of these: + +- every source row binds a normalized source-root-relative path to one exact, + content-addressed byte snapshot; +- every section has exact one-based line and UTF-8 byte ranges plus a quote + hash; +- every work row is `source_section_for_semantic_extraction`; +- `semantic_claim_extraction_performed=false`, + `database_write_performed=false`, and `admission_performed=false` throughout; +- the receipt says the directory snapshot was not atomic and only guarantees + the exact files copied from stable descriptors; +- the verifier recomputes snapshots, locators, cardinality, artifact hashes, + bundle-tree hash, and receipt hash; +- a second run over unchanged inputs produces byte-identical output. +- every inventory work item appears in exactly one shard selected by + `sha256(utf8(work_item_id)) mod shard_count`; +- each shard binds the exact inventory tree, source-manifest, work-index, and + inventory-receipt hashes without embedding source-body text or absolute paths; +- the shard verifier rejects wrong inventories, missing, duplicate, unknown, or + misplaced work IDs, symlinks, extras, incomplete output, and rehashed + side-effect overclaims; +- shard directories are `0700`, files are `0600`, and repeated runs with the + same inventory and shard count are byte-identical. + +Run focused checks: + +```bash +python3 -m pytest -q tests/test_run_local_corpus_knowledge_rebuild.py +python3 -m pytest -q tests/test_shard_corpus_extraction_work.py +python3 -m ruff check \ + ops/run_local_corpus_knowledge_rebuild.py \ + ops/shard_corpus_extraction_work.py \ + tests/test_run_local_corpus_knowledge_rebuild.py \ + tests/test_shard_corpus_extraction_work.py +``` + +## Safety Boundary + +The build reserves a new output directory atomically and never replaces an +existing destination. It publishes `receipt.json` while the incomplete marker +is still present, then removes that marker as the final completion step. It +opens source files descriptor-first without following +symlinks, streams exact bytes into private content-addressed snapshots, rejects +invalid UTF-8, binary controls, lone CR newlines, unsupported entries, path +collisions, and configured resource-limit overruns, and deduplicates identical +content. + +The defaults are bounded. Raise them only with the explicit `--max-*` options; +the command enforces hard upper bounds on every override. + +The sharder independently verifies the inventory before reading work metadata, +reserves a new output directory without replacement, writes through held +directory descriptors, keeps an incomplete marker through receipt publication, +and revalidates the inventory before publishing. It reads no snapshot bodies. +Hash shards can be empty; membership is stable only for the exact inventory and +the same shard count. + +## Semantic Handoff + +The inventory plus one verified shard is input to a separate semantic +extractor, not to a database writer. The extractor must verify both bundles, +then resolve each relative snapshot path and exact byte range from the supplied +inventory. Only after that extractor has produced explicit, source-backed V3 +manifests may the operator run: + +```text +scripts/compile_kb_source_packet.py +``` + +That compiler's output still requires separate review and guarded admission. +Do not fabricate V3 manifests from raw sections in this inventory helper. + +Use `teleo-reconstruction-recovery` for restore/replay selection and +`teleo-kb-db-change-workflow` only for an explicitly authorized later review or +apply task. + +## Claim Ceiling + +The output proves deterministic local inventory of the exact copied UTF-8 +files, bounded syntactic sectioning, bundle integrity, and an exact once-only +hash partition of extraction work for a fixed shard count. It does not prove an +atomic snapshot of concurrent directory changes, non-empty shards, +format-specific syntax, semantic extraction or extractor quality, claim +atomicity or correctness, database deduplication, review, admission, +application, binary-document extraction, or any live environment. diff --git a/.agents/skills/teleo-kb-db-change-workflow/SKILL.md b/.agents/skills/teleo-kb-db-change-workflow/SKILL.md index 34ce68e..06f2dfb 100644 --- a/.agents/skills/teleo-kb-db-change-workflow/SKILL.md +++ b/.agents/skills/teleo-kb-db-change-workflow/SKILL.md @@ -39,6 +39,40 @@ A useful composed change must retain: Do not flatten strategy, governance, concept maps, identity, and reasoning tools into generic claim rows merely because the input document discusses them. +## V3 Source-To-Rebuild Route + +For new reconstruction work, use this order: + +1. Run `ops/run_local_corpus_knowledge_rebuild.py` to retain exact UTF-8 + snapshots and section work items. Its output contains no semantic claims. +2. For a large corpus, run `ops/shard_corpus_extraction_work.py` to create + deterministic exact-once work partitions. Sharding schedules extraction; it + does not read source bodies or produce claims. +3. Have a separately pinned extractor produce an explicit + `livingip.kbSourceExtractionManifest.v3`. Every evidence quote must bind to + exact UTF-8 byte offsets in the retained extraction. +4. Run `scripts/compile_kb_source_packet.py` with the original artifact, exact + extracted text, manifest, and extractor specification. The result is + build-only and remains `pending_review`. +5. On a disposable PostgreSQL clone, stage with + `scripts/stage_normalized_proposal.py --compiled-v3-bundle ...`, then render + the exact rows with `scripts/kb_proposal_review_packet.py`. +6. Record a real review separately with `scripts/approve_proposal.py` and exact + `--reviewed-by m3taversal`. Never use test-entered review metadata as + evidence that `m3taversal` actually reviewed a production packet. +7. Apply only the exact UUID with `scripts/apply_worker.py`, `--enable`, + `--contract-version 3`, and `--proposal-id ...`; refuse rather than draining + another approved proposal from the queue. +8. Export the applied transition with + `scripts/export_kb_transition_replay_material.py`, assemble ordered V3 + materials with `ops/build_v3_genesis_ledger_bundle.py`, and independently + replay with `ops/run_local_genesis_ledger_rebuild.py`. + +This repository does not yet provide a general PDF/office extractor or an +automatic corpus-wide semantic extractor. For binary originals, retain their +exact bytes and pin the separately produced UTF-8 extraction plus extractor +specification before step 3. + Run the deterministic source-to-proposal canary: ```bash @@ -47,6 +81,17 @@ Run the deterministic source-to-proposal canary: --output /tmp/leo-local-ingestion-proposal-canary-current.json ``` +Run the complete disposable V3 source-to-clean-rebuild lifecycle: + +```bash +.venv/bin/python -m pytest -q tests/test_v3_source_rebuild_lifecycle_postgres.py +``` + +This lifecycle uses a synthetic pinned extractor manifest and synthetic +operator-supplied review metadata. It proves the local machinery and exact +replay boundary, not semantic extraction quality, real human approval, or a +live database mutation. + Require the source, claim, evidence, and proposal UUIDs to link exactly; claim and evidence text must be exact substrings of the source; status must remain `pending_review`; the container must use network `none` and leave no volume or @@ -54,7 +99,12 @@ container behind. ## Canonical Apply Contract -The strict `approve_claim` v2 payload may create: +The strict `approve_claim` v2 contract remains supported for its retained +proofs. V2/V3 payloads may create the supported canonical rows for their exact +contract version; V3 additionally carries explicit provenance, access, +assessment, and admission fields. + +The shared row families include: - `public.claims` - `public.sources` @@ -64,9 +114,19 @@ The strict `approve_claim` v2 payload may create: The lifecycle is split across: +- `scripts/compile_kb_source_packet.py`: hash-bound source plus V3 semantic + manifest to a deterministic build-only bundle; +- `scripts/stage_normalized_proposal.py`: revalidate and stage an exact bundle + only on the named disposable database; +- `scripts/kb_proposal_review_packet.py`: read-only review of exact proposed + canonical rows; - `scripts/kb_proposal_normalize.py`: fail-closed rich intent to strict payload; - `scripts/approve_proposal.py`: `kb_review` approves the exact type/payload; - `scripts/apply_proposal.py`: operator-only `kb_apply` writes and verifies; +- `scripts/apply_worker.py`: exact-ID, one-at-a-time V2/V3 worker wrapper; +- `scripts/export_kb_transition_replay_material.py`: read-only applied-transition + exporter; +- `ops/build_v3_genesis_ledger_bundle.py`: V3-only ordered ledger assembler; - `scripts/kb_apply_prereqs.sql`: roles, immutable approval row, gate functions, ownership, indexes, and ACLs; - `scripts/run_approve_claim_isolated_container_canary.sh`: disposable diff --git a/.agents/skills/teleo-leo-onboarding/SKILL.md b/.agents/skills/teleo-leo-onboarding/SKILL.md index 61fb45f..90dca18 100644 --- a/.agents/skills/teleo-leo-onboarding/SKILL.md +++ b/.agents/skills/teleo-leo-onboarding/SKILL.md @@ -90,6 +90,7 @@ Run the deterministic isolated acceptance canary with: | GCP and Cloud SQL | `teleo-gcp-parity-ops` | Current main evidence; keep open PR #148 candidate-only | | Hermes packaging/runtime | `nousresearch-hermes-agent` | `hermes-agent/` plus the live service-specific ops skill | | Database provenance | `teleo-infra-provenance` and `teleo-db-operator` | Separate Git, runtime, SQLite, Postgres, and retained proof | +| Local corpus inventory and extraction preparation | `teleo-corpus-knowledge-rebuild` | Exact UTF-8 snapshots and section work items; semantic claims are a separate reviewed step | | Source ingestion and proposal/apply | `teleo-kb-db-change-workflow` | Hash-bound source, review, guarded apply, receipt | | Reconstruction and recovery | `teleo-reconstruction-recovery` | `docs/kb-rebuild-and-recompile.md` | | Identity and rendered soul | `working-leo-m3taversal-outcomes` | Canonical identity rows before `SOUL.md` | diff --git a/.agents/skills/teleo-leo-onboarding/scripts/run_clean_context_canary.py b/.agents/skills/teleo-leo-onboarding/scripts/run_clean_context_canary.py index 3c36733..38587ae 100755 --- a/.agents/skills/teleo-leo-onboarding/scripts/run_clean_context_canary.py +++ b/.agents/skills/teleo-leo-onboarding/scripts/run_clean_context_canary.py @@ -48,6 +48,11 @@ SEMANTIC_CONTRACT = { "PR #148 is a separate, open GCP least-privilege candidate", "PR #148 open candidate only; not canonical main", ), + "local_corpus_source_inventory": ( + "teleo-corpus-knowledge-rebuild", + "It does not perform semantic claim extraction.", + "deterministic source snapshots and extraction work items; no claims, proposals, or DB writes", + ), "next_safe_action": ( "teleo-leo-onboarding", ".agents/skills/teleo-leo-onboarding/scripts/validate_skill_pack.py --root .", diff --git a/.agents/skills/teleo-reconstruction-recovery/SKILL.md b/.agents/skills/teleo-reconstruction-recovery/SKILL.md index e7e5f71..ac9c62a 100644 --- a/.agents/skills/teleo-reconstruction-recovery/SKILL.md +++ b/.agents/skills/teleo-reconstruction-recovery/SKILL.md @@ -30,7 +30,10 @@ as canonical `main` recovery instructions until it merges. | Need | Path | Honest result | |---|---|---| | Restore the latest verified state quickly | `ops/run_local_canonical_postgres_rebuild.py` | Exact snapshot recovery | +| Snapshot a Forgejo/text corpus for extraction | `ops/run_local_corpus_knowledge_rebuild.py` | Content-addressed UTF-8 sources and work items; no claims | +| Partition verified extraction work | `ops/shard_corpus_extraction_work.py` | Deterministic exact-once work shards; no source interpretation | | Prove strict post-genesis applies replay exactly | `ops/run_local_genesis_ledger_rebuild.py` | Isolated insert-only ledger replay | +| Assemble reviewed V3 replay materials | `ops/build_v3_genesis_ledger_bundle.py` | Hash-pinned V3 ledger; optional operator-supplied admission metadata | | Turn one retained source into a reviewed packet | `scripts/compile_kb_source_packet.py` | Deterministic proposal packet, not canonical knowledge | | Rebuild every row from original sources | Follow the semantic recompilation contract in `docs/kb-rebuild-and-recompile.md` | Partial until every row and receipt is accounted for | | Diagnose a live incident before recovery | Use `.agents/skills/teleo-vps-runtime-ops/SKILL.md` and `.agents/skills/teleo-infra-provenance/SKILL.md` | Fresh read-only incident map | @@ -40,12 +43,18 @@ as canonical `main` recovery instructions until it merges. 1. Identify the intended source database, snapshot, manifest, commit, and hashes. 2. Keep private dumps, row payloads, source excerpts, and replay material outside the repository and mode `0600`. -3. Run the selected path in an isolated, network-disabled local container. -4. Verify schema, constraints, roles, counts, row hashes, key queries, and exact +3. Inventory text sources before semantic extraction. For a binary original, + retain its exact bytes and provide text from a separately pinned extractor; + this repository does not yet supply a general PDF/office extractor. +4. Build and independently verify the V3 ledger before running genesis replay. + Admission metadata is operator-supplied and is not a cryptographic signature + from `m3taversal`. +5. Run the selected path in an isolated, network-disabled local container. +6. Verify schema, constraints, roles, counts, row hashes, key queries, and exact cleanup. -5. Run the relevant focused tests, then the unseen-reasoning verifier when the +7. Run the relevant focused tests, then the unseen-reasoning verifier when the restored state is meant to support Leo answers. -6. Separate local reconstruction from VPS restore, GCP restore, service restart, +8. Separate local reconstruction from VPS restore, GCP restore, service restart, Telegram delivery, promotion, and production apply. Each is its own proof row. ## Commands @@ -54,9 +63,16 @@ Use the repository virtual environment documented in `README.md`: ```bash .venv/bin/python ops/run_local_canonical_postgres_rebuild.py --help +.venv/bin/python ops/run_local_corpus_knowledge_rebuild.py --help +.venv/bin/python ops/shard_corpus_extraction_work.py --help +.venv/bin/python ops/build_v3_genesis_ledger_bundle.py --help .venv/bin/python ops/run_local_genesis_ledger_rebuild.py --help .venv/bin/python scripts/compile_kb_source_packet.py --help .venv/bin/python -m pytest -q \ + tests/test_run_local_corpus_knowledge_rebuild.py \ + tests/test_shard_corpus_extraction_work.py \ + tests/test_v3_source_rebuild_lifecycle_postgres.py \ + tests/test_build_v3_genesis_ledger_bundle.py \ tests/test_run_local_canonical_postgres_rebuild.py \ tests/test_run_local_genesis_ledger_rebuild.py \ tests/test_verify_leo_unseen_reasoning_chain.py @@ -76,6 +92,10 @@ would promote GCP or mutate production without exact authorization. ## Claim Ceiling - Exact snapshot recovery is working when its parity receipt passes. +- Corpus inventory proves exact copied text bytes and extraction work locations, + not semantic claims or binary extraction. +- The V3 ledger builder accepts only exact source-normalized applied materials, + selects every ordered entry, and never infers admission from proposal status. - The merged genesis-plus-ledger path proves an isolated insert-only slice. - Semantic source-to-blank-database recompilation remains incomplete until every canonical row traces to a genesis record or reviewed replay receipt. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f7c53c..a5a85d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,8 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/docs/kb-rebuild-and-recompile.md b/docs/kb-rebuild-and-recompile.md index b21eb56..acbf408 100644 --- a/docs/kb-rebuild-and-recompile.md +++ b/docs/kb-rebuild-and-recompile.md @@ -96,6 +96,104 @@ The remaining gaps are concrete rather than mysterious: This proves that most of the initial database came from the retained file-KB import path. It does not yet prove a clean blank-database recompile. +## Genesis-Backed Source Lifecycle: Working T2 Canary + +`scripts/run_leo_integrated_learning_lifecycle.py` can now start from one +retained real genesis dump rather than its legacy synthetic fixture schema. The +genesis mode is all-or-nothing: the dump, parity manifest, repository-pinned +trusted exporter receipt, and a digest-pinned PostgreSQL image are mandatory. + +Run: + +```bash +python3 scripts/run_leo_integrated_learning_lifecycle.py \ + --source-packet /private/path/source-packet \ + --genesis-dump /private/path/genesis.dump \ + --genesis-manifest /private/path/genesis-manifest.jsonl \ + --genesis-exporter-receipt /private/path/exporter-receipt.json \ + --image docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 \ + --review-action approve \ + --review-note "Approved exact retained payload for disposable T2 verification only." \ + --output /private/path/leo-genesis-lifecycle-receipt.json +``` + +Before the first Docker command, the runner copies the dump, manifest, exporter +receipt, source packet, controller, subprocess engines, and their transitive +repository imports into one mode `0500` private staging directory with mode +`0400` files. It validates the custom-dump signature; the exact +repository-pinned exporter-receipt hash, source identity, capture authorization, +and provenance binding; exporter-bound dump and manifest bytes; manifest +contract; image digest; executing Git commit; complete engine dependency hash +set; and a fresh deterministic compile against the retained +`proposal-packet.json`. Every enumerated engine file must be tracked at that +commit, its working-tree bytes must equal the commit blob, and its staged hash +must equal the same commit-bound hash. The compiler performs no model calls. Any +untrusted or incomplete exporter contract, moving image tag, engine drift, +staged-byte drift, invalid manifest, retained-packet drift, or output/input +collision fails before a container can start. + +The child tools consume staged paths, and both runs bind their consumed file +set, engine set, and commit back to the same preflight manifest. The already +running Python controller imports repository modules before it creates staging; +the proof therefore establishes commit-matched imported dependencies and +byte-identical staged copies, not a claim that the controller process itself was +re-executed from staging. Successful public receipts retain logical names, +hashes, counts, and cleanup evidence without host-specific absolute paths. + +Each of two independent runs then: + +1. starts a uniquely labeled, networkless PostgreSQL container with tmpfs data; +2. creates only manifest-declared global roles, restores the supplied custom + dump directly, replays captured database ACLs, and proves the full parity + manifest before installing the current guarded-apply prerequisites; +3. skips the synthetic schema bootstrap and performs no canonical preseed; +4. proves the strict proposal, immutable approval, and every payload-owned + claim/source/evidence/edge/tool target are absent before staging; +5. stages `pending_review`, explicitly approves as the independent review role, + and drives the real apply worker through an exact-candidate report-only tick, + an enabled pre-commit failure with persisted retry state, a successful + password-authenticated retry with a mode `0600` replay receipt, and a + post-apply no-op tick; +6. recompiles the retained packet identically and fingerprints only fields + controlled by its immutable apply payload; and +7. removes the container and private temporary root, then proves its exact + instance and shared two-run label have no remaining containers. + +The two-run comparison excludes generated evidence/edge IDs, timestamps, +container identity, and the PostgreSQL system identifier. It requires the +source hashes, proposal and payload identities, phase states, restored key +counts, row counts, recompile hash, and payload-controlled row projections to +match exactly. + +Older retained parity manifests may predate explicit +`application_roles[].settings` and `database_settings` arrays. Their raw bytes +and trusted exporter binding remain mandatory. For role creation and current +parity SQL, the runner narrowly projects missing arrays from the captured +`default_transaction_read_only` field (or an empty list), without changing the +retained bytes. Such a run may prove data/schema parity and captured role +attributes, but its receipt reports full role/global parity as `not_proven`. +Only a source manifest that actually captured both settings arrays can earn the +full role/global parity claim. + +The focused Docker regression constructs a real custom-format genesis with +nondefault role and per-database settings, restores it twice, executes the real +worker lifecycle, compares post-apply fingerprints, and verifies container and +staging cleanup: + +```bash +pytest -q \ + tests/test_run_leo_integrated_learning_lifecycle.py \ + -k test_real_docker_genesis_restore_worker_parity_and_repeatability +``` + +The detailed receipt can contain proposal payloads and canonical row excerpts. +It is written atomically as mode `0600` and must remain private. Standard output +contains only the artifact/status, receipt basename and SHA-256, aggregate +problem/check counts, and error type. A passing receipt proves T2 disposable +runtime reconstruction only. It does not prove or permit VPS/GCP mutation, +production access, live Telegram behavior, or a blank-schema rebuild of all +historical knowledge. + ## Target Compiler The durable rebuild model is: @@ -148,6 +246,24 @@ not retain every column of the proposal ledger, so exact reconstruction also needs the full approved proposal row, immutable approval snapshot, and final applied proposal row. +New approvals retain the exact post-approval `kb_proposals` row inside +`kb_proposal_approvals`, including `updated_at`. After the worker applies and +writes its replay receipt, export the seven-field private material without any +pre-apply filesystem capture: + +```bash +python3 scripts/export_kb_transition_replay_material.py \ + --replay-receipt /private/path/.json \ + --sequence 1 \ + --output /private/path/entry-0001.json +``` + +The exporter uses the restricted `kb_apply` read function in a read-only +transaction, validates exact approved/approval/applied identities and the +worker receipt, and publishes a mode-`0600` file once. An identical rerun is a +no-op; drift or output aliasing is refused. Already-applied legacy approvals +whose exact approved row was never retained remain explicitly unavailable. + ## Genesis Plus Strict Ledger: Working Deterministic Slice `ops/run_local_genesis_ledger_rebuild.py` now executes the first exact @@ -251,6 +367,24 @@ The exact v1 claim ceiling is intentionally bounded: - this proves only an isolated local reconstruction. It does not touch or prove VPS, GCP, Telegram, a live database, or blank-schema source recompilation. +V3 `approve_claim` receipts extend the same exact-ledger boundary to admitted +claim versions, content-addressed source versions, located evidence, typed +edges, and evidence assessments. Ledger order is also the schema-cutover order: +all V2 entries must replay before the first V3 entry, the hash-pinned V3 contract +is installed once at that boundary, guarded prerequisites are rerun, and any V2 +entry after cutover fails before replay. Historical V2 proposals already present +in genesis remain visible; the reconstruction does not silently apply, delete, +or reinterpret them. + +Receipt-pinned V3 rows are seeded before the idempotent guarded apply, then read +back exactly. PostgreSQL generates the proposal's apply timestamp, so the +disposable reconstruction takes an exclusive proposal-table lock, verifies the +exact V3 decision trigger is enabled, transactionally disables only that trigger, +normalizes the receipt-pinned proposal timestamps as `postgres`, re-enables the +trigger, and verifies it is enabled before commit. Any failure rolls the trigger +state and timestamp update back together. This is a disposable reconstruction +mechanism, not a production write path or a relaxation of the live V3 contract. + The transition contract for `revise_strategy` is final-state deterministic, not a claim that the v1 receipt independently contains a historical before-image: diff --git a/docs/reports/leo-working-state-20260709/skill-pack-manifest.json b/docs/reports/leo-working-state-20260709/skill-pack-manifest.json index 36de40a..2f55186 100644 --- a/docs/reports/leo-working-state-20260709/skill-pack-manifest.json +++ b/docs/reports/leo-working-state-20260709/skill-pack-manifest.json @@ -1,7 +1,7 @@ { "pack": "leo-teleo-skill-pack", "created_date": "2026-07-09", - "last_updated_utc": "2026-07-15T00:16:53Z", + "last_updated_utc": "2026-07-18T22:02:36Z", "status": "repo_native_path_validated", "contains_secrets": false, "production_mutation_authorized": false, @@ -46,6 +46,11 @@ "role": "Snapshot recovery, genesis-ledger replay, source recompilation, and incident recovery", "path": ".agents/skills/teleo-reconstruction-recovery/SKILL.md" }, + { + "name": "teleo-corpus-knowledge-rebuild", + "role": "Deterministic UTF-8 source snapshots and semantic-extraction work indexes; no claim generation", + "path": ".agents/skills/teleo-corpus-knowledge-rebuild/SKILL.md" + }, { "name": "leo-telegram-canary-ops", "role": "Live Telegram bot testing through authenticated Chrome and Computer Use", @@ -103,9 +108,9 @@ "gcp": ["teleo-gcp-parity-ops"], "hermes_runtime": ["nousresearch-hermes-agent", "teleo-vps-runtime-ops"], "database_provenance": ["teleo-infra-provenance", "teleo-db-operator"], - "ingestion": ["teleo-kb-db-change-workflow", "living-ip-kb-interop"], + "ingestion": ["teleo-corpus-knowledge-rebuild", "teleo-kb-db-change-workflow", "living-ip-kb-interop"], "proposal_apply": ["teleo-kb-db-change-workflow"], - "reconstruction": ["teleo-reconstruction-recovery"], + "reconstruction": ["teleo-corpus-knowledge-rebuild", "teleo-reconstruction-recovery"], "identity": ["teleo-leo-onboarding", "working-leo-m3taversal-outcomes"], "testing": ["teleo-leo-onboarding", "teleo-proof-handoff", "crabbox", "decision-engine-refinement"], "security": ["teleo-vps-runtime-ops", "teleo-gcp-parity-ops", "private-password-storage"], @@ -134,6 +139,9 @@ ".agents/skills/teleo-leo-onboarding/scripts/run_clean_context_canary.py", ".agents/skills/teleo-leo-onboarding/scripts/validate_skill_pack.py", "ops/run_local_canonical_postgres_rebuild.py", + "ops/run_local_corpus_knowledge_rebuild.py", + "ops/shard_corpus_extraction_work.py", + "ops/build_v3_genesis_ledger_bundle.py", "ops/run_local_genesis_ledger_rebuild.py", "ops/capture_vps_canonical_postgres_snapshot.py", "ops/restore_gcp_generated_postgres_snapshot.py", diff --git a/hermes-agent/leoclean-bin/kb_tool.py b/hermes-agent/leoclean-bin/kb_tool.py index f8df56f..db28ef5 100755 --- a/hermes-agent/leoclean-bin/kb_tool.py +++ b/hermes-agent/leoclean-bin/kb_tool.py @@ -374,9 +374,23 @@ def sql_array(values: list[str], cast: str = "text") -> str: def psql_json(args: argparse.Namespace, sql: str) -> list[dict[str, Any]]: if args.local: - command = ["docker", "exec", "-i", args.container, "psql", "-U", "postgres", "-d", args.db, "-At", "-q"] + command = [ + "docker", + "exec", + "-i", + args.container, + "psql", + "-U", + "postgres", + "-d", + args.db, + "-At", + "-q", + "-v", + "ON_ERROR_STOP=1", + ] else: - remote = f"docker exec -i {args.container} psql -U postgres -d {args.db} -At -q" + remote = f"docker exec -i {args.container} psql -U postgres -d {args.db} -At -q -v ON_ERROR_STOP=1" command = ["ssh", "-o", "ConnectTimeout=15", "-o", "BatchMode=yes", args.ssh, remote] result = subprocess.run(command, text=True, capture_output=True, input=sql, check=False) if result.returncode != 0: diff --git a/ops/build_v3_genesis_ledger_bundle.py b/ops/build_v3_genesis_ledger_bundle.py new file mode 100644 index 0000000..5e710f3 --- /dev/null +++ b/ops/build_v3_genesis_ledger_bundle.py @@ -0,0 +1,1101 @@ +#!/usr/bin/env python3 +"""Build or verify a private, hash-pinned V3 clean-genesis ledger bundle. + +This command is local and read-only with respect to PostgreSQL. It accepts only +exact exporter-shaped, source-normalized V3 ``approve_claim`` materials. An +admission manifest is never inferred from proposal approval: it is emitted only +when the operator supplies the explicit review flag and every review field. The +result is hash-pinned review metadata, not authenticated review authority, and +does not make a reconstruction canonically eligible by itself. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import stat +import sys +import tempfile +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +try: + from . import run_local_genesis_ledger_rebuild as genesis_rebuild +except ImportError: # pragma: no cover - direct script execution + import run_local_genesis_ledger_rebuild as genesis_rebuild + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPTS_DIR = REPO_ROOT / "scripts" +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +import export_kb_transition_replay_material as replay_exporter # noqa: E402 + +RECEIPT_ARTIFACT = "teleo_v3_genesis_ledger_bundle_builder_receipt" +CANONICAL_REVIEWER_HANDLE = "m3taversal" +REVIEW_METADATA_PROVENANCE = "operator_supplied_explicit_review_metadata" +NORMALIZATION_SCHEMA = "livingip.kbSourceProposalNormalization.v3" +EXTRACTION_MANIFEST_SCHEMA = "livingip.kbSourceExtractionManifest.v3" +MANIFEST_IDENTITY_BINDING = "sha256(exact duplicate-free UTF-8 JSON file bytes)" +NORMALIZATION_FIELDS = frozenset( + { + "schema", + "semantic_extraction_statement", + "semantic_extraction_is_retained_input", + "model_training_claim", + "manifest_schema", + "manifest_file_sha256", + "manifest_identity_binding", + "manifest_canonical_sha256", + "artifact_sha256", + "extracted_text_sha256", + "extractor", + "source_identity", + "source_locator", + "normalized_candidate_content_sha256", + "identity_material", + "packet_identity_sha256", + "parent_packet_sha256", + "quote_bindings", + } +) +IDENTITY_FIELDS = frozenset( + { + "artifact_sha256", + "extracted_text_sha256", + "extractor", + "manifest_file_sha256", + "source_identity", + "source_locator", + "normalized_candidate_content_sha256", + } +) +EXTRACTOR_FIELDS = frozenset({"name", "version", "spec_sha256"}) +QUOTE_BINDING_FIELDS = frozenset({"claim_key", "evidence_key", "kind", "quote", "quote_sha256", "unit", "start", "end"}) +LOCATOR_FIELDS = frozenset( + { + "schema", + "source_identity", + "source_locator", + "artifact_sha256", + "extracted_text_sha256", + "extractor", + "unit", + "start", + "end", + "quote_sha256", + } +) + + +class BundleError(RuntimeError): + """A fail-closed error whose text is safe for command output.""" + + def __init__(self, code: str, safe_message: str) -> None: + super().__init__(safe_message) + self.code = code + self.safe_message = safe_message + + +@dataclass(frozen=True) +class MaterialPin: + path: Path + sha256: str + replay_material_sha256: str + entry: genesis_rebuild.LedgerEntry + + +@dataclass(frozen=True) +class StagedFile: + destination: Path + temporary: Path + payload: bytes + + +@dataclass(frozen=True) +class PublishedFile: + destination: Path + payload: bytes + device: int + inode: int + + +def _canonical_json_bytes(value: Any) -> bytes: + try: + rendered = json.dumps( + value, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + except (TypeError, ValueError) as exc: + raise BundleError("invalid_json_value", "bundle data is not deterministic JSON") from exc + return (rendered + "\n").encode("utf-8") + + +def _compiler_canonical_bytes(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def _compiler_canonical_sha256(value: Any) -> str: + return hashlib.sha256(_compiler_canonical_bytes(value)).hexdigest() + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _stable_v3_uuid(row_kind: str, packet_identity_sha256: str, semantic_key: Any) -> str: + material = _compiler_canonical_bytes( + ["livingip", "kb-source-compiler", 3, packet_identity_sha256, row_kind, semantic_key] + ).decode("ascii") + return str(uuid.uuid5(uuid.NAMESPACE_URL, material)) + + +def _absolute_path(path: Path) -> Path: + expanded = path.expanduser() + return expanded if expanded.is_absolute() else Path.cwd() / expanded + + +def _regular_input(path: Path, *, label: str, private_json: bool = False) -> Path: + absolute = _absolute_path(path) + try: + metadata = absolute.lstat() + except OSError as exc: + raise BundleError("missing_input", f"{label} must be an existing regular non-symlink file") from exc + if absolute.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise BundleError("unsafe_input", f"{label} must be an existing regular non-symlink file") + if private_json and stat.S_IMODE(metadata.st_mode) != 0o600: + raise BundleError("unsafe_private_permissions", f"{label} permissions must be exactly 0600") + return absolute.resolve() + + +def _prepare_new_output(path: Path, *, label: str) -> Path: + absolute = _absolute_path(path) + if absolute.name in {"", ".", ".."}: + raise BundleError("unsafe_output", f"{label} must name a new file") + try: + absolute.lstat() + except FileNotFoundError: + pass + except OSError as exc: + raise BundleError("unsafe_output", f"{label} cannot be inspected safely") from exc + else: + raise BundleError("existing_output", f"{label} must not already exist") + + parent = absolute.parent + if parent.exists() and (parent.is_symlink() or not parent.is_dir()): + raise BundleError("unsafe_output_parent", f"{label} parent must be a real directory") + return parent.resolve(strict=False) / absolute.name + + +def _assert_distinct_paths(paths: list[tuple[Path, str]]) -> None: + seen: dict[Path, str] = {} + for path, label in paths: + resolved = path.resolve(strict=False) + if resolved in seen: + raise BundleError("path_alias", f"{label} must not alias another bundle path") + seen[resolved] = label + + +def _read_json_object(path: Path, *, label: str) -> dict[str, Any]: + try: + raw = path.read_text(encoding="utf-8") + return replay_exporter._parse_json_object(raw, label=label) + except replay_exporter.ExportError as exc: + raise BundleError("invalid_json_input", f"{label} is not exact duplicate-free UTF-8 JSON") from exc + except (OSError, UnicodeError) as exc: + raise BundleError("invalid_json_input", f"{label} is not exact duplicate-free UTF-8 JSON") from exc + + +def _require_exact_fields(value: Any, expected: frozenset[str], *, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or frozenset(value) != expected: + raise BundleError("invalid_v3_provenance", f"{label} does not match the current V3 provenance contract") + return value + + +def _require_sha256(value: Any, *, label: str) -> str: + if not isinstance(value, str) or not genesis_rebuild.SHA256_RE.fullmatch(value): + raise BundleError("invalid_v3_provenance", f"{label} must be a lowercase SHA-256 digest") + return value + + +def _canonical_uuid(value: Any, *, label: str) -> str: + if not isinstance(value, str): + raise BundleError("invalid_reviewer_identity", f"{label} must be a canonical UUID") + try: + normalized = str(uuid.UUID(value)) + except (AttributeError, TypeError, ValueError) as exc: + raise BundleError("invalid_reviewer_identity", f"{label} must be a canonical UUID") from exc + if value != normalized: + raise BundleError("invalid_reviewer_identity", f"{label} must be a canonical UUID") + return normalized + + +def _validate_source_normalization(entry: genesis_rebuild.LedgerEntry) -> None: + proposal = entry.applied_proposal + if entry.contract_version != 3 or proposal.get("proposal_type") != "approve_claim": + raise BundleError("legacy_material_forbidden", "every material must be a V3 approve_claim transition") + + payload = proposal.get("payload") + apply_payload = payload.get("apply_payload") if isinstance(payload, dict) else None + if not isinstance(apply_payload, dict) or apply_payload.get("contract_version") != 3: + raise BundleError("legacy_material_forbidden", "every material must be a V3 approve_claim transition") + + manifest = _require_exact_fields( + apply_payload.get("normalization_manifest"), + NORMALIZATION_FIELDS, + label="normalization manifest", + ) + if ( + manifest["schema"] != NORMALIZATION_SCHEMA + or manifest["manifest_schema"] != EXTRACTION_MANIFEST_SCHEMA + or manifest["manifest_identity_binding"] != MANIFEST_IDENTITY_BINDING + or manifest["semantic_extraction_is_retained_input"] is not True + or manifest["model_training_claim"] is not False + or not isinstance(manifest["semantic_extraction_statement"], str) + or "retained input" not in manifest["semantic_extraction_statement"].lower() + ): + raise BundleError( + "invalid_v3_provenance", + "normalization manifest does not retain the current source-backed V3 provenance statement", + ) + + digest_fields = ( + "manifest_file_sha256", + "manifest_canonical_sha256", + "artifact_sha256", + "extracted_text_sha256", + "normalized_candidate_content_sha256", + "packet_identity_sha256", + "parent_packet_sha256", + ) + digests = { + field: _require_sha256(manifest[field], label=f"normalization manifest {field}") for field in digest_fields + } + extractor = _require_exact_fields(manifest["extractor"], EXTRACTOR_FIELDS, label="normalization extractor") + if any(not isinstance(extractor[field], str) or not extractor[field].strip() for field in ("name", "version")): + raise BundleError("invalid_v3_provenance", "normalization extractor identity must be non-empty") + _require_sha256(extractor["spec_sha256"], label="normalization extractor spec_sha256") + + identity = _require_exact_fields( + manifest["identity_material"], + IDENTITY_FIELDS, + label="normalization identity material", + ) + for field in IDENTITY_FIELDS: + if identity[field] != manifest[field]: + raise BundleError("invalid_v3_provenance", "normalization identity material does not match its pins") + if _compiler_canonical_sha256(identity) != digests["packet_identity_sha256"]: + raise BundleError("invalid_v3_provenance", "normalization packet identity is not recoverable") + + artifact_sha256 = digests["artifact_sha256"] + source_identity = f"document:sha256:{artifact_sha256}" + source_locator = f"artifact://sha256/{artifact_sha256}" + if manifest["source_identity"] != source_identity or manifest["source_locator"] != source_locator: + raise BundleError("invalid_v3_provenance", "normalization source identity is not content-addressed") + + source_proposal_id = apply_payload.get("source_proposal_id") + try: + canonical_source_proposal_id = str(uuid.UUID(str(source_proposal_id))) + except (AttributeError, TypeError, ValueError) as exc: + raise BundleError("invalid_v3_provenance", "V3 source proposal identity is invalid") from exc + expected_source_proposal_id = _stable_v3_uuid( + "source_proposal", + digests["packet_identity_sha256"], + digests["normalized_candidate_content_sha256"], + ) + if source_proposal_id != canonical_source_proposal_id or source_proposal_id != expected_source_proposal_id: + raise BundleError("invalid_v3_provenance", "V3 source proposal identity is not recoverable") + + payload_sha256 = _compiler_canonical_sha256(proposal["payload"]) + expected_proposal_id = _stable_v3_uuid( + "pending_review_proposal", + digests["packet_identity_sha256"], + payload_sha256, + ) + expected_source_ref = f"source-compiler-v3:{digests['packet_identity_sha256'][:32]}:{payload_sha256[:16]}" + if proposal.get("id") != expected_proposal_id or proposal.get("source_ref") != expected_source_ref: + raise BundleError("invalid_v3_provenance", "V3 normalized proposal identity is not recoverable") + + sources = apply_payload.get("sources") + evidence_rows = apply_payload.get("evidence") + claims = apply_payload.get("claims") + assessments = apply_payload.get("assessments") + if ( + not isinstance(sources, list) + or len(sources) != 1 + or not isinstance(sources[0], dict) + or not isinstance(evidence_rows, list) + or not evidence_rows + or not isinstance(claims, list) + or not claims + or not isinstance(assessments, list) + or len(assessments) != len(claims) + ): + raise BundleError("invalid_v3_provenance", "V3 material is not an exact source-backed claim bundle") + + source = sources[0] + if ( + source.get("content_hash") != artifact_sha256 + or source.get("canonical_url") is not None + or source.get("storage_uri") != source_locator + ): + raise BundleError("invalid_v3_provenance", "V3 source row does not retain the normalized artifact pin") + + raw_bindings = manifest["quote_bindings"] + if not isinstance(raw_bindings, list) or not raw_bindings or len(raw_bindings) != len(evidence_rows): + raise BundleError("invalid_v3_provenance", "normalization quote bindings must select every evidence row") + + bindings: list[dict[str, Any]] = [] + binding_keys: list[tuple[str, str]] = [] + for raw_binding in raw_bindings: + binding = _require_exact_fields(raw_binding, QUOTE_BINDING_FIELDS, label="normalization quote binding") + claim_key = binding["claim_key"] + evidence_key = binding["evidence_key"] + quote = binding["quote"] + start = binding["start"] + end = binding["end"] + if ( + not isinstance(claim_key, str) + or not claim_key.strip() + or not isinstance(evidence_key, str) + or not evidence_key.strip() + or binding["kind"] != "evidence" + or not isinstance(quote, str) + or not quote + or binding["unit"] != "utf8_bytes" + or type(start) is not int + or type(end) is not int + or start < 0 + or end <= start + or end - start != len(quote.encode("utf-8")) + or _sha256_bytes(quote.encode("utf-8")) + != _require_sha256(binding["quote_sha256"], label="normalization quote SHA-256") + ): + raise BundleError("invalid_v3_provenance", "normalization quote binding is invalid") + binding_keys.append((claim_key, evidence_key)) + bindings.append(binding) + + if len(binding_keys) != len(set(binding_keys)) or binding_keys != sorted(binding_keys): + raise BundleError("invalid_v3_provenance", "normalization quote bindings are duplicate or ambiguously ordered") + + packet_identity = digests["packet_identity_sha256"] + claim_keys = sorted({claim_key for claim_key, _ in binding_keys}) + expected_claim_ids = {_stable_v3_uuid("claim", packet_identity, claim_key) for claim_key in claim_keys} + observed_claim_ids = {row.get("id") for row in claims if isinstance(row, dict)} + if len(observed_claim_ids) != len(claims) or observed_claim_ids != expected_claim_ids: + raise BundleError("invalid_v3_provenance", "normalized V3 claim identities are not recoverable") + + evidence_by_id = { + row.get("id"): row for row in evidence_rows if isinstance(row, dict) and isinstance(row.get("id"), str) + } + if len(evidence_by_id) != len(evidence_rows): + raise BundleError("invalid_v3_provenance", "normalized V3 evidence identities are not unique") + + source_id = source.get("id") + for binding in bindings: + claim_key = binding["claim_key"] + evidence_key = binding["evidence_key"] + expected_evidence_id = _stable_v3_uuid("evidence", packet_identity, [claim_key, evidence_key]) + evidence = evidence_by_id.get(expected_evidence_id) + expected_locator = { + "schema": "livingip.extractedTextQuoteLocator.v1", + "source_identity": source_identity, + "source_locator": source_locator, + "artifact_sha256": artifact_sha256, + "extracted_text_sha256": digests["extracted_text_sha256"], + "extractor": extractor, + "unit": "utf8_bytes", + "start": binding["start"], + "end": binding["end"], + "quote_sha256": binding["quote_sha256"], + } + if ( + evidence is None + or evidence.get("claim_id") != _stable_v3_uuid("claim", packet_identity, claim_key) + or evidence.get("source_id") != source_id + or evidence.get("excerpt") != binding["quote"] + or evidence.get("source_content_hash") != artifact_sha256 + or not isinstance(evidence.get("locator_json"), dict) + or frozenset(evidence["locator_json"]) != LOCATOR_FIELDS + or evidence["locator_json"] != expected_locator + ): + raise BundleError("invalid_v3_provenance", "normalized V3 evidence provenance is not recoverable") + + expected_assessments = { + _stable_v3_uuid("assessment", packet_identity, claim_key): _stable_v3_uuid("claim", packet_identity, claim_key) + for claim_key in claim_keys + } + observed_assessments = { + row.get("id"): row.get("claim_id") + for row in assessments + if isinstance(row, dict) and isinstance(row.get("id"), str) + } + if observed_assessments != expected_assessments: + raise BundleError("invalid_v3_provenance", "normalized V3 assessment identities are not recoverable") + + +def _validate_materials(paths: list[Path]) -> list[MaterialPin]: + if not paths: + raise BundleError("missing_material", "at least one ordered --material is required") + + pins: list[MaterialPin] = [] + proposal_ids: set[str] = set() + material_paths: set[Path] = set() + for expected_sequence, raw_path in enumerate(paths, 1): + path = _regular_input(raw_path, label=f"material {expected_sequence}") + if path in material_paths: + raise BundleError("duplicate_material", "ordered material paths must be unique") + material_paths.add(path) + raw_material = _read_json_object(path, label=f"material {expected_sequence}") + try: + material = replay_exporter.validate_material(raw_material, expected_sequence=expected_sequence) + except replay_exporter.ExportError as exc: + raise BundleError( + "invalid_exported_material", + f"material {expected_sequence} failed the exact exporter contract", + ) from exc + if _compiler_canonical_bytes(material) != _compiler_canonical_bytes(raw_material): + raise BundleError( + "noncanonical_exported_material", + f"material {expected_sequence} is not in exact exporter-canonical form", + ) + + material_sha256 = genesis_rebuild.canonical_rebuild.sha256_file(path) + hashes = material.get("replay_receipt", {}).get("hashes") + replay_material_sha256 = hashes.get("replay_material_sha256") if isinstance(hashes, dict) else None + _require_sha256(replay_material_sha256, label=f"material {expected_sequence} replay hash") + try: + entry = genesis_rebuild._validate_entry_material( + material, + expected_sequence=expected_sequence, + expected_replay_hash=replay_material_sha256, + material_path=path, + material_sha256=material_sha256, + ) + except genesis_rebuild.ReconstructionError as exc: + raise BundleError( + "invalid_genesis_material", + f"material {expected_sequence} failed the genesis replay contract", + ) from exc + _validate_source_normalization(entry) + proposal_id = entry.applied_proposal["id"] + if proposal_id in proposal_ids: + raise BundleError("duplicate_proposal", "ordered materials contain a duplicate proposal identity") + proposal_ids.add(proposal_id) + pins.append( + MaterialPin( + path=path, + sha256=material_sha256, + replay_material_sha256=replay_material_sha256, + entry=entry, + ) + ) + return pins + + +def _relative_artifact(path: Path, *, ledger_parent: Path) -> str: + rendered = Path(os.path.relpath(path, start=ledger_parent)).as_posix() + if Path(rendered).is_absolute(): # pragma: no cover - os.path.relpath is relative on one filesystem + raise BundleError("invalid_relative_path", "ledger artifact paths must be relative") + return rendered + + +def _validate_dump_and_manifests( + genesis_dump: Path, + genesis_manifest: Path, + final_manifest: Path, +) -> None: + try: + genesis_rebuild.canonical_rebuild.validate_custom_dump(genesis_dump) + except (OSError, ValueError) as exc: + raise BundleError("invalid_genesis_dump", "genesis dump failed the custom-format preflight") from exc + for path, label in ((genesis_manifest, "genesis parity manifest"), (final_manifest, "final parity manifest")): + try: + genesis_rebuild.load_manifest(path) + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise BundleError("invalid_parity_manifest", f"{label} failed the existing manifest validator") from exc + + +def _ledger_payload( + *, + ledger_path: Path, + genesis_dump: Path, + genesis_manifest: Path, + final_manifest: Path, + materials: list[MaterialPin], +) -> dict[str, Any]: + return { + "artifact": genesis_rebuild.LEDGER_ARTIFACT, + "contract_version": genesis_rebuild.LEDGER_CONTRACT_VERSION, + "engine": genesis_rebuild._engine_hashes(), + "genesis": { + "dump_sha256": genesis_rebuild.canonical_rebuild.sha256_file(genesis_dump), + "parity_manifest_sha256": genesis_rebuild.canonical_rebuild.sha256_file(genesis_manifest), + }, + "entries": [ + { + "sequence": pin.entry.sequence, + "material": _relative_artifact(pin.path, ledger_parent=ledger_path.parent), + "sha256": pin.sha256, + "replay_material_sha256": pin.replay_material_sha256, + } + for pin in materials + ], + "final_parity": { + "manifest": _relative_artifact(final_manifest, ledger_parent=ledger_path.parent), + "sha256": genesis_rebuild.canonical_rebuild.sha256_file(final_manifest), + }, + } + + +def _admission_payload( + args: argparse.Namespace, + *, + ledger_sha256: str, + materials: list[MaterialPin], + review_note_file: Path, +) -> dict[str, Any]: + if args.reviewed_by != CANONICAL_REVIEWER_HANDLE: + raise BundleError("invalid_reviewer_handle", "--reviewed-by must equal m3taversal exactly") + reviewer_id = _canonical_uuid(args.reviewed_by_agent_id, label="--reviewed-by-agent-id") + try: + reviewed_at = genesis_rebuild.replay_receipt.canonical_utc_timestamp( + args.reviewed_at, + path="--reviewed-at", + ) + except (TypeError, ValueError) as exc: + raise BundleError("invalid_reviewed_at", "--reviewed-at must be timezone-aware") from exc + try: + review_note = review_note_file.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise BundleError("invalid_review_note", "--review-note-file must contain readable UTF-8 text") from exc + if not review_note.strip(): + raise BundleError("invalid_review_note", "--review-note-file must be non-empty") + + return { + "artifact": genesis_rebuild.ADMISSION_ARTIFACT, + "contract_version": genesis_rebuild.ADMISSION_CONTRACT_VERSION, + "ledger_sha256": ledger_sha256, + "reviewed_by_handle": CANONICAL_REVIEWER_HANDLE, + "reviewed_by_agent_id": reviewer_id, + "reviewed_at": reviewed_at, + "review_note": review_note, + "entries": [ + { + "sequence": pin.entry.sequence, + "proposal_id": pin.entry.applied_proposal["id"], + "material_sha256": pin.sha256, + "replay_material_sha256": pin.replay_material_sha256, + "admission_scope": args.admission_scope, + "basis": args.admission_basis, + } + for pin in materials + ], + } + + +def _sync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _stage_private_json(destination: Path, payload: dict[str, Any]) -> StagedFile: + parent_preexisting = destination.parent.exists() + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if destination.parent.is_symlink() or not destination.parent.is_dir(): + raise BundleError("unsafe_output_parent", "output parent must be a real directory") + if not parent_preexisting: + os.chmod(destination.parent, 0o700) + try: + destination.lstat() + except FileNotFoundError: + pass + else: + raise BundleError("existing_output", "bundle output appeared before publication") + + payload_bytes = _canonical_json_bytes(payload) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{destination.name}.", dir=str(destination.parent)) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload_bytes) + handle.flush() + os.fsync(handle.fileno()) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + try: + temporary.unlink() + except FileNotFoundError: + pass + raise + return StagedFile(destination=destination, temporary=temporary, payload=payload_bytes) + + +def _unlink_if_owned(published: PublishedFile) -> bool: + try: + metadata = published.destination.lstat() + if ( + stat.S_ISREG(metadata.st_mode) + and metadata.st_dev == published.device + and metadata.st_ino == published.inode + and published.destination.read_bytes() == published.payload + ): + published.destination.unlink() + return True + except FileNotFoundError: + return True + except OSError: + return False + return False + + +def _rollback_published(files: list[PublishedFile]) -> None: + changed_directories: set[Path] = set() + for published in reversed(files): + if _unlink_if_owned(published): + changed_directories.add(published.destination.parent) + for directory in changed_directories: + try: + _sync_directory(directory) + except OSError: + pass + + +def _publish_no_replace(staged_files: list[StagedFile]) -> list[PublishedFile]: + published: list[PublishedFile] = [] + directories = {staged.destination.parent for staged in staged_files} + try: + for staged in staged_files: + temporary_metadata = staged.temporary.lstat() + try: + os.link(staged.temporary, staged.destination) + except FileExistsError as exc: + raise BundleError("existing_output", "bundle output appeared during no-replace publication") from exc + published_item = PublishedFile( + destination=staged.destination, + payload=staged.payload, + device=temporary_metadata.st_dev, + inode=temporary_metadata.st_ino, + ) + published.append(published_item) + metadata = staged.destination.lstat() + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_dev != temporary_metadata.st_dev + or metadata.st_ino != temporary_metadata.st_ino + or stat.S_IMODE(metadata.st_mode) != 0o600 + or staged.destination.read_bytes() != staged.payload + ): + raise BundleError("publication_readback_failed", "published bundle output failed exact readback") + for directory in directories: + _sync_directory(directory) + + for staged in staged_files: + staged.temporary.unlink() + for directory in directories: + _sync_directory(directory) + + for item in published: + metadata = item.destination.lstat() + if ( + item.destination.is_symlink() + or not stat.S_ISREG(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + or item.destination.read_bytes() != item.payload + ): + raise BundleError("publication_readback_failed", "published bundle output failed exact readback") + return published + except BaseException: + _rollback_published(published) + raise + finally: + for staged in staged_files: + try: + staged.temporary.unlink() + except FileNotFoundError: + pass + + +def _strict_admission_readback(path: Path) -> None: + manifest = _read_json_object(path, label="admission manifest") + if manifest.get("reviewed_by_handle") != CANONICAL_REVIEWER_HANDLE: + raise BundleError("invalid_reviewer_handle", "admission reviewer must equal m3taversal exactly") + _canonical_uuid(manifest.get("reviewed_by_agent_id"), label="admission reviewer agent id") + + +def _preflight_bundle( + *, + genesis_dump: Path, + genesis_manifest: Path, + ledger: Path, + ledger_sha256: str, + admission_manifest: Path | None, + admission_manifest_sha256: str | None, + expected_final_manifest: Path | None = None, + expected_materials: list[Path] | None = None, +) -> genesis_rebuild.ReconstructionBundle: + arguments = argparse.Namespace( + genesis_dump=genesis_dump, + genesis_manifest=genesis_manifest, + ledger=ledger, + ledger_sha256=ledger_sha256, + admission_manifest=admission_manifest, + admission_manifest_sha256=admission_manifest_sha256, + output=ledger.parent / f".{ledger.name}.preflight-receipt-{uuid.uuid4().hex}.json", + ) + try: + bundle = genesis_rebuild.load_bundle(arguments) + except genesis_rebuild.ReconstructionError as exc: + raise BundleError("genesis_preflight_refused", "existing genesis ledger preflight refused the bundle") from exc + for entry in bundle.entries: + _validate_source_normalization(entry) + if expected_final_manifest is not None and bundle.final_manifest_path != expected_final_manifest.resolve(): + raise BundleError("final_manifest_mismatch", "ledger does not select the supplied final parity manifest") + if expected_materials is not None: + observed = [entry.material_path for entry in bundle.entries] + expected = [path.resolve() for path in expected_materials] + if observed != expected: + raise BundleError("material_order_mismatch", "ledger does not select the supplied material order exactly") + if admission_manifest is not None: + _strict_admission_readback(admission_manifest) + return bundle + + +def _verify_deterministic_json(path: Path, *, label: str) -> dict[str, Any]: + value = _read_json_object(path, label=label) + if path.read_bytes() != _canonical_json_bytes(value): + raise BundleError("nondeterministic_json", f"{label} is not in deterministic builder encoding") + return value + + +def _admission_summary(*, emitted: bool, sha256: str | None = None) -> dict[str, Any]: + if not emitted: + return {"status": "not_requested", "emitted": False} + return { + "status": "explicit_review_metadata_supplied", + "emitted": True, + "sha256": sha256, + "review_metadata_provenance": REVIEW_METADATA_PROVENANCE, + "cryptographically_authenticated": False, + } + + +def _require_build_arguments(args: argparse.Namespace) -> None: + if args.output_ledger is None: + raise BundleError("missing_output", "build mode requires --output-ledger") + if args.final_manifest is None: + raise BundleError("missing_final_manifest", "build mode requires --final-manifest") + if not args.materials: + raise BundleError("missing_material", "build mode requires at least one ordered --material") + if ( + args.ledger_sha256 is not None + or args.admission_manifest is not None + or args.admission_manifest_sha256 is not None + ): + raise BundleError("invalid_build_option", "pre-existing ledger or admission pins are verify-only options") + + admission_values = ( + args.admission_output, + args.reviewed_by, + args.reviewed_by_agent_id, + args.reviewed_at, + args.review_note_file, + args.admission_scope, + args.admission_basis, + ) + if not args.emit_admission_manifest and any(value is not None for value in admission_values): + raise BundleError( + "implicit_admission_forbidden", + "admission metadata requires the explicit --emit-admission-manifest flag", + ) + if args.emit_admission_manifest and any(value is None for value in admission_values): + raise BundleError( + "incomplete_admission_review", + "explicit admission requires output, reviewer, timestamp, note file, scope, and basis", + ) + + +def _require_verify_arguments(args: argparse.Namespace) -> Path: + supplied = args.verify_ledger + ledger = Path(supplied) if supplied else args.verify_ledger_alias or args.output_ledger + if ledger is None: + raise BundleError("missing_ledger", "--verify-ledger requires a ledger path") + if args.ledger_sha256 is None or not genesis_rebuild.SHA256_RE.fullmatch(args.ledger_sha256): + raise BundleError("missing_ledger_pin", "--verify-ledger requires --ledger-sha256") + if args.emit_admission_manifest or any( + value is not None + for value in ( + args.admission_output, + args.reviewed_by, + args.reviewed_by_agent_id, + args.reviewed_at, + args.review_note_file, + args.admission_scope, + args.admission_basis, + ) + ): + raise BundleError("invalid_verify_option", "verify mode never emits or derives admission metadata") + if (args.admission_manifest is None) != (args.admission_manifest_sha256 is None): + raise BundleError( + "incomplete_admission_pin", + "verify mode requires both --admission-manifest and --admission-manifest-sha256", + ) + return ledger + + +def build(args: argparse.Namespace) -> dict[str, Any]: + _require_build_arguments(args) + ledger_output = _prepare_new_output(args.output_ledger, label="--output-ledger") + admission_output = ( + _prepare_new_output(args.admission_output, label="--admission-output") if args.emit_admission_manifest else None + ) + + genesis_dump = _regular_input(args.genesis_dump, label="--genesis-dump") + genesis_manifest = _regular_input(args.genesis_manifest, label="--genesis-manifest") + final_manifest = _regular_input(args.final_manifest, label="--final-manifest") + review_note_file = ( + _regular_input(args.review_note_file, label="--review-note-file") if args.emit_admission_manifest else None + ) + material_paths = [_regular_input(path, label=f"material {index}") for index, path in enumerate(args.materials, 1)] + + path_inventory = [ + (genesis_dump, "--genesis-dump"), + (genesis_manifest, "--genesis-manifest"), + (final_manifest, "--final-manifest"), + (ledger_output, "--output-ledger"), + *[(path, f"material {index}") for index, path in enumerate(material_paths, 1)], + ] + if review_note_file is not None: + path_inventory.append((review_note_file, "--review-note-file")) + if admission_output is not None: + path_inventory.append((admission_output, "--admission-output")) + _assert_distinct_paths(path_inventory) + + _validate_dump_and_manifests(genesis_dump, genesis_manifest, final_manifest) + materials = _validate_materials(material_paths) + ledger_payload = _ledger_payload( + ledger_path=ledger_output, + genesis_dump=genesis_dump, + genesis_manifest=genesis_manifest, + final_manifest=final_manifest, + materials=materials, + ) + + staged_files: list[StagedFile] = [] + published: list[PublishedFile] = [] + try: + staged_ledger = _stage_private_json(ledger_output, ledger_payload) + staged_files.append(staged_ledger) + ledger_sha256 = _sha256_bytes(staged_ledger.payload) + + staged_admission: StagedFile | None = None + admission_sha256: str | None = None + if admission_output is not None and review_note_file is not None: + admission_payload = _admission_payload( + args, + ledger_sha256=ledger_sha256, + materials=materials, + review_note_file=review_note_file, + ) + staged_admission = _stage_private_json(admission_output, admission_payload) + staged_files.append(staged_admission) + admission_sha256 = _sha256_bytes(staged_admission.payload) + + _preflight_bundle( + genesis_dump=genesis_dump, + genesis_manifest=genesis_manifest, + ledger=staged_ledger.temporary, + ledger_sha256=ledger_sha256, + admission_manifest=staged_admission.temporary if staged_admission is not None else None, + admission_manifest_sha256=admission_sha256, + expected_final_manifest=final_manifest, + expected_materials=material_paths, + ) + published = _publish_no_replace(staged_files) + _preflight_bundle( + genesis_dump=genesis_dump, + genesis_manifest=genesis_manifest, + ledger=ledger_output, + ledger_sha256=ledger_sha256, + admission_manifest=admission_output, + admission_manifest_sha256=admission_sha256, + expected_final_manifest=final_manifest, + expected_materials=material_paths, + ) + except BaseException: + if published: + _rollback_published(published) + raise + finally: + for staged in staged_files: + try: + staged.temporary.unlink() + except FileNotFoundError: + pass + + return { + "artifact": RECEIPT_ARTIFACT, + "status": "pass", + "action": "build", + "entry_count": len(materials), + "ledger_sha256": ledger_sha256, + "admission": _admission_summary( + emitted=admission_output is not None, + sha256=admission_sha256, + ), + } + + +def verify(args: argparse.Namespace) -> dict[str, Any]: + raw_ledger = _require_verify_arguments(args) + ledger = _regular_input(raw_ledger, label="ledger", private_json=True) + genesis_dump = _regular_input(args.genesis_dump, label="--genesis-dump") + genesis_manifest = _regular_input(args.genesis_manifest, label="--genesis-manifest") + expected_final_manifest = ( + _regular_input(args.final_manifest, label="--final-manifest") if args.final_manifest is not None else None + ) + expected_materials = [ + _regular_input(path, label=f"material {index}") for index, path in enumerate(args.materials, 1) + ] + admission_manifest = ( + _regular_input(args.admission_manifest, label="--admission-manifest", private_json=True) + if args.admission_manifest is not None + else None + ) + + _verify_deterministic_json(ledger, label="ledger") + actual_ledger_sha256 = genesis_rebuild.canonical_rebuild.sha256_file(ledger) + if actual_ledger_sha256 != args.ledger_sha256: + raise BundleError("ledger_hash_mismatch", "--ledger-sha256 does not match the exact ledger bytes") + if admission_manifest is not None: + _verify_deterministic_json(admission_manifest, label="admission manifest") + actual_admission_sha256 = genesis_rebuild.canonical_rebuild.sha256_file(admission_manifest) + if actual_admission_sha256 != args.admission_manifest_sha256: + raise BundleError( + "admission_hash_mismatch", + "--admission-manifest-sha256 does not match the exact admission bytes", + ) + else: + actual_admission_sha256 = None + + bundle = _preflight_bundle( + genesis_dump=genesis_dump, + genesis_manifest=genesis_manifest, + ledger=ledger, + ledger_sha256=args.ledger_sha256, + admission_manifest=admission_manifest, + admission_manifest_sha256=args.admission_manifest_sha256, + expected_final_manifest=expected_final_manifest, + expected_materials=expected_materials or None, + ) + return { + "artifact": RECEIPT_ARTIFACT, + "status": "pass", + "action": "verify", + "entry_count": len(bundle.entries), + "ledger_sha256": actual_ledger_sha256, + "admission": _admission_summary( + emitted=admission_manifest is not None, + sha256=actual_admission_sha256, + ), + } + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--genesis-dump", required=True, type=Path) + parser.add_argument("--genesis-manifest", required=True, type=Path) + parser.add_argument("--final-manifest", type=Path) + parser.add_argument("--material", dest="materials", action="append", default=[], type=Path) + parser.add_argument("--output-ledger", "--ledger-output", "--output", dest="output_ledger", type=Path) + parser.add_argument( + "--verify-ledger", + nargs="?", + const="", + metavar="LEDGER", + help="verify an existing ledger; path may instead be supplied with --ledger or --output-ledger", + ) + parser.add_argument("--ledger", dest="verify_ledger_alias", type=Path) + parser.add_argument("--ledger-sha256") + + parser.add_argument( + "--emit-admission-manifest", + "--write-admission-manifest", + dest="emit_admission_manifest", + action="store_true", + help="emit unauthenticated operator-supplied review metadata; this never grants canonical eligibility", + ) + parser.add_argument("--admission-output", "--output-admission-manifest", dest="admission_output", type=Path) + parser.add_argument("--reviewed-by") + parser.add_argument("--reviewed-by-agent-id", "--reviewer-agent-id", dest="reviewed_by_agent_id") + parser.add_argument("--reviewed-at") + parser.add_argument("--review-note-file", type=Path) + parser.add_argument("--admission-scope", choices=sorted(genesis_rebuild.ADMISSION_SCOPES)) + parser.add_argument("--admission-basis", choices=sorted(genesis_rebuild.ADMISSION_BASES)) + + parser.add_argument("--admission-manifest", type=Path) + parser.add_argument("--admission-manifest-sha256") + return parser.parse_args(argv) + + +def run(args: argparse.Namespace) -> dict[str, Any]: + verify_mode = args.verify_ledger is not None or args.verify_ledger_alias is not None + return verify(args) if verify_mode else build(args) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + payload = run(args) + except BundleError as exc: + print( + json.dumps( + { + "artifact": RECEIPT_ARTIFACT, + "status": "refused", + "code": exc.code, + "message": exc.safe_message, + }, + indent=2, + sort_keys=True, + ), + file=sys.stderr, + ) + return 1 + except OSError: + print( + json.dumps( + { + "artifact": RECEIPT_ARTIFACT, + "status": "refused", + "code": "private_file_operation_failed", + "message": "private bundle file operation failed; details withheld", + }, + indent=2, + sort_keys=True, + ), + file=sys.stderr, + ) + return 1 + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/rollback_teleo_v3_epistemic_contract.sql b/ops/rollback_teleo_v3_epistemic_contract.sql new file mode 100644 index 0000000..e17c23c --- /dev/null +++ b/ops/rollback_teleo_v3_epistemic_contract.sql @@ -0,0 +1,357 @@ +-- Schema-only rollback for ops/teleo_v3_epistemic_contract.sql. +-- +-- The rollback refuses to discard proposal-linked V3 data. A disposable +-- verifier may delete its own canary rows first; a real migration needs an +-- explicit export/port decision before this file can run. + +\set ON_ERROR_STOP on +set timezone = 'UTC'; + +begin; + +do $ownership_guard$ +declare + v_artifacts_present boolean; + v_ledger_owner_valid boolean; + v_ledger_row_valid boolean; + v_missing_objects text; +begin + select + pg_catalog.to_regclass('kb_stage.teleo_v3_installation_ownership') is not null + or pg_catalog.to_regclass('public.claim_evidence_assessments') is not null + or exists ( + select 1 + from information_schema.columns columns + where columns.table_schema = 'public' + and (columns.table_name, columns.column_name) in ( + ('claims', 'owner_agent_id'), + ('claims', 'proposition'), + ('claims', 'body_markdown'), + ('claims', 'scope'), + ('claims', 'access_scope'), + ('claims', 'admission_state'), + ('claims', 'revision_criteria'), + ('claims', 'revision_kind'), + ('claims', 'supersedes_id'), + ('claims', 'accepted_by_proposal_id'), + ('sources', 'canonical_url'), + ('sources', 'storage_uri'), + ('sources', 'content_hash'), + ('sources', 'access_scope'), + ('sources', 'ingestion_origin'), + ('sources', 'provenance_status'), + ('sources', 'source_class'), + ('sources', 'auto_promotion_policy'), + ('sources', 'accepted_by_proposal_id'), + ('claim_evidence', 'excerpt'), + ('claim_evidence', 'locator_json'), + ('claim_evidence', 'source_content_hash'), + ('claim_evidence', 'rationale'), + ('claim_evidence', 'polarity'), + ('claim_evidence', 'accepted_from_proposal_id'), + ('claim_edges', 'relation_type'), + ('claim_edges', 'rationale'), + ('claim_edges', 'scope_or_conditions'), + ('claim_edges', 'accepted_by_proposal_id') + ) + ) + or exists ( + select 1 + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + where namespace.nspname = 'kb_stage' + and procedure.proname like 'teleo\_v3\_%' escape '\' + ) + or exists ( + select 1 + from pg_catalog.pg_trigger trigger_row + where not trigger_row.tgisinternal + and trigger_row.tgname like 'teleo\_v3\_%' escape '\' + ) + or exists ( + select 1 + from pg_catalog.pg_constraint constraint_row + where constraint_row.conname like 'teleo\_v3\_%' escape '\' + ) + or exists ( + select 1 + from pg_catalog.pg_class index_row + where index_row.relkind = 'i' + and index_row.relname like 'teleo\_v3\_%' escape '\' + ) + into v_artifacts_present; + + if pg_catalog.to_regclass('kb_stage.teleo_v3_installation_ownership') is null then + if v_artifacts_present then + raise exception using + errcode = '55000', + message = 'teleo_v3 rollback refused: V3-named objects lack exact installation ownership'; + end if; + return; + end if; + + select owner_role.rolname = 'kb_gate_owner' + into v_ledger_owner_valid + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = relation.relowner + where namespace.nspname = 'kb_stage' + and relation.relname = 'teleo_v3_installation_ownership' + and relation.relkind in ('r', 'p'); + if not coalesce(v_ledger_owner_valid, false) then + raise exception using + errcode = '55000', + message = 'teleo_v3 rollback refused: installation ownership ledger owner is invalid'; + end if; + + execute $ownership_query$ + select count(*) = 1 + and bool_and( + contract_name = 'livingip.teleo-v3-epistemic-contract' + and contract_version = 1 + and installation_token = 'collision-free-install-v1' + ) + from kb_stage.teleo_v3_installation_ownership + $ownership_query$ into v_ledger_row_valid; + if not coalesce(v_ledger_row_valid, false) then + raise exception using + errcode = '55000', + message = 'teleo_v3 rollback refused: installation ownership ledger is not exact'; + end if; + + with expected_columns(table_name, column_name) as ( + values + ('claims', 'owner_agent_id'), + ('claims', 'proposition'), + ('claims', 'body_markdown'), + ('claims', 'scope'), + ('claims', 'access_scope'), + ('claims', 'admission_state'), + ('claims', 'revision_criteria'), + ('claims', 'revision_kind'), + ('claims', 'supersedes_id'), + ('claims', 'accepted_by_proposal_id'), + ('sources', 'canonical_url'), + ('sources', 'storage_uri'), + ('sources', 'content_hash'), + ('sources', 'access_scope'), + ('sources', 'ingestion_origin'), + ('sources', 'provenance_status'), + ('sources', 'source_class'), + ('sources', 'auto_promotion_policy'), + ('sources', 'accepted_by_proposal_id'), + ('claim_evidence', 'excerpt'), + ('claim_evidence', 'locator_json'), + ('claim_evidence', 'source_content_hash'), + ('claim_evidence', 'rationale'), + ('claim_evidence', 'polarity'), + ('claim_evidence', 'accepted_from_proposal_id'), + ('claim_edges', 'relation_type'), + ('claim_edges', 'rationale'), + ('claim_edges', 'scope_or_conditions'), + ('claim_edges', 'accepted_by_proposal_id') + ), missing(object_name) as ( + select pg_catalog.format('%I.%I', expected.table_name, expected.column_name) + from expected_columns expected + where not exists ( + select 1 + from information_schema.columns columns + where columns.table_schema = 'public' + and columns.table_name = expected.table_name + and columns.column_name = expected.column_name + ) + union all + select 'public.claim_evidence_assessments' + where pg_catalog.to_regclass('public.claim_evidence_assessments') is null + ) + select pg_catalog.string_agg(object_name, ', ' order by object_name) + into v_missing_objects + from missing; + if v_missing_objects is not null then + raise exception using + errcode = '55000', + message = pg_catalog.format( + 'teleo_v3 rollback refused: installation-owned object set is incomplete: %s', + v_missing_objects + ); + end if; +end +$ownership_guard$; + +do $data_guard$ +declare + contract_rows_exist boolean := false; +begin + if to_regclass('public.claim_evidence_assessments') is not null then + execute 'select exists (select 1 from public.claim_evidence_assessments)' + into contract_rows_exist; + if contract_rows_exist then + raise exception using + errcode = '55000', + message = 'teleo_v3 rollback refused: claim_evidence_assessments contains V3 data'; + end if; + end if; + + if exists ( + select 1 from information_schema.columns + where table_schema = 'public' and table_name = 'claim_edges' + and column_name = 'accepted_by_proposal_id' + ) then + execute 'select exists (select 1 from public.claim_edges where accepted_by_proposal_id is not null)' + into contract_rows_exist; + if contract_rows_exist then + raise exception using + errcode = '55000', + message = 'teleo_v3 rollback refused: claim_edges contains proposal-linked V3 data'; + end if; + end if; + + if exists ( + select 1 from information_schema.columns + where table_schema = 'public' and table_name = 'claim_evidence' + and column_name = 'accepted_from_proposal_id' + ) then + execute 'select exists (select 1 from public.claim_evidence where accepted_from_proposal_id is not null)' + into contract_rows_exist; + if contract_rows_exist then + raise exception using + errcode = '55000', + message = 'teleo_v3 rollback refused: claim_evidence contains proposal-linked V3 data'; + end if; + end if; + + if exists ( + select 1 from information_schema.columns + where table_schema = 'public' and table_name = 'sources' + and column_name = 'accepted_by_proposal_id' + ) then + execute 'select exists (select 1 from public.sources where accepted_by_proposal_id is not null)' + into contract_rows_exist; + if contract_rows_exist then + raise exception using + errcode = '55000', + message = 'teleo_v3 rollback refused: sources contains proposal-linked V3 data'; + end if; + end if; + + if exists ( + select 1 from information_schema.columns + where table_schema = 'public' and table_name = 'claims' + and column_name = 'admission_state' + ) then + execute $query$ + select exists ( + select 1 from public.claims + where admission_state = 'admitted' or accepted_by_proposal_id is not null + ) + $query$ into contract_rows_exist; + if contract_rows_exist then + raise exception using + errcode = '55000', + message = 'teleo_v3 rollback refused: claims contains admitted or proposal-linked V3 data'; + end if; + end if; +end +$data_guard$; + +drop trigger if exists teleo_v3_guard_linked_proposal_decision on kb_stage.kb_proposals; +drop trigger if exists teleo_v3_00_enforce_apply_contract_cutover on kb_stage.kb_proposals; +alter table kb_stage.kb_proposals + drop constraint if exists teleo_v3_proposal_apply_receipt_check; + +drop trigger if exists teleo_v3_edge_immutable on public.claim_edges; +drop trigger if exists teleo_v3_edge_reject_delete on public.claim_edges; +drop trigger if exists teleo_v3_edge_validate_endpoints on public.claim_edges; +drop trigger if exists teleo_v3_edge_require_accepted_proposal on public.claim_edges; +drop trigger if exists teleo_v3_00_edge_require_linked_insert on public.claim_edges; + +drop trigger if exists teleo_v3_evidence_immutable on public.claim_evidence; +drop trigger if exists teleo_v3_evidence_reject_delete on public.claim_evidence; +drop trigger if exists teleo_v3_evidence_validate_provenance on public.claim_evidence; +drop trigger if exists teleo_v3_evidence_require_accepted_proposal on public.claim_evidence; +drop trigger if exists teleo_v3_00_evidence_require_linked_insert on public.claim_evidence; + +drop trigger if exists teleo_v3_source_immutable on public.sources; +drop trigger if exists teleo_v3_source_reject_delete on public.sources; +drop trigger if exists teleo_v3_source_require_accepted_proposal on public.sources; +drop trigger if exists teleo_v3_00_source_require_linked_insert on public.sources; + +drop trigger if exists teleo_v3_claim_immutable on public.claims; +drop trigger if exists teleo_v3_claim_reject_delete on public.claims; +drop trigger if exists teleo_v3_claim_require_accepted_proposal on public.claims; +drop trigger if exists teleo_v3_00_claim_require_linked_insert on public.claims; + +drop index if exists public.teleo_v3_edge_typed_adjacency_idx; +drop index if exists public.teleo_v3_evidence_claim_polarity_idx; +drop index if exists public.teleo_v3_assessment_one_successor_idx; +drop index if exists public.teleo_v3_claim_one_admitted_successor_idx; + +drop table if exists public.claim_evidence_assessments; + +alter table public.claim_edges + drop constraint if exists teleo_v3_edge_contract_check, + drop constraint if exists teleo_v3_edge_accepted_proposal_fkey, + drop column if exists relation_type, + drop column if exists rationale, + drop column if exists scope_or_conditions, + drop column if exists accepted_by_proposal_id; + +alter table public.claim_evidence + drop constraint if exists teleo_v3_evidence_contract_check, + drop constraint if exists teleo_v3_evidence_accepted_proposal_fkey, + drop column if exists excerpt, + drop column if exists locator_json, + drop column if exists source_content_hash, + drop column if exists rationale, + drop column if exists polarity, + drop column if exists accepted_from_proposal_id; + +alter table public.sources + drop constraint if exists teleo_v3_source_contract_check, + drop constraint if exists teleo_v3_source_accepted_proposal_fkey, + drop column if exists canonical_url, + drop column if exists storage_uri, + drop column if exists content_hash, + drop column if exists access_scope, + drop column if exists ingestion_origin, + drop column if exists provenance_status, + drop column if exists source_class, + drop column if exists auto_promotion_policy, + drop column if exists accepted_by_proposal_id; + +alter table public.claims + drop constraint if exists teleo_v3_claim_admission_check, + drop constraint if exists teleo_v3_claim_accepted_proposal_fkey, + drop constraint if exists teleo_v3_claim_supersedes_fkey, + drop constraint if exists teleo_v3_claim_owner_agent_fkey, + drop column if exists owner_agent_id, + drop column if exists proposition, + drop column if exists body_markdown, + drop column if exists scope, + drop column if exists access_scope, + drop column if exists admission_state, + drop column if exists revision_criteria, + drop column if exists revision_kind, + drop column if exists supersedes_id, + drop column if exists accepted_by_proposal_id; + +drop function if exists kb_stage.teleo_v3_guard_linked_proposal_decision(); +drop function if exists kb_stage.teleo_v3_acquire_protected_relation_locks(); +drop function if exists kb_stage.teleo_v3_enforce_apply_contract_cutover(); +drop function if exists kb_stage.teleo_v3_validate_evidence_assessment(); +drop function if exists kb_stage.teleo_v3_validate_claim_edge(); +drop function if exists kb_stage.teleo_v3_validate_claim_evidence(); +drop function if exists kb_stage.teleo_v3_guard_immutable_fields(); +drop function if exists kb_stage.teleo_v3_reject_delete(); +drop function if exists kb_stage.teleo_v3_require_linked_insert(); +drop function if exists kb_stage.teleo_v3_require_accepted_proposal(); + +drop table if exists kb_stage.teleo_v3_installation_ownership; + +commit; + +select jsonb_build_object( + 'contract', 'livingip.teleo-v3-epistemic-contract', + 'version', 1, + 'migration', 'rolled_back' +)::text; diff --git a/ops/run_local_corpus_knowledge_rebuild.py b/ops/run_local_corpus_knowledge_rebuild.py new file mode 100755 index 0000000..d372d9c --- /dev/null +++ b/ops/run_local_corpus_knowledge_rebuild.py @@ -0,0 +1,1363 @@ +#!/usr/bin/env python3 +"""Snapshot a local UTF-8 corpus for later, explicit semantic extraction. + +Build mode writes only exact content-addressed source snapshots, syntactic +section locators, an extraction-work index, and a receipt. Verification mode is +read-only. Neither mode creates claims, proposals, admissions, or database +writes. +""" + +from __future__ import annotations + +import argparse +import codecs +import copy +import hashlib +import json +import os +import re +import stat +import unicodedata +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +SOURCE_MANIFEST_SCHEMA = "livingip.localCorpusSourceManifest.v2" +EXTRACTION_WORK_INDEX_SCHEMA = "livingip.localCorpusExtractionWorkIndex.v1" +RECEIPT_SCHEMA = "livingip.localCorpusInventoryReceipt.v2" +STATUS_SCHEMA = "livingip.localCorpusInventoryStatus.v2" + +SOURCE_MANIFEST_PATH = "source-manifest.json" +EXTRACTION_WORK_INDEX_PATH = "extraction-work-index.json" +RECEIPT_PATH = "receipt.json" +INCOMPLETE_MARKER = ".incomplete" +SNAPSHOT_ROOT = PurePosixPath("snapshots/sha256") +MODE_SOURCE_INVENTORY_ONLY = "source-inventory-only" +WORK_TYPE = "source_section_for_semantic_extraction" + +DEFAULT_MAX_SOURCE_BYTES = 16 * 1024 * 1024 +DEFAULT_MAX_TOTAL_BYTES = 1024 * 1024 * 1024 +DEFAULT_MAX_SOURCE_COUNT = 20_000 +DEFAULT_MAX_SECTIONS_PER_SOURCE = 10_000 +DEFAULT_MAX_WORK_ITEMS = 250_000 + +HARD_MAX_SOURCE_BYTES = 256 * 1024 * 1024 +HARD_MAX_TOTAL_BYTES = 16 * 1024 * 1024 * 1024 +HARD_MAX_SOURCE_COUNT = 100_000 +HARD_MAX_SECTIONS_PER_SOURCE = 100_000 +HARD_MAX_WORK_ITEMS = 2_000_000 +MAX_METADATA_FILE_BYTES = 512 * 1024 * 1024 +COPY_CHUNK_BYTES = 1024 * 1024 + +ALLOWED_EXTENSIONS = frozenset( + { + ".adoc", + ".csv", + ".htm", + ".html", + ".json", + ".jsonl", + ".markdown", + ".md", + ".rst", + ".text", + ".toml", + ".tsv", + ".txt", + ".xml", + ".yaml", + ".yml", + } +) +DOCUMENT_EXTENSIONS = frozenset({".adoc", ".markdown", ".md", ".rst"}) +MARKDOWN_HEADING_RE = re.compile(r"^(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$") +ASCIIDOC_HEADING_RE = re.compile(r"^(={1,6})[ \t]+(.+?)[ \t]*$") +RST_UNDERLINE_RE = re.compile(r'^[=\-~^"`:+*#<>_]{3,}$') +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + +NO_WRITE_FLAGS = { + "semantic_claim_extraction_performed": False, + "database_write_performed": False, + "admission_performed": False, +} + + +class RebuildError(RuntimeError): + """Fail-closed inventory error with a stable public code.""" + + def __init__(self, code: str, safe_message: str) -> None: + super().__init__(safe_message) + self.code = code + self.safe_message = safe_message + + +@dataclass(frozen=True) +class ResourceLimits: + max_source_bytes: int = DEFAULT_MAX_SOURCE_BYTES + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES + max_source_count: int = DEFAULT_MAX_SOURCE_COUNT + max_sections_per_source: int = DEFAULT_MAX_SECTIONS_PER_SOURCE + max_work_items: int = DEFAULT_MAX_WORK_ITEMS + + def as_dict(self) -> dict[str, int]: + return { + "max_sections_per_source": self.max_sections_per_source, + "max_source_bytes": self.max_source_bytes, + "max_source_count": self.max_source_count, + "max_total_bytes": self.max_total_bytes, + "max_work_items": self.max_work_items, + } + + +@dataclass(frozen=True) +class SourceEntry: + parent_fd: int + name: str + relative_path: str + extension: str + + +@dataclass(frozen=True) +class ReservedOutput: + path: Path + device: int + inode: int + + +def canonical_json_bytes(value: Any) -> bytes: + try: + rendered = json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + except ValueError as exc: + raise RebuildError("invalid_json_value", "inventory metadata contains a non-finite JSON number") from exc + return rendered.encode("utf-8") + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256(canonical_json_bytes(value)).hexdigest() + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path: Path, *, max_bytes: int | None = None) -> str: + digest = hashlib.sha256() + total = 0 + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise RebuildError("bundle_file_unavailable", "a bundle file could not be opened safely") from exc + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode): + raise RebuildError("unsafe_bundle_entry", "a bundle entry is not a regular file") + while True: + chunk = os.read(descriptor, COPY_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if max_bytes is not None and total > max_bytes: + raise RebuildError("metadata_limit_exceeded", "a bundle metadata file exceeds the verifier limit") + digest.update(chunk) + finally: + os.close(descriptor) + return digest.hexdigest() + + +def rendered_json_bytes(value: Any) -> bytes: + try: + rendered = json.dumps(value, allow_nan=False, ensure_ascii=True, indent=2, sort_keys=True) + except ValueError as exc: + raise RebuildError("invalid_json_value", "inventory metadata contains a non-finite JSON number") from exc + return (rendered + "\n").encode("utf-8") + + +def _absolute_path(path: Path) -> Path: + expanded = path.expanduser() + return expanded if expanded.is_absolute() else Path.cwd() / expanded + + +def _assert_no_symlink_components(path: Path, *, label: str, allow_missing_leaf: bool = False) -> None: + absolute = _absolute_path(path) + current = Path(absolute.anchor) + parts = absolute.parts[1:] + for index, part in enumerate(parts): + current /= part + try: + metadata = current.lstat() + except FileNotFoundError: + if allow_missing_leaf and index == len(parts) - 1: + return + raise RebuildError("path_unavailable", f"{label} contains a missing component") from None + except OSError as exc: + raise RebuildError("path_unavailable", f"{label} could not be inspected") from exc + if stat.S_ISLNK(metadata.st_mode): + raise RebuildError("symlink_rejected", f"{label} contains a symlink component") + + +def _paths_overlap(left: Path, right: Path) -> bool: + return left == right or left in right.parents or right in left.parents + + +def _validate_roots(source_root: Path, output: Path) -> tuple[Path, Path]: + source_absolute = _absolute_path(source_root) + _assert_no_symlink_components(source_absolute, label="source root") + try: + source_metadata = source_absolute.lstat() + except OSError as exc: + raise RebuildError("invalid_source_root", "source root is unavailable") from exc + if not stat.S_ISDIR(source_metadata.st_mode): + raise RebuildError("invalid_source_root", "source root must be a non-symlink directory") + source_resolved = source_absolute.resolve(strict=True) + + output_absolute = _absolute_path(output) + _assert_no_symlink_components(output_absolute, label="output path", allow_missing_leaf=True) + output_parent = output_absolute.parent + try: + parent_metadata = output_parent.lstat() + except OSError as exc: + raise RebuildError("invalid_output_parent", "output parent is unavailable") from exc + if not stat.S_ISDIR(parent_metadata.st_mode): + raise RebuildError("invalid_output_parent", "output parent must be a non-symlink directory") + output_resolved = output_parent.resolve(strict=True) / output_absolute.name + if _paths_overlap(source_resolved, output_resolved): + raise RebuildError("output_overlap", "source root and output directory must be disjoint") + return source_resolved, output_absolute + + +def _validate_bundle_root(bundle: Path) -> Path: + absolute = _absolute_path(bundle) + _assert_no_symlink_components(absolute, label="bundle root") + try: + metadata = absolute.lstat() + except OSError as exc: + raise RebuildError("invalid_bundle_root", "bundle root is unavailable") from exc + if not stat.S_ISDIR(metadata.st_mode): + raise RebuildError("invalid_bundle_root", "bundle root must be a non-symlink directory") + return absolute.resolve(strict=True) + + +def _validate_relative_path(relative_path: str) -> str: + try: + relative_path.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise RebuildError("unsafe_relative_path", "source paths must be strict UTF-8") from exc + parsed = PurePosixPath(relative_path) + if parsed.is_absolute() or not relative_path or any(part in {"", ".", ".."} for part in parsed.parts): + raise RebuildError("unsafe_relative_path", "source paths must be normalized source-root-relative paths") + if any(unicodedata.category(character) == "Cc" for character in relative_path): + raise RebuildError("unsafe_relative_path", "source paths must not contain control characters") + return relative_path + + +def _normalized_path_key(relative_path: str) -> str: + return unicodedata.normalize("NFC", _validate_relative_path(relative_path)).casefold() + + +def _directory_flags() -> int: + return os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + + +def _iter_source_entries(source_root: Path) -> Iterator[SourceEntry]: + try: + root_fd = os.open(source_root, _directory_flags()) + except OSError as exc: + raise RebuildError("source_inventory_failed", "source root could not be opened safely") from exc + + def visit(directory_fd: int, prefix: PurePosixPath) -> Iterator[SourceEntry]: + try: + with os.scandir(directory_fd) as stream: + inspected: list[tuple[os.DirEntry[str], os.stat_result]] = [] + for entry in stream: + try: + inspected.append((entry, entry.stat(follow_symlinks=False))) + except OSError as exc: + raise RebuildError( + "source_inventory_failed", + "a source entry could not be inspected", + ) from exc + entries = sorted( + inspected, + key=lambda row: ( + row[0].name.encode("utf-8", errors="surrogateescape") + + (b"/" if stat.S_ISDIR(row[1].st_mode) else b"") + ), + ) + except OSError as exc: + raise RebuildError("source_inventory_failed", "source tree could not be inventoried") from exc + for entry, metadata in entries: + relative = (prefix / entry.name).as_posix() + _validate_relative_path(relative) + if stat.S_ISLNK(metadata.st_mode): + raise RebuildError("symlink_rejected", f"source entry is a symlink: {relative}") + if stat.S_ISDIR(metadata.st_mode): + try: + child_fd = os.open(entry.name, _directory_flags(), dir_fd=directory_fd) + except OSError as exc: + raise RebuildError( + "source_inventory_failed", "a source directory changed during inventory" + ) from exc + try: + yield from visit(child_fd, prefix / entry.name) + finally: + os.close(child_fd) + continue + if not stat.S_ISREG(metadata.st_mode): + raise RebuildError("unsupported_source_entry", f"source entry is not a regular file: {relative}") + extension = PurePosixPath(relative).suffix.casefold() + if extension not in ALLOWED_EXTENSIONS: + raise RebuildError("unsupported_source_file", f"source extension is not allowed: {relative}") + yield SourceEntry(directory_fd, entry.name, relative, extension) + + try: + yield from visit(root_fd, PurePosixPath()) + finally: + os.close(root_fd) + + +def _validate_limits(limits: ResourceLimits) -> ResourceLimits: + hard_limits = { + "max_sections_per_source": HARD_MAX_SECTIONS_PER_SOURCE, + "max_source_bytes": HARD_MAX_SOURCE_BYTES, + "max_source_count": HARD_MAX_SOURCE_COUNT, + "max_total_bytes": HARD_MAX_TOTAL_BYTES, + "max_work_items": HARD_MAX_WORK_ITEMS, + } + for name, value in limits.as_dict().items(): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise RebuildError("invalid_resource_limit", f"{name} must be a positive integer") + if value > hard_limits[name]: + raise RebuildError("invalid_resource_limit", f"{name} exceeds its hard safety bound") + if limits.max_source_bytes > limits.max_total_bytes: + raise RebuildError("invalid_resource_limit", "max_source_bytes must not exceed max_total_bytes") + if limits.max_sections_per_source > limits.max_work_items: + raise RebuildError("invalid_resource_limit", "max_sections_per_source must not exceed max_work_items") + return limits + + +def _limits_from_mapping(value: Mapping[str, Any]) -> ResourceLimits: + expected = { + "max_sections_per_source", + "max_source_bytes", + "max_source_count", + "max_total_bytes", + "max_work_items", + } + if set(value) != expected: + raise RebuildError("invalid_bundle_schema", "receipt resource limits do not match the supported schema") + return _validate_limits(ResourceLimits(**{name: value[name] for name in expected})) + + +def _assert_output_binding(reserved: ReservedOutput) -> None: + try: + metadata = reserved.path.lstat() + except OSError as exc: + raise RebuildError("output_binding_changed", "reserved output directory is no longer available") from exc + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_dev != reserved.device + or metadata.st_ino != reserved.inode + ): + raise RebuildError("output_binding_changed", "reserved output directory identity changed") + + +def _reserve_output(output: Path) -> ReservedOutput: + try: + os.mkdir(output, mode=0o700) + except FileExistsError as exc: + raise RebuildError("output_exists", "output directory must not already exist") from exc + except OSError as exc: + raise RebuildError("output_reservation_failed", "output directory could not be reserved") from exc + metadata = output.lstat() + reserved = ReservedOutput(output, metadata.st_dev, metadata.st_ino) + try: + _write_private_bytes(output / INCOMPLETE_MARKER, b"source inventory build incomplete\n") + except Exception: + _cleanup_reserved_output(reserved) + raise + return reserved + + +def _remove_tree_contents(directory_fd: int) -> None: + for name in os.listdir(directory_fd): + try: + metadata = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + continue + if stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode): + child_fd = os.open(name, _directory_flags(), dir_fd=directory_fd) + try: + _remove_tree_contents(child_fd) + finally: + os.close(child_fd) + os.rmdir(name, dir_fd=directory_fd) + else: + os.unlink(name, dir_fd=directory_fd) + + +def _cleanup_reserved_output(reserved: ReservedOutput) -> None: + try: + _assert_output_binding(reserved) + directory_fd = os.open(reserved.path, _directory_flags()) + except (OSError, RebuildError): + return + try: + opened = os.fstat(directory_fd) + if opened.st_dev != reserved.device or opened.st_ino != reserved.inode: + return + _remove_tree_contents(directory_fd) + except OSError: + return + finally: + os.close(directory_fd) + try: + _assert_output_binding(reserved) + os.rmdir(reserved.path) + except (OSError, RebuildError): + return + + +def _mkdir_private(path: Path) -> None: + try: + os.mkdir(path, mode=0o700) + except FileExistsError: + metadata = path.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise RebuildError("unsafe_output_entry", "an output directory path is occupied unsafely") from None + + +def _write_private_bytes(path: Path, content: bytes) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags, 0o600) + except OSError as exc: + raise RebuildError("output_write_failed", "a bundle artifact could not be created exclusively") from exc + try: + view = memoryview(content) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise RebuildError("output_write_failed", "a bundle artifact write did not complete") + view = view[written:] + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _write_private_json(path: Path, value: Any) -> None: + _write_private_bytes(path, rendered_json_bytes(value)) + + +def _validate_text(text: str) -> str: + if not text.strip(): + raise RebuildError("empty_source_file", "source file contains no non-whitespace UTF-8 text") + for character in text: + if unicodedata.category(character) == "Cc" and character not in {"\t", "\n", "\r"}: + raise RebuildError("binary_control_character", "source contains a disallowed control character") + for index, character in enumerate(text): + if character == "\r" and (index + 1 >= len(text) or text[index + 1] != "\n"): + raise RebuildError("unsupported_newline", "CR-only or lone-CR newlines are not supported") + return text + + +def _newline_style(text: str) -> str: + crlf = text.count("\r\n") + lf = text.count("\n") - crlf + if crlf and lf: + return "mixed-lf-crlf" + if crlf: + return "crlf" + return "lf" if lf else "none" + + +def _descriptor_signature(metadata: os.stat_result) -> tuple[int, int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mode, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def _copy_source_to_partial( + entry: SourceEntry, + partial_path: Path, + *, + limits: ResourceLimits, + total_before: int, +) -> tuple[str, int]: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + source_fd = os.open(entry.name, flags, dir_fd=entry.parent_fd) + except OSError as exc: + raise RebuildError("source_open_failed", f"source could not be opened safely: {entry.relative_path}") from exc + output_fd: int | None = None + try: + before = os.fstat(source_fd) + if not stat.S_ISREG(before.st_mode): + raise RebuildError("source_changed", f"source is no longer a regular file: {entry.relative_path}") + if before.st_size > limits.max_source_bytes: + raise RebuildError("source_byte_limit_exceeded", f"source exceeds max_source_bytes: {entry.relative_path}") + if total_before + before.st_size > limits.max_total_bytes: + raise RebuildError("total_byte_limit_exceeded", "corpus exceeds max_total_bytes") + write_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + output_fd = os.open(partial_path, write_flags, 0o600) + digest = hashlib.sha256() + decoder = codecs.getincrementaldecoder("utf-8")("strict") + copied = 0 + try: + while True: + chunk = os.read(source_fd, COPY_CHUNK_BYTES) + if not chunk: + break + copied += len(chunk) + if copied > limits.max_source_bytes: + raise RebuildError( + "source_byte_limit_exceeded", f"source exceeds max_source_bytes: {entry.relative_path}" + ) + if total_before + copied > limits.max_total_bytes: + raise RebuildError("total_byte_limit_exceeded", "corpus exceeds max_total_bytes") + decoder.decode(chunk) + digest.update(chunk) + view = memoryview(chunk) + while view: + written = os.write(output_fd, view) + if written <= 0: + raise RebuildError("output_write_failed", "source snapshot write did not complete") + view = view[written:] + decoder.decode(b"", final=True) + except UnicodeDecodeError as exc: + raise RebuildError("invalid_utf8_source", f"source is not strict UTF-8: {entry.relative_path}") from exc + os.fsync(output_fd) + after = os.fstat(source_fd) + if _descriptor_signature(before) != _descriptor_signature(after) or copied != after.st_size: + raise RebuildError( + "source_changed_during_read", f"source changed while being copied: {entry.relative_path}" + ) + return digest.hexdigest(), copied + finally: + if output_fd is not None: + os.close(output_fd) + os.close(source_fd) + + +def _snapshot_path_for_hash(digest: str) -> PurePosixPath: + if not SHA256_RE.fullmatch(digest): + raise RebuildError("invalid_snapshot_hash", "snapshot SHA-256 is malformed") + return SNAPSHOT_ROOT / digest[:2] / f"{digest}.utf8" + + +def _install_snapshot(output: Path, partial_path: Path, digest: str, byte_count: int) -> tuple[Path, bool]: + relative = _snapshot_path_for_hash(digest) + prefix_dir = output / Path(relative.parent) + _mkdir_private(prefix_dir) + destination = output / Path(relative) + try: + os.link(partial_path, destination, follow_symlinks=False) + except FileExistsError: + metadata = destination.lstat() + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise RebuildError("snapshot_collision", "snapshot hash path is occupied by a non-regular entry") from None + if metadata.st_size != byte_count or sha256_file(destination) != digest: + raise RebuildError("snapshot_collision", "snapshot hash path contains mismatched bytes") from None + deduplicated = True + except OSError as exc: + raise RebuildError("snapshot_publish_failed", "content-addressed snapshot could not be installed") from exc + else: + deduplicated = False + try: + partial_path.unlink() + except OSError as exc: + raise RebuildError("snapshot_cleanup_failed", "temporary snapshot could not be removed") from exc + return destination, deduplicated + + +def _read_snapshot_text(snapshot: Path, *, expected_sha256: str, expected_bytes: int) -> str: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(snapshot, flags) + except OSError as exc: + raise RebuildError("snapshot_read_failed", "source snapshot could not be reopened safely") from exc + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != expected_bytes: + raise RebuildError("snapshot_mismatch", "source snapshot size changed") + raw = bytearray() + digest = hashlib.sha256() + while True: + chunk = os.read(descriptor, COPY_CHUNK_BYTES) + if not chunk: + break + digest.update(chunk) + raw.extend(chunk) + finally: + os.close(descriptor) + if digest.hexdigest() != expected_sha256: + raise RebuildError("snapshot_mismatch", "source snapshot hash changed") + try: + text = bytes(raw).decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: # defensive: copy path already decoded incrementally + raise RebuildError("invalid_utf8_source", "source snapshot is not strict UTF-8") from exc + return _validate_text(text) + + +def _heading_rows(text: str, extension: str) -> list[tuple[int, str]]: + lines = text.splitlines(keepends=True) + rows: list[tuple[int, str]] = [] + offset = 0 + for index, line in enumerate(lines): + content = line.rstrip("\r\n") + match: re.Match[str] | None = None + if extension in {".markdown", ".md"}: + match = MARKDOWN_HEADING_RE.fullmatch(content) + elif extension == ".adoc": + match = ASCIIDOC_HEADING_RE.fullmatch(content) + if match is not None: + rows.append((offset, match.group(2).strip())) + elif extension == ".rst" and index + 1 < len(lines): + underline = lines[index + 1].rstrip("\r\n").strip() + title = content.strip() + if title and RST_UNDERLINE_RE.fullmatch(underline): + rows.append((offset, title)) + offset += len(line) + return rows + + +def _position_map(text: str, positions: Sequence[int]) -> dict[int, tuple[int, int]]: + result: dict[int, tuple[int, int]] = {} + cursor = 0 + byte_offset = 0 + line = 1 + for position in sorted(set(positions)): + fragment = text[cursor:position] + byte_offset += len(fragment.encode("utf-8")) + line += fragment.count("\n") + result[position] = (byte_offset, line) + cursor = position + return result + + +def _build_sections(text: str, extension: str, *, max_sections: int) -> list[dict[str, Any]]: + headings = _heading_rows(text, extension) if extension in DOCUMENT_EXTENSIONS else [] + boundaries: list[int] = [] + if headings and headings[0][0] > 0 and text[: headings[0][0]].strip(): + boundaries.append(0) + boundaries.extend(position for position, _heading in headings) + if not boundaries: + boundaries = [0] + + character_ranges: list[tuple[int, int]] = [] + for index, raw_start in enumerate(boundaries): + raw_end = boundaries[index + 1] if index + 1 < len(boundaries) else len(text) + raw_segment = text[raw_start:raw_end] + stripped = raw_segment.strip() + if not stripped: + continue + leading = len(raw_segment) - len(raw_segment.lstrip()) + start = raw_start + leading + end = start + len(stripped) + character_ranges.append((start, end)) + if not character_ranges: + raise RebuildError("empty_source_file", "source file contains no non-whitespace UTF-8 text") + if len(character_ranges) > max_sections: + raise RebuildError("section_limit_exceeded", "source exceeds max_sections_per_source") + + positions = [position for start, end in character_ranges for position in (start, end - 1)] + mapped = _position_map(text, positions) + sections: list[dict[str, Any]] = [] + for ordinal, (start, end) in enumerate(character_ranges, start=1): + start_byte, line_start = mapped[start] + last_byte, line_end = mapped[end - 1] + encoded_last = text[end - 1].encode("utf-8") + end_byte = last_byte + len(encoded_last) + quote = text[start:end].encode("utf-8") + if start_byte + len(quote) != end_byte: + raise RebuildError("locator_construction_failed", "UTF-8 locator construction was inconsistent") + sections.append( + { + "line_end": line_end, + "line_start": line_start, + "quote_sha256": sha256_bytes(quote), + "section_ordinal": ordinal, + "utf8_byte_end": end_byte, + "utf8_byte_start": start_byte, + } + ) + return sections + + +def _sectioning_strategy(extension: str) -> str: + if extension in {".markdown", ".md"}: + return "markdown_atx_heading_or_whole_source" + if extension == ".adoc": + return "asciidoc_heading_or_whole_source" + if extension == ".rst": + return "rst_underline_heading_or_whole_source" + return "whole_utf8_source" + + +def _work_item(source: Mapping[str, Any], locator: Mapping[str, Any]) -> dict[str, Any]: + identity = { + "artifact_sha256": source["artifact_sha256"], + "locator": dict(locator), + "relative_path": source["relative_path"], + "snapshot_path": source["snapshot_path"], + "work_type": WORK_TYPE, + } + return { + "artifact_sha256": source["artifact_sha256"], + "locator": dict(locator), + "relative_path": source["relative_path"], + "snapshot_path": source["snapshot_path"], + "work_item_id": canonical_sha256(identity), + "work_type": WORK_TYPE, + **NO_WRITE_FLAGS, + } + + +def _artifact_records(root: Path, paths: Sequence[str]) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for relative in sorted(paths, key=lambda value: value.encode("utf-8")): + validated = _validate_relative_path(relative) + path = root / Path(validated) + metadata = path.lstat() + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise RebuildError("unsafe_bundle_entry", "bundle artifact is not a regular file") + records.append({"bytes": metadata.st_size, "path": validated, "sha256": sha256_file(path)}) + return records + + +def _after_source_snapshot(_relative_path: str, _source_root: Path, _snapshot: Path) -> None: + """Test seam after descriptor copy; production intentionally does nothing.""" + + +def _after_output_reservation(_reserved: ReservedOutput) -> None: + """Test seam after atomic reservation; production intentionally does nothing.""" + + +def build_source_inventory_bundle( + source_root: Path, + output: Path, + *, + mode: str = MODE_SOURCE_INVENTORY_ONLY, + limits: ResourceLimits | None = None, +) -> dict[str, Any]: + if mode != MODE_SOURCE_INVENTORY_ONLY: + raise RebuildError("unsupported_mode", "this command supports source-inventory-only mode") + checked_limits = _validate_limits(limits or ResourceLimits()) + source_resolved, output_absolute = _validate_roots(source_root, output) + reserved = _reserve_output(output_absolute) + try: + _after_output_reservation(reserved) + _assert_output_binding(reserved) + _mkdir_private(output_absolute / "snapshots") + _mkdir_private(output_absolute / "snapshots" / "sha256") + + sources: list[dict[str, Any]] = [] + work_items: list[dict[str, Any]] = [] + seen_paths: set[str] = set() + snapshot_paths: set[str] = set() + total_bytes = 0 + + for ordinal, entry in enumerate(_iter_source_entries(source_resolved), start=1): + if ordinal > checked_limits.max_source_count: + raise RebuildError("source_count_limit_exceeded", "corpus exceeds max_source_count") + normalized = _normalized_path_key(entry.relative_path) + if normalized in seen_paths: + raise RebuildError( + "duplicate_relative_path", + "source tree contains duplicate case-folded or Unicode-normalized relative paths", + ) + seen_paths.add(normalized) + + partial = output_absolute / "snapshots" / f".partial-{ordinal:08d}" + digest, byte_count = _copy_source_to_partial( + entry, + partial, + limits=checked_limits, + total_before=total_bytes, + ) + total_bytes += byte_count + snapshot, _deduplicated = _install_snapshot(output_absolute, partial, digest, byte_count) + snapshot_relative = snapshot.relative_to(output_absolute).as_posix() + snapshot_paths.add(snapshot_relative) + _after_source_snapshot(entry.relative_path, source_resolved, snapshot) + + text = _read_snapshot_text(snapshot, expected_sha256=digest, expected_bytes=byte_count) + sections = _build_sections( + text, + entry.extension, + max_sections=checked_limits.max_sections_per_source, + ) + if len(work_items) + len(sections) > checked_limits.max_work_items: + raise RebuildError("work_item_limit_exceeded", "corpus exceeds max_work_items") + source_row: dict[str, Any] = { + "artifact_sha256": digest, + "bytes": byte_count, + "content_validation": "strict_utf8_no_binary_controls", + "extension": entry.extension, + "format_syntax_validated": False, + "newline_style": _newline_style(text), + "relative_path": entry.relative_path, + "section_count": len(sections), + "sectioning_strategy": _sectioning_strategy(entry.extension), + "sections": sections, + "snapshot_path": snapshot_relative, + "source_path_binding": "source_root_relative_posix_path_plus_exact_snapshot_sha256", + } + sources.append(source_row) + work_items.extend(_work_item(source_row, locator) for locator in sections) + del text, sections + + if not sources: + raise RebuildError("empty_source_root", "source root contains no supported source files") + _assert_output_binding(reserved) + + source_manifest = { + "schema": SOURCE_MANIFEST_SCHEMA, + "mode": MODE_SOURCE_INVENTORY_ONLY, + **NO_WRITE_FLAGS, + "source_snapshot_semantics": { + "concurrent_directory_additions_or_removals_may_be_absent": True, + "directory_snapshot_atomic": False, + "guarantee": "each listed path is bound to the exact bytes copied from one stable file descriptor", + }, + "resource_limits": checked_limits.as_dict(), + "source_count": len(sources), + "total_source_bytes": total_bytes, + "unique_snapshot_count": len(snapshot_paths), + "duplicate_content_source_count": len(sources) - len(snapshot_paths), + "unsupported_files_ignored": False, + "symlinks_followed": False, + "sources": sources, + } + source_manifest_file = output_absolute / SOURCE_MANIFEST_PATH + _write_private_json(source_manifest_file, source_manifest) + + extraction_index = { + "schema": EXTRACTION_WORK_INDEX_SCHEMA, + "mode": MODE_SOURCE_INVENTORY_ONLY, + "work_type": WORK_TYPE, + **NO_WRITE_FLAGS, + "source_manifest": { + "path": SOURCE_MANIFEST_PATH, + "sha256": sha256_file(source_manifest_file), + }, + "source_count": len(sources), + "work_item_count": len(work_items), + "items": work_items, + } + extraction_index_file = output_absolute / EXTRACTION_WORK_INDEX_PATH + _write_private_json(extraction_index_file, extraction_index) + del work_items + + artifact_paths = [SOURCE_MANIFEST_PATH, EXTRACTION_WORK_INDEX_PATH, *snapshot_paths] + artifacts = _artifact_records(output_absolute, artifact_paths) + receipt = { + "schema": RECEIPT_SCHEMA, + "status": "pass", + "result": "source_inventory_bundle_ready", + "mode": MODE_SOURCE_INVENTORY_ONLY, + "deterministic": True, + **NO_WRITE_FLAGS, + "source_snapshot_atomic": False, + "source_count": len(sources), + "work_item_count": extraction_index["work_item_count"], + "total_source_bytes": total_bytes, + "unique_snapshot_count": len(snapshot_paths), + "resource_limits": checked_limits.as_dict(), + "outputs": { + "artifact_count": len(artifacts), + "artifacts": artifacts, + "extraction_work_index_path": EXTRACTION_WORK_INDEX_PATH, + "source_manifest_path": SOURCE_MANIFEST_PATH, + }, + "hashes": { + "bundle_tree_sha256": canonical_sha256(artifacts), + "extraction_work_index_file_sha256": sha256_file(extraction_index_file), + "source_manifest_file_sha256": sha256_file(source_manifest_file), + }, + } + receipt["hashes"]["receipt_content_sha256"] = canonical_sha256(receipt) + + _assert_output_binding(reserved) + _write_private_json(output_absolute / RECEIPT_PATH, receipt) + _assert_output_binding(reserved) + (output_absolute / INCOMPLETE_MARKER).unlink() + _assert_output_binding(reserved) + verified = verify_bundle(output_absolute) + return { + "bundle_path": output_absolute, + "receipt": receipt, + "receipt_file_sha256": sha256_file(output_absolute / RECEIPT_PATH), + "verification": verified, + } + except Exception: + _cleanup_reserved_output(reserved) + raise + + +def _load_json(path: Path) -> dict[str, Any]: + try: + metadata = path.lstat() + except OSError as exc: + raise RebuildError("missing_bundle_artifact", "a required bundle artifact is missing") from exc + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise RebuildError("unsafe_bundle_entry", "a required bundle artifact is not a regular file") + if metadata.st_size > MAX_METADATA_FILE_BYTES: + raise RebuildError("metadata_limit_exceeded", "a bundle metadata file exceeds the verifier limit") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + chunks: list[bytes] = [] + total = 0 + while True: + chunk = os.read(descriptor, COPY_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > MAX_METADATA_FILE_BYTES: + raise RebuildError("metadata_limit_exceeded", "a bundle metadata file exceeds the verifier limit") + chunks.append(chunk) + finally: + os.close(descriptor) + + def reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, child in pairs: + if key in value: + raise RebuildError("duplicate_json_key", "a bundle metadata file contains a duplicate JSON key") + value[key] = child + return value + + def reject_nonfinite_number(_value: str) -> None: + raise RebuildError("invalid_json_value", "a bundle metadata file contains a non-finite JSON number") + + try: + value = json.loads( + b"".join(chunks).decode("utf-8", errors="strict"), + object_pairs_hook=reject_duplicate_pairs, + parse_constant=reject_nonfinite_number, + ) + except RebuildError: + raise + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RebuildError("invalid_bundle_json", "a bundle metadata file is not valid UTF-8 JSON") from exc + if not isinstance(value, dict): + raise RebuildError("invalid_bundle_schema", "a bundle metadata root must be an object") + return value + + +def _assert_no_write_flags(value: Mapping[str, Any], *, label: str) -> None: + for key, expected in NO_WRITE_FLAGS.items(): + if value.get(key) is not expected: + raise RebuildError("unsafe_bundle_flags", f"{label} does not retain the required no-write flags") + + +def _assert_exact_keys(value: Mapping[str, Any], expected: set[str], *, label: str) -> None: + if set(value) != expected: + raise RebuildError("invalid_bundle_schema", f"{label} contains missing or unsupported fields") + + +def _actual_bundle_tree(root: Path) -> tuple[set[str], set[str]]: + files: set[str] = set() + directories: set[str] = set() + + def visit(directory: Path, prefix: PurePosixPath) -> None: + try: + with os.scandir(directory) as stream: + entries = list(stream) + except OSError as exc: + raise RebuildError("bundle_scan_failed", "bundle tree could not be scanned") from exc + for entry in entries: + relative = (prefix / entry.name).as_posix() + _validate_relative_path(relative) + metadata = entry.stat(follow_symlinks=False) + if stat.S_ISLNK(metadata.st_mode): + raise RebuildError("unsafe_bundle_entry", "bundle contains a symlink") + if stat.S_ISDIR(metadata.st_mode): + directories.add(relative) + visit(Path(entry.path), prefix / entry.name) + elif stat.S_ISREG(metadata.st_mode): + files.add(relative) + else: + raise RebuildError("unsafe_bundle_entry", "bundle contains a special file") + + visit(root, PurePosixPath()) + return files, directories + + +def verify_bundle(bundle: Path) -> dict[str, Any]: + root = _validate_bundle_root(bundle) + if (root / INCOMPLETE_MARKER).exists() or (root / INCOMPLETE_MARKER).is_symlink(): + raise RebuildError("incomplete_bundle", "bundle retains its incomplete marker") + + source_manifest = _load_json(root / SOURCE_MANIFEST_PATH) + extraction_index = _load_json(root / EXTRACTION_WORK_INDEX_PATH) + receipt = _load_json(root / RECEIPT_PATH) + if source_manifest.get("schema") != SOURCE_MANIFEST_SCHEMA: + raise RebuildError("invalid_bundle_schema", "source manifest schema is unsupported") + if extraction_index.get("schema") != EXTRACTION_WORK_INDEX_SCHEMA: + raise RebuildError("invalid_bundle_schema", "extraction-work index schema is unsupported") + if receipt.get("schema") != RECEIPT_SCHEMA: + raise RebuildError("invalid_bundle_schema", "receipt schema is unsupported") + for label, value in ( + ("source manifest", source_manifest), + ("extraction-work index", extraction_index), + ("receipt", receipt), + ): + if value.get("mode") != MODE_SOURCE_INVENTORY_ONLY: + raise RebuildError("invalid_bundle_schema", f"{label} mode is unsupported") + _assert_no_write_flags(value, label=label) + _assert_exact_keys( + source_manifest, + { + "admission_performed", + "database_write_performed", + "duplicate_content_source_count", + "mode", + "resource_limits", + "schema", + "semantic_claim_extraction_performed", + "source_count", + "source_snapshot_semantics", + "sources", + "symlinks_followed", + "total_source_bytes", + "unique_snapshot_count", + "unsupported_files_ignored", + }, + label="source manifest", + ) + _assert_exact_keys( + extraction_index, + { + "admission_performed", + "database_write_performed", + "items", + "mode", + "schema", + "semantic_claim_extraction_performed", + "source_count", + "source_manifest", + "work_item_count", + "work_type", + }, + label="extraction-work index", + ) + _assert_exact_keys( + receipt, + { + "admission_performed", + "database_write_performed", + "deterministic", + "hashes", + "mode", + "outputs", + "resource_limits", + "result", + "schema", + "semantic_claim_extraction_performed", + "source_count", + "source_snapshot_atomic", + "status", + "total_source_bytes", + "unique_snapshot_count", + "work_item_count", + }, + label="receipt", + ) + if receipt.get("status") != "pass" or receipt.get("result") != "source_inventory_bundle_ready": + raise RebuildError("invalid_bundle_schema", "receipt does not represent a completed inventory bundle") + if receipt.get("source_snapshot_atomic") is not False: + raise RebuildError("false_snapshot_claim", "receipt must not claim an atomic source-directory snapshot") + semantics = source_manifest.get("source_snapshot_semantics") + if semantics != { + "concurrent_directory_additions_or_removals_may_be_absent": True, + "directory_snapshot_atomic": False, + "guarantee": "each listed path is bound to the exact bytes copied from one stable file descriptor", + }: + raise RebuildError("false_snapshot_claim", "source manifest must bound concurrent-directory snapshot claims") + if receipt.get("deterministic") is not True: + raise RebuildError("invalid_bundle_schema", "receipt deterministic flag is unsupported") + if source_manifest.get("unsupported_files_ignored") is not False: + raise RebuildError("invalid_bundle_schema", "source manifest must not silently ignore unsupported files") + if source_manifest.get("symlinks_followed") is not False: + raise RebuildError("invalid_bundle_schema", "source manifest must not claim that symlinks were followed") + if extraction_index.get("work_type") != WORK_TYPE: + raise RebuildError("invalid_work_index", "extraction-work type is unsupported") + + limits_value = receipt.get("resource_limits") + if not isinstance(limits_value, dict): + raise RebuildError("invalid_bundle_schema", "receipt resource limits are missing") + limits = _limits_from_mapping(limits_value) + if source_manifest.get("resource_limits") != limits.as_dict(): + raise RebuildError("invalid_bundle_schema", "source manifest resource limits differ from the receipt") + + sources = source_manifest.get("sources") + items = extraction_index.get("items") + if not isinstance(sources, list) or not isinstance(items, list): + raise RebuildError("invalid_bundle_schema", "source and work-item collections must be arrays") + if not sources or len(sources) > limits.max_source_count: + raise RebuildError("invalid_bundle_cardinality", "source count is empty or exceeds its limit") + if len(items) > limits.max_work_items: + raise RebuildError("invalid_bundle_cardinality", "work-item count exceeds its limit") + + expected_items: list[dict[str, Any]] = [] + snapshot_paths: set[str] = set() + path_keys: set[str] = set() + total_bytes = 0 + previous_path: str | None = None + for source in sources: + if not isinstance(source, dict): + raise RebuildError("invalid_bundle_schema", "source rows must be objects") + _assert_exact_keys( + source, + { + "artifact_sha256", + "bytes", + "content_validation", + "extension", + "format_syntax_validated", + "newline_style", + "relative_path", + "section_count", + "sectioning_strategy", + "sections", + "snapshot_path", + "source_path_binding", + }, + label="source row", + ) + relative_path = source.get("relative_path") + extension = source.get("extension") + digest = source.get("artifact_sha256") + byte_count = source.get("bytes") + snapshot_relative = source.get("snapshot_path") + if not isinstance(relative_path, str) or not isinstance(extension, str): + raise RebuildError("invalid_bundle_schema", "source path or extension is malformed") + _validate_relative_path(relative_path) + if previous_path is not None and relative_path.encode("utf-8") <= previous_path.encode("utf-8"): + raise RebuildError("invalid_bundle_order", "source rows are not in deterministic path order") + previous_path = relative_path + path_key = _normalized_path_key(relative_path) + if path_key in path_keys: + raise RebuildError("duplicate_relative_path", "source rows contain a normalized path collision") + path_keys.add(path_key) + if extension != PurePosixPath(relative_path).suffix.casefold() or extension not in ALLOWED_EXTENSIONS: + raise RebuildError("invalid_bundle_schema", "source extension binding is invalid") + if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest): + raise RebuildError("invalid_bundle_schema", "source hash is malformed") + if isinstance(byte_count, bool) or not isinstance(byte_count, int) or byte_count <= 0: + raise RebuildError("invalid_bundle_schema", "source byte count is malformed") + if byte_count > limits.max_source_bytes: + raise RebuildError("source_byte_limit_exceeded", "bundle source exceeds its recorded limit") + total_bytes += byte_count + if total_bytes > limits.max_total_bytes: + raise RebuildError("total_byte_limit_exceeded", "bundle exceeds its recorded total-byte limit") + if snapshot_relative != _snapshot_path_for_hash(digest).as_posix(): + raise RebuildError("invalid_bundle_schema", "source snapshot path is not content addressed") + snapshot_paths.add(snapshot_relative) + snapshot = root / Path(snapshot_relative) + text = _read_snapshot_text(snapshot, expected_sha256=digest, expected_bytes=byte_count) + expected_sections = _build_sections(text, extension, max_sections=limits.max_sections_per_source) + if source.get("sections") != expected_sections or source.get("section_count") != len(expected_sections): + raise RebuildError("invalid_section_locator", "source section locators do not match snapshot bytes") + if source.get("newline_style") != _newline_style(text): + raise RebuildError("invalid_bundle_schema", "source newline style does not match snapshot bytes") + if source.get("content_validation") != "strict_utf8_no_binary_controls": + raise RebuildError("invalid_bundle_schema", "source content-validation label is unsupported") + if source.get("format_syntax_validated") is not False: + raise RebuildError("invalid_bundle_schema", "inventory must not claim format-level validation") + if source.get("sectioning_strategy") != _sectioning_strategy(extension): + raise RebuildError("invalid_bundle_schema", "source sectioning strategy is unsupported") + if source.get("source_path_binding") != "source_root_relative_posix_path_plus_exact_snapshot_sha256": + raise RebuildError("invalid_bundle_schema", "source path-binding label is unsupported") + expected_items.extend(_work_item(source, locator) for locator in expected_sections) + del text, expected_sections + + if items != expected_items: + raise RebuildError("invalid_work_index", "extraction-work rows do not exactly match source section locators") + if any(item.get("work_type") != WORK_TYPE for item in items if isinstance(item, dict)): + raise RebuildError("invalid_work_index", "extraction-work type is unsupported") + if any(not isinstance(item, dict) for item in items): + raise RebuildError("invalid_work_index", "extraction-work rows must be objects") + if len({item["work_item_id"] for item in items}) != len(items): + raise RebuildError("invalid_work_index", "extraction-work IDs are not unique") + + expected_counts = { + "source_count": len(sources), + "total_source_bytes": total_bytes, + "unique_snapshot_count": len(snapshot_paths), + } + for key, expected in expected_counts.items(): + if source_manifest.get(key) != expected or receipt.get(key) != expected: + raise RebuildError("invalid_bundle_cardinality", f"{key} does not match verified content") + if source_manifest.get("duplicate_content_source_count") != len(sources) - len(snapshot_paths): + raise RebuildError("invalid_bundle_cardinality", "duplicate-content source count is incorrect") + if source_manifest.get("source_count") != extraction_index.get("source_count"): + raise RebuildError("invalid_bundle_cardinality", "source counts differ between bundle indexes") + if extraction_index.get("work_item_count") != len(items) or receipt.get("work_item_count") != len(items): + raise RebuildError("invalid_bundle_cardinality", "work-item counts differ from verified content") + manifest_binding = extraction_index.get("source_manifest") + if manifest_binding != {"path": SOURCE_MANIFEST_PATH, "sha256": sha256_file(root / SOURCE_MANIFEST_PATH)}: + raise RebuildError("invalid_bundle_hash", "extraction-work index does not bind the source manifest") + + artifact_paths = [SOURCE_MANIFEST_PATH, EXTRACTION_WORK_INDEX_PATH, *snapshot_paths] + artifacts = _artifact_records(root, artifact_paths) + outputs = receipt.get("outputs") + hashes = receipt.get("hashes") + if not isinstance(outputs, dict) or not isinstance(hashes, dict): + raise RebuildError("invalid_bundle_schema", "receipt outputs or hashes are malformed") + _assert_exact_keys( + outputs, + {"artifact_count", "artifacts", "extraction_work_index_path", "source_manifest_path"}, + label="receipt outputs", + ) + _assert_exact_keys( + hashes, + { + "bundle_tree_sha256", + "extraction_work_index_file_sha256", + "receipt_content_sha256", + "source_manifest_file_sha256", + }, + label="receipt hashes", + ) + if outputs.get("artifacts") != artifacts or outputs.get("artifact_count") != len(artifacts): + raise RebuildError("invalid_bundle_hash", "receipt artifact inventory does not match bundle bytes") + if outputs.get("source_manifest_path") != SOURCE_MANIFEST_PATH: + raise RebuildError("invalid_bundle_schema", "receipt source-manifest path is incorrect") + if outputs.get("extraction_work_index_path") != EXTRACTION_WORK_INDEX_PATH: + raise RebuildError("invalid_bundle_schema", "receipt extraction-work path is incorrect") + expected_hashes = { + "bundle_tree_sha256": canonical_sha256(artifacts), + "extraction_work_index_file_sha256": sha256_file(root / EXTRACTION_WORK_INDEX_PATH), + "source_manifest_file_sha256": sha256_file(root / SOURCE_MANIFEST_PATH), + } + for key, expected in expected_hashes.items(): + if hashes.get(key) != expected: + raise RebuildError("invalid_bundle_hash", f"receipt {key} does not match bundle bytes") + receipt_preimage = copy.deepcopy(receipt) + preimage_hashes = receipt_preimage.get("hashes") + if not isinstance(preimage_hashes, dict): + raise RebuildError("invalid_bundle_schema", "receipt hash object is malformed") + recorded_receipt_hash = preimage_hashes.pop("receipt_content_sha256", None) + if recorded_receipt_hash != canonical_sha256(receipt_preimage): + raise RebuildError("invalid_receipt_hash", "receipt content hash does not verify") + + actual_files, actual_directories = _actual_bundle_tree(root) + expected_files = {RECEIPT_PATH, *artifact_paths} + if actual_files != expected_files: + raise RebuildError("unexpected_bundle_entry", "bundle contains missing or unexpected files") + expected_directories = {"snapshots", "snapshots/sha256"} + expected_directories.update(str(PurePosixPath(path).parent) for path in snapshot_paths) + if actual_directories != expected_directories: + raise RebuildError("unexpected_bundle_entry", "bundle contains missing or unexpected directories") + + return { + "bundle_path": root, + "bundle_tree_sha256": expected_hashes["bundle_tree_sha256"], + "receipt_file_sha256": sha256_file(root / RECEIPT_PATH), + "source_count": len(sources), + "status": "pass", + "unique_snapshot_count": len(snapshot_paths), + "work_item_count": len(items), + } + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, help="directory of strict UTF-8 source files") + parser.add_argument("--output", type=Path, help="new bundle directory; must not already exist") + parser.add_argument("--mode", default=MODE_SOURCE_INVENTORY_ONLY, help="must be source-inventory-only") + parser.add_argument("--verify-bundle", type=Path, help="read-only validation of an existing bundle") + parser.add_argument("--max-source-bytes", type=int, default=DEFAULT_MAX_SOURCE_BYTES) + parser.add_argument("--max-total-bytes", type=int, default=DEFAULT_MAX_TOTAL_BYTES) + parser.add_argument("--max-source-count", type=int, default=DEFAULT_MAX_SOURCE_COUNT) + parser.add_argument("--max-sections-per-source", type=int, default=DEFAULT_MAX_SECTIONS_PER_SOURCE) + parser.add_argument("--max-work-items", type=int, default=DEFAULT_MAX_WORK_ITEMS) + return parser.parse_args(argv) + + +def _print_status( + status: str, + *, + operation: str, + reason: str | None = None, + message: str | None = None, + result: Mapping[str, Any] | None = None, +) -> None: + payload: dict[str, Any] = { + "schema": STATUS_SCHEMA, + "status": status, + "operation": operation, + "mode": MODE_SOURCE_INVENTORY_ONLY, + **NO_WRITE_FLAGS, + } + if reason is not None: + payload["reason"] = reason + if message is not None: + payload["message"] = message + if result is not None: + payload.update( + { + "bundle_path": str(result["bundle_path"]), + "bundle_tree_sha256": result["bundle_tree_sha256"], + "receipt_file_sha256": result["receipt_file_sha256"], + "source_count": result["source_count"], + "unique_snapshot_count": result["unique_snapshot_count"], + "work_item_count": result["work_item_count"], + } + ) + print(json.dumps(payload, ensure_ascii=True, sort_keys=True)) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + operation = "verify" if args.verify_bundle is not None else "build" + try: + if args.verify_bundle is not None: + if args.source_root is not None or args.output is not None: + raise RebuildError("invalid_arguments", "verify mode does not accept source-root or output") + result = verify_bundle(args.verify_bundle) + else: + if args.source_root is None or args.output is None: + raise RebuildError("invalid_arguments", "build mode requires source-root and output") + limits = ResourceLimits( + max_source_bytes=args.max_source_bytes, + max_total_bytes=args.max_total_bytes, + max_source_count=args.max_source_count, + max_sections_per_source=args.max_sections_per_source, + max_work_items=args.max_work_items, + ) + built = build_source_inventory_bundle(args.source_root, args.output, mode=args.mode, limits=limits) + result = built["verification"] + except RebuildError as exc: + _print_status("rejected", operation=operation, reason=exc.code, message=exc.safe_message) + return 2 + except OSError: + _print_status( + "rejected", + operation=operation, + reason="filesystem_operation_failed", + message="a fail-closed filesystem operation did not complete", + ) + return 2 + _print_status("pass", operation=operation, result=result) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/run_local_genesis_ledger_rebuild.py b/ops/run_local_genesis_ledger_rebuild.py index a500fc6..74accfb 100644 --- a/ops/run_local_genesis_ledger_rebuild.py +++ b/ops/run_local_genesis_ledger_rebuild.py @@ -2,7 +2,7 @@ """Deterministically rebuild a disposable Postgres from genesis plus a strict ledger. The v1 replay boundary supports strict ``add_edge``, ``attach_evidence``, -``approve_claim``, and ``revise_strategy`` receipts. Insert-only actions seed +V2/V3 ``approve_claim``, and ``revise_strategy`` receipts. Insert-only actions seed their receipt-pinned rows before the guarded transition. ``revise_strategy`` instead captures the exact prestate identities, executes the exact hash-matched guarded SQL, validates the generated delta, and normalizes only that delta to the @@ -41,20 +41,34 @@ if str(SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(SCRIPTS_DIR)) import apply_proposal as apply_engine # noqa: E402, I001 +import export_kb_transition_replay_material as replay_exporter # noqa: E402 import kb_apply_replay_receipt as replay_receipt # noqa: E402 LEDGER_ARTIFACT = "teleo_genesis_plus_ledger" MATERIAL_ARTIFACT = "teleo_strict_proposal_replay_material" +ADMISSION_ARTIFACT = "teleo_ledger_admission_manifest" LEDGER_CONTRACT_VERSION = 1 MATERIAL_CONTRACT_VERSION = 1 +ADMISSION_CONTRACT_VERSION = 1 +CANONICAL_REVIEWER_HANDLE = "m3taversal" +REVIEW_METADATA_PROVENANCE = "operator_supplied_explicit_review_metadata" +ADMISSION_BASES = frozenset( + { + "forgejo_primary_source", + "reviewed_telegram_steering", + "reviewed_primary_source", + } +) +ADMISSION_SCOPES = frozenset({"collective_core", "agent_identity", "knowledge_commons"}) DEFAULT_REBUILD_IMAGE = ( "docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777" ) SUPPORTED_PROPOSAL_TYPES = frozenset({"add_edge", "attach_evidence", "approve_claim", "revise_strategy"}) CLAIM_CEILING = ( "exact genesis plus ordered strict proposal replay; supported types are add_edge, " - "attach_evidence, approve_claim, and revise_strategy; mutating strategy replay " + "attach_evidence, V2/V3 approve_claim, and revise_strategy; V3 cutover is ordered and exact; " + "mutating strategy replay " "captures prestate identities and normalizes the exact receipt-pinned poststate; " "full approved/applied proposal rows and immutable approval snapshots are required" ) @@ -67,14 +81,17 @@ NOT_PROVEN = ( APPLY_ENGINE_PATH = SCRIPTS_DIR / "apply_proposal.py" REPLAY_RECEIPT_ENGINE_PATH = SCRIPTS_DIR / "kb_apply_replay_receipt.py" GUARD_PREREQUISITES_PATH = SCRIPTS_DIR / "kb_apply_prereqs.sql" +V3_CONTRACT_PATH = REPO_ROOT / "ops" / "teleo_v3_epistemic_contract.sql" PARITY_SQL_PATH = REPO_ROOT / "ops" / "postgres_parity_manifest.sql" BASE_REBUILD_PATH = REPO_ROOT / "ops" / "run_local_canonical_postgres_rebuild.py" GUARD_PREREQUISITES_DESTINATION = "/tmp/kb-apply-prereqs.sql" +V3_CONTRACT_DESTINATION = "/tmp/teleo-v3-epistemic-contract.sql" FIXED_ENGINE_PATHS = frozenset( { APPLY_ENGINE_PATH.resolve(), REPLAY_RECEIPT_ENGINE_PATH.resolve(), GUARD_PREREQUISITES_PATH.resolve(), + V3_CONTRACT_PATH.resolve(), PARITY_SQL_PATH.resolve(), BASE_REBUILD_PATH.resolve(), Path(__file__).resolve(), @@ -83,6 +100,7 @@ FIXED_ENGINE_PATHS = frozenset( SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") IMAGE_DIGEST_RE = re.compile(r"\S+@sha256:[0-9a-f]{64}\Z") +TYPED_TIMESTAMP_FIELDS = frozenset({"captured_at", "created_at", "updated_at", "reviewed_at", "applied_at"}) PROPOSAL_TRANSITION_FIELDS = frozenset( {"status", "applied_by_handle", "applied_by_agent_id", "applied_at", "updated_at"} ) @@ -125,6 +143,7 @@ CANONICAL_TABLE_ORDER = ( "sources", "reasoning_tools", "claim_evidence", + "claim_evidence_assessments", "claim_edges", ) STRATEGY_REQUIRED_FIELDS = frozenset( @@ -173,6 +192,7 @@ class LedgerEntry: material_path: Path material_sha256: str replay_material_sha256: str + contract_version: int approved_proposal: dict[str, Any] approval_snapshot: dict[str, Any] applied_proposal: dict[str, Any] @@ -196,6 +216,10 @@ class ReconstructionBundle: final_manifest_bytes: int engine_hashes: dict[str, str] entries: tuple[LedgerEntry, ...] + admission: dict[str, Any] + admission_manifest_path: Path | None + admission_manifest_sha256: str | None + admission_manifest_bytes: int | None protected_paths: frozenset[Path] @@ -228,12 +252,18 @@ def _require_exact_keys(value: Any, expected: frozenset[str], field: str) -> dic def _load_json_object(path: Path, field: str) -> dict[str, Any]: try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError) as exc: - raise ReconstructionError("invalid_json_artifact", f"{field} is not valid UTF-8 JSON") from exc - if not isinstance(value, dict): - raise ReconstructionError("invalid_json_artifact", f"{field} must contain one JSON object") - return value + raw = path.read_text(encoding="utf-8") + return replay_exporter._parse_json_object(raw, label=field) + except replay_exporter.ExportError as exc: + raise ReconstructionError( + "invalid_json_artifact", + f"{field} is not exact duplicate-free UTF-8 JSON", + ) from exc + except (OSError, UnicodeError) as exc: + raise ReconstructionError( + "invalid_json_artifact", + f"{field} is not exact duplicate-free UTF-8 JSON", + ) from exc def _resolve_artifact(ledger_path: Path, declared: Any, field: str) -> Path: @@ -263,11 +293,183 @@ def _engine_hashes() -> dict[str, str]: "apply_engine_sha256": APPLY_ENGINE_PATH, "replay_receipt_engine_sha256": REPLAY_RECEIPT_ENGINE_PATH, "guard_prerequisites_sha256": GUARD_PREREQUISITES_PATH, + "v3_contract_sha256": V3_CONTRACT_PATH, "parity_sql_sha256": PARITY_SQL_PATH, } return {name: canonical_rebuild.sha256_file(path) for name, path in paths.items()} +def _proposal_contract_version(proposal: dict[str, Any]) -> int: + payload = proposal.get("payload") + apply_payload = payload.get("apply_payload") if isinstance(payload, dict) else None + if not isinstance(apply_payload, dict): + raise ReconstructionError("invalid_material", "proposal is missing a strict apply payload") + raw_version = apply_payload.get("contract_version", 2) + if type(raw_version) is not int or raw_version not in {2, 3}: + raise ReconstructionError("unsupported_contract_version", "proposal contract version is unsupported") + if raw_version == 3 and proposal.get("proposal_type") != "approve_claim": + raise ReconstructionError("unsupported_contract_version", "V3 replay currently supports approve_claim only") + return raw_version + + +def _validate_contract_order(entries: list[LedgerEntry]) -> None: + v3_started = False + for entry in entries: + if entry.contract_version == 3: + v3_started = True + elif v3_started: + raise ReconstructionError( + "contract_order_mismatch", + "V2 ledger entries cannot appear after the ordered V3 cutover", + ) + + +def _load_admission_manifest( + args: argparse.Namespace, + *, + ledger_sha256: str, + entries: list[LedgerEntry], +) -> tuple[dict[str, Any], Path | None, str | None, int | None]: + manifest_path = getattr(args, "admission_manifest", None) + manifest_pin = getattr(args, "admission_manifest_sha256", None) + if manifest_path is None and manifest_pin is None: + return ( + { + "status": "forensic_only", + "canonical_retrieval_eligible": False, + "reason": "no explicit admission manifest was supplied", + "entry_count": len(entries), + }, + None, + None, + None, + ) + if manifest_path is None or manifest_pin is None: + raise ReconstructionError( + "incomplete_admission_pin", + "admission requires both a manifest and its SHA-256 pin", + ) + if not isinstance(manifest_path, Path) or not manifest_path.is_file(): + raise ReconstructionError("missing_admission_manifest", "admission manifest is not an existing regular file") + + manifest_path = manifest_path.resolve() + manifest_sha256 = _assert_file_hash(manifest_path, manifest_pin, "--admission-manifest-sha256") + manifest = _require_exact_keys( + _load_json_object(manifest_path, "admission manifest"), + frozenset( + { + "artifact", + "contract_version", + "ledger_sha256", + "reviewed_by_handle", + "reviewed_by_agent_id", + "reviewed_at", + "review_note", + "entries", + } + ), + "admission manifest", + ) + if manifest["artifact"] != ADMISSION_ARTIFACT or manifest["contract_version"] != ADMISSION_CONTRACT_VERSION: + raise ReconstructionError( + "unsupported_admission_contract", + "admission manifest is not the supported v1 contract", + ) + if _validate_sha256(manifest["ledger_sha256"], "admission manifest ledger_sha256") != ledger_sha256: + raise ReconstructionError("admission_ledger_mismatch", "admission manifest does not select this exact ledger") + + reviewer_handle = manifest["reviewed_by_handle"] + review_note = manifest["review_note"] + if reviewer_handle != CANONICAL_REVIEWER_HANDLE: + raise ReconstructionError( + "invalid_admission_review", + "admission reviewer handle must equal m3taversal exactly", + ) + if not isinstance(review_note, str) or not review_note.strip(): + raise ReconstructionError("invalid_admission_review", "admission review note must be non-empty") + try: + reviewer_agent_id = str(uuid.UUID(str(manifest["reviewed_by_agent_id"]))) + reviewed_at = replay_receipt.canonical_utc_timestamp( + manifest["reviewed_at"], + path="admission manifest reviewed_at", + ) + except (AttributeError, TypeError, ValueError) as exc: + raise ReconstructionError( + "invalid_admission_review", + "admission review requires one reviewer UUID and timezone-aware timestamp", + ) from exc + + raw_entries = manifest["entries"] + if not isinstance(raw_entries, list) or len(raw_entries) != len(entries): + raise ReconstructionError( + "admission_selection_mismatch", + "admission manifest must select every entry in the supplied clean-genesis ledger exactly once", + ) + selected: list[dict[str, Any]] = [] + for index, (raw_entry, entry) in enumerate(zip(raw_entries, entries, strict=True), 1): + selected_entry = _require_exact_keys( + raw_entry, + frozenset( + { + "sequence", + "proposal_id", + "material_sha256", + "replay_material_sha256", + "admission_scope", + "basis", + } + ), + f"admission manifest entries[{index - 1}]", + ) + if entry.contract_version != 3: + raise ReconstructionError( + "legacy_admission_forbidden", + "only source-backed V3 entries may enter an admitted clean genesis", + ) + expected = { + "sequence": entry.sequence, + "proposal_id": entry.applied_proposal["id"], + "material_sha256": entry.material_sha256, + "replay_material_sha256": entry.replay_material_sha256, + } + if any(selected_entry[field] != value for field, value in expected.items()): + raise ReconstructionError( + "admission_selection_mismatch", + f"admission manifest entry {index} does not pin the exact ledger material", + ) + scope = selected_entry["admission_scope"] + basis = selected_entry["basis"] + if scope not in ADMISSION_SCOPES or basis not in ADMISSION_BASES: + raise ReconstructionError( + "invalid_admission_classification", + f"admission manifest entry {index} has an unsupported scope or basis", + ) + selected.append({**expected, "admission_scope": scope, "basis": basis}) + + return ( + { + "status": "review_metadata_supplied", + "canonical_retrieval_eligible": False, + "reason": "operator-supplied review metadata is hash-pinned but not authenticated", + "artifact": ADMISSION_ARTIFACT, + "contract_version": ADMISSION_CONTRACT_VERSION, + "review_metadata_provenance": REVIEW_METADATA_PROVENANCE, + "cryptographically_authenticated": False, + "manifest_sha256": manifest_sha256, + "ledger_sha256": ledger_sha256, + "reviewed_by_handle": reviewer_handle.strip(), + "reviewed_by_agent_id": reviewer_agent_id, + "reviewed_at": reviewed_at, + "review_note_sha256": _sha256_text(review_note), + "entry_count": len(selected), + "entries": selected, + }, + manifest_path, + manifest_sha256, + manifest_path.stat().st_size, + ) + + def _validate_proposal_rows( approved: dict[str, Any], applied: dict[str, Any], @@ -337,7 +539,25 @@ def _validate_proposal_rows( "applied_by_agent_id", "applied_at", ) - if any(receipt_proposal.get(field) != applied[field] for field in envelope_fields): + try: + envelope_mismatch = any( + replay_receipt.normalize_typed_value( + field, + receipt_proposal.get(field), + path=f"replay_receipt.proposal.{field}", + ) + != replay_receipt.normalize_typed_value( + field, + applied[field], + path=f"material.applied_proposal.{field}", + ) + for field in envelope_fields + ) + except ValueError as exc: + raise ReconstructionError( + "invalid_material", "replay receipt proposal envelope contains an invalid typed value" + ) from exc + if envelope_mismatch: raise ReconstructionError( "material_mismatch", "replay receipt proposal envelope does not match applied proposal row" ) @@ -552,6 +772,7 @@ def _validate_entry_material( material_path=material_path, material_sha256=material_sha256, replay_material_sha256=actual_replay_hash, + contract_version=_proposal_contract_version(proposal), approved_proposal=approved, approval_snapshot=approval, applied_proposal=applied, @@ -644,6 +865,15 @@ def load_bundle(args: argparse.Namespace) -> ReconstructionBundle: entries.append(entry) protected_paths.add(material_path) + _validate_contract_order(entries) + admission, admission_path, admission_sha256, admission_bytes = _load_admission_manifest( + args, + ledger_sha256=ledger_sha256, + entries=entries, + ) + if admission_path is not None: + protected_paths.add(admission_path) + if args.output.resolve() in protected_paths: raise ReconstructionError("unsafe_output_path", "--output must not overwrite any reconstruction input") @@ -661,6 +891,10 @@ def load_bundle(args: argparse.Namespace) -> ReconstructionBundle: final_manifest_bytes=final_manifest_path.stat().st_size, engine_hashes=actual_engine_hashes, entries=tuple(entries), + admission=admission, + admission_manifest_path=admission_path, + admission_manifest_sha256=admission_sha256, + admission_manifest_bytes=admission_bytes, protected_paths=frozenset(protected_paths), ) @@ -669,7 +903,15 @@ def _jsonb_literal(value: Any) -> str: return apply_engine.sql_literal(_canonical_json(value)) + "::jsonb" +def _approval_seed_row(approved_proposal: dict[str, Any], approval_snapshot: dict[str, Any]) -> dict[str, Any]: + return { + **approval_snapshot, + "approved_proposal_snapshot": approved_proposal, + } + + def build_seed_sql(entry: LedgerEntry) -> str: + approval_seed = _approval_seed_row(entry.approved_proposal, entry.approval_snapshot) statements = [ "begin;", "set local standard_conforming_strings = on;", @@ -679,25 +921,30 @@ def build_seed_sql(entry: LedgerEntry) -> str: ") seeded", "on conflict (id) do nothing;", "insert into kb_stage.kb_proposal_approvals", - "select seeded.* from jsonb_populate_record(", - f" null::kb_stage.kb_proposal_approvals, {_jsonb_literal(entry.approval_snapshot)}", + "select seeded.*", + "from kb_stage.kb_proposals proposal", + "cross join lateral jsonb_populate_record(", + " null::kb_stage.kb_proposal_approvals,", + f" jsonb_set({_jsonb_literal(approval_seed)}, '{{approved_proposal_snapshot}}', to_jsonb(proposal), true)", ") seeded", + f"where proposal.id = {apply_engine.sql_literal(entry.approved_proposal['id'])}::uuid", "on conflict (proposal_id) do nothing;", ] - rows = entry.receipt["canonical_rows"] - for table in CANONICAL_TABLE_ORDER: - table_rows = rows.get(table) - if not table_rows: - continue - statements.extend( - [ - f"insert into public.{table}", - "select seeded.* from jsonb_populate_recordset(", - f" null::public.{table}, {_jsonb_literal(table_rows)}", - ") seeded", - "on conflict do nothing;", - ] - ) + if entry.contract_version != 3: + rows = entry.receipt["canonical_rows"] + for table in CANONICAL_TABLE_ORDER: + table_rows = rows.get(table) + if not table_rows: + continue + statements.extend( + [ + f"insert into public.{table}", + "select seeded.* from jsonb_populate_recordset(", + f" null::public.{table}, {_jsonb_literal(table_rows)}", + ") seeded", + "on conflict do nothing;", + ] + ) statements.append("commit;") return "\n".join(statements) + "\n" @@ -967,14 +1214,163 @@ def build_proposal_readback_sql(proposal_id: str) -> str: return f"""select jsonb_build_object( 'proposal', (select to_jsonb(p) from kb_stage.kb_proposals p where p.id = {literal}::uuid), - 'approval_snapshot', (select to_jsonb(a) from kb_stage.kb_proposal_approvals a - where a.proposal_id = {literal}::uuid) + 'approval_snapshot', (select to_jsonb(a) - 'approved_proposal_snapshot' + from kb_stage.kb_proposal_approvals a + where a.proposal_id = {literal}::uuid), + 'approved_proposal_snapshot', (select a.approved_proposal_snapshot + from kb_stage.kb_proposal_approvals a + where a.proposal_id = {literal}::uuid) )::text; """ +def build_v3_canonical_timestamp_normalization_sql(entry: LedgerEntry) -> str: + if entry.contract_version != 3: + raise ReconstructionError( + "invalid_v3_normalization", + "canonical V3 timestamp normalization requires a V3 ledger entry", + ) + proposal_id = entry.applied_proposal["id"] + table_contracts = { + "claims": ("accepted_by_proposal_id", "teleo_v3_claim_immutable"), + "sources": ("accepted_by_proposal_id", "teleo_v3_source_immutable"), + "claim_evidence": ("accepted_from_proposal_id", "teleo_v3_evidence_immutable"), + "claim_evidence_assessments": ( + "accepted_by_proposal_id", + "teleo_v3_assessment_immutable", + ), + "claim_edges": ("accepted_by_proposal_id", "teleo_v3_edge_immutable"), + } + statements = ["begin;", "set local standard_conforming_strings = on;"] + for table in CANONICAL_TABLE_ORDER: + rows = entry.receipt["canonical_rows"].get(table) or [] + if not rows: + continue + if table not in table_contracts or any("id" not in row for row in rows): + raise ReconstructionError( + "invalid_v3_normalization", + "V3 canonical timestamp rows lack an exact proposal-owned identity", + ) + timestamp_fields = sorted(TYPED_TIMESTAMP_FIELDS.intersection(*(set(row) for row in rows))) + if not timestamp_fields: + continue + proposal_column, immutable_trigger = table_contracts[table] + assignments = ",\n ".join(f"{field} = pinned.{field}" for field in timestamp_fields) + statements.append( + f"""lock table public.{table} in access exclusive mode; +do $preflight_{table}$ +begin + if not exists ( + select 1 + from pg_catalog.pg_trigger trigger + where trigger.tgrelid = 'public.{table}'::regclass + and trigger.tgname = '{immutable_trigger}' + and trigger.tgfoid = 'kb_stage.teleo_v3_guard_immutable_fields()'::regprocedure + and not trigger.tgisinternal + and trigger.tgenabled = 'O' + ) then + raise exception 'V3 {table} immutable guard is not enabled before deterministic normalization'; + end if; +end +$preflight_{table}$; +alter table public.{table} disable trigger {immutable_trigger}; +do $normalize_{table}$ +declare + changed_rows integer; +begin + update public.{table} target + set {assignments} + from jsonb_populate_recordset( + null::public.{table}, {_jsonb_literal(rows)} + ) pinned + where target.id = pinned.id + and target.{proposal_column} = {apply_engine.sql_literal(proposal_id)}::uuid; + get diagnostics changed_rows = row_count; + if changed_rows <> {len(rows)} then + raise exception 'deterministic V3 {table} timestamp normalization failed'; + end if; +end +$normalize_{table}$; +alter table public.{table} enable trigger {immutable_trigger}; +do $postflight_{table}$ +begin + if not exists ( + select 1 + from pg_catalog.pg_trigger trigger + where trigger.tgrelid = 'public.{table}'::regclass + and trigger.tgname = '{immutable_trigger}' + and trigger.tgfoid = 'kb_stage.teleo_v3_guard_immutable_fields()'::regprocedure + and not trigger.tgisinternal + and trigger.tgenabled = 'O' + ) then + raise exception 'V3 {table} immutable guard was not restored after deterministic normalization'; + end if; +end +$postflight_{table}$;""" + ) + statements.append("commit;") + return "\n".join(statements) + "\n" + + def build_applied_timestamp_normalization_sql(entry: LedgerEntry) -> str: applied = entry.applied_proposal + if entry.contract_version == 3: + return f"""begin; +lock table kb_stage.kb_proposals in access exclusive mode; +do $preflight$ +begin + if not exists ( + select 1 + from pg_catalog.pg_trigger trigger + where trigger.tgrelid = 'kb_stage.kb_proposals'::regclass + and trigger.tgname = 'teleo_v3_guard_linked_proposal_decision' + and trigger.tgfoid = 'kb_stage.teleo_v3_guard_linked_proposal_decision()'::regprocedure + and not trigger.tgisinternal + and trigger.tgenabled = 'O' + ) then + raise exception 'V3 proposal decision guard is not enabled before deterministic normalization'; + end if; +end +$preflight$; +alter table kb_stage.kb_proposals + disable trigger teleo_v3_guard_linked_proposal_decision; +do $normalization$ +declare + changed_rows integer; +begin + update kb_stage.kb_proposals + set applied_at = {apply_engine.sql_literal(applied["applied_at"])}::timestamptz, + updated_at = {apply_engine.sql_literal(applied["updated_at"])}::timestamptz + where id = {apply_engine.sql_literal(applied["id"])}::uuid + and status = 'applied' + and applied_by_handle = {apply_engine.sql_literal(applied["applied_by_handle"])} + and applied_by_agent_id is not distinct from + {apply_engine.sql_literal(applied["applied_by_agent_id"])}::uuid; + get diagnostics changed_rows = row_count; + if changed_rows <> 1 then + raise exception 'deterministic V3 proposal timestamp normalization failed'; + end if; +end +$normalization$; +alter table kb_stage.kb_proposals + enable trigger teleo_v3_guard_linked_proposal_decision; +do $postflight$ +begin + if not exists ( + select 1 + from pg_catalog.pg_trigger trigger + where trigger.tgrelid = 'kb_stage.kb_proposals'::regclass + and trigger.tgname = 'teleo_v3_guard_linked_proposal_decision' + and trigger.tgfoid = 'kb_stage.teleo_v3_guard_linked_proposal_decision()'::regprocedure + and not trigger.tgisinternal + and trigger.tgenabled = 'O' + ) then + raise exception 'V3 proposal decision guard was not restored after deterministic normalization'; + end if; +end +$postflight$; +commit; +""" return f"""do $reconstruction$ declare changed_rows integer; @@ -1089,17 +1485,191 @@ def _read_json( raise ReconstructionError(code, f"{action} returned an invalid private JSON readback") from exc +def install_v3_contract(args: argparse.Namespace, container: str) -> dict[str, Any]: + completed = canonical_rebuild._run( + [ + args.docker_bin, + "cp", + str(V3_CONTRACT_PATH), + f"{container}:{V3_CONTRACT_DESTINATION}", + ], + timeout=args.command_timeout, + ) + _require_command_success( + completed, + code="v3_contract_copy_failed", + action="V3 epistemic contract copy", + ) + + cutover = _read_json( + args, + container, + """begin; +alter role kb_apply nologin; +do $cutover$ +begin + if exists ( + select 1 + from pg_catalog.pg_stat_activity activity + where activity.datid = (select oid from pg_catalog.pg_database where datname = current_database()) + and activity.usename = 'kb_apply' + and activity.pid <> pg_catalog.pg_backend_pid() + ) then + raise exception 'V3 reconstruction cutover found an active kb_apply session'; + end if; +end +$cutover$; +commit; +select jsonb_build_object( + 'apply_role_login_disabled', not (select role.rolcanlogin from pg_catalog.pg_roles role where role.rolname = 'kb_apply'), + 'active_apply_sessions', ( + select count(*) + from pg_catalog.pg_stat_activity activity + where activity.datid = (select oid from pg_catalog.pg_database where datname = current_database()) + and activity.usename = 'kb_apply' + and activity.pid <> pg_catalog.pg_backend_pid() + ) +)::text; +""", + code="v3_cutover_preflight_failed", + action="V3 disposable writer cutover preflight", + ) + if cutover != {"apply_role_login_disabled": True, "active_apply_sessions": 0}: + raise ReconstructionError( + "v3_cutover_preflight_failed", + "V3 disposable writer cutover did not disable login with zero active apply sessions", + ) + + for destination, label in ( + (V3_CONTRACT_DESTINATION, "V3 epistemic contract"), + (GUARD_PREREQUISITES_DESTINATION, "guarded apply prerequisites after V3"), + ): + completed = canonical_rebuild._run( + [ + args.docker_bin, + "exec", + container, + "psql", + "-X", + "-U", + "postgres", + "-d", + args.database, + "-Atq", + "-v", + "ON_ERROR_STOP=1", + "-f", + destination, + ], + timeout=args.apply_timeout, + ) + _require_command_success( + completed, + code="v3_contract_install_failed", + action=label, + ) + + catalog = _read_json( + args, + container, + """select jsonb_build_object( + 'assessment_table_present', to_regclass('public.claim_evidence_assessments') is not null, + 'apply_role_login_restored', (select role.rolcanlogin from pg_catalog.pg_roles role where role.rolname = 'kb_apply'), + 'cutover_guard_enabled', exists ( + select 1 from pg_catalog.pg_trigger trigger + where trigger.tgrelid = 'kb_stage.kb_proposals'::regclass + and trigger.tgname = 'teleo_v3_00_enforce_apply_contract_cutover' + and trigger.tgfoid = 'kb_stage.teleo_v3_enforce_apply_contract_cutover()'::regprocedure + and not trigger.tgisinternal + and trigger.tgenabled = 'O' + ), + 'decision_guard_enabled', exists ( + select 1 from pg_catalog.pg_trigger trigger + where trigger.tgrelid = 'kb_stage.kb_proposals'::regclass + and trigger.tgname = 'teleo_v3_guard_linked_proposal_decision' + and trigger.tgfoid = 'kb_stage.teleo_v3_guard_linked_proposal_decision()'::regprocedure + and not trigger.tgisinternal + and trigger.tgenabled = 'O' + ) +)::text; +""", + code="v3_contract_readback_failed", + action="V3 epistemic contract catalog readback", + ) + if catalog != { + "assessment_table_present": True, + "apply_role_login_restored": True, + "cutover_guard_enabled": True, + "decision_guard_enabled": True, + }: + raise ReconstructionError("v3_contract_readback_failed", "V3 contract catalog readback was incomplete") + return { + "status": "pass", + "sha256": canonical_rebuild.sha256_file(V3_CONTRACT_PATH), + "cutover": cutover, + "guard_prerequisites_rerun": True, + "catalog": catalog, + } + + def _assert_exact_json(actual: Any, expected: Any, *, code: str, action: str) -> None: if _canonical_json(actual) != _canonical_json(expected): raise ReconstructionError(code, f"{action} did not match the hash-pinned material") +def _assert_exact_canonical_rows(actual: Any, expected: Any, *, code: str, action: str) -> None: + def normalize(value: Any, path: str) -> dict[str, list[dict[str, Any]]]: + if not isinstance(value, dict): + raise ValueError(f"{path} must be an object") + normalized: dict[str, list[dict[str, Any]]] = {} + for table, rows in value.items(): + if ( + not isinstance(table, str) + or not isinstance(rows, list) + or any(not isinstance(row, dict) for row in rows) + ): + raise ValueError(f"{path} must map table names to row arrays") + normalized[table] = [ + replay_receipt.normalize_typed_row(row, path=f"{path}.{table}[{index}]") + for index, row in enumerate(rows) + ] + return normalized + + try: + normalized_actual = normalize(actual, "actual canonical rows") + normalized_expected = normalize(expected, "expected canonical rows") + except ValueError as exc: + raise ReconstructionError(code, f"{action} returned invalid typed canonical rows") from exc + _assert_exact_json(normalized_actual, normalized_expected, code=code, action=action) + + +def _assert_exact_proposal_readback(actual: Any, expected: Any, *, code: str, action: str) -> None: + def normalize(value: Any, path: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError(f"{path} must be an object") + normalized = dict(value) + for field in ("proposal", "approval_snapshot", "approved_proposal_snapshot"): + row = normalized.get(field) + if not isinstance(row, dict): + raise ValueError(f"{path}.{field} must be an object") + normalized[field] = replay_receipt.normalize_typed_row(row, path=f"{path}.{field}") + return normalized + + try: + normalized_actual = normalize(actual, "actual proposal readback") + normalized_expected = normalize(expected, "expected proposal readback") + except ValueError as exc: + raise ReconstructionError(code, f"{action} returned invalid typed proposal rows") from exc + _assert_exact_json(normalized_actual, normalized_expected, code=code, action=action) + + def _entry_summary(entry: LedgerEntry) -> dict[str, Any]: proposal = entry.applied_proposal return { "sequence": entry.sequence, "proposal_id": proposal["id"], "proposal_type": proposal["proposal_type"], + "contract_version": entry.contract_version, "material_sha256": entry.material_sha256, "replay_material_sha256": entry.replay_material_sha256, "original_apply_sql_sha256": entry.receipt["apply_engine"]["apply_sql_sha256"], @@ -1142,11 +1712,12 @@ def apply_entry( code="entry_seed_readback_failed", action=f"ledger entry {entry.sequence} proposal seed readback", ) - _assert_exact_json( + _assert_exact_proposal_readback( proposal_readback, { "proposal": entry.approved_proposal, "approval_snapshot": entry.approval_snapshot, + "approved_proposal_snapshot": entry.approved_proposal, }, code="entry_seed_mismatch", action=f"ledger entry {entry.sequence} proposal seed", @@ -1163,6 +1734,21 @@ def apply_entry( ) ) summary["mutating_prestate_captured"] = True + elif entry.contract_version == 3: + seeded_rows = _read_json( + args, + container, + replay_receipt.build_postflight_sql(entry.receipt["proposal"]), + code="entry_seed_readback_failed", + action=f"ledger entry {entry.sequence} V3 empty canonical prestate readback", + ) + _assert_exact_canonical_rows( + seeded_rows, + {table: [] for table in entry.receipt["canonical_rows"]}, + code="entry_seed_mismatch", + action=f"ledger entry {entry.sequence} V3 empty canonical prestate", + ) + summary["canonical_seed_exact"] = True else: seeded_rows = _read_json( args, @@ -1171,7 +1757,7 @@ def apply_entry( code="entry_seed_readback_failed", action=f"ledger entry {entry.sequence} canonical seed readback", ) - _assert_exact_json( + _assert_exact_canonical_rows( seeded_rows, entry.receipt["canonical_rows"], code="entry_seed_mismatch", @@ -1211,6 +1797,17 @@ def apply_entry( action=f"ledger entry {entry.sequence} revise_strategy exact poststate normalization", ) summary["mutating_poststate_normalized"] = True + elif entry.contract_version == 3: + _psql( + args, + container, + build_v3_canonical_timestamp_normalization_sql(entry), + role="postgres", + timeout=args.command_timeout, + code="v3_canonical_timestamp_normalization_failed", + action=f"ledger entry {entry.sequence} V3 canonical timestamp normalization", + ) + summary["mutating_poststate_normalized"] = True _psql( args, @@ -1228,11 +1825,12 @@ def apply_entry( code="entry_postflight_failed", action=f"ledger entry {entry.sequence} applied proposal readback", ) - _assert_exact_json( + _assert_exact_proposal_readback( proposal_readback, { "proposal": entry.applied_proposal, "approval_snapshot": entry.approval_snapshot, + "approved_proposal_snapshot": entry.approved_proposal, }, code="entry_postflight_mismatch", action=f"ledger entry {entry.sequence} applied proposal", @@ -1246,7 +1844,7 @@ def apply_entry( code="entry_postflight_failed", action=f"ledger entry {entry.sequence} exact row readback", ) - _assert_exact_json( + _assert_exact_canonical_rows( final_rows, entry.receipt["canonical_rows"], code="entry_postflight_mismatch", @@ -1314,6 +1912,7 @@ def _initial_receipt(args: argparse.Namespace, container: str) -> dict[str, Any] "genesis_dump": {"bytes": None, "sha256": None, "pin_verified": False}, "genesis_manifest": {"bytes": None, "sha256": None, "pin_verified": False}, "ledger": {"bytes": None, "sha256": None, "pin_verified": False}, + "admission_manifest": {"bytes": None, "sha256": None, "pin_verified": False}, "final_manifest": {"bytes": None, "sha256": None, "pin_verified": False}, "engine_hashes": {}, }, @@ -1327,9 +1926,14 @@ def _initial_receipt(args: argparse.Namespace, container: str) -> dict[str, Any] "isolation": None, }, "guard_prerequisites": {"status": "pending", "sha256": None}, + "v3_contract": {"status": "not_required", "sha256": None}, "genesis_parity": {"status": "pending"}, "genesis_key_counts": {}, "ledger": {"entry_count": None, "entries": []}, + "admission": { + "status": "pending", + "canonical_retrieval_eligible": False, + }, "final_parity": {"status": "pending"}, "final_key_counts": {}, "cleanup": { @@ -1375,6 +1979,11 @@ def run_reconstruction(args: argparse.Namespace) -> dict[str, Any]: "sha256": bundle.ledger_sha256, "pin_verified": True, }, + "admission_manifest": { + "bytes": bundle.admission_manifest_bytes, + "sha256": bundle.admission_manifest_sha256, + "pin_verified": bundle.admission_manifest_sha256 is not None, + }, "final_manifest": { "bytes": bundle.final_manifest_bytes, "sha256": bundle.final_manifest_sha256, @@ -1384,6 +1993,14 @@ def run_reconstruction(args: argparse.Namespace) -> dict[str, Any]: } receipt["ledger"]["entry_count"] = len(bundle.entries) receipt["ledger"]["entries"] = [_entry_summary(entry) for entry in bundle.entries] + receipt["admission"] = bundle.admission + if bundle.admission["canonical_retrieval_eligible"] is not True: + receipt["claim_ceiling"] = f"forensic-only reconstruction; {CLAIM_CEILING}" + if any(entry.contract_version == 3 for entry in bundle.entries): + receipt["v3_contract"] = { + "status": "pending", + "sha256": bundle.engine_hashes["v3_contract_sha256"], + } phase = "container_name_preflight" absent, absence_error = canonical_rebuild.inspect_container_absence( @@ -1548,7 +2165,13 @@ def run_reconstruction(args: argparse.Namespace) -> dict[str, Any]: ) phase = "ordered_ledger_replay" + v3_installed = False for entry, summary in zip(bundle.entries, receipt["ledger"]["entries"], strict=True): + if entry.contract_version == 3 and not v3_installed: + phase = "v3_contract_install" + receipt["v3_contract"] = install_v3_contract(args, container) + v3_installed = True + phase = "ordered_ledger_replay" apply_entry(args, container, entry, summary) phase = "final_parity" @@ -1651,6 +2274,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--genesis-manifest", required=True, type=Path) parser.add_argument("--ledger", required=True, type=Path) parser.add_argument("--ledger-sha256", required=True) + parser.add_argument("--admission-manifest", type=Path) + parser.add_argument("--admission-manifest-sha256") parser.add_argument("--database", default="teleo") parser.add_argument("--image", default=DEFAULT_REBUILD_IMAGE) parser.add_argument("--container-prefix", default="teleo-genesis-ledger-rebuild") @@ -1698,6 +2323,13 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error(f"{flag} must be an existing regular file") if not SHA256_RE.fullmatch(args.ledger_sha256): parser.error("--ledger-sha256 must be a lowercase SHA-256 digest") + if (args.admission_manifest is None) != (args.admission_manifest_sha256 is None): + parser.error("--admission-manifest and --admission-manifest-sha256 must be supplied together") + if args.admission_manifest is not None: + if not args.admission_manifest.is_file(): + parser.error("--admission-manifest must be an existing regular file") + if not SHA256_RE.fullmatch(args.admission_manifest_sha256): + parser.error("--admission-manifest-sha256 must be a lowercase SHA-256 digest") try: protected = { args.genesis_dump.resolve(), @@ -1705,6 +2337,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: *_declared_artifact_paths(args.ledger), *FIXED_ENGINE_PATHS, } + if args.admission_manifest is not None: + protected.add(args.admission_manifest.resolve()) except ReconstructionError as exc: parser.error(exc.safe_message) if args.output.resolve() in protected: diff --git a/ops/shard_corpus_extraction_work.py b/ops/shard_corpus_extraction_work.py new file mode 100644 index 0000000..a0a3ddb --- /dev/null +++ b/ops/shard_corpus_extraction_work.py @@ -0,0 +1,1023 @@ +#!/usr/bin/env python3 +"""Build or verify deterministic shards for a verified corpus work index. + +This command reads only inventory metadata. It does not read source snapshots, +extract claims, create proposals, admit knowledge, or write to a database. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import re +import stat +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +try: + from . import run_local_corpus_knowledge_rebuild as inventory +except ImportError: # pragma: no cover - direct script execution + import run_local_corpus_knowledge_rebuild as inventory + + +SHARD_SCHEMA = "livingip.localCorpusExtractionWorkShard.v1" +MANIFEST_SCHEMA = "livingip.localCorpusExtractionShardManifest.v1" +RECEIPT_SCHEMA = "livingip.localCorpusExtractionShardReceipt.v1" +STATUS_SCHEMA = "livingip.localCorpusExtractionShardStatus.v1" +INVENTORY_BINDING_SCHEMA = "livingip.localCorpusInventoryBinding.v1" + +MODE = "semantic-extraction-work-sharding" +ASSIGNMENT_ALGORITHM = "sha256_utf8_work_item_id_modulo_v1" +SHARDS_DIR = "shards" +MANIFEST_PATH = "shard-manifest.json" +RECEIPT_PATH = "receipt.json" +INCOMPLETE_MARKER = ".incomplete" +MIN_SHARD_COUNT = 1 +MAX_SHARD_COUNT = 1024 +MAX_METADATA_BYTES = inventory.MAX_METADATA_FILE_BYTES +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + +NO_WRITE_FLAGS = dict(inventory.NO_WRITE_FLAGS) +WORK_ITEM_KEYS = { + "admission_performed", + "artifact_sha256", + "database_write_performed", + "locator", + "relative_path", + "semantic_claim_extraction_performed", + "snapshot_path", + "work_item_id", + "work_type", +} +LOCATOR_KEYS = { + "line_end", + "line_start", + "quote_sha256", + "section_ordinal", + "utf8_byte_end", + "utf8_byte_start", +} + + +class ShardError(RuntimeError): + """Fail-closed sharding error with a stable, non-sensitive code.""" + + def __init__(self, code: str, safe_message: str) -> None: + super().__init__(safe_message) + self.code = code + self.safe_message = safe_message + + +@dataclass(frozen=True) +class InventoryView: + root: Path + binding: dict[str, Any] + items: list[dict[str, Any]] + + +@dataclass(frozen=True) +class ReservedOutput: + path: Path + name: str + parent_fd: int + root_fd: int + device: int + inode: int + + +def canonical_json_bytes(value: Any) -> bytes: + try: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise ShardError("invalid_json_value", "shard data is not deterministic JSON") from exc + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256(canonical_json_bytes(value)).hexdigest() + + +def rendered_json_bytes(value: Any) -> bytes: + try: + rendered = json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + indent=2, + sort_keys=True, + ) + except (TypeError, ValueError) as exc: + raise ShardError("invalid_json_value", "shard data is not deterministic JSON") from exc + return (rendered + "\n").encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _absolute_path(path: Path) -> Path: + expanded = path.expanduser() + return expanded if expanded.is_absolute() else Path.cwd() / expanded + + +def _directory_flags() -> int: + return os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + + +def _assert_exact_keys(value: Mapping[str, Any], expected: set[str], *, label: str) -> None: + if set(value) != expected: + raise ShardError("invalid_shard_schema", f"{label} fields do not match the supported schema") + + +def _assert_no_write_flags(value: Mapping[str, Any], *, label: str) -> None: + for key, expected in NO_WRITE_FLAGS.items(): + if value.get(key) is not expected: + raise ShardError("false_sharding_claim", f"{label} contains an unsupported side-effect claim") + + +def _assert_no_absolute_strings(value: Any) -> None: + if isinstance(value, dict): + for child in value.values(): + _assert_no_absolute_strings(child) + elif isinstance(value, list): + for child in value: + _assert_no_absolute_strings(child) + elif isinstance(value, str) and (value.startswith(("/", "\\")) or re.match(r"^[A-Za-z]:[\\/]", value)): + raise ShardError("absolute_path_disclosure", "shard metadata must not contain absolute paths") + + +def _duplicate_free_json_object(raw: bytes, *, label: str) -> dict[str, Any]: + def pairs_hook(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, child in pairs: + if key in value: + raise ShardError("duplicate_json_key", f"{label} contains a duplicate JSON key") + value[key] = child + return value + + def reject_constant(_value: str) -> None: + raise ShardError("invalid_json_value", f"{label} contains a non-finite JSON number") + + try: + parsed = json.loads(raw.decode("utf-8"), object_pairs_hook=pairs_hook, parse_constant=reject_constant) + except ShardError: + raise + except (UnicodeError, json.JSONDecodeError) as exc: + raise ShardError("invalid_json", f"{label} is not valid duplicate-free UTF-8 JSON") from exc + if not isinstance(parsed, dict): + raise ShardError("invalid_json", f"{label} must be a JSON object") + return parsed + + +def _read_regular_bytes(path: Path, *, label: str) -> bytes: + try: + before = path.lstat() + except OSError as exc: + raise ShardError("missing_bundle_entry", f"{label} is missing") from exc + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise ShardError("unsafe_bundle_entry", f"{label} must be a regular non-symlink file") + if before.st_size > MAX_METADATA_BYTES: + raise ShardError("metadata_limit_exceeded", f"{label} exceeds the metadata byte limit") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise ShardError("unsafe_bundle_entry", f"{label} could not be opened safely") from exc + try: + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or opened.st_dev != before.st_dev or opened.st_ino != before.st_ino: + raise ShardError("bundle_entry_changed", f"{label} changed while it was opened") + chunks: list[bytes] = [] + total = 0 + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > MAX_METADATA_BYTES: + raise ShardError("metadata_limit_exceeded", f"{label} exceeds the metadata byte limit") + chunks.append(chunk) + after = os.fstat(descriptor) + before_signature = (before.st_size, before.st_mtime_ns, before.st_ctime_ns) + after_signature = (after.st_size, after.st_mtime_ns, after.st_ctime_ns) + if before_signature != after_signature or total != after.st_size: + raise ShardError("bundle_entry_changed", f"{label} changed while it was read") + return b"".join(chunks) + finally: + os.close(descriptor) + + +def _inventory_status_signature(value: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + str(value.get("bundle_path")), + value.get("bundle_tree_sha256"), + value.get("receipt_file_sha256"), + value.get("source_count"), + value.get("unique_snapshot_count"), + value.get("work_item_count"), + ) + + +def _load_verified_inventory(bundle: Path) -> InventoryView: + try: + first = inventory.verify_bundle(bundle) + except inventory.RebuildError as exc: + raise ShardError("inventory_verification_failed", exc.safe_message) from exc + root = Path(first["bundle_path"]) + source_raw = _read_regular_bytes(root / inventory.SOURCE_MANIFEST_PATH, label="inventory source manifest") + work_raw = _read_regular_bytes( + root / inventory.EXTRACTION_WORK_INDEX_PATH, + label="inventory extraction-work index", + ) + receipt_raw = _read_regular_bytes(root / inventory.RECEIPT_PATH, label="inventory receipt") + source_manifest = _duplicate_free_json_object(source_raw, label="inventory source manifest") + work_index = _duplicate_free_json_object(work_raw, label="inventory extraction-work index") + receipt = _duplicate_free_json_object(receipt_raw, label="inventory receipt") + try: + second = inventory.verify_bundle(root) + except inventory.RebuildError as exc: + raise ShardError("inventory_verification_failed", exc.safe_message) from exc + if _inventory_status_signature(first) != _inventory_status_signature(second): + raise ShardError("inventory_changed", "inventory metadata changed during verification") + + items = work_index.get("items") + if not isinstance(items, list) or not items: + raise ShardError("invalid_inventory_work", "inventory must contain at least one extraction work item") + for item in items: + _validate_work_item(item, label="inventory work item") + + source_hash = sha256_bytes(source_raw) + work_hash = sha256_bytes(work_raw) + receipt_hash = sha256_bytes(receipt_raw) + receipt_hashes = receipt.get("hashes") + if not isinstance(receipt_hashes, dict): + raise ShardError("inventory_hash_mismatch", "inventory receipt hashes are malformed") + if source_hash != receipt_hashes.get("source_manifest_file_sha256"): + raise ShardError("inventory_hash_mismatch", "inventory source-manifest hash does not match") + if work_hash != receipt_hashes.get("extraction_work_index_file_sha256"): + raise ShardError("inventory_hash_mismatch", "inventory extraction-work hash does not match") + if receipt_hash != first.get("receipt_file_sha256"): + raise ShardError("inventory_hash_mismatch", "inventory receipt file hash does not match") + + binding_core = { + "schema": INVENTORY_BINDING_SCHEMA, + "mode": inventory.MODE_SOURCE_INVENTORY_ONLY, + "work_type": inventory.WORK_TYPE, + "source_manifest_schema": source_manifest.get("schema"), + "extraction_work_index_schema": work_index.get("schema"), + "inventory_receipt_schema": receipt.get("schema"), + "source_count": first.get("source_count"), + "work_item_count": first.get("work_item_count"), + "inventory_bundle_tree_sha256": first.get("bundle_tree_sha256"), + "source_manifest_file_sha256": source_hash, + "extraction_work_index_file_sha256": work_hash, + "inventory_receipt_file_sha256": receipt_hash, + } + binding = { + **binding_core, + "inventory_bundle_binding_sha256": canonical_sha256( + { + "inventory_bundle_tree_sha256": binding_core["inventory_bundle_tree_sha256"], + "inventory_receipt_file_sha256": receipt_hash, + } + ), + } + binding["inventory_metadata_sha256"] = canonical_sha256(binding) + _validate_inventory_binding(binding, expected=binding) + return InventoryView(root=root, binding=binding, items=items) + + +def _validate_inventory_binding(value: Any, *, expected: Mapping[str, Any]) -> dict[str, Any]: + if not isinstance(value, dict): + raise ShardError("inventory_mismatch", "inventory binding must be an object") + expected_keys = { + "schema", + "mode", + "work_type", + "source_manifest_schema", + "extraction_work_index_schema", + "inventory_receipt_schema", + "source_count", + "work_item_count", + "inventory_bundle_tree_sha256", + "source_manifest_file_sha256", + "extraction_work_index_file_sha256", + "inventory_receipt_file_sha256", + "inventory_bundle_binding_sha256", + "inventory_metadata_sha256", + } + _assert_exact_keys(value, expected_keys, label="inventory binding") + preimage = dict(value) + metadata_hash = preimage.pop("inventory_metadata_sha256", None) + if metadata_hash != canonical_sha256(preimage): + raise ShardError("inventory_mismatch", "inventory metadata hash does not verify") + bundle_binding = canonical_sha256( + { + "inventory_bundle_tree_sha256": value.get("inventory_bundle_tree_sha256"), + "inventory_receipt_file_sha256": value.get("inventory_receipt_file_sha256"), + } + ) + if value.get("inventory_bundle_binding_sha256") != bundle_binding: + raise ShardError("inventory_mismatch", "inventory bundle binding hash does not verify") + if value != expected: + raise ShardError("inventory_mismatch", "shards do not bind the supplied inventory bundle") + return value + + +def _validate_work_item(value: Any, *, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ShardError("invalid_work_item", f"{label} must be an object") + _assert_exact_keys(value, WORK_ITEM_KEYS, label=label) + _assert_no_write_flags(value, label=label) + if value.get("work_type") != inventory.WORK_TYPE: + raise ShardError("invalid_work_item", f"{label} has an unsupported work type") + work_id = value.get("work_item_id") + artifact_hash = value.get("artifact_sha256") + if not isinstance(work_id, str) or not SHA256_RE.fullmatch(work_id): + raise ShardError("invalid_work_item", f"{label} has a malformed work-item ID") + if not isinstance(artifact_hash, str) or not SHA256_RE.fullmatch(artifact_hash): + raise ShardError("invalid_work_item", f"{label} has a malformed artifact hash") + locator = value.get("locator") + if not isinstance(locator, dict): + raise ShardError("invalid_work_item", f"{label} has a malformed locator") + _assert_exact_keys(locator, LOCATOR_KEYS, label=f"{label} locator") + if not isinstance(locator.get("quote_sha256"), str) or not SHA256_RE.fullmatch(locator["quote_sha256"]): + raise ShardError("invalid_work_item", f"{label} has a malformed quote hash") + relative_path = value.get("relative_path") + snapshot_path = value.get("snapshot_path") + for candidate in (relative_path, snapshot_path): + if not isinstance(candidate, str): + raise ShardError("invalid_work_item", f"{label} has a malformed relative path") + path = PurePosixPath(candidate) + if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise ShardError("absolute_path_disclosure", f"{label} must contain only safe relative paths") + return value + + +def assignment_for(work_item_id: str, shard_count: int) -> int: + digest = hashlib.sha256(work_item_id.encode("utf-8")).digest() + return int.from_bytes(digest, "big") % shard_count + + +def _assignment_contract(shard_count: int) -> dict[str, Any]: + return { + "algorithm": ASSIGNMENT_ALGORITHM, + "hash": "sha256", + "input": "utf8(work_item_id)", + "reduction": "unsigned_big_endian_modulo_shard_count", + "shard_count": shard_count, + } + + +def _validate_shard_count(value: Any, *, work_item_count: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ShardError("invalid_shard_count", "shard count must be an integer") + if not MIN_SHARD_COUNT <= value <= MAX_SHARD_COUNT: + raise ShardError("invalid_shard_count", "shard count must be between 1 and 1024") + if value > work_item_count: + raise ShardError("invalid_shard_count", "shard count must not exceed the work-item count") + return value + + +def _prepare_output(output: Path, *, inventory_root: Path) -> Path: + absolute = _absolute_path(output) + if absolute.name in {"", ".", ".."}: + raise ShardError("unsafe_output", "output-dir must name a new directory") + try: + parent = absolute.parent.resolve(strict=True) + except OSError as exc: + raise ShardError("unsafe_output_parent", "output-dir parent must be an existing directory") from exc + try: + parent_metadata = parent.lstat() + except OSError as exc: + raise ShardError("unsafe_output_parent", "output-dir parent could not be inspected") from exc + if stat.S_ISLNK(parent_metadata.st_mode) or not stat.S_ISDIR(parent_metadata.st_mode): + raise ShardError("unsafe_output_parent", "output-dir parent must be a real directory") + prepared = parent / absolute.name + try: + prepared.lstat() + except FileNotFoundError: + pass + except OSError as exc: + raise ShardError("unsafe_output", "output-dir could not be inspected safely") from exc + else: + raise ShardError("output_exists", "output-dir must not already exist") + inventory_resolved = inventory_root.resolve() + common = Path(os.path.commonpath((str(prepared), str(inventory_resolved)))) + if common in {prepared, inventory_resolved}: + raise ShardError("overlapping_paths", "output-dir and inventory-bundle must not overlap") + return prepared + + +def _write_bytes_at(directory_fd: int, name: str, payload: bytes) -> None: + if not name or "/" in name or name in {".", ".."}: + raise ShardError("unsafe_output_entry", "output artifact name is unsafe") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(name, flags, 0o600, dir_fd=directory_fd) + except OSError as exc: + raise ShardError("output_write_failed", "output artifact could not be created exclusively") from exc + try: + view = memoryview(payload) + while view: + written = os.write(descriptor, view) + if written <= 0: + raise ShardError("output_write_failed", "output artifact write did not complete") + view = view[written:] + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _write_json_at(directory_fd: int, name: str, value: Any) -> bytes: + payload = rendered_json_bytes(value) + _write_bytes_at(directory_fd, name, payload) + return payload + + +def _reserve_output(output: Path) -> ReservedOutput: + try: + parent_fd = os.open(output.parent, _directory_flags()) + except OSError as exc: + raise ShardError("output_reservation_failed", "output parent could not be opened safely") from exc + try: + os.mkdir(output.name, mode=0o700, dir_fd=parent_fd) + except FileExistsError as exc: + os.close(parent_fd) + raise ShardError("output_exists", "output-dir must not already exist") from exc + except OSError as exc: + os.close(parent_fd) + raise ShardError("output_reservation_failed", "output-dir could not be reserved") from exc + root_fd: int | None = None + try: + root_fd = os.open(output.name, _directory_flags(), dir_fd=parent_fd) + metadata = os.fstat(root_fd) + reserved = ReservedOutput( + path=output, + name=output.name, + parent_fd=parent_fd, + root_fd=root_fd, + device=metadata.st_dev, + inode=metadata.st_ino, + ) + _write_bytes_at(root_fd, INCOMPLETE_MARKER, b"corpus extraction sharding incomplete\n") + return reserved + except Exception: + if root_fd is not None: + try: + _remove_tree_contents(root_fd) + except OSError: + pass + os.close(root_fd) + try: + os.rmdir(output.name, dir_fd=parent_fd) + except OSError: + pass + os.close(parent_fd) + raise + + +def _assert_output_binding(reserved: ReservedOutput) -> None: + try: + opened = os.fstat(reserved.root_fd) + linked = os.stat(reserved.name, dir_fd=reserved.parent_fd, follow_symlinks=False) + except OSError as exc: + raise ShardError("output_binding_changed", "reserved output directory is no longer available") from exc + for metadata in (opened, linked): + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_ISLNK(metadata.st_mode) + or metadata.st_dev != reserved.device + or metadata.st_ino != reserved.inode + ): + raise ShardError("output_binding_changed", "reserved output directory identity changed") + if stat.S_IMODE(opened.st_mode) != 0o700: + raise ShardError("unsafe_private_permissions", "reserved output directory permissions changed") + + +def _remove_tree_contents(directory_fd: int) -> None: + for name in os.listdir(directory_fd): + try: + metadata = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + continue + if stat.S_ISDIR(metadata.st_mode) and not stat.S_ISLNK(metadata.st_mode): + child_fd = os.open(name, _directory_flags(), dir_fd=directory_fd) + try: + _remove_tree_contents(child_fd) + finally: + os.close(child_fd) + os.rmdir(name, dir_fd=directory_fd) + else: + os.unlink(name, dir_fd=directory_fd) + + +def _release_reserved(reserved: ReservedOutput) -> None: + os.close(reserved.root_fd) + os.close(reserved.parent_fd) + + +def _cleanup_reserved(reserved: ReservedOutput) -> None: + try: + _remove_tree_contents(reserved.root_fd) + except OSError: + pass + try: + linked = os.stat(reserved.name, dir_fd=reserved.parent_fd, follow_symlinks=False) + if ( + stat.S_ISDIR(linked.st_mode) + and not stat.S_ISLNK(linked.st_mode) + and linked.st_dev == reserved.device + and linked.st_ino == reserved.inode + ): + os.rmdir(reserved.name, dir_fd=reserved.parent_fd) + except OSError: + pass + _release_reserved(reserved) + + +def _mkdir_at(directory_fd: int, name: str) -> int: + try: + os.mkdir(name, mode=0o700, dir_fd=directory_fd) + return os.open(name, _directory_flags(), dir_fd=directory_fd) + except OSError as exc: + raise ShardError("output_write_failed", "private shard directory could not be created") from exc + + +def _shard_path(index: int, shard_count: int) -> str: + return f"{SHARDS_DIR}/shard-{index:04d}-of-{shard_count:04d}.json" + + +def _artifact_records(root: Path, paths: Sequence[str]) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for relative in sorted(paths, key=lambda value: value.encode("utf-8")): + posix = PurePosixPath(relative) + if posix.is_absolute() or any(part in {"", ".", ".."} for part in posix.parts): + raise ShardError("unsafe_bundle_entry", "bundle artifact path is unsafe") + raw = _read_regular_bytes(root / Path(relative), label="shard artifact") + records.append({"bytes": len(raw), "path": relative, "sha256": sha256_bytes(raw)}) + return records + + +def _after_output_reservation(_reserved: ReservedOutput) -> None: + """Test seam after exclusive reservation; production does nothing.""" + + +def _before_final_publication(_inventory_root: Path) -> None: + """Test seam before final inventory revalidation; production does nothing.""" + + +def _before_receipt_publication(_reserved: ReservedOutput) -> None: + """Test seam while the incomplete marker must still exist.""" + + +def build_shard_bundle(inventory_bundle: Path, output_dir: Path, shard_count: int) -> dict[str, Any]: + view = _load_verified_inventory(inventory_bundle) + checked_count = _validate_shard_count(shard_count, work_item_count=len(view.items)) + output = _prepare_output(output_dir, inventory_root=view.root) + reserved = _reserve_output(output) + success = False + try: + _after_output_reservation(reserved) + _assert_output_binding(reserved) + assignments: list[list[dict[str, Any]]] = [[] for _ in range(checked_count)] + for item in sorted(view.items, key=lambda row: row["work_item_id"].encode("utf-8")): + assignments[assignment_for(item["work_item_id"], checked_count)].append(item) + + shards_fd = _mkdir_at(reserved.root_fd, SHARDS_DIR) + shard_records: list[dict[str, Any]] = [] + try: + for index, items in enumerate(assignments): + _assert_output_binding(reserved) + work_ids = [item["work_item_id"] for item in items] + shard_document = { + "schema": SHARD_SCHEMA, + "mode": MODE, + "deterministic": True, + **NO_WRITE_FLAGS, + "work_type": inventory.WORK_TYPE, + "inventory": view.binding, + "assignment": {**_assignment_contract(checked_count), "shard_index": index}, + "work_item_count": len(items), + "work_item_ids_sha256": canonical_sha256(work_ids), + "items": items, + } + filename = Path(_shard_path(index, checked_count)).name + payload = _write_json_at(shards_fd, filename, shard_document) + shard_records.append( + { + "bytes": len(payload), + "index": index, + "path": _shard_path(index, checked_count), + "sha256": sha256_bytes(payload), + "work_item_count": len(items), + "work_item_ids_sha256": canonical_sha256(work_ids), + } + ) + finally: + os.close(shards_fd) + + assignment_contract = _assignment_contract(checked_count) + manifest = { + "schema": MANIFEST_SCHEMA, + "mode": MODE, + "deterministic": True, + **NO_WRITE_FLAGS, + "work_type": inventory.WORK_TYPE, + "inventory": view.binding, + "assignment": assignment_contract, + "shard_count": checked_count, + "work_item_count": len(view.items), + "shards": shard_records, + } + _assert_output_binding(reserved) + _write_json_at(reserved.root_fd, MANIFEST_PATH, manifest) + + _before_final_publication(view.root) + fresh_view = _load_verified_inventory(view.root) + if fresh_view.binding != view.binding or fresh_view.items != view.items: + raise ShardError("inventory_changed", "inventory changed before shard publication") + + artifact_paths = [MANIFEST_PATH, *(record["path"] for record in shard_records)] + artifacts = _artifact_records(output, artifact_paths) + receipt = { + "schema": RECEIPT_SCHEMA, + "status": "pass", + "result": "extraction_work_shards_ready", + "mode": MODE, + "deterministic": True, + **NO_WRITE_FLAGS, + "work_type": inventory.WORK_TYPE, + "inventory": view.binding, + "assignment": assignment_contract, + "shard_count": checked_count, + "work_item_count": len(view.items), + "outputs": { + "artifact_count": len(artifacts), + "artifacts": artifacts, + "manifest_path": MANIFEST_PATH, + }, + "hashes": { + "bundle_tree_sha256": canonical_sha256(artifacts), + "manifest_file_sha256": next( + record["sha256"] for record in artifacts if record["path"] == MANIFEST_PATH + ), + }, + } + receipt["hashes"]["receipt_content_sha256"] = canonical_sha256(receipt) + _before_receipt_publication(reserved) + _assert_output_binding(reserved) + _write_json_at(reserved.root_fd, RECEIPT_PATH, receipt) + os.fsync(reserved.root_fd) + os.unlink(INCOMPLETE_MARKER, dir_fd=reserved.root_fd) + os.fsync(reserved.root_fd) + _assert_output_binding(reserved) + verified = verify_shard_bundle(view.root, output) + _assert_output_binding(reserved) + success = True + return {"bundle_path": output, "verification": verified} + finally: + if success: + _release_reserved(reserved) + else: + _cleanup_reserved(reserved) + + +def _validate_shard_root(bundle: Path) -> Path: + absolute = _absolute_path(bundle) + try: + metadata = absolute.lstat() + except OSError as exc: + raise ShardError("missing_shard_bundle", "shard bundle must be an existing directory") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise ShardError("unsafe_shard_bundle", "shard bundle must be a real non-symlink directory") + root = absolute.resolve() + if stat.S_IMODE(metadata.st_mode) != 0o700: + raise ShardError("unsafe_private_permissions", "shard bundle directory permissions must be 0700") + marker = root / INCOMPLETE_MARKER + if marker.exists() or marker.is_symlink(): + raise ShardError("incomplete_shard_bundle", "shard bundle retains its incomplete marker") + return root + + +def _scan_bundle_tree(root: Path) -> tuple[set[str], set[str]]: + files: set[str] = set() + directories: set[str] = set() + + def visit(directory: Path, relative_parent: PurePosixPath) -> None: + for child in sorted(directory.iterdir(), key=lambda path: path.name.encode("utf-8")): + relative = (relative_parent / child.name).as_posix() + metadata = child.lstat() + if stat.S_ISLNK(metadata.st_mode): + raise ShardError("unsafe_bundle_entry", "shard bundle contains a symlink") + if stat.S_ISDIR(metadata.st_mode): + if stat.S_IMODE(metadata.st_mode) != 0o700: + raise ShardError("unsafe_private_permissions", "shard directory permissions must be 0700") + directories.add(relative) + visit(child, relative_parent / child.name) + elif stat.S_ISREG(metadata.st_mode): + if stat.S_IMODE(metadata.st_mode) != 0o600: + raise ShardError("unsafe_private_permissions", "shard file permissions must be 0600") + files.add(relative) + else: + raise ShardError("unsafe_bundle_entry", "shard bundle contains a special file") + + visit(root, PurePosixPath()) + return files, directories + + +def _validate_common_document( + value: Any, + *, + expected_keys: set[str], + label: str, + expected_inventory: Mapping[str, Any], +) -> dict[str, Any]: + if not isinstance(value, dict): + raise ShardError("invalid_shard_schema", f"{label} must be an object") + _assert_exact_keys(value, expected_keys, label=label) + _assert_no_write_flags(value, label=label) + if value.get("mode") != MODE or value.get("deterministic") is not True: + raise ShardError("invalid_shard_schema", f"{label} mode or deterministic flag is unsupported") + if value.get("work_type") != inventory.WORK_TYPE: + raise ShardError("invalid_shard_schema", f"{label} work type is unsupported") + _validate_inventory_binding(value.get("inventory"), expected=expected_inventory) + _assert_no_absolute_strings(value) + return value + + +def verify_shard_bundle(inventory_bundle: Path, shard_bundle: Path) -> dict[str, Any]: + view = _load_verified_inventory(inventory_bundle) + root = _validate_shard_root(shard_bundle) + manifest_raw = _read_regular_bytes(root / MANIFEST_PATH, label="shard manifest") + receipt_raw = _read_regular_bytes(root / RECEIPT_PATH, label="shard receipt") + manifest = _duplicate_free_json_object(manifest_raw, label="shard manifest") + receipt = _duplicate_free_json_object(receipt_raw, label="shard receipt") + + manifest_keys = { + "schema", + "mode", + "deterministic", + *NO_WRITE_FLAGS, + "work_type", + "inventory", + "assignment", + "shard_count", + "work_item_count", + "shards", + } + receipt_keys = { + "schema", + "status", + "result", + "mode", + "deterministic", + *NO_WRITE_FLAGS, + "work_type", + "inventory", + "assignment", + "shard_count", + "work_item_count", + "outputs", + "hashes", + } + _validate_common_document( + manifest, + expected_keys=manifest_keys, + label="shard manifest", + expected_inventory=view.binding, + ) + _validate_common_document( + receipt, + expected_keys=receipt_keys, + label="shard receipt", + expected_inventory=view.binding, + ) + if manifest.get("schema") != MANIFEST_SCHEMA or receipt.get("schema") != RECEIPT_SCHEMA: + raise ShardError("invalid_shard_schema", "shard manifest or receipt schema is unsupported") + if receipt.get("status") != "pass" or receipt.get("result") != "extraction_work_shards_ready": + raise ShardError("invalid_shard_schema", "receipt does not represent a completed shard bundle") + + shard_count = _validate_shard_count(manifest.get("shard_count"), work_item_count=len(view.items)) + if receipt.get("shard_count") != shard_count: + raise ShardError("invalid_shard_cardinality", "receipt shard count differs from the manifest") + assignment = _assignment_contract(shard_count) + if manifest.get("assignment") != assignment or receipt.get("assignment") != assignment: + raise ShardError("invalid_assignment", "shard assignment contract is unsupported") + if manifest.get("work_item_count") != len(view.items) or receipt.get("work_item_count") != len(view.items): + raise ShardError("invalid_shard_cardinality", "shard work-item count differs from the inventory") + + shard_records = manifest.get("shards") + if not isinstance(shard_records, list) or len(shard_records) != shard_count: + raise ShardError("invalid_shard_cardinality", "manifest must contain one record per shard") + expected_files = {MANIFEST_PATH, RECEIPT_PATH} + expected_files.update(_shard_path(index, shard_count) for index in range(shard_count)) + actual_files, actual_directories = _scan_bundle_tree(root) + if actual_directories != {SHARDS_DIR} or actual_files != expected_files: + raise ShardError("unexpected_bundle_entry", "shard bundle contains missing or unexpected entries") + + expected_by_id = {item["work_item_id"]: item for item in view.items} + seen: set[str] = set() + expected_shard_records: list[dict[str, Any]] = [] + shard_paths: list[str] = [] + for index, record in enumerate(shard_records): + if not isinstance(record, dict): + raise ShardError("invalid_shard_schema", "shard record must be an object") + _assert_exact_keys( + record, + {"bytes", "index", "path", "sha256", "work_item_count", "work_item_ids_sha256"}, + label="shard record", + ) + path = _shard_path(index, shard_count) + if record.get("index") != index or record.get("path") != path: + raise ShardError("invalid_shard_order", "shard records are not in deterministic index order") + raw = _read_regular_bytes(root / Path(path), label="shard file") + document = _duplicate_free_json_object(raw, label="shard file") + shard_keys = { + "schema", + "mode", + "deterministic", + *NO_WRITE_FLAGS, + "work_type", + "inventory", + "assignment", + "work_item_count", + "work_item_ids_sha256", + "items", + } + _validate_common_document( + document, + expected_keys=shard_keys, + label="shard file", + expected_inventory=view.binding, + ) + if document.get("schema") != SHARD_SCHEMA: + raise ShardError("invalid_shard_schema", "shard file schema is unsupported") + expected_assignment = {**assignment, "shard_index": index} + if document.get("assignment") != expected_assignment: + raise ShardError("invalid_assignment", "shard file assignment metadata is unsupported") + items = document.get("items") + if not isinstance(items, list): + raise ShardError("invalid_shard_schema", "shard items must be an array") + ids: list[str] = [] + previous_id: str | None = None + for item in items: + _validate_work_item(item, label="shard work item") + work_id = item["work_item_id"] + if previous_id is not None and work_id.encode("utf-8") <= previous_id.encode("utf-8"): + raise ShardError("invalid_shard_order", "shard work items are not in deterministic order") + previous_id = work_id + if work_id in seen: + raise ShardError("duplicate_work_item", "a work-item ID appears more than once across shards") + if work_id not in expected_by_id: + raise ShardError("unknown_work_item", "a shard contains a work-item ID absent from the inventory") + if item != expected_by_id[work_id]: + raise ShardError("inventory_mismatch", "a shard work item differs from the verified inventory row") + if assignment_for(work_id, shard_count) != index: + raise ShardError("invalid_assignment", "a work item is assigned to the wrong hash shard") + seen.add(work_id) + ids.append(work_id) + ids_hash = canonical_sha256(ids) + if document.get("work_item_count") != len(items) or document.get("work_item_ids_sha256") != ids_hash: + raise ShardError("invalid_shard_cardinality", "shard item count or ID hash is incorrect") + expected_record = { + "bytes": len(raw), + "index": index, + "path": path, + "sha256": sha256_bytes(raw), + "work_item_count": len(items), + "work_item_ids_sha256": ids_hash, + } + if record != expected_record: + raise ShardError("invalid_shard_hash", "manifest shard record does not match shard bytes") + expected_shard_records.append(expected_record) + shard_paths.append(path) + + missing = set(expected_by_id) - seen + if missing: + raise ShardError("missing_work_item", "one or more inventory work items are absent from all shards") + if shard_records != expected_shard_records: + raise ShardError("invalid_shard_hash", "manifest shard records are not reproducible") + + artifacts = _artifact_records(root, [MANIFEST_PATH, *shard_paths]) + outputs = receipt.get("outputs") + hashes = receipt.get("hashes") + if not isinstance(outputs, dict) or not isinstance(hashes, dict): + raise ShardError("invalid_shard_schema", "receipt outputs or hashes are malformed") + _assert_exact_keys(outputs, {"artifact_count", "artifacts", "manifest_path"}, label="receipt outputs") + _assert_exact_keys( + hashes, + {"bundle_tree_sha256", "manifest_file_sha256", "receipt_content_sha256"}, + label="receipt hashes", + ) + if outputs.get("manifest_path") != MANIFEST_PATH: + raise ShardError("invalid_shard_schema", "receipt manifest path is unsupported") + if outputs.get("artifacts") != artifacts or outputs.get("artifact_count") != len(artifacts): + raise ShardError("invalid_shard_hash", "receipt artifact inventory does not match bundle bytes") + expected_hashes = { + "bundle_tree_sha256": canonical_sha256(artifacts), + "manifest_file_sha256": sha256_bytes(manifest_raw), + } + for key, expected in expected_hashes.items(): + if hashes.get(key) != expected: + raise ShardError("invalid_shard_hash", f"receipt {key} does not match bundle bytes") + receipt_preimage = copy.deepcopy(receipt) + preimage_hashes = receipt_preimage.get("hashes") + if not isinstance(preimage_hashes, dict): + raise ShardError("invalid_shard_schema", "receipt hash object is malformed") + recorded_receipt_hash = preimage_hashes.pop("receipt_content_sha256", None) + if recorded_receipt_hash != canonical_sha256(receipt_preimage): + raise ShardError("invalid_receipt_hash", "receipt content hash does not verify") + + return { + "bundle_path": root, + "bundle_tree_sha256": expected_hashes["bundle_tree_sha256"], + "inventory_bundle_binding_sha256": view.binding["inventory_bundle_binding_sha256"], + "receipt_file_sha256": sha256_bytes(receipt_raw), + "shard_count": shard_count, + "status": "pass", + "work_item_count": len(seen), + } + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inventory-bundle", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, help="new directory for deterministic shard artifacts") + parser.add_argument("--shard-count", type=int, help="number of stable hash shards, from 1 through 1024") + parser.add_argument("--verify-bundle", type=Path, help="read-only validation of an existing shard bundle") + return parser.parse_args(argv) + + +def _print_status( + status: str, + *, + operation: str, + reason: str | None = None, + message: str | None = None, + result: Mapping[str, Any] | None = None, +) -> None: + payload: dict[str, Any] = { + "schema": STATUS_SCHEMA, + "status": status, + "operation": operation, + "mode": MODE, + **NO_WRITE_FLAGS, + } + if reason is not None: + payload["reason"] = reason + if message is not None: + payload["message"] = message + if result is not None: + payload.update( + { + "bundle_tree_sha256": result["bundle_tree_sha256"], + "inventory_bundle_binding_sha256": result["inventory_bundle_binding_sha256"], + "receipt_file_sha256": result["receipt_file_sha256"], + "shard_count": result["shard_count"], + "work_item_count": result["work_item_count"], + } + ) + print(json.dumps(payload, ensure_ascii=True, sort_keys=True)) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + operation = "verify" if args.verify_bundle is not None else "build" + try: + if args.verify_bundle is not None: + if args.output_dir is not None or args.shard_count is not None: + raise ShardError("invalid_arguments", "verify mode accepts only inventory-bundle and verify-bundle") + result = verify_shard_bundle(args.inventory_bundle, args.verify_bundle) + else: + if args.output_dir is None or args.shard_count is None: + raise ShardError("invalid_arguments", "build mode requires output-dir and shard-count") + built = build_shard_bundle(args.inventory_bundle, args.output_dir, args.shard_count) + result = built["verification"] + except ShardError as exc: + _print_status("rejected", operation=operation, reason=exc.code, message=exc.safe_message) + return 2 + except OSError: + _print_status( + "rejected", + operation=operation, + reason="filesystem_operation_failed", + message="a fail-closed filesystem operation did not complete", + ) + return 2 + _print_status("pass", operation=operation, result=result) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/teleo_v3_epistemic_contract.sql b/ops/teleo_v3_epistemic_contract.sql new file mode 100644 index 0000000..f2e6fa8 --- /dev/null +++ b/ops/teleo_v3_epistemic_contract.sql @@ -0,0 +1,2293 @@ +-- Minimum additive LivingIP V3 epistemic contract for the canonical KB tables. +-- +-- This is a compatibility migration, not a production rollout. Existing rows +-- remain in the visible legacy tier. Only rows explicitly linked to an accepted +-- proposal are required to satisfy the modern provenance contract. +-- +-- CUTOVER ORDER: deploy the cutover-aware writer, stop the default V2 worker, +-- ALTER ROLE kb_apply NOLOGIN, and end every kb_apply session before running this +-- migration. Approved historical V2 proposals remain review-visible, but the V3 +-- contract blocks their execution until each packet is migrated or rejected. + +\set ON_ERROR_STOP on +set timezone = 'UTC'; + +begin; + +-- Stable bigint key: controlled V3 function/trigger DDL is exclusive while +-- authorized applies hold the shared transaction form through commit/rollback. +select pg_catalog.pg_advisory_xact_lock(6072343533146485505); + +-- The worker mirrors this declaration, and tests require byte-equivalent +-- catalog predicates in preflight/postflight. SHA-256 pins every guard body. +do $trigger_contract_declaration$ +begin + perform pg_catalog.set_config( + 'teleo.v3_expected_trigger_contract', + $trigger_contract$ +[ + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_enforce_apply_contract_cutover", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": true, + "function_schema": "kb_stage", + "function_security_definer": true, + "function_source_sha256": "3ab2d5e1d5d1d18b13579fe2c0cae487036c7b147cf7f1b6b9a3633274756c18", + "schema_name": "kb_stage", + "table_name": "kb_proposals", + "trigger_name": "teleo_v3_00_enforce_apply_contract_cutover", + "trigger_type": 23, + "update_columns": [ + "payload", + "proposal_type", + "status" + ], + "when_expression": null + }, + { + "argument_count": 2, + "argument_hex": "61636365707465645f62795f70726f706f73616c5f69640061646d697373696f6e5f737461746500", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_linked_insert", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d8c190bd2fc90159b9a4824015c10a73a701a263bba90b67f8ccce75e454e6f8", + "schema_name": "public", + "table_name": "claims", + "trigger_name": "teleo_v3_00_claim_require_linked_insert", + "trigger_type": 7, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 1, + "argument_hex": "61636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_accepted_proposal", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": true, + "function_schema": "kb_stage", + "function_security_definer": true, + "function_source_sha256": "ca398ef2f185488c0712bd5ebc22402ab9e41e5dfd2f5823b367e6a382e8a64f", + "schema_name": "public", + "table_name": "claims", + "trigger_name": "teleo_v3_claim_require_accepted_proposal", + "trigger_type": 23, + "update_columns": [ + "accepted_by_proposal_id" + ], + "when_expression": null + }, + { + "argument_count": 1, + "argument_hex": "61636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_linked_insert", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d8c190bd2fc90159b9a4824015c10a73a701a263bba90b67f8ccce75e454e6f8", + "schema_name": "public", + "table_name": "sources", + "trigger_name": "teleo_v3_00_source_require_linked_insert", + "trigger_type": 7, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 20, + "argument_hex": "6964006f776e65725f6167656e745f6964007479706500746578740070726f706f736974696f6e00626f64795f6d61726b646f776e0073636f7065006163636573735f73636f7065007374617475730061646d697373696f6e5f7374617465007265766973696f6e5f6372697465726961007265766973696f6e5f6b696e6400737570657273656465735f696400737570657273656465645f627900636f6e666964656e6365007461677300637265617465645f627900637265617465645f617400757064617465645f61740061636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_guard_immutable_fields", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d243741e552fe8c232440ffa5f853cf261f4b86b6f32d76f375734a2235c2d13", + "schema_name": "public", + "table_name": "claims", + "trigger_name": "teleo_v3_claim_immutable", + "trigger_type": 19, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_reject_delete", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "46190723ab4520954858dbd4bb6c9230e68e41c68b5aee3b8c2719e1416de43c", + "schema_name": "public", + "table_name": "claims", + "trigger_name": "teleo_v3_claim_reject_delete", + "trigger_type": 11, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 1, + "argument_hex": "61636365707465645f66726f6d5f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_linked_insert", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d8c190bd2fc90159b9a4824015c10a73a701a263bba90b67f8ccce75e454e6f8", + "schema_name": "public", + "table_name": "claim_evidence", + "trigger_name": "teleo_v3_00_evidence_require_linked_insert", + "trigger_type": 7, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 1, + "argument_hex": "61636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_accepted_proposal", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": true, + "function_schema": "kb_stage", + "function_security_definer": true, + "function_source_sha256": "ca398ef2f185488c0712bd5ebc22402ab9e41e5dfd2f5823b367e6a382e8a64f", + "schema_name": "public", + "table_name": "sources", + "trigger_name": "teleo_v3_source_require_accepted_proposal", + "trigger_type": 23, + "update_columns": [ + "accepted_by_proposal_id" + ], + "when_expression": null + }, + { + "argument_count": 1, + "argument_hex": "61636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_linked_insert", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d8c190bd2fc90159b9a4824015c10a73a701a263bba90b67f8ccce75e454e6f8", + "schema_name": "public", + "table_name": "claim_edges", + "trigger_name": "teleo_v3_00_edge_require_linked_insert", + "trigger_type": 7, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 18, + "argument_hex": "696400736f757263655f747970650075726c0073746f726167655f70617468006578636572707400686173680063616e6f6e6963616c5f75726c0073746f726167655f75726900636f6e74656e745f68617368006163636573735f73636f706500696e67657374696f6e5f6f726967696e0070726f76656e616e63655f73746174757300736f757263655f636c617373006175746f5f70726f6d6f74696f6e5f706f6c69637900637265617465645f62790063617074757265645f617400637265617465645f61740061636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_guard_immutable_fields", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d243741e552fe8c232440ffa5f853cf261f4b86b6f32d76f375734a2235c2d13", + "schema_name": "public", + "table_name": "sources", + "trigger_name": "teleo_v3_source_immutable", + "trigger_type": 19, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_reject_delete", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "46190723ab4520954858dbd4bb6c9230e68e41c68b5aee3b8c2719e1416de43c", + "schema_name": "public", + "table_name": "sources", + "trigger_name": "teleo_v3_source_reject_delete", + "trigger_type": 11, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 1, + "argument_hex": "61636365707465645f66726f6d5f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_accepted_proposal", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": true, + "function_schema": "kb_stage", + "function_security_definer": true, + "function_source_sha256": "ca398ef2f185488c0712bd5ebc22402ab9e41e5dfd2f5823b367e6a382e8a64f", + "schema_name": "public", + "table_name": "claim_evidence", + "trigger_name": "teleo_v3_evidence_require_accepted_proposal", + "trigger_type": 23, + "update_columns": [ + "accepted_from_proposal_id" + ], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_validate_claim_evidence", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "4c697c497699760214aca99f255c215e393319fd4482d4b36efac839280c3733", + "schema_name": "public", + "table_name": "claim_evidence", + "trigger_name": "teleo_v3_evidence_validate_provenance", + "trigger_type": 23, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 13, + "argument_hex": "696400636c61696d5f696400736f757263655f696400726f6c65007765696768740065786365727074006c6f6361746f725f6a736f6e00736f757263655f636f6e74656e745f6861736800726174696f6e616c6500706f6c617269747900637265617465645f627900637265617465645f61740061636365707465645f66726f6d5f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_guard_immutable_fields", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d243741e552fe8c232440ffa5f853cf261f4b86b6f32d76f375734a2235c2d13", + "schema_name": "public", + "table_name": "claim_evidence", + "trigger_name": "teleo_v3_evidence_immutable", + "trigger_type": 19, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_reject_delete", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "46190723ab4520954858dbd4bb6c9230e68e41c68b5aee3b8c2719e1416de43c", + "schema_name": "public", + "table_name": "claim_evidence", + "trigger_name": "teleo_v3_evidence_reject_delete", + "trigger_type": 11, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 1, + "argument_hex": "61636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_accepted_proposal", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": true, + "function_schema": "kb_stage", + "function_security_definer": true, + "function_source_sha256": "ca398ef2f185488c0712bd5ebc22402ab9e41e5dfd2f5823b367e6a382e8a64f", + "schema_name": "public", + "table_name": "claim_edges", + "trigger_name": "teleo_v3_edge_require_accepted_proposal", + "trigger_type": 23, + "update_columns": [ + "accepted_by_proposal_id" + ], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_validate_claim_edge", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "6a7a3e9d1d5ac6e9490abc9b7b1f321bb2e0948aea2671415aad28101f4ee936", + "schema_name": "public", + "table_name": "claim_edges", + "trigger_name": "teleo_v3_edge_validate_endpoints", + "trigger_type": 23, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 11, + "argument_hex": "69640066726f6d5f636c61696d00746f5f636c61696d00656467655f74797065007765696768740072656c6174696f6e5f7479706500726174696f6e616c650073636f70655f6f725f636f6e646974696f6e7300637265617465645f627900637265617465645f61740061636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_guard_immutable_fields", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d243741e552fe8c232440ffa5f853cf261f4b86b6f32d76f375734a2235c2d13", + "schema_name": "public", + "table_name": "claim_edges", + "trigger_name": "teleo_v3_edge_immutable", + "trigger_type": 19, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_reject_delete", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "46190723ab4520954858dbd4bb6c9230e68e41c68b5aee3b8c2719e1416de43c", + "schema_name": "public", + "table_name": "claim_edges", + "trigger_name": "teleo_v3_edge_reject_delete", + "trigger_type": 11, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 1, + "argument_hex": "61636365707465645f62795f70726f706f73616c5f696400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_require_accepted_proposal", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": true, + "function_schema": "kb_stage", + "function_security_definer": true, + "function_source_sha256": "ca398ef2f185488c0712bd5ebc22402ab9e41e5dfd2f5823b367e6a382e8a64f", + "schema_name": "public", + "table_name": "claim_evidence_assessments", + "trigger_name": "teleo_v3_assessment_require_accepted_proposal", + "trigger_type": 23, + "update_columns": [ + "accepted_by_proposal_id" + ], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_validate_evidence_assessment", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "860f939b4878abe8e2e604e03be9eeb6c23b4878af4b19f9ec42de4b821fac61", + "schema_name": "public", + "table_name": "claim_evidence_assessments", + "trigger_name": "teleo_v3_assessment_validate", + "trigger_type": 23, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 10, + "argument_hex": "696400636c61696d5f696400737570706f72745f74696572006368616c6c656e67655f74696572006f766572616c6c5f737461746500726174696f6e616c650065766964656e63655f7365745f686173680061636365707465645f62795f70726f706f73616c5f696400737570657273656465735f6173736573736d656e745f696400637265617465645f617400", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_guard_immutable_fields", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "d243741e552fe8c232440ffa5f853cf261f4b86b6f32d76f375734a2235c2d13", + "schema_name": "public", + "table_name": "claim_evidence_assessments", + "trigger_name": "teleo_v3_assessment_immutable", + "trigger_type": 19, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_reject_delete", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "46190723ab4520954858dbd4bb6c9230e68e41c68b5aee3b8c2719e1416de43c", + "schema_name": "public", + "table_name": "claim_evidence_assessments", + "trigger_name": "teleo_v3_assessment_reject_delete", + "trigger_type": 11, + "update_columns": [], + "when_expression": null + }, + { + "argument_count": 0, + "argument_hex": "", + "function_config": [ + "search_path=pg_catalog, pg_temp" + ], + "function_identity_arguments": "", + "function_name": "teleo_v3_guard_linked_proposal_decision", + "function_owner": "kb_gate_owner", + "function_owner_only_execute": null, + "function_schema": "kb_stage", + "function_security_definer": false, + "function_source_sha256": "602ee8f81f4515b0237f09a503ecbfe4fce2dc4a799e13f833554103b0e83e32", + "schema_name": "kb_stage", + "table_name": "kb_proposals", + "trigger_name": "teleo_v3_guard_linked_proposal_decision", + "trigger_type": 19, + "update_columns": [ + "applied_at", + "applied_by_agent_id", + "applied_by_handle", + "payload", + "proposal_type", + "rationale", + "review_note", + "reviewed_at", + "reviewed_by_agent_id", + "reviewed_by_handle", + "status" + ], + "when_expression": null + } +] +$trigger_contract$, + true + ); +end +$trigger_contract_declaration$; + + +do $preflight$ +declare + v_contract_state text; + v_apply_can_login boolean; + v_legacy_queue_count bigint; + v_preexisting_objects text; + v_ownership_valid boolean; +begin + if to_regclass('public.agents') is null then + raise exception 'teleo_v3_epistemic_contract: missing public.agents'; + end if; + if to_regclass('public.claims') is null then + raise exception 'teleo_v3_epistemic_contract: missing public.claims'; + end if; + if to_regclass('public.sources') is null then + raise exception 'teleo_v3_epistemic_contract: missing public.sources'; + end if; + if to_regclass('public.claim_evidence') is null then + raise exception 'teleo_v3_epistemic_contract: missing public.claim_evidence'; + end if; + if to_regclass('public.claim_edges') is null then + raise exception 'teleo_v3_epistemic_contract: missing public.claim_edges'; + end if; + if to_regclass('kb_stage.kb_proposals') is null then + raise exception 'teleo_v3_epistemic_contract: missing kb_stage.kb_proposals'; + end if; + if not exists ( + select 1 from pg_catalog.pg_roles role + where role.rolname = 'kb_gate_owner' + and not role.rolcanlogin + and not role.rolinherit + and not role.rolsuper + and not role.rolcreatedb + and not role.rolcreaterole + and not role.rolreplication + and not role.rolbypassrls + ) then + raise exception using + errcode = '55000', + message = 'teleo_v3_epistemic_contract: kb_gate_owner must be NOLOGIN NOINHERIT and unprivileged'; + end if; + if not exists (select 1 from pg_catalog.pg_roles where rolname = 'kb_apply') then + raise exception 'teleo_v3_epistemic_contract: missing runtime role kb_apply'; + end if; + if not exists (select 1 from pg_catalog.pg_roles where rolname = 'kb_review') then + raise exception 'teleo_v3_epistemic_contract: missing runtime role kb_review'; + end if; + if exists ( + select 1 + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles member_role on member_role.oid = membership.member + join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid + where member_role.rolname in ('kb_gate_owner', 'kb_review', 'kb_apply') + or granted_role.rolname in ('kb_gate_owner', 'kb_review', 'kb_apply') + ) then + raise exception using + errcode = '55000', + message = 'teleo_v3_epistemic_contract: protected role membership drift must be removed'; + end if; + if not pg_catalog.has_schema_privilege('kb_gate_owner', 'kb_stage', 'USAGE') + or not pg_catalog.has_table_privilege('kb_gate_owner', 'kb_stage.kb_proposals', 'SELECT') + or not pg_catalog.has_table_privilege('kb_gate_owner', 'kb_stage.kb_proposals', 'UPDATE') then + raise exception 'teleo_v3_epistemic_contract: kb_gate_owner lacks narrow proposal-ledger lock access'; + end if; + + -- TELEO_V3_CONTRACT_STATE_QUERY_BEGIN +with expected_triggers as ( + select expected.* + from pg_catalog.jsonb_to_recordset(pg_catalog.current_setting('teleo.v3_expected_trigger_contract')::jsonb) as expected( + schema_name text, + table_name text, + trigger_name text, + trigger_type integer, + update_columns jsonb, + when_expression text, + argument_count integer, + argument_hex text, + function_schema text, + function_name text, + function_identity_arguments text, + function_owner text, + function_security_definer boolean, + function_config jsonb, + function_source_sha256 text, + function_owner_only_execute boolean + ) +), expected_functions as ( + select distinct function_schema, function_name, function_identity_arguments, + function_owner, function_security_definer, function_config, + function_source_sha256, function_owner_only_execute + from expected_triggers +), actual_v3_functions as ( + select procedure.oid, namespace.nspname as function_schema, + procedure.proname as function_name, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + owner_role.rolname as function_owner, + procedure.prosecdef as function_security_definer, + pg_catalog.to_jsonb(procedure.proconfig) as function_config, + pg_catalog.encode( + pg_catalog.sha256(pg_catalog.convert_to(procedure.prosrc, 'UTF8')), + 'hex' + ) as function_source_sha256, + not exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + ) as function_owner_only_execute, + language.lanname as function_language, + procedure.prokind as function_kind, + procedure.pronargs as function_argument_count, + procedure.prorettype + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner + join pg_catalog.pg_language language on language.oid = procedure.prolang + where namespace.nspname = 'kb_stage' + and pg_catalog.left(procedure.proname, 9) = E'teleo_v3_' + and procedure.proname <> E'teleo_v3_acquire_protected_relation_locks' +), actual_v3_catalog_lock_functions as ( + select procedure.oid, + owner_role.rolname as function_owner, + procedure.prosecdef as function_security_definer, + pg_catalog.to_jsonb(procedure.proconfig) as function_config, + pg_catalog.encode( + pg_catalog.sha256(pg_catalog.convert_to(procedure.prosrc, 'UTF8')), + 'hex' + ) as function_source_sha256, + language.lanname as function_language, + procedure.prokind as function_kind, + procedure.pronargs as function_argument_count, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + procedure.prorettype, + exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee + where acl.privilege_type = 'EXECUTE' + and grantee.rolname = 'kb_apply' + ) as apply_execute, + not exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + and acl.grantee <> ( + select role.oid from pg_catalog.pg_roles role where role.rolname = 'kb_apply' + ) + ) as execute_acl_exact + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner + join pg_catalog.pg_language language on language.oid = procedure.prolang + where namespace.nspname = 'kb_stage' + and procedure.proname = E'teleo_v3_acquire_protected_relation_locks' +), actual_v3_triggers as ( + select trigger_row.*, table_namespace.nspname as schema_name, + relation.relname as table_name, + function_namespace.nspname as function_schema, + procedure.proname as function_name, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + coalesce(( + select pg_catalog.jsonb_agg(attribute.attname order by attribute.attname) + from pg_catalog.unnest(trigger_row.tgattr::smallint[]) update_column(attnum) + join pg_catalog.pg_attribute attribute + on attribute.attrelid = trigger_row.tgrelid + and attribute.attnum = update_column.attnum + and not attribute.attisdropped + ), '[]'::jsonb) as update_columns, + pg_catalog.pg_get_expr(trigger_row.tgqual, trigger_row.tgrelid, true) as when_expression, + pg_catalog.encode(trigger_row.tgargs, 'hex') as argument_hex + from pg_catalog.pg_trigger trigger_row + join pg_catalog.pg_class relation on relation.oid = trigger_row.tgrelid + join pg_catalog.pg_namespace table_namespace on table_namespace.oid = relation.relnamespace + join pg_catalog.pg_proc procedure on procedure.oid = trigger_row.tgfoid + join pg_catalog.pg_namespace function_namespace on function_namespace.oid = procedure.pronamespace + where pg_catalog.left(trigger_row.tgname, 9) = E'teleo_v3_' +), function_contract as ( + select + (select pg_catalog.count(*) from actual_v3_functions) = + (select pg_catalog.count(*) from expected_functions) + and not exists ( + select 1 + from expected_functions expected + where not exists ( + select 1 + from actual_v3_functions actual + where actual.function_schema = expected.function_schema + and actual.function_name = expected.function_name + and actual.function_identity_arguments = expected.function_identity_arguments + and actual.function_owner = expected.function_owner + and actual.function_security_definer = expected.function_security_definer + and actual.function_config = expected.function_config + and actual.function_source_sha256 = expected.function_source_sha256 + and ( + expected.function_owner_only_execute is null + or actual.function_owner_only_execute = expected.function_owner_only_execute + ) + and actual.function_language = 'plpgsql' + and actual.function_kind = 'f' + and actual.function_argument_count = 0 + and actual.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype + ) + ) as valid +), catalog_lock_function_contract as ( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + function_owner = E'kb_gate_owner' + and function_security_definer + and function_config = '["search_path=pg_catalog, pg_temp"]'::jsonb + and function_source_sha256 = E'897345944ba0f1b2a5c01b098fb17de6f881a122e906277086fa7889e76219e8' + and function_language = 'plpgsql' + and function_kind = 'f' + and function_argument_count = 0 + and function_identity_arguments = '' + and prorettype = 'pg_catalog.void'::pg_catalog.regtype + and apply_execute + and execute_acl_exact + ) as valid + from actual_v3_catalog_lock_functions +), trigger_contract as ( + select + (select pg_catalog.count(*) from actual_v3_triggers) = + (select pg_catalog.count(*) from expected_triggers) + and not exists ( + select 1 + from expected_triggers expected + where not exists ( + select 1 + from actual_v3_triggers actual + where actual.schema_name = expected.schema_name + and actual.table_name = expected.table_name + and actual.tgname = expected.trigger_name + and not actual.tgisinternal + and actual.tgenabled = 'O' + and actual.tgtype = expected.trigger_type + and actual.update_columns = expected.update_columns + and actual.when_expression is not distinct from expected.when_expression + and actual.tgnargs = expected.argument_count + and actual.argument_hex = expected.argument_hex + and actual.tgconstraint = 0::pg_catalog.oid + and not actual.tgdeferrable + and not actual.tginitdeferred + and actual.tgconstrrelid = 0::pg_catalog.oid + and actual.tgparentid = 0::pg_catalog.oid + and actual.tgoldtable is null + and actual.tgnewtable is null + and actual.function_schema = expected.function_schema + and actual.function_name = expected.function_name + and actual.function_identity_arguments = expected.function_identity_arguments + and exists ( + select 1 + from actual_v3_functions function_row + where function_row.oid = actual.tgfoid + and function_row.function_schema = expected.function_schema + and function_row.function_name = expected.function_name + and function_row.function_identity_arguments = expected.function_identity_arguments + ) + ) + ) as valid +), catalog_state as ( + select + exists (select 1 from actual_v3_functions) + or exists (select 1 from actual_v3_catalog_lock_functions) + or exists (select 1 from actual_v3_triggers) as artifacts_present, + (select valid from function_contract) as functions_valid, + (select valid from catalog_lock_function_contract) as catalog_lock_function_valid, + (select valid from trigger_contract) as triggers_valid, + exists ( + select 1 + from pg_catalog.pg_roles owner_role + where owner_role.rolname = E'kb_gate_owner' + and not owner_role.rolcanlogin + and not owner_role.rolinherit + and not owner_role.rolsuper + and not owner_role.rolcreatedb + and not owner_role.rolcreaterole + and not owner_role.rolreplication + and not owner_role.rolbypassrls + ) as owner_role_valid, + not exists ( + select 1 + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles member_role on member_role.oid = membership.member + join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid + where member_role.rolname in (E'kb_gate_owner', E'kb_review', E'kb_apply') + or granted_role.rolname in (E'kb_gate_owner', E'kb_review', E'kb_apply') + ) as memberships_valid +) +select case + when not artifacts_present then 'V2' + when functions_valid and catalog_lock_function_valid and triggers_valid + and owner_role_valid and memberships_valid then 'V3' + else 'INVALID' +end + into v_contract_state +from catalog_state; + -- TELEO_V3_CONTRACT_STATE_QUERY_END + + if v_contract_state <> 'V3' then + select role.rolcanlogin into v_apply_can_login + from pg_catalog.pg_roles role where role.rolname = 'kb_apply'; + if v_apply_can_login then + raise exception using + errcode = '55000', + message = 'teleo_v3 cutover refused before DDL: stop the V2 worker and ALTER ROLE kb_apply NOLOGIN'; + end if; + if exists ( + select 1 from pg_catalog.pg_stat_activity activity + where activity.datid = (select oid from pg_catalog.pg_database where datname = current_database()) + and activity.usename = 'kb_apply' + and activity.pid <> pg_catalog.pg_backend_pid() + ) then + raise exception using + errcode = '55000', + message = 'teleo_v3 cutover refused before DDL: active kb_apply sessions must exit'; + end if; + end if; + + select pg_catalog.count(*) into v_legacy_queue_count + from kb_stage.kb_proposals proposal + where proposal.status::text in ('pending_review', 'approved') + and proposal.proposal_type in ('revise_strategy', 'add_edge', 'attach_evidence', 'approve_claim') + and not ( + proposal.proposal_type = 'approve_claim' + and pg_catalog.jsonb_typeof(proposal.payload) = 'object' + and pg_catalog.jsonb_typeof(proposal.payload->'apply_payload') = 'object' + and pg_catalog.jsonb_typeof(proposal.payload->'apply_payload'->'contract_version') = 'number' + and proposal.payload->'apply_payload'->>'contract_version' = '3' + ); + if v_legacy_queue_count > 0 then + raise notice 'teleo_v3 cutover retains % pending/approved V2 proposal(s) for explicit migration or rejection', + v_legacy_queue_count; + end if; + + if not exists ( + select 1 from information_schema.columns + where table_schema = 'public' and table_name = 'claims' and column_name = 'text' + ) then + raise exception 'teleo_v3_epistemic_contract: public.claims.text compatibility column is required'; + end if; + if not exists ( + select 1 from information_schema.columns + where table_schema = 'public' and table_name = 'sources' and column_name = 'hash' + ) then + raise exception 'teleo_v3_epistemic_contract: public.sources.hash compatibility column is required'; + end if; + if not exists ( + select 1 from information_schema.columns + where table_schema = 'public' and table_name = 'claim_edges' and column_name = 'edge_type' + ) then + raise exception 'teleo_v3_epistemic_contract: public.claim_edges.edge_type compatibility column is required'; + end if; + + if v_contract_state = 'V2' then + with expected_columns(table_name, column_name) as ( + values + ('claims', 'owner_agent_id'), + ('claims', 'proposition'), + ('claims', 'body_markdown'), + ('claims', 'scope'), + ('claims', 'access_scope'), + ('claims', 'admission_state'), + ('claims', 'revision_criteria'), + ('claims', 'revision_kind'), + ('claims', 'supersedes_id'), + ('claims', 'accepted_by_proposal_id'), + ('sources', 'canonical_url'), + ('sources', 'storage_uri'), + ('sources', 'content_hash'), + ('sources', 'access_scope'), + ('sources', 'ingestion_origin'), + ('sources', 'provenance_status'), + ('sources', 'source_class'), + ('sources', 'auto_promotion_policy'), + ('sources', 'accepted_by_proposal_id'), + ('claim_evidence', 'excerpt'), + ('claim_evidence', 'locator_json'), + ('claim_evidence', 'source_content_hash'), + ('claim_evidence', 'rationale'), + ('claim_evidence', 'polarity'), + ('claim_evidence', 'accepted_from_proposal_id'), + ('claim_edges', 'relation_type'), + ('claim_edges', 'rationale'), + ('claim_edges', 'scope_or_conditions'), + ('claim_edges', 'accepted_by_proposal_id') + ), present_objects(object_name) as ( + select pg_catalog.format('%I.%I', columns.table_name, columns.column_name) + from information_schema.columns columns + join expected_columns expected + on expected.table_name = columns.table_name + and expected.column_name = columns.column_name + where columns.table_schema = 'public' + union all + select 'public.claim_evidence_assessments' + where pg_catalog.to_regclass('public.claim_evidence_assessments') is not null + union all + select 'kb_stage.teleo_v3_installation_ownership' + where pg_catalog.to_regclass('kb_stage.teleo_v3_installation_ownership') is not null + ) + select pg_catalog.string_agg(object_name, ', ' order by object_name) + into v_preexisting_objects + from present_objects; + + if v_preexisting_objects is not null then + raise exception using + errcode = '55000', + message = pg_catalog.format( + 'teleo_v3_epistemic_contract: preexisting V3 schema objects lack installation ownership: %s', + v_preexisting_objects + ); + end if; + elsif v_contract_state in ('V3', 'INVALID') then + if pg_catalog.to_regclass('kb_stage.teleo_v3_installation_ownership') is null then + if v_contract_state = 'INVALID' then + raise exception using + errcode = '55000', + message = 'teleo_v3_epistemic_contract: partial or altered V3 contract lacks exact installation ownership'; + end if; + raise exception using + errcode = '55000', + message = 'teleo_v3_epistemic_contract: existing V3 contract lacks exact installation ownership'; + end if; + + select owner_role.rolname = 'kb_gate_owner' + into v_ownership_valid + from pg_catalog.pg_class relation + join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = relation.relowner + where namespace.nspname = 'kb_stage' + and relation.relname = 'teleo_v3_installation_ownership' + and relation.relkind in ('r', 'p'); + + if not coalesce(v_ownership_valid, false) then + raise exception using + errcode = '55000', + message = 'teleo_v3_epistemic_contract: installation ownership ledger owner is invalid'; + end if; + + execute $ownership_query$ + select count(*) = 1 + and bool_and( + contract_name = 'livingip.teleo-v3-epistemic-contract' + and contract_version = 1 + and installation_token = 'collision-free-install-v1' + ) + from kb_stage.teleo_v3_installation_ownership + $ownership_query$ into v_ownership_valid; + + if not coalesce(v_ownership_valid, false) then + raise exception using + errcode = '55000', + message = 'teleo_v3_epistemic_contract: installation ownership ledger is not exact'; + end if; + end if; +end +$preflight$; + +create table if not exists kb_stage.teleo_v3_installation_ownership ( + contract_name text primary key, + contract_version integer not null, + installation_token text not null, + constraint teleo_v3_installation_contract_check check ( + contract_name = 'livingip.teleo-v3-epistemic-contract' and contract_version = 1 + ), + constraint teleo_v3_installation_token_check check ( + installation_token = 'collision-free-install-v1' + ) +); +alter table kb_stage.teleo_v3_installation_ownership owner to kb_gate_owner; +revoke all privileges on table kb_stage.teleo_v3_installation_ownership + from public, kb_apply, kb_review; +insert into kb_stage.teleo_v3_installation_ownership ( + contract_name, contract_version, installation_token +) values ( + 'livingip.teleo-v3-epistemic-contract', + 1, + 'collision-free-install-v1' +) +on conflict (contract_name) do nothing; + +alter table public.claims + add column if not exists owner_agent_id uuid, + add column if not exists proposition text, + add column if not exists body_markdown text, + add column if not exists scope text, + add column if not exists access_scope text not null default 'collective_internal', + add column if not exists admission_state text not null default 'legacy', + add column if not exists revision_criteria text, + add column if not exists revision_kind text, + add column if not exists supersedes_id uuid, + add column if not exists accepted_by_proposal_id uuid; + +alter table public.sources + add column if not exists canonical_url text, + add column if not exists storage_uri text, + add column if not exists content_hash text, + add column if not exists access_scope text, + add column if not exists ingestion_origin text, + add column if not exists provenance_status text, + add column if not exists source_class text, + add column if not exists auto_promotion_policy text, + add column if not exists accepted_by_proposal_id uuid; + +alter table public.claim_evidence + add column if not exists excerpt text, + add column if not exists locator_json jsonb, + add column if not exists source_content_hash text, + add column if not exists rationale text, + add column if not exists polarity text, + add column if not exists accepted_from_proposal_id uuid; + +alter table public.claim_edges + add column if not exists relation_type text, + add column if not exists rationale text, + add column if not exists scope_or_conditions text, + add column if not exists accepted_by_proposal_id uuid; + +create table if not exists public.claim_evidence_assessments ( + id uuid primary key default gen_random_uuid(), + claim_id uuid not null references public.claims(id), + support_tier text, + challenge_tier text, + overall_state text not null, + rationale text not null, + evidence_set_hash text not null, + accepted_by_proposal_id uuid not null references kb_stage.kb_proposals(id), + supersedes_assessment_id uuid references public.claim_evidence_assessments(id), + created_at timestamptz not null default now(), + constraint teleo_v3_assessment_support_tier_check check ( + support_tier is null or support_tier in ('none', 'weak', 'moderate', 'strong') + ), + constraint teleo_v3_assessment_challenge_tier_check check ( + challenge_tier is null or challenge_tier in ('none', 'weak', 'moderate', 'strong') + ), + constraint teleo_v3_assessment_overall_state_check check ( + overall_state in ('insufficient', 'support_dominant', 'challenge_dominant', 'mixed') + ), + constraint teleo_v3_assessment_rationale_check check (btrim(rationale) <> ''), + constraint teleo_v3_assessment_evidence_set_hash_check check ( + evidence_set_hash ~ '^[0-9a-f]{64}$' + ), + constraint teleo_v3_assessment_not_self_superseding_check check ( + supersedes_assessment_id is null or supersedes_assessment_id <> id + ) +); + +do $constraints$ +begin + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.claims'::regclass + and conname = 'teleo_v3_claim_owner_agent_fkey' + ) then + alter table public.claims + add constraint teleo_v3_claim_owner_agent_fkey + foreign key (owner_agent_id) references public.agents(id); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.claims'::regclass + and conname = 'teleo_v3_claim_supersedes_fkey' + ) then + alter table public.claims + add constraint teleo_v3_claim_supersedes_fkey + foreign key (supersedes_id) references public.claims(id); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.claims'::regclass + and conname = 'teleo_v3_claim_accepted_proposal_fkey' + ) then + alter table public.claims + add constraint teleo_v3_claim_accepted_proposal_fkey + foreign key (accepted_by_proposal_id) references kb_stage.kb_proposals(id); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.claims'::regclass + and conname = 'teleo_v3_claim_admission_check' + ) then + alter table public.claims + add constraint teleo_v3_claim_admission_check check ( + admission_state = 'legacy' + or ( + admission_state = 'admitted' + and type in ('empirical', 'predictive', 'causal', 'normative', 'conceptual') + and proposition is not null and btrim(proposition) <> '' + and text = proposition + and body_markdown is not null and btrim(body_markdown) <> '' + and scope is not null and btrim(scope) <> '' + and access_scope in ('agent_private', 'collective_internal', 'public') + and (access_scope <> 'agent_private' or owner_agent_id is not null) + and revision_criteria is not null and btrim(revision_criteria) <> '' + and revision_kind is not null + and revision_kind in ('original', 'editorial', 'semantic', 'metadata_correction') + and status in ('active', 'contested', 'retracted', 'superseded') + and accepted_by_proposal_id is not null + and ( + (revision_kind = 'original' and supersedes_id is null) + or (revision_kind <> 'original' and supersedes_id is not null) + ) + ) + ); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.sources'::regclass + and conname = 'teleo_v3_source_accepted_proposal_fkey' + ) then + alter table public.sources + add constraint teleo_v3_source_accepted_proposal_fkey + foreign key (accepted_by_proposal_id) references kb_stage.kb_proposals(id); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.sources'::regclass + and conname = 'teleo_v3_source_contract_check' + ) then + alter table public.sources + add constraint teleo_v3_source_contract_check check ( + accepted_by_proposal_id is null + or ( + content_hash is not null + and content_hash ~ '^[0-9a-f]{64}$' + and hash = content_hash + and ( + (canonical_url is not null and btrim(canonical_url) <> '') + or (storage_uri is not null and btrim(storage_uri) <> '') + ) + and access_scope is not null + and access_scope in ('agent_private', 'collective_internal', 'public') + and ingestion_origin is not null + and ingestion_origin in ('operator_upload', 'agent_fetch', 'contributor', 'system_import', 'unknown') + and provenance_status is not null + and provenance_status in ('verified', 'unverified', 'disputed') + and source_class is not null + and source_class in ( + 'primary_record', 'peer_reviewed', 'institutional', 'journalistic', 'self_published', 'unknown' + ) + and auto_promotion_policy is not null + and auto_promotion_policy in ('eligible', 'human_required', 'blocked') + ) + ); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.claim_evidence'::regclass + and conname = 'teleo_v3_evidence_accepted_proposal_fkey' + ) then + alter table public.claim_evidence + add constraint teleo_v3_evidence_accepted_proposal_fkey + foreign key (accepted_from_proposal_id) references kb_stage.kb_proposals(id); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.claim_evidence'::regclass + and conname = 'teleo_v3_evidence_contract_check' + ) then + alter table public.claim_evidence + add constraint teleo_v3_evidence_contract_check check ( + accepted_from_proposal_id is null + or ( + polarity is not null + and polarity in ('supports', 'challenges', 'illustrates') + and ( + (polarity = 'supports' and role::text = 'grounds') + or (polarity = 'challenges' and role::text = 'contradicts') + or (polarity = 'illustrates' and role::text = 'illustrates') + ) + and locator_json is not null + and jsonb_typeof(locator_json) = 'object' + and locator_json <> '{}'::jsonb + and source_content_hash is not null + and source_content_hash ~ '^[0-9a-f]{64}$' + and rationale is not null and btrim(rationale) <> '' + and weight is null + ) + ); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.claim_edges'::regclass + and conname = 'teleo_v3_edge_accepted_proposal_fkey' + ) then + alter table public.claim_edges + add constraint teleo_v3_edge_accepted_proposal_fkey + foreign key (accepted_by_proposal_id) references kb_stage.kb_proposals(id); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'public.claim_edges'::regclass + and conname = 'teleo_v3_edge_contract_check' + ) then + alter table public.claim_edges + add constraint teleo_v3_edge_contract_check check ( + accepted_by_proposal_id is null + or ( + relation_type is not null + and relation_type in ('supports', 'challenges', 'depends_on', 'contradicts') + and ( + relation_type = edge_type::text + or (relation_type = 'depends_on' and edge_type::text = 'requires') + ) + and rationale is not null and btrim(rationale) <> '' + and from_claim <> to_claim + and weight is null + ) + ); + end if; + + if not exists ( + select 1 from pg_constraint + where conrelid = 'kb_stage.kb_proposals'::regclass + and conname = 'teleo_v3_proposal_apply_receipt_check' + ) then + alter table kb_stage.kb_proposals + add constraint teleo_v3_proposal_apply_receipt_check check ( + ( + status::text = 'applied' + and applied_at is not null + and ( + applied_by_agent_id is not null + or nullif(btrim(applied_by_handle), '') is not null + ) + ) + or ( + status::text <> 'applied' + and applied_at is null + and applied_by_agent_id is null + and nullif(btrim(applied_by_handle), '') is null + ) + ) not valid; + end if; +end +$constraints$; + +create unique index if not exists teleo_v3_claim_one_admitted_successor_idx + on public.claims (supersedes_id) + where admission_state = 'admitted' and supersedes_id is not null; + +create unique index if not exists teleo_v3_assessment_one_successor_idx + on public.claim_evidence_assessments (supersedes_assessment_id) + where supersedes_assessment_id is not null; + +create index if not exists teleo_v3_evidence_claim_polarity_idx + on public.claim_evidence (claim_id, polarity) + where accepted_from_proposal_id is not null; + +create index if not exists teleo_v3_edge_typed_adjacency_idx + on public.claim_edges (from_claim, relation_type, to_claim) + where accepted_by_proposal_id is not null; + +-- kb_apply intentionally has SELECT-only access to the proposal ledger, so a +-- fixed SECURITY DEFINER helper takes only that table's transaction-held lock. +-- The apply takes the remaining relation locks directly under its narrow DML +-- grants. The exact helper body, owner, search_path, return type, and ACL are +-- part of the fail-closed catalog predicate above and below. +create or replace function kb_stage.teleo_v3_acquire_protected_relation_locks() +returns void +language plpgsql +security definer +set search_path = pg_catalog, pg_temp +as $function$ +begin + lock table kb_stage.kb_proposals in row exclusive mode; +end +$function$; + +alter function kb_stage.teleo_v3_acquire_protected_relation_locks() owner to kb_gate_owner; +revoke all on function kb_stage.teleo_v3_acquire_protected_relation_locks() + from public, kb_apply, kb_review; +grant execute on function kb_stage.teleo_v3_acquire_protected_relation_locks() + to kb_apply; + +create or replace function kb_stage.teleo_v3_enforce_apply_contract_cutover() +returns trigger +language plpgsql +security definer +set search_path = pg_catalog, pg_temp +as $function$ +declare + v_exact_v3 boolean; +begin + v_exact_v3 := + new.proposal_type = 'approve_claim' + and pg_catalog.jsonb_typeof(new.payload) = 'object' + and pg_catalog.jsonb_typeof(new.payload->'apply_payload') = 'object' + and pg_catalog.jsonb_typeof(new.payload->'apply_payload'->'contract_version') = 'number' + and new.payload->'apply_payload'->>'contract_version' = '3'; + + if v_exact_v3 then + return new; + end if; + if tg_op = 'UPDATE' + and old.status::text in ('pending_review', 'approved') + and new.status::text in ('rejected', 'canceled') + and new.proposal_type is not distinct from old.proposal_type + and new.payload is not distinct from old.payload then + return new; + end if; + if new.status::text in ('pending_review', 'approved', 'applied') + and new.proposal_type in ('revise_strategy', 'add_edge', 'attach_evidence', 'approve_claim') then + raise exception using + errcode = '55000', + message = pg_catalog.format( + 'teleo_v3 cutover: proposal %s uses a disabled V2 or malformed apply contract; migrate or reject it', + new.id + ); + end if; + return new; +end +$function$; + +alter function kb_stage.teleo_v3_enforce_apply_contract_cutover() owner to kb_gate_owner; +revoke all on function kb_stage.teleo_v3_enforce_apply_contract_cutover() + from public, kb_apply, kb_review; + +create or replace function kb_stage.teleo_v3_require_accepted_proposal() +returns trigger +language plpgsql +security definer +set search_path = pg_catalog, pg_temp +as $function$ +declare + proposal_id uuid; + proposal_status text; + proposal_reviewed_by_handle text; + proposal_reviewed_by_agent_id uuid; + proposal_reviewed_at timestamptz; + proposal_review_note text; + proposal_applied_by_handle text; + proposal_applied_by_agent_id uuid; + proposal_applied_at timestamptz; +begin + proposal_id := nullif(pg_catalog.to_jsonb(new) ->> tg_argv[0], '')::uuid; + if proposal_id is null then + return new; + end if; + + select status::text, reviewed_by_handle, reviewed_by_agent_id, reviewed_at, review_note, + applied_by_handle, applied_by_agent_id, applied_at + into proposal_status, proposal_reviewed_by_handle, proposal_reviewed_by_agent_id, + proposal_reviewed_at, proposal_review_note, proposal_applied_by_handle, + proposal_applied_by_agent_id, proposal_applied_at + from kb_stage.kb_proposals + where id = proposal_id + for share; + + if not found then + raise exception using + errcode = '23503', + message = pg_catalog.format('teleo_v3: proposal %s does not exist', proposal_id); + end if; + if proposal_status not in ('approved', 'accepted', 'applied') + or proposal_reviewed_at is null + or (proposal_reviewed_by_agent_id is null and nullif(pg_catalog.btrim(proposal_reviewed_by_handle), '') is null) + or nullif(pg_catalog.btrim(proposal_review_note), '') is null then + raise exception using + errcode = '23514', + message = pg_catalog.format('teleo_v3: proposal %s is not an accepted reviewed decision', proposal_id); + end if; + if proposal_status = 'applied' + and ( + proposal_applied_at is null + or ( + proposal_applied_by_agent_id is null + and nullif(pg_catalog.btrim(proposal_applied_by_handle), '') is null + ) + ) then + raise exception using + errcode = '23514', + message = pg_catalog.format('teleo_v3: applied proposal %s lacks immutable apply metadata', proposal_id); + end if; + if proposal_status <> 'applied' + and ( + proposal_applied_at is not null + or proposal_applied_by_agent_id is not null + or nullif(pg_catalog.btrim(proposal_applied_by_handle), '') is not null + ) then + raise exception using + errcode = '23514', + message = pg_catalog.format('teleo_v3: non-applied proposal %s carries invalid apply metadata', proposal_id); + end if; + return new; +end +$function$; + +-- The trigger takes FOR SHARE on the proposal ledger so a decision cannot be +-- downgraded between validation and canonical insertion. Runtime writers do +-- not receive direct UPDATE/row-lock privilege on that ledger; the fixed, +-- trigger-only SECURITY DEFINER body is the sole narrow lock path. +alter function kb_stage.teleo_v3_require_accepted_proposal() owner to kb_gate_owner; +revoke all on function kb_stage.teleo_v3_require_accepted_proposal() + from public, kb_apply, kb_review; + +do $security_definer_invariants$ +declare + v_drift text; +begin + select pg_catalog.format( + 'owner=%I security_definer=%s config=%s acl=%s', + owner_role.rolname, + procedure.prosecdef, + procedure.proconfig, + procedure.proacl + ) + into v_drift + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner + where namespace.nspname = 'kb_stage' + and procedure.proname in ( + 'teleo_v3_enforce_apply_contract_cutover', + 'teleo_v3_require_accepted_proposal' + ) + and ( + owner_role.rolname <> 'kb_gate_owner' + or not procedure.prosecdef + or pg_catalog.array_to_string(procedure.proconfig, ',') is distinct from + 'search_path=pg_catalog, pg_temp' + or exists ( + select 1 + from pg_catalog.aclexplode( + coalesce( + procedure.proacl, + pg_catalog.acldefault('f', procedure.proowner) + ) + ) acl + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + ) + ); + if v_drift is not null then + raise exception 'teleo_v3_epistemic_contract: SECURITY DEFINER trigger invariants failed: %', v_drift; + end if; +end +$security_definer_invariants$; + +create or replace function kb_stage.teleo_v3_require_linked_insert() +returns trigger +language plpgsql +security invoker +set search_path = pg_catalog, pg_temp +as $function$ +declare + proposal_field text := tg_argv[0]; + admission_field text := case when array_length(tg_argv, 1) > 1 then tg_argv[1] else null end; + proposal_value text; + admission_value text; +begin + proposal_value := pg_catalog.to_jsonb(new) ->> proposal_field; + if nullif(pg_catalog.btrim(proposal_value), '') is null then + raise exception using + errcode = '23514', + message = pg_catalog.format( + 'teleo_v3: new %s.%s rows must be proposal-linked through %s', + tg_table_schema, tg_table_name, proposal_field + ); + end if; + if admission_field is not null then + admission_value := pg_catalog.to_jsonb(new) ->> admission_field; + if admission_value is distinct from 'admitted' then + raise exception using + errcode = '23514', + message = pg_catalog.format( + 'teleo_v3: new %s.%s rows must use %s=admitted; legacy is reserved for rows present at migration', + tg_table_schema, tg_table_name, admission_field + ); + end if; + end if; + return new; +end +$function$; + +create or replace function kb_stage.teleo_v3_guard_immutable_fields() +returns trigger +language plpgsql +security invoker +set search_path = pg_catalog, pg_temp +as $function$ +declare + field_name text; +begin + if ( + tg_table_schema = 'public' + and tg_table_name = 'claims' + and (pg_catalog.to_jsonb(old) ->> 'admission_state') is distinct from 'admitted' + ) or ( + tg_table_schema = 'public' + and tg_table_name = 'sources' + and (pg_catalog.to_jsonb(old) ->> 'accepted_by_proposal_id') is null + ) or ( + tg_table_schema = 'public' + and tg_table_name = 'claim_evidence' + and (pg_catalog.to_jsonb(old) ->> 'accepted_from_proposal_id') is null + ) or ( + tg_table_schema = 'public' + and tg_table_name = 'claim_edges' + and (pg_catalog.to_jsonb(old) ->> 'accepted_by_proposal_id') is null + ) then + return new; + end if; + + foreach field_name in array tg_argv loop + if (pg_catalog.to_jsonb(new) -> field_name) is distinct from (pg_catalog.to_jsonb(old) -> field_name) then + raise exception using + errcode = '55000', + message = pg_catalog.format( + 'teleo_v3: immutable %s.%s field %s requires a new version', tg_table_schema, tg_table_name, field_name + ); + end if; + end loop; + return new; +end +$function$; + +create or replace function kb_stage.teleo_v3_reject_delete() +returns trigger +language plpgsql +security invoker +set search_path = pg_catalog, pg_temp +as $function$ +begin + if ( + tg_table_schema = 'public' + and tg_table_name = 'claims' + and (pg_catalog.to_jsonb(old) ->> 'admission_state') is distinct from 'admitted' + and (pg_catalog.to_jsonb(old) ->> 'accepted_by_proposal_id') is null + ) or ( + tg_table_schema = 'public' + and tg_table_name = 'sources' + and (pg_catalog.to_jsonb(old) ->> 'accepted_by_proposal_id') is null + ) or ( + tg_table_schema = 'public' + and tg_table_name = 'claim_evidence' + and (pg_catalog.to_jsonb(old) ->> 'accepted_from_proposal_id') is null + ) or ( + tg_table_schema = 'public' + and tg_table_name = 'claim_edges' + and (pg_catalog.to_jsonb(old) ->> 'accepted_by_proposal_id') is null + ) then + return old; + end if; + + raise exception using + errcode = '55000', + message = pg_catalog.format( + 'teleo_v3: proposal-linked %s.%s rows are append-only; supersede or retract through a new reviewed version', + tg_table_schema, tg_table_name + ); +end +$function$; + +create or replace function kb_stage.teleo_v3_validate_claim_evidence() +returns trigger +language plpgsql +security invoker +set search_path = pg_catalog, pg_temp +as $function$ +declare + claim_admission_state text; + source_hash text; + source_proposal_id uuid; +begin + if new.accepted_from_proposal_id is null then + return new; + end if; + + select admission_state into claim_admission_state + from public.claims where id = new.claim_id; + select content_hash, accepted_by_proposal_id into source_hash, source_proposal_id + from public.sources where id = new.source_id; + + if claim_admission_state is distinct from 'admitted' then + raise exception using + errcode = '23514', + message = 'teleo_v3: accepted evidence must target an admitted claim version'; + end if; + if source_proposal_id is null then + raise exception using + errcode = '23514', + message = 'teleo_v3: accepted evidence must reference a proposal-linked source version'; + end if; + if source_hash is distinct from new.source_content_hash then + raise exception using + errcode = '23514', + message = 'teleo_v3: evidence source_content_hash does not match the immutable source version'; + end if; + return new; +end +$function$; + +create or replace function kb_stage.teleo_v3_validate_claim_edge() +returns trigger +language plpgsql +security invoker +set search_path = pg_catalog, pg_temp +as $function$ +declare + from_admission_state text; + to_admission_state text; +begin + if new.accepted_by_proposal_id is null then + return new; + end if; + + select admission_state into from_admission_state + from public.claims where id = new.from_claim; + select admission_state into to_admission_state + from public.claims where id = new.to_claim; + if from_admission_state is distinct from 'admitted' or to_admission_state is distinct from 'admitted' then + raise exception using + errcode = '23514', + message = 'teleo_v3: accepted typed edges require two admitted claim versions'; + end if; + return new; +end +$function$; + +create or replace function kb_stage.teleo_v3_validate_evidence_assessment() +returns trigger +language plpgsql +security invoker +set search_path = pg_catalog, pg_temp +as $function$ +declare + claim_type text; + claim_admission_state text; +begin + select type, admission_state into claim_type, claim_admission_state + from public.claims where id = new.claim_id; + if claim_admission_state is distinct from 'admitted' then + raise exception using + errcode = '23514', + message = 'teleo_v3: evidence assessments require an admitted claim version'; + end if; + if claim_type in ('empirical', 'predictive', 'causal') + and (new.support_tier is null or new.challenge_tier is null) then + raise exception using + errcode = '23514', + message = 'teleo_v3: truth-apt claim assessments require support and challenge tiers'; + end if; + if not exists ( + select 1 from public.claim_evidence + where claim_id = new.claim_id and accepted_from_proposal_id is not null + ) then + raise exception using + errcode = '23514', + message = 'teleo_v3: evidence assessments require accepted claim-specific evidence'; + end if; + return new; +end +$function$; + +create or replace function kb_stage.teleo_v3_guard_linked_proposal_decision() +returns trigger +language plpgsql +security invoker +set search_path = pg_catalog, pg_temp +as $function$ +declare + proposal_is_linked boolean; + valid_apply_transition boolean; +begin + select + exists (select 1 from public.claims where accepted_by_proposal_id = old.id) + or exists (select 1 from public.sources where accepted_by_proposal_id = old.id) + or exists (select 1 from public.claim_evidence where accepted_from_proposal_id = old.id) + or exists (select 1 from public.claim_edges where accepted_by_proposal_id = old.id) + or exists (select 1 from public.claim_evidence_assessments where accepted_by_proposal_id = old.id) + into proposal_is_linked; + + valid_apply_transition := + old.status::text in ('approved', 'accepted') + and new.status::text = 'applied' + and old.applied_by_handle is null + and old.applied_by_agent_id is null + and old.applied_at is null + and new.applied_at is not null + and ( + new.applied_by_agent_id is not null + or nullif(pg_catalog.btrim(new.applied_by_handle), '') is not null + ); + + if new.status::text = 'applied' and ( + new.applied_at is null + or ( + new.applied_by_agent_id is null + and nullif(pg_catalog.btrim(new.applied_by_handle), '') is null + ) + ) then + raise exception using + errcode = '55000', + message = pg_catalog.format('teleo_v3: applied proposal %s requires immutable apply metadata', old.id); + end if; + + if new.status::text <> 'applied' and ( + new.applied_at is not null + or new.applied_by_agent_id is not null + or nullif(pg_catalog.btrim(new.applied_by_handle), '') is not null + ) then + raise exception using + errcode = '55000', + message = pg_catalog.format('teleo_v3: non-applied proposal %s cannot carry apply metadata', old.id); + end if; + + if old.status::text = 'applied' and ( + new.status::text <> 'applied' + or new.proposal_type is distinct from old.proposal_type + or new.payload is distinct from old.payload + or new.rationale is distinct from old.rationale + or new.reviewed_by_handle is distinct from old.reviewed_by_handle + or new.reviewed_by_agent_id is distinct from old.reviewed_by_agent_id + or new.reviewed_at is distinct from old.reviewed_at + or new.review_note is distinct from old.review_note + or new.applied_by_handle is distinct from old.applied_by_handle + or new.applied_by_agent_id is distinct from old.applied_by_agent_id + or new.applied_at is distinct from old.applied_at + ) then + raise exception using + errcode = '55000', + message = pg_catalog.format('teleo_v3: applied proposal %s is immutable', old.id); + end if; + + if ( + new.applied_by_handle is distinct from old.applied_by_handle + or new.applied_by_agent_id is distinct from old.applied_by_agent_id + or new.applied_at is distinct from old.applied_at + ) and not valid_apply_transition then + raise exception using + errcode = '55000', + message = pg_catalog.format('teleo_v3: proposal %s apply metadata may only be set during apply', old.id); + end if; + + if proposal_is_linked and ( + new.status::text not in ('approved', 'accepted', 'applied') + or ( + old.status::text in ('approved', 'accepted') + and new.status::text not in (old.status::text, 'applied') + ) + or (old.status::text = 'applied' and new.status::text <> 'applied') + or new.proposal_type is distinct from old.proposal_type + or new.payload is distinct from old.payload + or new.rationale is distinct from old.rationale + or new.reviewed_by_handle is distinct from old.reviewed_by_handle + or new.reviewed_by_agent_id is distinct from old.reviewed_by_agent_id + or new.reviewed_at is distinct from old.reviewed_at + or new.review_note is distinct from old.review_note + or new.reviewed_at is null + or (new.reviewed_by_agent_id is null and nullif(pg_catalog.btrim(new.reviewed_by_handle), '') is null) + or nullif(pg_catalog.btrim(new.review_note), '') is null + ) then + raise exception using + errcode = '55000', + message = pg_catalog.format('teleo_v3: linked accepted proposal %s cannot lose its decision metadata', old.id); + end if; + return new; +end +$function$; + +do $trigger_function_owners$ +declare + v_function record; +begin + for v_function in + select distinct expected.function_schema, expected.function_name, + expected.function_identity_arguments + from pg_catalog.jsonb_to_recordset( + pg_catalog.current_setting('teleo.v3_expected_trigger_contract')::jsonb + ) as expected( + function_schema text, + function_name text, + function_identity_arguments text + ) + loop + if v_function.function_identity_arguments <> '' then + raise exception 'teleo_v3_epistemic_contract: trigger functions must have zero identity arguments'; + end if; + execute pg_catalog.format( + 'alter function %I.%I() owner to kb_gate_owner', + v_function.function_schema, + v_function.function_name + ); + end loop; +end +$trigger_function_owners$; + +do $triggers$ +declare + v_trigger record; +begin + execute $ddl$ + create or replace trigger teleo_v3_00_enforce_apply_contract_cutover + before insert or update of proposal_type, payload, status on kb_stage.kb_proposals + for each row execute function kb_stage.teleo_v3_enforce_apply_contract_cutover() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_00_claim_require_linked_insert + before insert on public.claims + for each row execute function kb_stage.teleo_v3_require_linked_insert( + 'accepted_by_proposal_id', 'admission_state' + ) + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_claim_require_accepted_proposal + before insert or update of accepted_by_proposal_id on public.claims + for each row execute function kb_stage.teleo_v3_require_accepted_proposal('accepted_by_proposal_id') + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_00_source_require_linked_insert + before insert on public.sources + for each row execute function kb_stage.teleo_v3_require_linked_insert('accepted_by_proposal_id') + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_claim_immutable + before update on public.claims + for each row execute function kb_stage.teleo_v3_guard_immutable_fields( + 'id', 'owner_agent_id', 'type', 'text', 'proposition', 'body_markdown', 'scope', 'access_scope', + 'status', 'admission_state', 'revision_criteria', 'revision_kind', 'supersedes_id', 'superseded_by', + 'confidence', 'tags', 'created_by', 'created_at', 'updated_at', 'accepted_by_proposal_id' + ) + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_claim_reject_delete + before delete on public.claims + for each row execute function kb_stage.teleo_v3_reject_delete() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_00_evidence_require_linked_insert + before insert on public.claim_evidence + for each row execute function kb_stage.teleo_v3_require_linked_insert('accepted_from_proposal_id') + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_source_require_accepted_proposal + before insert or update of accepted_by_proposal_id on public.sources + for each row execute function kb_stage.teleo_v3_require_accepted_proposal('accepted_by_proposal_id') + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_00_edge_require_linked_insert + before insert on public.claim_edges + for each row execute function kb_stage.teleo_v3_require_linked_insert('accepted_by_proposal_id') + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_source_immutable + before update on public.sources + for each row execute function kb_stage.teleo_v3_guard_immutable_fields( + 'id', 'source_type', 'url', 'storage_path', 'excerpt', 'hash', 'canonical_url', 'storage_uri', + 'content_hash', 'access_scope', 'ingestion_origin', 'provenance_status', 'source_class', + 'auto_promotion_policy', 'created_by', 'captured_at', 'created_at', 'accepted_by_proposal_id' + ) + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_source_reject_delete + before delete on public.sources + for each row execute function kb_stage.teleo_v3_reject_delete() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_evidence_require_accepted_proposal + before insert or update of accepted_from_proposal_id on public.claim_evidence + for each row execute function kb_stage.teleo_v3_require_accepted_proposal('accepted_from_proposal_id') + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_evidence_validate_provenance + before insert or update on public.claim_evidence + for each row execute function kb_stage.teleo_v3_validate_claim_evidence() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_evidence_immutable + before update on public.claim_evidence + for each row execute function kb_stage.teleo_v3_guard_immutable_fields( + 'id', 'claim_id', 'source_id', 'role', 'weight', 'excerpt', 'locator_json', 'source_content_hash', + 'rationale', 'polarity', 'created_by', 'created_at', 'accepted_from_proposal_id' + ) + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_evidence_reject_delete + before delete on public.claim_evidence + for each row execute function kb_stage.teleo_v3_reject_delete() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_edge_require_accepted_proposal + before insert or update of accepted_by_proposal_id on public.claim_edges + for each row execute function kb_stage.teleo_v3_require_accepted_proposal('accepted_by_proposal_id') + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_edge_validate_endpoints + before insert or update on public.claim_edges + for each row execute function kb_stage.teleo_v3_validate_claim_edge() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_edge_immutable + before update on public.claim_edges + for each row execute function kb_stage.teleo_v3_guard_immutable_fields( + 'id', 'from_claim', 'to_claim', 'edge_type', 'weight', 'relation_type', 'rationale', + 'scope_or_conditions', 'created_by', 'created_at', 'accepted_by_proposal_id' + ) + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_edge_reject_delete + before delete on public.claim_edges + for each row execute function kb_stage.teleo_v3_reject_delete() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_assessment_require_accepted_proposal + before insert or update of accepted_by_proposal_id on public.claim_evidence_assessments + for each row execute function kb_stage.teleo_v3_require_accepted_proposal('accepted_by_proposal_id') + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_assessment_validate + before insert or update on public.claim_evidence_assessments + for each row execute function kb_stage.teleo_v3_validate_evidence_assessment() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_assessment_immutable + before update on public.claim_evidence_assessments + for each row execute function kb_stage.teleo_v3_guard_immutable_fields( + 'id', 'claim_id', 'support_tier', 'challenge_tier', 'overall_state', 'rationale', + 'evidence_set_hash', 'accepted_by_proposal_id', 'supersedes_assessment_id', 'created_at' + ) + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_assessment_reject_delete + before delete on public.claim_evidence_assessments + for each row execute function kb_stage.teleo_v3_reject_delete() + $ddl$; + + execute $ddl$ + create or replace trigger teleo_v3_guard_linked_proposal_decision + before update of proposal_type, status, rationale, payload, reviewed_by_handle, + reviewed_by_agent_id, reviewed_at, review_note, applied_by_handle, applied_by_agent_id, applied_at + on kb_stage.kb_proposals + for each row execute function kb_stage.teleo_v3_guard_linked_proposal_decision() + $ddl$; + + for v_trigger in + select expected.schema_name, expected.table_name, expected.trigger_name + from pg_catalog.jsonb_to_recordset( + pg_catalog.current_setting('teleo.v3_expected_trigger_contract')::jsonb + ) as expected(schema_name text, table_name text, trigger_name text) + loop + execute pg_catalog.format( + 'alter table %I.%I enable trigger %I', + v_trigger.schema_name, + v_trigger.table_name, + v_trigger.trigger_name + ); + end loop; +end +$triggers$; + +do $contract_postflight$ +declare + v_contract_state text; +begin + -- TELEO_V3_CONTRACT_POSTFLIGHT_QUERY_BEGIN +with expected_triggers as ( + select expected.* + from pg_catalog.jsonb_to_recordset(pg_catalog.current_setting('teleo.v3_expected_trigger_contract')::jsonb) as expected( + schema_name text, + table_name text, + trigger_name text, + trigger_type integer, + update_columns jsonb, + when_expression text, + argument_count integer, + argument_hex text, + function_schema text, + function_name text, + function_identity_arguments text, + function_owner text, + function_security_definer boolean, + function_config jsonb, + function_source_sha256 text, + function_owner_only_execute boolean + ) +), expected_functions as ( + select distinct function_schema, function_name, function_identity_arguments, + function_owner, function_security_definer, function_config, + function_source_sha256, function_owner_only_execute + from expected_triggers +), actual_v3_functions as ( + select procedure.oid, namespace.nspname as function_schema, + procedure.proname as function_name, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + owner_role.rolname as function_owner, + procedure.prosecdef as function_security_definer, + pg_catalog.to_jsonb(procedure.proconfig) as function_config, + pg_catalog.encode( + pg_catalog.sha256(pg_catalog.convert_to(procedure.prosrc, 'UTF8')), + 'hex' + ) as function_source_sha256, + not exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + ) as function_owner_only_execute, + language.lanname as function_language, + procedure.prokind as function_kind, + procedure.pronargs as function_argument_count, + procedure.prorettype + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner + join pg_catalog.pg_language language on language.oid = procedure.prolang + where namespace.nspname = 'kb_stage' + and pg_catalog.left(procedure.proname, 9) = E'teleo_v3_' + and procedure.proname <> E'teleo_v3_acquire_protected_relation_locks' +), actual_v3_catalog_lock_functions as ( + select procedure.oid, + owner_role.rolname as function_owner, + procedure.prosecdef as function_security_definer, + pg_catalog.to_jsonb(procedure.proconfig) as function_config, + pg_catalog.encode( + pg_catalog.sha256(pg_catalog.convert_to(procedure.prosrc, 'UTF8')), + 'hex' + ) as function_source_sha256, + language.lanname as function_language, + procedure.prokind as function_kind, + procedure.pronargs as function_argument_count, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + procedure.prorettype, + exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee + where acl.privilege_type = 'EXECUTE' + and grantee.rolname = 'kb_apply' + ) as apply_execute, + not exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + and acl.grantee <> ( + select role.oid from pg_catalog.pg_roles role where role.rolname = 'kb_apply' + ) + ) as execute_acl_exact + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner + join pg_catalog.pg_language language on language.oid = procedure.prolang + where namespace.nspname = 'kb_stage' + and procedure.proname = E'teleo_v3_acquire_protected_relation_locks' +), actual_v3_triggers as ( + select trigger_row.*, table_namespace.nspname as schema_name, + relation.relname as table_name, + function_namespace.nspname as function_schema, + procedure.proname as function_name, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + coalesce(( + select pg_catalog.jsonb_agg(attribute.attname order by attribute.attname) + from pg_catalog.unnest(trigger_row.tgattr::smallint[]) update_column(attnum) + join pg_catalog.pg_attribute attribute + on attribute.attrelid = trigger_row.tgrelid + and attribute.attnum = update_column.attnum + and not attribute.attisdropped + ), '[]'::jsonb) as update_columns, + pg_catalog.pg_get_expr(trigger_row.tgqual, trigger_row.tgrelid, true) as when_expression, + pg_catalog.encode(trigger_row.tgargs, 'hex') as argument_hex + from pg_catalog.pg_trigger trigger_row + join pg_catalog.pg_class relation on relation.oid = trigger_row.tgrelid + join pg_catalog.pg_namespace table_namespace on table_namespace.oid = relation.relnamespace + join pg_catalog.pg_proc procedure on procedure.oid = trigger_row.tgfoid + join pg_catalog.pg_namespace function_namespace on function_namespace.oid = procedure.pronamespace + where pg_catalog.left(trigger_row.tgname, 9) = E'teleo_v3_' +), function_contract as ( + select + (select pg_catalog.count(*) from actual_v3_functions) = + (select pg_catalog.count(*) from expected_functions) + and not exists ( + select 1 + from expected_functions expected + where not exists ( + select 1 + from actual_v3_functions actual + where actual.function_schema = expected.function_schema + and actual.function_name = expected.function_name + and actual.function_identity_arguments = expected.function_identity_arguments + and actual.function_owner = expected.function_owner + and actual.function_security_definer = expected.function_security_definer + and actual.function_config = expected.function_config + and actual.function_source_sha256 = expected.function_source_sha256 + and ( + expected.function_owner_only_execute is null + or actual.function_owner_only_execute = expected.function_owner_only_execute + ) + and actual.function_language = 'plpgsql' + and actual.function_kind = 'f' + and actual.function_argument_count = 0 + and actual.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype + ) + ) as valid +), catalog_lock_function_contract as ( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + function_owner = E'kb_gate_owner' + and function_security_definer + and function_config = '["search_path=pg_catalog, pg_temp"]'::jsonb + and function_source_sha256 = E'897345944ba0f1b2a5c01b098fb17de6f881a122e906277086fa7889e76219e8' + and function_language = 'plpgsql' + and function_kind = 'f' + and function_argument_count = 0 + and function_identity_arguments = '' + and prorettype = 'pg_catalog.void'::pg_catalog.regtype + and apply_execute + and execute_acl_exact + ) as valid + from actual_v3_catalog_lock_functions +), trigger_contract as ( + select + (select pg_catalog.count(*) from actual_v3_triggers) = + (select pg_catalog.count(*) from expected_triggers) + and not exists ( + select 1 + from expected_triggers expected + where not exists ( + select 1 + from actual_v3_triggers actual + where actual.schema_name = expected.schema_name + and actual.table_name = expected.table_name + and actual.tgname = expected.trigger_name + and not actual.tgisinternal + and actual.tgenabled = 'O' + and actual.tgtype = expected.trigger_type + and actual.update_columns = expected.update_columns + and actual.when_expression is not distinct from expected.when_expression + and actual.tgnargs = expected.argument_count + and actual.argument_hex = expected.argument_hex + and actual.tgconstraint = 0::pg_catalog.oid + and not actual.tgdeferrable + and not actual.tginitdeferred + and actual.tgconstrrelid = 0::pg_catalog.oid + and actual.tgparentid = 0::pg_catalog.oid + and actual.tgoldtable is null + and actual.tgnewtable is null + and actual.function_schema = expected.function_schema + and actual.function_name = expected.function_name + and actual.function_identity_arguments = expected.function_identity_arguments + and exists ( + select 1 + from actual_v3_functions function_row + where function_row.oid = actual.tgfoid + and function_row.function_schema = expected.function_schema + and function_row.function_name = expected.function_name + and function_row.function_identity_arguments = expected.function_identity_arguments + ) + ) + ) as valid +), catalog_state as ( + select + exists (select 1 from actual_v3_functions) + or exists (select 1 from actual_v3_catalog_lock_functions) + or exists (select 1 from actual_v3_triggers) as artifacts_present, + (select valid from function_contract) as functions_valid, + (select valid from catalog_lock_function_contract) as catalog_lock_function_valid, + (select valid from trigger_contract) as triggers_valid, + exists ( + select 1 + from pg_catalog.pg_roles owner_role + where owner_role.rolname = E'kb_gate_owner' + and not owner_role.rolcanlogin + and not owner_role.rolinherit + and not owner_role.rolsuper + and not owner_role.rolcreatedb + and not owner_role.rolcreaterole + and not owner_role.rolreplication + and not owner_role.rolbypassrls + ) as owner_role_valid, + not exists ( + select 1 + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles member_role on member_role.oid = membership.member + join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid + where member_role.rolname in (E'kb_gate_owner', E'kb_review', E'kb_apply') + or granted_role.rolname in (E'kb_gate_owner', E'kb_review', E'kb_apply') + ) as memberships_valid +) +select case + when not artifacts_present then 'V2' + when functions_valid and catalog_lock_function_valid and triggers_valid + and owner_role_valid and memberships_valid then 'V3' + else 'INVALID' +end + into v_contract_state +from catalog_state; + -- TELEO_V3_CONTRACT_POSTFLIGHT_QUERY_END + if v_contract_state <> 'V3' then + raise exception using + errcode = '55000', + message = pg_catalog.format( + 'teleo_v3_epistemic_contract: installed trigger catalog is %s', + v_contract_state + ); + end if; +end +$contract_postflight$; + +commit; + +select jsonb_build_object( + 'contract', 'livingip.teleo-v3-epistemic-contract', + 'version', 1, + 'migration', 'installed', + 'legacy_claims', (select count(*) from public.claims where admission_state = 'legacy'), + 'admitted_claims', (select count(*) from public.claims where admission_state = 'admitted') +)::text; diff --git a/ops/verify_teleo_v3_epistemic_contract.py b/ops/verify_teleo_v3_epistemic_contract.py new file mode 100644 index 0000000..9bed6dc --- /dev/null +++ b/ops/verify_teleo_v3_epistemic_contract.py @@ -0,0 +1,1926 @@ +#!/usr/bin/env python3 +"""Verify the minimum V3 epistemic contract in disposable PostgreSQL 16. + +The verifier accepts no database address or credentials. It creates its own +networkless, tmpfs-backed Docker container, installs the additive migration, +round-trips proposal-linked epistemic rows, exercises adversarial failures, +rolls the contract back, reinstalls it, rolls it back again, and removes the +container. Production and persistent databases are unreachable by design. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import secrets +import shutil +import subprocess +import time +import uuid +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +MIGRATION_SQL = REPO_ROOT / "ops" / "teleo_v3_epistemic_contract.sql" +ROLLBACK_SQL = REPO_ROOT / "ops" / "rollback_teleo_v3_epistemic_contract.sql" + +RECEIPT_SCHEMA = "livingip.teleoV3EpistemicContractReceipt.v1" +POSTGRES_IMAGE = os.environ.get("LEOCLEAN_RUNTIME_POSTGRES_IMAGE", "postgres:16-alpine") +DATABASE = "teleo_v3_contract" +CONTAINER_LABEL = "livingip.canary=teleo-v3-epistemic-contract" +TRIGGER_CONTRACT_PATTERN = re.compile( + r"\$trigger_contract\$\s*(\[.*?\])\s*\$trigger_contract\$", + re.DOTALL, +) +CATALOG_LOCK_FUNCTION_NAME = "teleo_v3_acquire_protected_relation_locks" +CATALOG_LOCK_FUNCTION_SOURCE_SHA256 = "897345944ba0f1b2a5c01b098fb17de6f881a122e906277086fa7889e76219e8" + +AGENT_ID = "11111111-1111-4111-8111-111111111111" +LEGACY_CLAIM_ID = "20000000-0000-4000-8000-000000000001" +LEGACY_RELATED_CLAIM_ID = "20000000-0000-4000-8000-000000000002" +LEGACY_SOURCE_ID = "20000000-0000-4000-8000-000000000003" +LEGACY_EVIDENCE_ID = "20000000-0000-4000-8000-000000000004" +LEGACY_EDGE_ID = "20000000-0000-4000-8000-000000000005" + +PROPOSAL_ID = "30000000-0000-4000-8000-000000000001" +PENDING_PROPOSAL_ID = "30000000-0000-4000-8000-000000000002" +APPLIED_WITHOUT_RECEIPT_PROPOSAL_ID = "30000000-0000-4000-8000-000000000003" +PRIMARY_CLAIM_ID = "40000000-0000-4000-8000-000000000001" +SUPPORT_CLAIM_ID = "40000000-0000-4000-8000-000000000002" +SOURCE_ID = "40000000-0000-4000-8000-000000000003" +EVIDENCE_ID = "40000000-0000-4000-8000-000000000004" +ASSESSMENT_ID = "40000000-0000-4000-8000-000000000005" +EDGE_ID = "40000000-0000-4000-8000-000000000006" + +SOURCE_PAYLOAD = b"LivingIP V3 source payload: immutable epistemic provenance.\n" +SOURCE_CONTENT_HASH = hashlib.sha256(SOURCE_PAYLOAD).hexdigest() +EVIDENCE_SET_HASH = hashlib.sha256( + json.dumps( + { + "claim_id": PRIMARY_CLAIM_ID, + "evidence": [ + { + "source_id": SOURCE_ID, + "source_content_hash": SOURCE_CONTENT_HASH, + "polarity": "supports", + "locator": {"paragraph": 1, "section": "epistemic-contract"}, + } + ], + }, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") +).hexdigest() + +PRIMARY_PROPOSITION = "An admitted V3 claim preserves exact evidence provenance." +PRIMARY_BODY = ( + "The claim records a proposition and explanatory body together, pins a content-hashed source locator, " + "and remains attributable to the reviewed proposal that admitted it." +) +PRIMARY_SCOPE = "Minimum disposable PostgreSQL V3 contract verification." +PRIMARY_REVISION_CRITERIA = ( + "Revise if the stored source hash, locator, proposal decision, or evidence assessment cannot be reproduced." +) + + +def load_expected_trigger_contract() -> list[dict[str, Any]]: + match = TRIGGER_CONTRACT_PATTERN.search(MIGRATION_SQL.read_text(encoding="utf-8")) + if match is None: + raise VerificationError("migration does not declare the V3 trigger catalog contract") + contract = json.loads(match.group(1)) + if not isinstance(contract, list): + raise VerificationError("migration V3 trigger catalog contract is not a JSON array") + return contract + + +def trigger_catalog_is_exact(catalog: dict[str, Any]) -> bool: + expected_triggers = load_expected_trigger_contract() + actual_triggers = catalog["triggers"] + if len(actual_triggers) != len(expected_triggers): + return False + + expected_by_key = {(row["schema_name"], row["table_name"], row["trigger_name"]): row for row in expected_triggers} + actual_by_key = {(row["schema_name"], row["table_name"], row["trigger_name"]): row for row in actual_triggers} + if len(expected_by_key) != len(expected_triggers) or len(actual_by_key) != len(actual_triggers): + return False + if actual_by_key.keys() != expected_by_key.keys(): + return False + + for key, expected in expected_by_key.items(): + actual = actual_by_key[key] + if actual["enabled_state"] != "O" or actual["is_internal"]: + return False + if actual["trigger_type"] != expected["trigger_type"]: + return False + if actual["update_columns"] != expected["update_columns"]: + return False + if actual["when_expression"] != expected["when_expression"]: + return False + if actual["argument_count"] != expected["argument_count"]: + return False + if actual["argument_hex"] != expected["argument_hex"]: + return False + if any( + ( + actual["constraint_oid"] != "0", + actual["is_deferrable"], + actual["is_initially_deferred"], + actual["constraint_relation_oid"] != "0", + actual["parent_trigger_oid"] != "0", + actual["old_transition_table"] is not None, + actual["new_transition_table"] is not None, + ) + ): + return False + if actual["function_schema"] != expected["function_schema"]: + return False + if actual["function_name"] != expected["function_name"]: + return False + if actual["function_identity_arguments"] != expected["function_identity_arguments"]: + return False + + expected_functions = { + ( + row["function_schema"], + row["function_name"], + row["function_identity_arguments"], + ): row + for row in expected_triggers + } + actual_functions = { + (row["schema_name"], row["function_name"], row["arguments"]): row for row in catalog["functions"] + } + if len(actual_functions) != len(catalog["functions"]): + return False + catalog_lock_function = actual_functions.pop( + ("kb_stage", CATALOG_LOCK_FUNCTION_NAME, ""), + None, + ) + if catalog_lock_function is None or any( + ( + catalog_lock_function["owner_name"] != "kb_gate_owner", + not catalog_lock_function["security_definer"], + catalog_lock_function["config"] != ["search_path=pg_catalog, pg_temp"], + catalog_lock_function["source_sha256"] != CATALOG_LOCK_FUNCTION_SOURCE_SHA256, + catalog_lock_function["language_name"] != "plpgsql", + catalog_lock_function["function_kind"] != "f", + catalog_lock_function["argument_count"] != 0, + catalog_lock_function["return_type"] != "void", + catalog_lock_function["non_owner_execute_grantees"] != ["kb_apply"], + ) + ): + return False + if actual_functions.keys() != expected_functions.keys(): + return False + for key, expected in expected_functions.items(): + actual = actual_functions[key] + if actual["owner_name"] != expected["function_owner"]: + return False + if actual["security_definer"] != expected["function_security_definer"]: + return False + if actual["config"] != expected["function_config"]: + return False + if actual["source_sha256"] != expected["function_source_sha256"]: + return False + if expected["function_owner_only_execute"] is not None and ( + actual["owner_only_execute"] != expected["function_owner_only_execute"] + ): + return False + if actual["language_name"] != "plpgsql": + return False + if actual["function_kind"] != "f" or actual["argument_count"] != 0: + return False + if actual["return_type"] != "trigger": + return False + return True + + +class VerificationError(RuntimeError): + """Raised when the disposable runtime cannot prove the contract.""" + + +def canonical_sha256(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def parse_last_json_line(output: str) -> Any: + for line in reversed(output.splitlines()): + candidate = line.strip() + if not candidate: + continue + try: + return json.loads(candidate) + except json.JSONDecodeError: + continue + return None + + +def safe_error_text(value: str, *, limit: int = 1600) -> str: + return " ".join(value.split())[-limit:] + + +def sql_literal(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace("'", "''") + return f"E'{escaped}'" + + +def docker_available() -> bool: + docker = shutil.which("docker") + if not docker: + return False + result = subprocess.run( + [docker, "info", "--format", "{{.ServerVersion}}"], + text=True, + capture_output=True, + check=False, + timeout=15, + ) + return result.returncode == 0 + + +BOOTSTRAP_SQL = f""" +\\set ON_ERROR_STOP on +create schema kb_stage; +create role kb_gate_owner nologin noinherit; +create role kb_apply nologin noinherit; +create role kb_review login noinherit; +create type evidence_role as enum ('grounds', 'illustrates', 'contradicts'); +create type edge_type as enum ( + 'supports', 'challenges', 'requires', 'relates', 'contradicts', 'supersedes', + 'derives_from', 'cites', 'causes', 'constrains', 'accelerates' +); + +create table public.agents ( + id uuid primary key, + handle text not null unique, + kind text not null, + created_at timestamptz not null default now() +); + +create table public.claims ( + id uuid primary key, + type text not null, + text text not null, + status text not null, + confidence numeric, + tags text[] not null default '{{}}', + created_by uuid references public.agents(id), + superseded_by uuid references public.claims(id), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +create table public.sources ( + id uuid primary key, + source_type text not null, + url text, + storage_path text, + excerpt text, + hash text not null, + created_by uuid references public.agents(id), + captured_at timestamptz, + created_at timestamptz not null default now() +); + +create table public.claim_evidence ( + id uuid primary key default gen_random_uuid(), + claim_id uuid not null references public.claims(id), + source_id uuid not null references public.sources(id), + role evidence_role not null, + weight numeric, + created_by uuid references public.agents(id), + created_at timestamptz not null default now() +); + +create table public.claim_edges ( + id uuid primary key, + from_claim uuid not null references public.claims(id), + to_claim uuid not null references public.claims(id), + edge_type edge_type not null, + weight numeric, + created_by uuid references public.agents(id), + created_at timestamptz not null default now() +); + +create table kb_stage.kb_proposals ( + id uuid primary key, + proposal_type text not null, + status text not null, + proposed_by_handle text, + proposed_by_agent_id uuid references public.agents(id), + channel text, + source_ref text, + rationale text, + payload jsonb not null, + reviewed_by_handle text, + reviewed_by_agent_id uuid references public.agents(id), + reviewed_at timestamptz, + review_note text, + applied_by_handle text, + applied_by_agent_id uuid references public.agents(id), + applied_at timestamptz, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + constraint kb_proposals_status_check check ( + status in ('pending_review', 'approved', 'rejected', 'applied', 'canceled') + ) +); + +grant usage on schema kb_stage to kb_gate_owner; +grant select, update on kb_stage.kb_proposals to kb_gate_owner; + +insert into public.agents (id, handle, kind, created_at) +values ('{AGENT_ID}', 'v3-contract-reviewer', 'human', '2026-07-18T08:00:00Z'); + +insert into public.claims + (id, type, text, status, confidence, tags, created_by, created_at, updated_at) +values + ('{LEGACY_CLAIM_ID}', 'structural', 'Legacy claim without a body.', 'open', 0.4, + array['legacy'], '{AGENT_ID}', '2026-07-18T08:01:00Z', '2026-07-18T08:01:00Z'), + ('{LEGACY_RELATED_CLAIM_ID}', 'structural', 'Second legacy claim.', 'open', null, + array['legacy'], '{AGENT_ID}', '2026-07-18T08:02:00Z', '2026-07-18T08:02:00Z'); + +insert into public.sources + (id, source_type, url, storage_path, excerpt, hash, created_by, captured_at, created_at) +values + ('{LEGACY_SOURCE_ID}', 'legacy_document', null, '/legacy/source.txt', 'Legacy excerpt.', 'legacy-hash', + '{AGENT_ID}', '2026-07-18T08:03:00Z', '2026-07-18T08:03:00Z'); + +insert into public.claim_evidence + (id, claim_id, source_id, role, weight, created_by, created_at) +values + ('{LEGACY_EVIDENCE_ID}', '{LEGACY_CLAIM_ID}', '{LEGACY_SOURCE_ID}', 'grounds', 0.25, + '{AGENT_ID}', '2026-07-18T08:04:00Z'); + +insert into public.claim_edges + (id, from_claim, to_claim, edge_type, weight, created_by, created_at) +values + ('{LEGACY_EDGE_ID}', '{LEGACY_CLAIM_ID}', '{LEGACY_RELATED_CLAIM_ID}', 'relates', 0.5, + '{AGENT_ID}', '2026-07-18T08:05:00Z'); + +-- This malformed historical row proves the NOT VALID proposal constraint is +-- additive while the canonical-row trigger still refuses it as authority. +insert into kb_stage.kb_proposals ( + id, proposal_type, status, proposed_by_handle, proposed_by_agent_id, channel, + rationale, payload, reviewed_by_handle, reviewed_by_agent_id, reviewed_at, review_note, + created_at, updated_at +) values ( + '{APPLIED_WITHOUT_RECEIPT_PROPOSAL_ID}', 'approve_claim', 'applied', 'leo', '{AGENT_ID}', + 'legacy_fixture', 'Historical applied status without a receipt.', + '{{"contract_version": 2}}'::jsonb, 'v3-contract-reviewer', '{AGENT_ID}', + '2026-07-18T08:06:00Z', 'Historical malformed receipt retained for compatibility testing.', + '2026-07-18T08:06:00Z', '2026-07-18T08:06:00Z' +); +""" + + +LEGACY_STATE_SQL = """ +select jsonb_build_object( + 'claims', coalesce(( + select jsonb_agg(to_jsonb(row_data) order by row_data.id) + from ( + select id, type, text, status, confidence, tags, created_by, superseded_by, created_at, updated_at + from public.claims + where id in ( + '20000000-0000-4000-8000-000000000001'::uuid, + '20000000-0000-4000-8000-000000000002'::uuid + ) + ) row_data + ), '[]'::jsonb), + 'sources', coalesce(( + select jsonb_agg(to_jsonb(row_data) order by row_data.id) + from ( + select id, source_type, url, storage_path, excerpt, hash, created_by, captured_at, created_at + from public.sources + where id = '20000000-0000-4000-8000-000000000003'::uuid + ) row_data + ), '[]'::jsonb), + 'claim_evidence', coalesce(( + select jsonb_agg(to_jsonb(row_data) order by row_data.id) + from ( + select id, claim_id, source_id, role, weight, created_by, created_at + from public.claim_evidence + where id = '20000000-0000-4000-8000-000000000004'::uuid + ) row_data + ), '[]'::jsonb), + 'claim_edges', coalesce(( + select jsonb_agg(to_jsonb(row_data) order by row_data.id) + from ( + select id, from_claim, to_claim, edge_type, weight, created_by, created_at + from public.claim_edges + where id = '20000000-0000-4000-8000-000000000005'::uuid + ) row_data + ), '[]'::jsonb) +)::text; +""" + + +CONTRACT_CATALOG_SQL = """ +with contract_columns as ( + select table_schema, table_name, column_name, data_type, udt_name, is_nullable, column_default + from information_schema.columns + where table_schema = 'public' + and ( + (table_name = 'claims' and column_name in ( + 'owner_agent_id', 'proposition', 'body_markdown', 'scope', 'access_scope', 'admission_state', + 'revision_criteria', 'revision_kind', 'supersedes_id', 'accepted_by_proposal_id' + )) + or (table_name = 'sources' and column_name in ( + 'canonical_url', 'storage_uri', 'content_hash', 'access_scope', 'ingestion_origin', + 'provenance_status', 'source_class', 'auto_promotion_policy', 'accepted_by_proposal_id' + )) + or (table_name = 'claim_evidence' and column_name in ( + 'excerpt', 'locator_json', 'source_content_hash', 'rationale', 'polarity', 'accepted_from_proposal_id' + )) + or (table_name = 'claim_edges' and column_name in ( + 'relation_type', 'rationale', 'scope_or_conditions', 'accepted_by_proposal_id' + )) + ) +), contract_constraints as ( + select namespace.nspname as schema_name, relation.relname as table_name, + constraint_row.conname as constraint_name, + pg_get_constraintdef(constraint_row.oid, true) as definition + from pg_constraint constraint_row + join pg_class relation on relation.oid = constraint_row.conrelid + join pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname in ('public', 'kb_stage') + and constraint_row.conname like 'teleo_v3_%' +), contract_triggers as ( + select namespace.nspname as schema_name, relation.relname as table_name, + trigger_row.tgname as trigger_name, + trigger_row.tgenabled as enabled_state, + trigger_row.tgisinternal as is_internal, + trigger_row.tgtype as trigger_type, + coalesce(( + select jsonb_agg(attribute.attname order by attribute.attname) + from unnest(trigger_row.tgattr::smallint[]) update_column(attnum) + join pg_attribute attribute + on attribute.attrelid = trigger_row.tgrelid + and attribute.attnum = update_column.attnum + and not attribute.attisdropped + ), '[]'::jsonb) as update_columns, + pg_get_expr(trigger_row.tgqual, trigger_row.tgrelid, true) as when_expression, + trigger_row.tgnargs as argument_count, + encode(trigger_row.tgargs, 'hex') as argument_hex, + trigger_row.tgconstraint::text as constraint_oid, + trigger_row.tgdeferrable as is_deferrable, + trigger_row.tginitdeferred as is_initially_deferred, + trigger_row.tgconstrrelid::text as constraint_relation_oid, + trigger_row.tgparentid::text as parent_trigger_oid, + trigger_row.tgoldtable as old_transition_table, + trigger_row.tgnewtable as new_transition_table, + function_namespace.nspname as function_schema, + trigger_function.proname as function_name, + pg_get_function_identity_arguments(trigger_function.oid) as function_identity_arguments, + pg_get_triggerdef(trigger_row.oid, true) as definition + from pg_trigger trigger_row + join pg_class relation on relation.oid = trigger_row.tgrelid + join pg_namespace namespace on namespace.oid = relation.relnamespace + join pg_proc trigger_function on trigger_function.oid = trigger_row.tgfoid + join pg_namespace function_namespace on function_namespace.oid = trigger_function.pronamespace + where left(trigger_row.tgname, 9) = 'teleo_v3_' +), contract_functions as ( + select namespace.nspname as schema_name, procedure.proname as function_name, + pg_get_function_identity_arguments(procedure.oid) as arguments, + owner_role.rolname as owner_name, + procedure.prosecdef as security_definer, + procedure.proconfig as config, + encode(sha256(convert_to(procedure.prosrc, 'UTF8')), 'hex') as source_sha256, + language.lanname as language_name, + procedure.prokind as function_kind, + procedure.pronargs as argument_count, + procedure.prorettype::regtype::text as return_type, + not exists ( + select 1 + from pg_catalog.aclexplode( + coalesce( + procedure.proacl, + pg_catalog.acldefault('f', procedure.proowner) + ) + ) acl + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + ) as owner_only_execute, + coalesce(( + select jsonb_agg(coalesce(grantee.rolname, 'PUBLIC') order by coalesce(grantee.rolname, 'PUBLIC')) + from pg_catalog.aclexplode( + coalesce( + procedure.proacl, + pg_catalog.acldefault('f', procedure.proowner) + ) + ) acl + left join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + ), '[]'::jsonb) as non_owner_execute_grantees + from pg_proc procedure + join pg_namespace namespace on namespace.oid = procedure.pronamespace + join pg_roles owner_role on owner_role.oid = procedure.proowner + join pg_language language on language.oid = procedure.prolang + where namespace.nspname = 'kb_stage' + and left(procedure.proname, 9) = 'teleo_v3_' +), contract_indexes as ( + select schemaname, tablename, indexname, indexdef + from pg_indexes + where schemaname = 'public' and indexname like 'teleo_v3_%' +) +select jsonb_build_object( + 'assessment_table', to_regclass('public.claim_evidence_assessments')::text, + 'columns', coalesce(( + select jsonb_agg(to_jsonb(contract_columns) order by table_name, column_name) from contract_columns + ), '[]'::jsonb), + 'constraints', coalesce(( + select jsonb_agg(to_jsonb(contract_constraints) order by schema_name, table_name, constraint_name) + from contract_constraints + ), '[]'::jsonb), + 'triggers', coalesce(( + select jsonb_agg(to_jsonb(contract_triggers) order by schema_name, table_name, trigger_name) + from contract_triggers + ), '[]'::jsonb), + 'functions', coalesce(( + select jsonb_agg(to_jsonb(contract_functions) order by schema_name, function_name, arguments) + from contract_functions + ), '[]'::jsonb), + 'indexes', coalesce(( + select jsonb_agg(to_jsonb(contract_indexes) order by schemaname, tablename, indexname) + from contract_indexes + ), '[]'::jsonb) +)::text; +""" + + +POSTGRES_READY_LOG_LINE = "database system is ready to accept connections" +POSTGRES_STARTUP_TIMEOUT_SECONDS = 45.0 +POSTGRES_STARTUP_PROBE_TIMEOUT_SECONDS = 10.0 +POSTGRES_STARTUP_POLL_INTERVAL_SECONDS = 0.25 +POSTGRES_CLEANUP_COMMAND_TIMEOUT_SECONDS = 10.0 +POSTGRES_CLEANUP_MAX_ATTEMPTS = 5 +POSTGRES_CLEANUP_POLL_INTERVAL_SECONDS = 0.25 +DOCKER_CONTAINER_ID_PATTERN = re.compile(r"[0-9a-f]{12,64}") + + +def postgres_final_startup_ready( + logs: str, + database_probe: subprocess.CompletedProcess[str], +) -> bool: + return ( + logs.count(POSTGRES_READY_LOG_LINE) >= 2 + and database_probe.returncode == 0 + and database_probe.stdout.strip() == "1" + ) + + +class DisposablePostgres: + def __init__(self) -> None: + self.docker = shutil.which("docker") + if not self.docker: + raise VerificationError("docker executable is unavailable") + self.name = f"teleo-v3-contract-{uuid.uuid4().hex[:12]}" + self.password = secrets.token_urlsafe(32) + self.started = False + self.cleanup_required = False + self.environment: dict[str, Any] = {} + + def run( + self, + arguments: list[str], + *, + input_text: str | None = None, + check: bool = True, + timeout: float = 120, + environment: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [self.docker, *arguments], + input=input_text, + text=True, + capture_output=True, + check=False, + timeout=timeout, + env=environment, + ) + if check and result.returncode != 0: + detail = safe_error_text(result.stderr or result.stdout) + raise VerificationError(f"docker {arguments[0]} failed ({result.returncode}): {detail}") + return result + + def _redacted_startup_detail(self, value: str | bytes | None) -> str: + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + raw_detail = value or "" + if self.password: + raw_detail = raw_detail.replace(self.password, "") + detail = safe_error_text(raw_detail, limit=500) + return detail or "" + + def _startup_result_diagnostic( + self, + stage: str, + result: subprocess.CompletedProcess[str], + ) -> str: + detail = self._redacted_startup_detail(result.stderr or result.stdout) + return f"stage={stage} kind=nonzero returncode={result.returncode} detail={detail}" + + def _startup_exception_diagnostic( + self, + stage: str, + exc: OSError | subprocess.TimeoutExpired, + ) -> str: + if isinstance(exc, subprocess.TimeoutExpired): + detail = self._redacted_startup_detail(exc.stderr or exc.output) + return f"stage={stage} kind=timeout timeout_seconds={exc.timeout:g} detail={detail}" + return f"stage={stage} kind=launch_error detail={self._redacted_startup_detail(str(exc))}" + + def _wait_for_final_startup( + self, + *, + startup_timeout: float = POSTGRES_STARTUP_TIMEOUT_SECONDS, + probe_timeout: float = POSTGRES_STARTUP_PROBE_TIMEOUT_SECONDS, + poll_interval: float = POSTGRES_STARTUP_POLL_INTERVAL_SECONDS, + ) -> None: + if startup_timeout <= 0 or probe_timeout <= 0 or poll_interval < 0: + raise ValueError("PostgreSQL startup timing values must be positive") + + deadline = time.monotonic() + startup_timeout + ready_streak = 0 + attempts = 0 + last_diagnostic = "stage=startup kind=no_probe_completed" + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + + attempts += 1 + try: + logs = self.run( + ["logs", self.name], + check=False, + timeout=min(probe_timeout, remaining), + ) + except (OSError, subprocess.TimeoutExpired) as exc: + ready_streak = 0 + last_diagnostic = self._startup_exception_diagnostic("docker_logs", exc) + else: + if logs.returncode != 0: + ready_streak = 0 + last_diagnostic = self._startup_result_diagnostic("docker_logs", logs) + else: + remaining = deadline - time.monotonic() + if remaining <= 0: + last_diagnostic = "stage=target_database_probe kind=deadline_exhausted_after_logs" + break + try: + database_probe = self.psql( + "select 1;", + check=False, + timeout=min(probe_timeout, remaining), + ) + except (OSError, subprocess.TimeoutExpired) as exc: + ready_streak = 0 + last_diagnostic = self._startup_exception_diagnostic("target_database_probe", exc) + else: + if postgres_final_startup_ready(logs.stdout + logs.stderr, database_probe): + ready_streak += 1 + if ready_streak == 2: + return + else: + ready_streak = 0 + if database_probe.returncode != 0: + last_diagnostic = self._startup_result_diagnostic( + "target_database_probe", + database_probe, + ) + else: + ready_transitions = (logs.stdout + logs.stderr).count(POSTGRES_READY_LOG_LINE) + probe_output = self._redacted_startup_detail(database_probe.stdout) + last_diagnostic = ( + "stage=readiness_gate kind=not_ready " + f"ready_transitions={ready_transitions} " + f"database_probe_returncode={database_probe.returncode} " + f"database_probe_output={probe_output}" + ) + + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(poll_interval, remaining)) + + raise VerificationError( + f"disposable PostgreSQL did not become stably ready within {startup_timeout:g}s " + f"after {attempts} attempt(s); last_probe={last_diagnostic}" + ) + + def _launch_container(self) -> subprocess.CompletedProcess[str]: + arguments = [ + "run", + "--detach", + "--rm", + "--name", + self.name, + "--network", + "none", + "--label", + CONTAINER_LABEL, + "--label", + f"livingip.canary.instance={self.name}", + "--tmpfs", + "/var/lib/postgresql/data:rw,nosuid,nodev,size=384m", + "--env", + "POSTGRES_PASSWORD", + "--env", + f"POSTGRES_DB={DATABASE}", + POSTGRES_IMAGE, + ] + launch_environment = os.environ.copy() + launch_environment["POSTGRES_PASSWORD"] = self.password + self.cleanup_required = True + try: + launched = self.run( + arguments, + check=False, + environment=launch_environment, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + diagnostic = self._startup_exception_diagnostic("docker_run", exc) + raise VerificationError( + "disposable PostgreSQL launch state is ambiguous; cleanup_required=true; " + f"{diagnostic}" + ) from None + if launched.returncode != 0: + diagnostic = self._startup_result_diagnostic("docker_run", launched) + raise VerificationError( + "disposable PostgreSQL launch failed; cleanup_required=true; " + f"{diagnostic}" + ) + self.started = True + return launched + + def start(self) -> dict[str, Any]: + launched = self._launch_container() + + self._wait_for_final_startup() + + inspected = json.loads(self.run(["inspect", self.name]).stdout)[0] + labels = inspected.get("Config", {}).get("Labels", {}) + mounts = inspected.get("Mounts", []) + tmpfs = inspected.get("HostConfig", {}).get("Tmpfs", {}) + self.environment = { + "container_id_sha256": hashlib.sha256(launched.stdout.strip().encode("utf-8")).hexdigest(), + "container_name": self.name, + "image": inspected.get("Config", {}).get("Image"), + "image_id": inspected.get("Image"), + "network_mode": inspected.get("HostConfig", {}).get("NetworkMode"), + "tmpfs_data_dir": "/var/lib/postgresql/data" in tmpfs, + "docker_volume_mounts": [ + {"type": mount.get("Type"), "destination": mount.get("Destination")} for mount in mounts + ], + "canary_label": labels.get("livingip.canary"), + "instance_label": labels.get("livingip.canary.instance"), + } + if self.environment["network_mode"] != "none": + raise VerificationError("disposable PostgreSQL is not networkless") + if not self.environment["tmpfs_data_dir"] or mounts: + raise VerificationError("disposable PostgreSQL must use tmpfs without Docker volumes") + if self.environment["canary_label"] != "teleo-v3-epistemic-contract": + raise VerificationError("disposable PostgreSQL canary label is missing") + if self.environment["instance_label"] != self.name: + raise VerificationError("disposable PostgreSQL instance label is missing") + return self.environment + + def psql( + self, + sql: str, + *, + check: bool = True, + timeout: float = 120, + ) -> subprocess.CompletedProcess[str]: + return self.run( + [ + "exec", + "--interactive", + self.name, + "psql", + "--no-psqlrc", + "--no-password", + "--set=ON_ERROR_STOP=1", + "--set=VERBOSITY=verbose", + "--tuples-only", + "--no-align", + "--username", + "postgres", + "--dbname", + DATABASE, + ], + input_text=sql, + check=check, + timeout=timeout, + ) + + def psql_json(self, sql: str) -> Any: + result = self.psql(sql) + parsed = parse_last_json_line(result.stdout) + if parsed is None: + raise VerificationError("PostgreSQL readback did not contain JSON") + return parsed + + def apply_file(self, path: Path, *, check: bool = True) -> subprocess.CompletedProcess[str]: + return self.psql(path.read_text(encoding="utf-8"), check=check) + + def _cleanup_run( + self, + arguments: list[str], + *, + stage: str, + ) -> tuple[subprocess.CompletedProcess[str] | None, str | None]: + try: + return ( + self.run( + arguments, + check=False, + timeout=POSTGRES_CLEANUP_COMMAND_TIMEOUT_SECONDS, + ), + None, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + return None, self._startup_exception_diagnostic(stage, exc) + + def cleanup(self) -> dict[str, Any]: + ownership_filters = [ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + f"name=^/{self.name}$", + "--filter", + "label=livingip.canary=teleo-v3-epistemic-contract", + "--filter", + f"label=livingip.canary.instance={self.name}", + "--format", + "{{.ID}}", + ] + name_readback = [ + "container", + "ls", + "--all", + "--filter", + f"name=^/{self.name}$", + "--format", + "{{.Names}}", + ] + label_readback = [ + "container", + "ls", + "--all", + "--filter", + "label=livingip.canary=teleo-v3-epistemic-contract", + "--filter", + f"label=livingip.canary.instance={self.name}", + "--format", + "{{.Names}}", + ] + errors: list[str] = [] + observed_owned_ids: set[str] = set() + remove_returncodes: list[int] = [] + remove_attempted = False + stable_clear_observations = 0 + inspect: subprocess.CompletedProcess[str] | None = None + names: subprocess.CompletedProcess[str] | None = None + labels: subprocess.CompletedProcess[str] | None = None + attempts = 0 + + for attempts in range(1, POSTGRES_CLEANUP_MAX_ATTEMPTS + 1): + ownership, ownership_error = self._cleanup_run( + ownership_filters, + stage="cleanup_ownership_readback", + ) + if ownership_error: + errors.append(ownership_error) + owned_ids: list[str] = [] + if ownership is not None and ownership.returncode == 0: + candidates = [line.strip() for line in ownership.stdout.splitlines() if line.strip()] + if all(DOCKER_CONTAINER_ID_PATTERN.fullmatch(candidate) for candidate in candidates): + owned_ids = candidates + observed_owned_ids.update(candidates) + else: + errors.append("stage=cleanup_ownership_readback kind=invalid_container_id") + elif ownership is not None: + errors.append(self._startup_result_diagnostic("cleanup_ownership_readback", ownership)) + + if self.cleanup_required: + for container_id in owned_ids: + remove_attempted = True + removed, remove_error = self._cleanup_run( + ["rm", "--force", container_id], + stage="cleanup_remove", + ) + if remove_error: + errors.append(remove_error) + elif removed is not None: + remove_returncodes.append(removed.returncode) + if removed.returncode != 0: + errors.append(self._startup_result_diagnostic("cleanup_remove", removed)) + + inspect, inspect_error = self._cleanup_run( + ["inspect", self.name], + stage="cleanup_inspect_readback", + ) + names, names_error = self._cleanup_run(name_readback, stage="cleanup_name_readback") + labels, labels_error = self._cleanup_run(label_readback, stage="cleanup_label_readback") + errors.extend(error for error in (inspect_error, names_error, labels_error) if error) + if names is not None and names.returncode != 0: + errors.append(self._startup_result_diagnostic("cleanup_name_readback", names)) + if labels is not None and labels.returncode != 0: + errors.append(self._startup_result_diagnostic("cleanup_label_readback", labels)) + + name_clear = names is not None and names.returncode == 0 and not names.stdout.strip() + label_clear = labels is not None and labels.returncode == 0 and not labels.stdout.strip() + inspect_clear = inspect is not None and inspect.returncode != 0 + if inspect_clear and name_clear and label_clear: + stable_clear_observations += 1 + if stable_clear_observations == 2: + break + else: + stable_clear_observations = 0 + if attempts < POSTGRES_CLEANUP_MAX_ATTEMPTS: + time.sleep(POSTGRES_CLEANUP_POLL_INTERVAL_SECONDS) + + name_readback_clear = names is not None and names.returncode == 0 and not names.stdout.strip() + instance_label_readback_clear = labels is not None and labels.returncode == 0 and not labels.stdout.strip() + container_absent = ( + inspect is not None + and inspect.returncode != 0 + and name_readback_clear + and instance_label_readback_clear + ) + return { + "cleanup_required": self.cleanup_required, + "cleanup_reconciliation_attempted": self.cleanup_required, + "remove_attempted": remove_attempted, + "remove_returncode": remove_returncodes[-1] if remove_returncodes else None, + "remove_returncodes": remove_returncodes, + "owned_container_ids_observed": sorted(observed_owned_ids), + "container_absent": container_absent, + "name_readback_clear": name_readback_clear, + "instance_label_readback_clear": instance_label_readback_clear, + "stable_absence_readback": stable_clear_observations >= 2, + "readback_attempts": attempts, + "foreign_name_collision_detected": not name_readback_clear and instance_label_readback_clear, + "errors": errors, + "network_mode_was_none": self.environment.get("network_mode") == "none", + "tmpfs_data_dir_was_present": self.environment.get("tmpfs_data_dir") is True, + "docker_volume_mounts_observed": self.environment.get("docker_volume_mounts", []), + } + + +def catalog_readback(postgres: DisposablePostgres) -> dict[str, Any]: + value = postgres.psql_json(CONTRACT_CATALOG_SQL) + if not isinstance(value, dict): + raise VerificationError("contract catalog readback was not an object") + return value + + +def catalog_is_absent(catalog: dict[str, Any]) -> bool: + return catalog.get("assessment_table") is None and all( + catalog.get(key) == [] for key in ("columns", "constraints", "triggers", "functions", "indexes") + ) + + +def failure_receipt(result: subprocess.CompletedProcess[str], expected_sqlstate: str) -> dict[str, Any]: + match = re.search(r"(?:ERROR|FATAL):\s+([0-9A-Z]{5}):", result.stderr) + observed_sqlstate = match.group(1) if match else None + return { + "refused": result.returncode != 0, + "returncode": result.returncode, + "expected_sqlstate": expected_sqlstate, + "observed_sqlstate": observed_sqlstate, + "expected_sqlstate_observed": observed_sqlstate == expected_sqlstate, + "stderr_tail": safe_error_text(result.stderr, limit=500), + } + + +def expect_failure( + postgres: DisposablePostgres, + sql: str, + *, + expected_sqlstate: str, +) -> dict[str, Any]: + result = postgres.psql(sql, check=False) + receipt = failure_receipt(result, expected_sqlstate) + if not receipt["refused"] or not receipt["expected_sqlstate_observed"]: + raise VerificationError( + f"adversarial statement did not fail with SQLSTATE {expected_sqlstate}: {receipt['stderr_tail']}" + ) + return receipt + + +def proposal_decision_lock_race_check(postgres: DisposablePostgres) -> dict[str, Any]: + """Prove a canonical writer locks the accepted decision against concurrent downgrade.""" + + assert postgres.docker is not None + command = [ + postgres.docker, + "exec", + "--interactive", + postgres.name, + "psql", + "--no-psqlrc", + "--no-password", + "--set=ON_ERROR_STOP=1", + "--set=VERBOSITY=verbose", + "--tuples-only", + "--no-align", + "--username", + "postgres", + "--dbname", + DATABASE, + ] + holder = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert holder.stdin is not None + holder.stdin.write( + f""" +set application_name = 'teleo_v3_proposal_lock_holder'; +begin; +select status from kb_stage.kb_proposals where id = '{PROPOSAL_ID}'::uuid for share; +select pg_sleep(4); +rollback; +""" + ) + holder.stdin.close() + holder.stdin = None + try: + deadline = time.monotonic() + 8 + lock_held = False + while time.monotonic() < deadline: + lock_held = postgres.psql_json( + """ +select jsonb_build_object( + 'held', exists ( + select 1 from pg_stat_activity + where application_name = 'teleo_v3_proposal_lock_holder' + and state = 'active' + and wait_event = 'PgSleep' + ) +)::text; +""" + )["held"] + if lock_held: + break + time.sleep(0.1) + if not lock_held: + raise VerificationError("proposal lock holder did not reach the protected transaction") + downgrade = postgres.psql( + f""" +set statement_timeout = '750ms'; +update kb_stage.kb_proposals set status = 'rejected' where id = '{PROPOSAL_ID}'::uuid; +""", + check=False, + ) + downgrade_receipt = failure_receipt(downgrade, "57014") + holder_stdout, holder_stderr = holder.communicate(timeout=8) + status = postgres.psql_json( + f"select jsonb_build_object('status', status)::text from kb_stage.kb_proposals " + f"where id = '{PROPOSAL_ID}'::uuid;" + )["status"] + return { + "lock_observed": lock_held, + "concurrent_downgrade": downgrade_receipt, + "holder_returncode": holder.returncode, + "holder_stderr_tail": safe_error_text(holder_stderr, limit=300), + "holder_output_observed": bool(holder_stdout.strip()), + "proposal_status_after": status, + } + finally: + if holder.poll() is None: + holder.terminate() + try: + holder.wait(timeout=3) + except subprocess.TimeoutExpired: + holder.kill() + holder.wait(timeout=3) + + +def accepted_rows_sql() -> str: + return f""" +insert into kb_stage.kb_proposals ( + id, proposal_type, status, proposed_by_handle, proposed_by_agent_id, channel, + source_ref, rationale, payload, reviewed_by_handle, reviewed_by_agent_id, + reviewed_at, review_note, created_at, updated_at +) values ( + '{PROPOSAL_ID}', 'approve_claim', 'approved', 'leo', '{AGENT_ID}', 'disposable_verifier', + 'fixture://livingip-v3-contract', 'Admit exact hash-bound V3 epistemic rows.', + '{{"apply_payload": {{"contract_version": 3}}}}'::jsonb, 'v3-contract-reviewer', '{AGENT_ID}', + '2026-07-18T09:00:00Z', 'Accepted only for the disposable PostgreSQL contract verifier.', + '2026-07-18T08:59:00Z', '2026-07-18T09:00:00Z' +); + +insert into kb_stage.kb_proposals ( + id, proposal_type, status, proposed_by_handle, proposed_by_agent_id, channel, + rationale, payload, created_at, updated_at +) values ( + '{PENDING_PROPOSAL_ID}', 'approve_claim', 'pending_review', 'leo', '{AGENT_ID}', + 'disposable_verifier', 'Pending proposal must not authorize canonical rows.', + '{{"apply_payload": {{"contract_version": 3}}}}'::jsonb, + '2026-07-18T09:01:00Z', '2026-07-18T09:01:00Z' +); + +insert into public.sources ( + id, source_type, url, storage_path, excerpt, hash, created_by, captured_at, created_at, + canonical_url, storage_uri, content_hash, access_scope, ingestion_origin, + provenance_status, source_class, auto_promotion_policy, accepted_by_proposal_id +) values ( + '{SOURCE_ID}', 'document', 'https://example.invalid/livingip-v3-source', null, + 'Immutable epistemic provenance.', '{SOURCE_CONTENT_HASH}', '{AGENT_ID}', + '2026-07-18T09:02:00Z', '2026-07-18T09:02:00Z', + 'https://example.invalid/livingip-v3-source', null, '{SOURCE_CONTENT_HASH}', + 'collective_internal', 'operator_upload', 'verified', 'primary_record', 'human_required', + '{PROPOSAL_ID}' +); + +insert into public.claims ( + id, type, text, status, confidence, tags, created_by, created_at, updated_at, + owner_agent_id, proposition, body_markdown, scope, access_scope, admission_state, + revision_criteria, revision_kind, supersedes_id, accepted_by_proposal_id +) values ( + '{PRIMARY_CLAIM_ID}', 'empirical', {sql_literal(PRIMARY_PROPOSITION)}, 'active', null, + array['v3-contract'], '{AGENT_ID}', '2026-07-18T09:03:00Z', '2026-07-18T09:03:00Z', + null, {sql_literal(PRIMARY_PROPOSITION)}, {sql_literal(PRIMARY_BODY)}, {sql_literal(PRIMARY_SCOPE)}, + 'collective_internal', 'admitted', {sql_literal(PRIMARY_REVISION_CRITERIA)}, 'semantic', + '{LEGACY_CLAIM_ID}', '{PROPOSAL_ID}' +), ( + '{SUPPORT_CLAIM_ID}', 'conceptual', + 'Content-hashed locators make source versions reproducible.', 'active', null, + array['v3-contract'], '{AGENT_ID}', '2026-07-18T09:04:00Z', '2026-07-18T09:04:00Z', + null, 'Content-hashed locators make source versions reproducible.', + 'A locator and immutable source hash identify the exact source payload used by an evidence link.', + 'Minimum disposable PostgreSQL V3 contract verification.', 'collective_internal', 'admitted', + 'Revise if the hash and locator no longer identify one exact source payload.', 'original', null, + '{PROPOSAL_ID}' +); + +insert into public.claim_evidence ( + id, claim_id, source_id, role, weight, created_by, created_at, + excerpt, locator_json, source_content_hash, rationale, polarity, accepted_from_proposal_id +) values ( + '{EVIDENCE_ID}', '{PRIMARY_CLAIM_ID}', '{SOURCE_ID}', 'grounds', null, '{AGENT_ID}', + '2026-07-18T09:05:00Z', 'Immutable epistemic provenance.', + '{{"paragraph": 1, "section": "epistemic-contract"}}'::jsonb, '{SOURCE_CONTENT_HASH}', + 'The cited passage directly supports reproducible source provenance.', 'supports', '{PROPOSAL_ID}' +); + +insert into public.claim_evidence_assessments ( + id, claim_id, support_tier, challenge_tier, overall_state, rationale, + evidence_set_hash, accepted_by_proposal_id, created_at +) values ( + '{ASSESSMENT_ID}', '{PRIMARY_CLAIM_ID}', 'strong', 'none', 'support_dominant', + 'One exact primary-record source supports the bounded contract claim and no challenge is linked.', + '{EVIDENCE_SET_HASH}', '{PROPOSAL_ID}', '2026-07-18T09:06:00Z' +); + +insert into public.claim_edges ( + id, from_claim, to_claim, edge_type, weight, created_by, created_at, + relation_type, rationale, scope_or_conditions, accepted_by_proposal_id +) values ( + '{EDGE_ID}', '{SUPPORT_CLAIM_ID}', '{PRIMARY_CLAIM_ID}', 'supports', null, '{AGENT_ID}', + '2026-07-18T09:07:00Z', 'supports', + 'Reproducible source versions are a prerequisite for exact evidence provenance.', + 'Within the minimum V3 epistemic storage contract.', '{PROPOSAL_ID}' +); +""" + + +def round_trip_sql() -> str: + return f""" +select jsonb_build_object( + 'proposal', (select to_jsonb(row_data) from ( + select id, proposal_type, status, reviewed_by_handle, reviewed_by_agent_id, + reviewed_at, review_note + from kb_stage.kb_proposals where id = '{PROPOSAL_ID}'::uuid + ) row_data), + 'claim', (select to_jsonb(row_data) from ( + select id, type, text, proposition, body_markdown, scope, access_scope, + admission_state, revision_criteria, revision_kind, supersedes_id, + accepted_by_proposal_id + from public.claims where id = '{PRIMARY_CLAIM_ID}'::uuid + ) row_data), + 'support_claim', (select to_jsonb(row_data) from ( + select id, admission_state, revision_kind, accepted_by_proposal_id + from public.claims where id = '{SUPPORT_CLAIM_ID}'::uuid + ) row_data), + 'source', (select to_jsonb(row_data) from ( + select id, canonical_url, storage_uri, hash, content_hash, access_scope, + ingestion_origin, provenance_status, source_class, auto_promotion_policy, + accepted_by_proposal_id + from public.sources where id = '{SOURCE_ID}'::uuid + ) row_data), + 'evidence', (select to_jsonb(row_data) from ( + select id, claim_id, source_id, role, excerpt, locator_json, source_content_hash, + rationale, polarity, accepted_from_proposal_id + from public.claim_evidence where id = '{EVIDENCE_ID}'::uuid + ) row_data), + 'assessment', (select to_jsonb(row_data) from ( + select id, claim_id, support_tier, challenge_tier, overall_state, rationale, + evidence_set_hash, accepted_by_proposal_id + from public.claim_evidence_assessments where id = '{ASSESSMENT_ID}'::uuid + ) row_data), + 'edge', (select to_jsonb(row_data) from ( + select id, from_claim, to_claim, relation_type, rationale, scope_or_conditions, + accepted_by_proposal_id + from public.claim_edges where id = '{EDGE_ID}'::uuid + ) row_data) +)::text; +""" + + +def adversarial_checks(postgres: DisposablePostgres) -> dict[str, Any]: + valid_source_columns = """ + source_type, url, hash, canonical_url, content_hash, access_scope, ingestion_origin, + provenance_status, source_class, auto_promotion_policy, accepted_by_proposal_id + """ + valid_source_values = f""" + 'document', 'https://example.invalid/pending', '{SOURCE_CONTENT_HASH}', + 'https://example.invalid/pending', '{SOURCE_CONTENT_HASH}', 'collective_internal', + 'operator_upload', 'verified', 'primary_record', 'human_required', '{PENDING_PROPOSAL_ID}' + """ + checks = { + "new_legacy_claim_cannot_bypass_v3_contract": expect_failure( + postgres, + """ +insert into public.claims (id, type, text, status) +values ('50000000-0000-4000-8000-000000000011', 'structural', 'Legacy bypass.', 'open'); +""", + expected_sqlstate="23514", + ), + "new_unlinked_source_cannot_bypass_v3_contract": expect_failure( + postgres, + f""" +insert into public.sources (id, source_type, hash) +values ('50000000-0000-4000-8000-000000000012', 'document', '{SOURCE_CONTENT_HASH}'); +""", + expected_sqlstate="23514", + ), + "new_unlinked_evidence_cannot_bypass_v3_contract": expect_failure( + postgres, + f""" +insert into public.claim_evidence (id, claim_id, source_id, role) +values ( + '50000000-0000-4000-8000-000000000013', '{LEGACY_CLAIM_ID}', '{LEGACY_SOURCE_ID}', 'grounds' +); +""", + expected_sqlstate="23514", + ), + "new_unlinked_edge_cannot_bypass_v3_contract": expect_failure( + postgres, + f""" +insert into public.claim_edges (id, from_claim, to_claim, edge_type) +values ( + '50000000-0000-4000-8000-000000000014', + '{LEGACY_CLAIM_ID}', '{LEGACY_RELATED_CLAIM_ID}', 'relates' +); +""", + expected_sqlstate="23514", + ), + "pending_proposal_cannot_authorize_source": expect_failure( + postgres, + f"insert into public.sources (id, {valid_source_columns}) " + f"values ('50000000-0000-4000-8000-000000000001', {valid_source_values});", + expected_sqlstate="23514", + ), + "applied_status_without_apply_receipt_cannot_authorize_source": expect_failure( + postgres, + f""" +insert into public.sources (id, {valid_source_columns}) values ( + '50000000-0000-4000-8000-000000000015', + 'document', 'https://example.invalid/missing-apply-receipt', '{SOURCE_CONTENT_HASH}', + 'https://example.invalid/missing-apply-receipt', '{SOURCE_CONTENT_HASH}', 'collective_internal', + 'operator_upload', 'verified', 'primary_record', 'human_required', + '{APPLIED_WITHOUT_RECEIPT_PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "new_applied_proposal_without_receipt_is_rejected": expect_failure( + postgres, + """ +insert into kb_stage.kb_proposals ( + id, proposal_type, status, rationale, payload, reviewed_by_handle, reviewed_at, review_note +) values ( + '30000000-0000-4000-8000-000000000004', 'approve_claim', 'applied', + 'Invalid new apply receipt.', '{"apply_payload": {"contract_version": 3}}'::jsonb, + 'v3-contract-reviewer', '2026-07-18T09:00:00Z', 'Missing applicator and applied_at.' +); +""", + expected_sqlstate="23514", + ), + "non_applied_proposal_cannot_carry_apply_metadata": expect_failure( + postgres, + f""" +update kb_stage.kb_proposals + set applied_by_handle = 'premature-applicator', applied_at = '2026-07-18T09:00:00Z' + where id = '{PENDING_PROPOSAL_ID}'::uuid; +""", + expected_sqlstate="55000", + ), + "admitted_claim_requires_body_scope_and_revision_criteria": expect_failure( + postgres, + f""" +insert into public.claims ( + id, type, text, status, proposition, body_markdown, scope, access_scope, + admission_state, revision_criteria, revision_kind, accepted_by_proposal_id +) values ( + '50000000-0000-4000-8000-000000000002', 'empirical', 'Incomplete admitted claim.', 'active', + 'Incomplete admitted claim.', null, 'Verifier scope.', 'collective_internal', + 'admitted', 'Revise when incomplete.', 'original', '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "null_revision_kind_cannot_bypass_claim_contract": expect_failure( + postgres, + f""" +insert into public.claims ( + id, type, text, status, proposition, body_markdown, scope, access_scope, + admission_state, revision_criteria, revision_kind, accepted_by_proposal_id +) values ( + '50000000-0000-4000-8000-000000000007', 'empirical', 'Null revision kind claim.', 'active', + 'Null revision kind claim.', 'A complete body cannot compensate for a null required revision kind.', + 'Verifier scope.', 'collective_internal', 'admitted', 'Revise when null checks fail.', null, '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "null_source_classification_cannot_bypass_source_contract": expect_failure( + postgres, + f""" +insert into public.sources ( + id, source_type, url, hash, canonical_url, content_hash, access_scope, ingestion_origin, + provenance_status, source_class, auto_promotion_policy, accepted_by_proposal_id +) values ( + '50000000-0000-4000-8000-000000000008', 'document', 'https://example.invalid/null-class', + '{SOURCE_CONTENT_HASH}', 'https://example.invalid/null-class', '{SOURCE_CONTENT_HASH}', + 'collective_internal', 'operator_upload', 'verified', null, 'human_required', '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "evidence_hash_must_match_source_version": expect_failure( + postgres, + f""" +insert into public.claim_evidence ( + id, claim_id, source_id, role, weight, locator_json, source_content_hash, + rationale, polarity, accepted_from_proposal_id +) values ( + '50000000-0000-4000-8000-000000000003', '{PRIMARY_CLAIM_ID}', '{SOURCE_ID}', 'grounds', null, + '{{"paragraph": 1}}'::jsonb, '{"0" * 64}', 'Mismatched source hash.', 'supports', '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "evidence_requires_locator_and_rationale": expect_failure( + postgres, + f""" +insert into public.claim_evidence ( + id, claim_id, source_id, role, weight, locator_json, source_content_hash, + rationale, polarity, accepted_from_proposal_id +) values ( + '50000000-0000-4000-8000-000000000004', '{PRIMARY_CLAIM_ID}', '{SOURCE_ID}', 'grounds', null, + '{{}}'::jsonb, '{SOURCE_CONTENT_HASH}', '', 'supports', '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "null_polarity_cannot_bypass_evidence_contract": expect_failure( + postgres, + f""" +insert into public.claim_evidence ( + id, claim_id, source_id, role, weight, locator_json, source_content_hash, + rationale, polarity, accepted_from_proposal_id +) values ( + '50000000-0000-4000-8000-000000000009', '{PRIMARY_CLAIM_ID}', '{SOURCE_ID}', 'grounds', null, + '{{"paragraph": 1}}'::jsonb, '{SOURCE_CONTENT_HASH}', 'Null polarity must fail.', null, '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "generic_relates_edge_is_not_admitted": expect_failure( + postgres, + f""" +insert into public.claim_edges ( + id, from_claim, to_claim, edge_type, weight, relation_type, rationale, accepted_by_proposal_id +) values ( + '50000000-0000-4000-8000-000000000005', '{SUPPORT_CLAIM_ID}', '{PRIMARY_CLAIM_ID}', + 'relates', null, 'relates', 'Generic relation must fail.', '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "null_relation_type_cannot_bypass_edge_contract": expect_failure( + postgres, + f""" +insert into public.claim_edges ( + id, from_claim, to_claim, edge_type, weight, relation_type, rationale, accepted_by_proposal_id +) values ( + '50000000-0000-4000-8000-000000000010', '{SUPPORT_CLAIM_ID}', '{PRIMARY_CLAIM_ID}', + 'supports', null, null, 'Null relation type must fail.', '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "truth_apt_assessment_requires_both_tiers": expect_failure( + postgres, + f""" +insert into public.claim_evidence_assessments ( + id, claim_id, support_tier, challenge_tier, overall_state, rationale, + evidence_set_hash, accepted_by_proposal_id +) values ( + '50000000-0000-4000-8000-000000000006', '{PRIMARY_CLAIM_ID}', 'strong', null, + 'support_dominant', 'Missing challenge tier must fail.', '{EVIDENCE_SET_HASH}', '{PROPOSAL_ID}' +); +""", + expected_sqlstate="23514", + ), + "source_hash_is_immutable": expect_failure( + postgres, + f"update public.sources set content_hash = '{'f' * 64}' where id = '{SOURCE_ID}'::uuid;", + expected_sqlstate="55000", + ), + "source_locator_is_immutable": expect_failure( + postgres, + f"update public.sources set canonical_url = 'https://example.invalid/mutated' " + f"where id = '{SOURCE_ID}'::uuid;", + expected_sqlstate="55000", + ), + "source_classification_is_immutable": expect_failure( + postgres, + f"update public.sources set source_class = 'journalistic' where id = '{SOURCE_ID}'::uuid;", + expected_sqlstate="55000", + ), + "admitted_claim_semantics_are_immutable": expect_failure( + postgres, + f"update public.claims set body_markdown = 'Mutated body.' where id = '{PRIMARY_CLAIM_ID}'::uuid;", + expected_sqlstate="55000", + ), + "admitted_claim_status_is_immutable": expect_failure( + postgres, + f"update public.claims set status = 'retracted' where id = '{PRIMARY_CLAIM_ID}'::uuid;", + expected_sqlstate="55000", + ), + "admitted_claim_update_timestamp_is_immutable": expect_failure( + postgres, + f"update public.claims set updated_at = updated_at + interval '1 second' " + f"where id = '{PRIMARY_CLAIM_ID}'::uuid;", + expected_sqlstate="55000", + ), + "admitted_claim_cannot_be_deleted": expect_failure( + postgres, + f"delete from public.claims where id = '{PRIMARY_CLAIM_ID}'::uuid;", + expected_sqlstate="55000", + ), + "proposal_linked_source_cannot_be_deleted": expect_failure( + postgres, + f"delete from public.sources where id = '{SOURCE_ID}'::uuid;", + expected_sqlstate="55000", + ), + "proposal_linked_evidence_cannot_be_deleted": expect_failure( + postgres, + f"delete from public.claim_evidence where id = '{EVIDENCE_ID}'::uuid;", + expected_sqlstate="55000", + ), + "proposal_linked_edge_cannot_be_deleted": expect_failure( + postgres, + f"delete from public.claim_edges where id = '{EDGE_ID}'::uuid;", + expected_sqlstate="55000", + ), + "evidence_assessment_cannot_be_deleted": expect_failure( + postgres, + f"delete from public.claim_evidence_assessments where id = '{ASSESSMENT_ID}'::uuid;", + expected_sqlstate="55000", + ), + "linked_proposal_cannot_lose_accepted_decision": expect_failure( + postgres, + f"update kb_stage.kb_proposals set status = 'rejected' where id = '{PROPOSAL_ID}'::uuid;", + expected_sqlstate="55000", + ), + "linked_proposal_decision_metadata_is_immutable": expect_failure( + postgres, + f"update kb_stage.kb_proposals set review_note = 'Rewritten decision.' where id = '{PROPOSAL_ID}'::uuid;", + expected_sqlstate="55000", + ), + } + return checks + + +def applied_receipt_adversarial_checks(postgres: DisposablePostgres) -> dict[str, Any]: + return { + "applied_proposal_status_is_immutable": expect_failure( + postgres, + f"update kb_stage.kb_proposals set status = 'approved' where id = '{PROPOSAL_ID}'::uuid;", + expected_sqlstate="55000", + ), + "applied_proposal_timestamp_is_immutable": expect_failure( + postgres, + f"update kb_stage.kb_proposals set applied_at = applied_at + interval '1 second' " + f"where id = '{PROPOSAL_ID}'::uuid;", + expected_sqlstate="55000", + ), + "applied_proposal_applicator_is_immutable": expect_failure( + postgres, + f"update kb_stage.kb_proposals set applied_by_handle = 'rewritten-applicator' " + f"where id = '{PROPOSAL_ID}'::uuid;", + expected_sqlstate="55000", + ), + } + + +def cleanup_canary_rows_sql() -> str: + return f""" +begin; +-- This verifier owns an isolated disposable database and runs as its superuser. +-- Production writers must never receive a role capable of disabling triggers. +set local session_replication_role = replica; +delete from public.claim_evidence_assessments where id = '{ASSESSMENT_ID}'::uuid; +delete from public.claim_edges where id = '{EDGE_ID}'::uuid; +delete from public.claim_evidence where id = '{EVIDENCE_ID}'::uuid; +delete from public.claims where id in ('{PRIMARY_CLAIM_ID}'::uuid, '{SUPPORT_CLAIM_ID}'::uuid); +delete from public.sources where id = '{SOURCE_ID}'::uuid; +delete from kb_stage.kb_proposals where id in ( + '{PROPOSAL_ID}'::uuid, + '{PENDING_PROPOSAL_ID}'::uuid, + '{APPLIED_WITHOUT_RECEIPT_PROPOSAL_ID}'::uuid +); +commit; +""" + + +def build_checks( + receipt: dict[str, Any], + *, + first_catalog: dict[str, Any], + second_catalog: dict[str, Any], + reinstall_catalog: dict[str, Any], + rollback_catalog: dict[str, Any], + final_catalog: dict[str, Any], + baseline_legacy: dict[str, Any], + after_install_legacy: dict[str, Any], + after_cleanup_legacy: dict[str, Any], + after_rollback_legacy: dict[str, Any], +) -> dict[str, bool]: + round_trip = receipt["round_trip"] + claim = round_trip["claim"] + source = round_trip["source"] + evidence = round_trip["evidence"] + assessment = round_trip["assessment"] + edge = round_trip["edge"] + proposal = round_trip["proposal"] + linked_ids = { + claim["accepted_by_proposal_id"], + source["accepted_by_proposal_id"], + evidence["accepted_from_proposal_id"], + assessment["accepted_by_proposal_id"], + edge["accepted_by_proposal_id"], + } + adversarial = receipt["adversarial"] + accepted_proposal_gate = next( + ( + function + for function in first_catalog["functions"] + if function["function_name"] == "teleo_v3_require_accepted_proposal" and function["arguments"] == "" + ), + {}, + ) + cutover_gate = next( + ( + function + for function in first_catalog["functions"] + if function["function_name"] == "teleo_v3_enforce_apply_contract_cutover" and function["arguments"] == "" + ), + {}, + ) + cutover_trigger = next( + ( + trigger + for trigger in first_catalog["triggers"] + if trigger["trigger_name"] == "teleo_v3_00_enforce_apply_contract_cutover" + ), + {}, + ) + return { + "container_network_is_none": receipt["environment"]["network_mode"] == "none", + "container_data_is_tmpfs": receipt["environment"]["tmpfs_data_dir"] is True, + "container_has_no_volume_mounts": receipt["environment"]["docker_volume_mounts"] == [], + "canary_cleanup_bypass_is_disposable_superuser_only": receipt["canary_cleanup"] + == { + "trigger_bypass": "set local session_replication_role=replica", + "scope": "isolated-disposable-superuser-transaction", + }, + "legacy_rows_unchanged_by_install": baseline_legacy == after_install_legacy, + "legacy_rows_unchanged_by_canary_cleanup": baseline_legacy == after_cleanup_legacy, + "legacy_rows_unchanged_by_rollback": baseline_legacy == after_rollback_legacy, + "migration_is_idempotent": first_catalog == second_catalog, + "security_definer_trigger_owner_and_acl_are_narrow": accepted_proposal_gate + == { + "schema_name": "kb_stage", + "function_name": "teleo_v3_require_accepted_proposal", + "arguments": "", + "owner_name": "kb_gate_owner", + "security_definer": True, + "config": ["search_path=pg_catalog, pg_temp"], + "source_sha256": "ca398ef2f185488c0712bd5ebc22402ab9e41e5dfd2f5823b367e6a382e8a64f", + "language_name": "plpgsql", + "function_kind": "f", + "argument_count": 0, + "return_type": "trigger", + "owner_only_execute": True, + "non_owner_execute_grantees": [], + }, + "cutover_trigger_function_owner_and_acl_are_narrow": cutover_gate + == { + "schema_name": "kb_stage", + "function_name": "teleo_v3_enforce_apply_contract_cutover", + "arguments": "", + "owner_name": "kb_gate_owner", + "security_definer": True, + "config": ["search_path=pg_catalog, pg_temp"], + "source_sha256": "3ab2d5e1d5d1d18b13579fe2c0cae487036c7b147cf7f1b6b9a3633274756c18", + "language_name": "plpgsql", + "function_kind": "f", + "argument_count": 0, + "return_type": "trigger", + "owner_only_execute": True, + "non_owner_execute_grantees": [], + }, + "full_trigger_catalog_is_exact": all( + trigger_catalog_is_exact(catalog) for catalog in (first_catalog, second_catalog, reinstall_catalog) + ), + "cutover_trigger_is_enabled_and_exactly_bound": ( + cutover_trigger.get("schema_name") == "kb_stage" + and cutover_trigger.get("table_name") == "kb_proposals" + and cutover_trigger.get("enabled_state") == "O" + and cutover_trigger.get("function_schema") == "kb_stage" + and cutover_trigger.get("function_name") == "teleo_v3_enforce_apply_contract_cutover" + and cutover_trigger.get("function_identity_arguments") == "" + ), + "claim_contract_round_trip": claim + == { + "id": PRIMARY_CLAIM_ID, + "type": "empirical", + "text": PRIMARY_PROPOSITION, + "proposition": PRIMARY_PROPOSITION, + "body_markdown": PRIMARY_BODY, + "scope": PRIMARY_SCOPE, + "access_scope": "collective_internal", + "admission_state": "admitted", + "revision_criteria": PRIMARY_REVISION_CRITERIA, + "revision_kind": "semantic", + "supersedes_id": LEGACY_CLAIM_ID, + "accepted_by_proposal_id": PROPOSAL_ID, + }, + "source_contract_round_trip": source["content_hash"] == SOURCE_CONTENT_HASH + and source["hash"] == SOURCE_CONTENT_HASH + and source["canonical_url"] == "https://example.invalid/livingip-v3-source" + and source["provenance_status"] == "verified" + and source["source_class"] == "primary_record", + "evidence_contract_round_trip": evidence["claim_id"] == PRIMARY_CLAIM_ID + and evidence["source_id"] == SOURCE_ID + and evidence["source_content_hash"] == SOURCE_CONTENT_HASH + and evidence["locator_json"] == {"paragraph": 1, "section": "epistemic-contract"} + and evidence["polarity"] == "supports" + and bool(evidence["rationale"]), + "assessment_contract_round_trip": assessment["claim_id"] == PRIMARY_CLAIM_ID + and assessment["support_tier"] == "strong" + and assessment["challenge_tier"] == "none" + and assessment["overall_state"] == "support_dominant" + and assessment["evidence_set_hash"] == EVIDENCE_SET_HASH, + "typed_edge_contract_round_trip": edge["from_claim"] == SUPPORT_CLAIM_ID + and edge["to_claim"] == PRIMARY_CLAIM_ID + and edge["relation_type"] == "supports" + and bool(edge["rationale"]), + "accepted_proposal_linkage_round_trip": linked_ids == {PROPOSAL_ID} + and proposal["status"] == "approved" + and proposal["reviewed_at"] is not None + and bool(proposal["review_note"]), + "accepted_proposal_lock_blocks_concurrent_downgrade": ( + receipt["proposal_decision_lock_race"]["lock_observed"] is True + and receipt["proposal_decision_lock_race"]["concurrent_downgrade"]["refused"] is True + and receipt["proposal_decision_lock_race"]["concurrent_downgrade"]["expected_sqlstate_observed"] is True + and receipt["proposal_decision_lock_race"]["holder_returncode"] == 0 + and receipt["proposal_decision_lock_race"]["proposal_status_after"] == "approved" + ), + "all_adversarial_writes_refused": all( + check["refused"] and check["expected_sqlstate_observed"] for check in adversarial.values() + ), + "approved_to_applied_transition_remains_compatible": ( + receipt["proposal_applied_transition"]["status"] == "applied" + and receipt["proposal_applied_transition"]["review_note"] == proposal["review_note"] + and receipt["proposal_applied_transition"]["applied_at"] is not None + ), + "rollback_refuses_v3_data_loss": receipt["rollback_with_data"]["refused"] + and receipt["rollback_with_data"]["expected_sqlstate_observed"], + "rollback_removes_contract": catalog_is_absent(rollback_catalog), + "reinstall_matches_original_contract": reinstall_catalog == first_catalog, + "final_rollback_removes_contract": catalog_is_absent(final_catalog), + "rollback_is_idempotent": receipt["idempotent_rollback_returncode"] == 0, + } + + +def write_receipt(output: Path, receipt: dict[str, Any]) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_name(f".{output.name}.{uuid.uuid4().hex}.tmp") + temporary.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, output) + + +def run_verification(output: Path) -> dict[str, Any]: + receipt: dict[str, Any] = { + "schema": RECEIPT_SCHEMA, + "status": "fail", + "required_tier": "T2_runtime", + "current_tier": "not_verified", + "production_access_attempted": False, + "production_migration_shipped": False, + "database_target_mode": "internally-created-disposable-container-only", + "migration_files": { + "install": str(MIGRATION_SQL.relative_to(REPO_ROOT)), + "rollback": str(ROLLBACK_SQL.relative_to(REPO_ROOT)), + "install_sha256": hashlib.sha256(MIGRATION_SQL.read_bytes()).hexdigest(), + "rollback_sha256": hashlib.sha256(ROLLBACK_SQL.read_bytes()).hexdigest(), + }, + "not_shipped": [ + "production migration execution", + "live apply-worker wiring", + "full V3 proposal/evaluation/identity architecture", + ], + } + postgres = DisposablePostgres() + error: str | None = None + try: + receipt["environment"] = postgres.start() + postgres.psql(BOOTSTRAP_SQL) + baseline_legacy = postgres.psql_json(LEGACY_STATE_SQL) + receipt["legacy_baseline"] = baseline_legacy + + first_install = postgres.apply_file(MIGRATION_SQL) + first_catalog = catalog_readback(postgres) + after_install_legacy = postgres.psql_json(LEGACY_STATE_SQL) + receipt["install_first"] = { + "returncode": first_install.returncode, + "catalog_sha256": canonical_sha256(first_catalog), + "catalog": first_catalog, + } + + second_install = postgres.apply_file(MIGRATION_SQL) + second_catalog = catalog_readback(postgres) + receipt["install_second"] = { + "returncode": second_install.returncode, + "catalog_sha256": canonical_sha256(second_catalog), + } + + postgres.psql(accepted_rows_sql()) + receipt["round_trip"] = postgres.psql_json(round_trip_sql()) + receipt["proposal_decision_lock_race"] = proposal_decision_lock_race_check(postgres) + receipt["adversarial"] = adversarial_checks(postgres) + postgres.psql( + f""" +update kb_stage.kb_proposals + set status = 'applied', + applied_by_handle = 'v3-contract-apply', + applied_at = '2026-07-18T09:08:00Z', + updated_at = '2026-07-18T09:08:00Z' + where id = '{PROPOSAL_ID}'::uuid; +""" + ) + receipt["proposal_applied_transition"] = postgres.psql_json( + f""" +select jsonb_build_object( + 'id', id, + 'status', status, + 'review_note', review_note, + 'applied_by_handle', applied_by_handle, + 'applied_at', applied_at +)::text +from kb_stage.kb_proposals +where id = '{PROPOSAL_ID}'::uuid; +""" + ) + receipt["adversarial"].update(applied_receipt_adversarial_checks(postgres)) + + rollback_with_data = postgres.apply_file(ROLLBACK_SQL, check=False) + receipt["rollback_with_data"] = failure_receipt(rollback_with_data, "55000") + if not ( + receipt["rollback_with_data"]["refused"] and receipt["rollback_with_data"]["expected_sqlstate_observed"] + ): + raise VerificationError("rollback did not refuse proposal-linked V3 data") + + receipt["canary_cleanup"] = { + "trigger_bypass": "set local session_replication_role=replica", + "scope": "isolated-disposable-superuser-transaction", + } + postgres.psql(cleanup_canary_rows_sql()) + after_cleanup_legacy = postgres.psql_json(LEGACY_STATE_SQL) + + rollback = postgres.apply_file(ROLLBACK_SQL) + rollback_catalog = catalog_readback(postgres) + after_rollback_legacy = postgres.psql_json(LEGACY_STATE_SQL) + receipt["rollback"] = { + "returncode": rollback.returncode, + "catalog": rollback_catalog, + "legacy_sha256": canonical_sha256(after_rollback_legacy), + } + + idempotent_rollback = postgres.apply_file(ROLLBACK_SQL) + receipt["idempotent_rollback_returncode"] = idempotent_rollback.returncode + + reinstall = postgres.apply_file(MIGRATION_SQL) + reinstall_catalog = catalog_readback(postgres) + receipt["reinstall"] = { + "returncode": reinstall.returncode, + "catalog_sha256": canonical_sha256(reinstall_catalog), + } + + final_rollback = postgres.apply_file(ROLLBACK_SQL) + final_catalog = catalog_readback(postgres) + receipt["final_rollback"] = { + "returncode": final_rollback.returncode, + "catalog": final_catalog, + } + + receipt["checks"] = build_checks( + receipt, + first_catalog=first_catalog, + second_catalog=second_catalog, + reinstall_catalog=reinstall_catalog, + rollback_catalog=rollback_catalog, + final_catalog=final_catalog, + baseline_legacy=baseline_legacy, + after_install_legacy=after_install_legacy, + after_cleanup_legacy=after_cleanup_legacy, + after_rollback_legacy=after_rollback_legacy, + ) + except Exception as exc: # retain a bounded failure receipt before returning non-zero + error = safe_error_text(str(exc)) + receipt["error"] = error + finally: + cleanup = postgres.cleanup() + receipt["cleanup"] = cleanup + cleanup_ok = ( + cleanup["container_absent"] + and cleanup["name_readback_clear"] + and cleanup["instance_label_readback_clear"] + and cleanup["stable_absence_readback"] + and cleanup["network_mode_was_none"] + and cleanup["tmpfs_data_dir_was_present"] + and cleanup["docker_volume_mounts_observed"] == [] + ) + receipt.setdefault("checks", {})["disposable_container_cleanup_verified"] = cleanup_ok + all_checks_pass = bool(receipt["checks"]) and all(receipt["checks"].values()) + if error is None and all_checks_pass: + receipt["status"] = "pass" + receipt["current_tier"] = "T2_runtime_disposable_postgresql" + receipt["strongest_claim"] = ( + "The additive minimum V3 epistemic contract installed, round-tripped accepted provenance, " + "rejected adversarial writes, rolled back, reinstalled, and cleaned up in networkless disposable PostgreSQL." + if receipt["status"] == "pass" + else "The disposable V3 epistemic contract verification did not pass every runtime check." + ) + write_receipt(output, receipt) + return receipt + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="retained JSON receipt path") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + receipt = run_verification(args.output) + print( + json.dumps( + { + "status": receipt["status"], + "current_tier": receipt["current_tier"], + "receipt": str(args.output.resolve()), + "checks_passed": sum(1 for value in receipt.get("checks", {}).values() if value), + "checks_total": len(receipt.get("checks", {})), + "cleanup": receipt.get("checks", {}).get("disposable_container_cleanup_verified"), + "production_access_attempted": receipt["production_access_attempted"], + }, + sort_keys=True, + ) + ) + return 0 if receipt["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/apply_proposal.py b/scripts/apply_proposal.py index 90aaaf2..ed0ad38 100644 --- a/scripts/apply_proposal.py +++ b/scripts/apply_proposal.py @@ -84,6 +84,7 @@ the approved proposal ledger can flip to ``applied``. from __future__ import annotations import argparse +import hashlib import json import shutil import subprocess @@ -99,6 +100,26 @@ DEFAULT_CONTAINER = "teleo-pg" DEFAULT_DB = "teleo" DEFAULT_HOST = "127.0.0.1" DEFAULT_ROLE = "kb_apply" +V3_CUTOVER_FUNCTION = "kb_stage.teleo_v3_enforce_apply_contract_cutover()" +V3_CUTOVER_FUNCTION_NAME = "teleo_v3_enforce_apply_contract_cutover" +V3_CUTOVER_TRIGGER = "teleo_v3_00_enforce_apply_contract_cutover" +V3_GATE_OWNER = "kb_gate_owner" +V3_PROTECTED_ROLES = ("kb_gate_owner", "kb_review", "kb_apply") +V3_TRIGGER_PREFIX = "teleo_v3_" +V3_CATALOG_DDL_ADVISORY_LOCK_KEY = 6072343533146485505 +V3_CATALOG_LOCK_FUNCTION = "teleo_v3_acquire_protected_relation_locks" +V3_CATALOG_LOCK_FUNCTION_SOURCE_SHA256 = "897345944ba0f1b2a5c01b098fb17de6f881a122e906277086fa7889e76219e8" +V3_TRIGGER_FUNCTION_SOURCE_SHA256 = { + "teleo_v3_enforce_apply_contract_cutover": "3ab2d5e1d5d1d18b13579fe2c0cae487036c7b147cf7f1b6b9a3633274756c18", + "teleo_v3_guard_immutable_fields": "d243741e552fe8c232440ffa5f853cf261f4b86b6f32d76f375734a2235c2d13", + "teleo_v3_guard_linked_proposal_decision": "602ee8f81f4515b0237f09a503ecbfe4fce2dc4a799e13f833554103b0e83e32", + "teleo_v3_reject_delete": "46190723ab4520954858dbd4bb6c9230e68e41c68b5aee3b8c2719e1416de43c", + "teleo_v3_require_accepted_proposal": "ca398ef2f185488c0712bd5ebc22402ab9e41e5dfd2f5823b367e6a382e8a64f", + "teleo_v3_require_linked_insert": "d8c190bd2fc90159b9a4824015c10a73a701a263bba90b67f8ccce75e454e6f8", + "teleo_v3_validate_claim_edge": "6a7a3e9d1d5ac6e9490abc9b7b1f321bb2e0948aea2671415aad28101f4ee936", + "teleo_v3_validate_claim_evidence": "4c697c497699760214aca99f255c215e393319fd4482d4b36efac839280c3733", + "teleo_v3_validate_evidence_assessment": "860f939b4878abe8e2e604e03be9eeb6c23b4878af4b19f9ec42de4b821fac61", +} # Handle of the service-agent row that performs canonical writes. Bootstrapped # once by superuser (scripts/kb_apply_prereqs.sql); kb_apply holds SELECT-only on @@ -129,10 +150,623 @@ MAX_BUNDLE_ROWS = { "sources": 500, "evidence": 1000, "edges": 1000, + "assessments": 100, "reasoning_tools": 50, } +def _trigger_argument_hex(arguments: tuple[str, ...]) -> str: + return b"".join(argument.encode("utf-8") + b"\0" for argument in arguments).hex() + + +def _v3_trigger_contract( + schema_name: str, + table_name: str, + trigger_name: str, + trigger_type: int, + function_name: str, + *, + update_columns: tuple[str, ...] = (), + arguments: tuple[str, ...] = (), + security_definer: bool = False, +) -> dict[str, Any]: + return { + "schema_name": schema_name, + "table_name": table_name, + "trigger_name": trigger_name, + "trigger_type": trigger_type, + "update_columns": sorted(update_columns), + "when_expression": None, + "argument_count": len(arguments), + "argument_hex": _trigger_argument_hex(arguments), + "function_schema": "kb_stage", + "function_name": function_name, + "function_identity_arguments": "", + "function_owner": V3_GATE_OWNER, + "function_security_definer": security_definer, + "function_config": ["search_path=pg_catalog, pg_temp"], + "function_source_sha256": V3_TRIGGER_FUNCTION_SOURCE_SHA256[function_name], + "function_owner_only_execute": True if security_definer else None, + } + + +# PostgreSQL pg_trigger.tgtype is a bitmask: ROW=1, BEFORE=2, INSERT=4, +# DELETE=8, UPDATE=16. Every V3 trigger is an ordinary row-level BEFORE trigger. +V3_EXPECTED_TRIGGER_CONTRACT = ( + _v3_trigger_contract( + "kb_stage", + "kb_proposals", + V3_CUTOVER_TRIGGER, + 23, + V3_CUTOVER_FUNCTION_NAME, + update_columns=("proposal_type", "payload", "status"), + security_definer=True, + ), + _v3_trigger_contract( + "public", + "claims", + "teleo_v3_00_claim_require_linked_insert", + 7, + "teleo_v3_require_linked_insert", + arguments=("accepted_by_proposal_id", "admission_state"), + ), + _v3_trigger_contract( + "public", + "claims", + "teleo_v3_claim_require_accepted_proposal", + 23, + "teleo_v3_require_accepted_proposal", + update_columns=("accepted_by_proposal_id",), + arguments=("accepted_by_proposal_id",), + security_definer=True, + ), + _v3_trigger_contract( + "public", + "sources", + "teleo_v3_00_source_require_linked_insert", + 7, + "teleo_v3_require_linked_insert", + arguments=("accepted_by_proposal_id",), + ), + _v3_trigger_contract( + "public", + "claims", + "teleo_v3_claim_immutable", + 19, + "teleo_v3_guard_immutable_fields", + arguments=( + "id", + "owner_agent_id", + "type", + "text", + "proposition", + "body_markdown", + "scope", + "access_scope", + "status", + "admission_state", + "revision_criteria", + "revision_kind", + "supersedes_id", + "superseded_by", + "confidence", + "tags", + "created_by", + "created_at", + "updated_at", + "accepted_by_proposal_id", + ), + ), + _v3_trigger_contract("public", "claims", "teleo_v3_claim_reject_delete", 11, "teleo_v3_reject_delete"), + _v3_trigger_contract( + "public", + "claim_evidence", + "teleo_v3_00_evidence_require_linked_insert", + 7, + "teleo_v3_require_linked_insert", + arguments=("accepted_from_proposal_id",), + ), + _v3_trigger_contract( + "public", + "sources", + "teleo_v3_source_require_accepted_proposal", + 23, + "teleo_v3_require_accepted_proposal", + update_columns=("accepted_by_proposal_id",), + arguments=("accepted_by_proposal_id",), + security_definer=True, + ), + _v3_trigger_contract( + "public", + "claim_edges", + "teleo_v3_00_edge_require_linked_insert", + 7, + "teleo_v3_require_linked_insert", + arguments=("accepted_by_proposal_id",), + ), + _v3_trigger_contract( + "public", + "sources", + "teleo_v3_source_immutable", + 19, + "teleo_v3_guard_immutable_fields", + arguments=( + "id", + "source_type", + "url", + "storage_path", + "excerpt", + "hash", + "canonical_url", + "storage_uri", + "content_hash", + "access_scope", + "ingestion_origin", + "provenance_status", + "source_class", + "auto_promotion_policy", + "created_by", + "captured_at", + "created_at", + "accepted_by_proposal_id", + ), + ), + _v3_trigger_contract("public", "sources", "teleo_v3_source_reject_delete", 11, "teleo_v3_reject_delete"), + _v3_trigger_contract( + "public", + "claim_evidence", + "teleo_v3_evidence_require_accepted_proposal", + 23, + "teleo_v3_require_accepted_proposal", + update_columns=("accepted_from_proposal_id",), + arguments=("accepted_from_proposal_id",), + security_definer=True, + ), + _v3_trigger_contract( + "public", + "claim_evidence", + "teleo_v3_evidence_validate_provenance", + 23, + "teleo_v3_validate_claim_evidence", + ), + _v3_trigger_contract( + "public", + "claim_evidence", + "teleo_v3_evidence_immutable", + 19, + "teleo_v3_guard_immutable_fields", + arguments=( + "id", + "claim_id", + "source_id", + "role", + "weight", + "excerpt", + "locator_json", + "source_content_hash", + "rationale", + "polarity", + "created_by", + "created_at", + "accepted_from_proposal_id", + ), + ), + _v3_trigger_contract( + "public", + "claim_evidence", + "teleo_v3_evidence_reject_delete", + 11, + "teleo_v3_reject_delete", + ), + _v3_trigger_contract( + "public", + "claim_edges", + "teleo_v3_edge_require_accepted_proposal", + 23, + "teleo_v3_require_accepted_proposal", + update_columns=("accepted_by_proposal_id",), + arguments=("accepted_by_proposal_id",), + security_definer=True, + ), + _v3_trigger_contract( + "public", "claim_edges", "teleo_v3_edge_validate_endpoints", 23, "teleo_v3_validate_claim_edge" + ), + _v3_trigger_contract( + "public", + "claim_edges", + "teleo_v3_edge_immutable", + 19, + "teleo_v3_guard_immutable_fields", + arguments=( + "id", + "from_claim", + "to_claim", + "edge_type", + "weight", + "relation_type", + "rationale", + "scope_or_conditions", + "created_by", + "created_at", + "accepted_by_proposal_id", + ), + ), + _v3_trigger_contract("public", "claim_edges", "teleo_v3_edge_reject_delete", 11, "teleo_v3_reject_delete"), + _v3_trigger_contract( + "public", + "claim_evidence_assessments", + "teleo_v3_assessment_require_accepted_proposal", + 23, + "teleo_v3_require_accepted_proposal", + update_columns=("accepted_by_proposal_id",), + arguments=("accepted_by_proposal_id",), + security_definer=True, + ), + _v3_trigger_contract( + "public", + "claim_evidence_assessments", + "teleo_v3_assessment_validate", + 23, + "teleo_v3_validate_evidence_assessment", + ), + _v3_trigger_contract( + "public", + "claim_evidence_assessments", + "teleo_v3_assessment_immutable", + 19, + "teleo_v3_guard_immutable_fields", + arguments=( + "id", + "claim_id", + "support_tier", + "challenge_tier", + "overall_state", + "rationale", + "evidence_set_hash", + "accepted_by_proposal_id", + "supersedes_assessment_id", + "created_at", + ), + ), + _v3_trigger_contract( + "public", + "claim_evidence_assessments", + "teleo_v3_assessment_reject_delete", + 11, + "teleo_v3_reject_delete", + ), + _v3_trigger_contract( + "kb_stage", + "kb_proposals", + "teleo_v3_guard_linked_proposal_decision", + 19, + "teleo_v3_guard_linked_proposal_decision", + update_columns=( + "proposal_type", + "status", + "rationale", + "payload", + "reviewed_by_handle", + "reviewed_by_agent_id", + "reviewed_at", + "review_note", + "applied_by_handle", + "applied_by_agent_id", + "applied_at", + ), + ), +) + +V3_PROTECTED_RELATIONS = ( + ("kb_stage", "kb_proposals"), + ("public", "claim_edges"), + ("public", "claim_evidence"), + ("public", "claim_evidence_assessments"), + ("public", "claims"), + ("public", "sources"), +) + + +def v3_expected_trigger_contract_json() -> str: + return json.dumps(V3_EXPECTED_TRIGGER_CONTRACT, separators=(",", ":"), sort_keys=True) + + +def build_v3_contract_state_query( + *, + into: str | None = None, + expected_contract_expression: str | None = None, +) -> str: + """Return the exact catalog query shared by worker and transaction guards.""" + protected_roles = ", ".join(sql_literal(role) for role in V3_PROTECTED_ROLES) + expected_contract = expected_contract_expression or sql_literal(v3_expected_trigger_contract_json()) + into_clause = f"\n into {into}" if into is not None else "" + return f"""with expected_triggers as ( + select expected.* + from pg_catalog.jsonb_to_recordset({expected_contract}::jsonb) as expected( + schema_name text, + table_name text, + trigger_name text, + trigger_type integer, + update_columns jsonb, + when_expression text, + argument_count integer, + argument_hex text, + function_schema text, + function_name text, + function_identity_arguments text, + function_owner text, + function_security_definer boolean, + function_config jsonb, + function_source_sha256 text, + function_owner_only_execute boolean + ) +), expected_functions as ( + select distinct function_schema, function_name, function_identity_arguments, + function_owner, function_security_definer, function_config, + function_source_sha256, function_owner_only_execute + from expected_triggers +), actual_v3_functions as ( + select procedure.oid, namespace.nspname as function_schema, + procedure.proname as function_name, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + owner_role.rolname as function_owner, + procedure.prosecdef as function_security_definer, + pg_catalog.to_jsonb(procedure.proconfig) as function_config, + pg_catalog.encode( + pg_catalog.sha256(pg_catalog.convert_to(procedure.prosrc, 'UTF8')), + 'hex' + ) as function_source_sha256, + not exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + ) as function_owner_only_execute, + language.lanname as function_language, + procedure.prokind as function_kind, + procedure.pronargs as function_argument_count, + procedure.prorettype + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner + join pg_catalog.pg_language language on language.oid = procedure.prolang + where namespace.nspname = 'kb_stage' + and pg_catalog.left(procedure.proname, {len(V3_TRIGGER_PREFIX)}) = {sql_literal(V3_TRIGGER_PREFIX)} + and procedure.proname <> {sql_literal(V3_CATALOG_LOCK_FUNCTION)} +), actual_v3_catalog_lock_functions as ( + select procedure.oid, + owner_role.rolname as function_owner, + procedure.prosecdef as function_security_definer, + pg_catalog.to_jsonb(procedure.proconfig) as function_config, + pg_catalog.encode( + pg_catalog.sha256(pg_catalog.convert_to(procedure.prosrc, 'UTF8')), + 'hex' + ) as function_source_sha256, + language.lanname as function_language, + procedure.prokind as function_kind, + procedure.pronargs as function_argument_count, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + procedure.prorettype, + exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + join pg_catalog.pg_roles grantee on grantee.oid = acl.grantee + where acl.privilege_type = 'EXECUTE' + and grantee.rolname = 'kb_apply' + ) as apply_execute, + not exists ( + select 1 + from pg_catalog.aclexplode( + coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner)) + ) acl + where acl.privilege_type = 'EXECUTE' + and acl.grantee <> procedure.proowner + and acl.grantee <> ( + select role.oid from pg_catalog.pg_roles role where role.rolname = 'kb_apply' + ) + ) as execute_acl_exact + from pg_catalog.pg_proc procedure + join pg_catalog.pg_namespace namespace on namespace.oid = procedure.pronamespace + join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner + join pg_catalog.pg_language language on language.oid = procedure.prolang + where namespace.nspname = 'kb_stage' + and procedure.proname = {sql_literal(V3_CATALOG_LOCK_FUNCTION)} +), actual_v3_triggers as ( + select trigger_row.*, table_namespace.nspname as schema_name, + relation.relname as table_name, + function_namespace.nspname as function_schema, + procedure.proname as function_name, + pg_catalog.pg_get_function_identity_arguments(procedure.oid) as function_identity_arguments, + coalesce(( + select pg_catalog.jsonb_agg(attribute.attname order by attribute.attname) + from pg_catalog.unnest(trigger_row.tgattr::smallint[]) update_column(attnum) + join pg_catalog.pg_attribute attribute + on attribute.attrelid = trigger_row.tgrelid + and attribute.attnum = update_column.attnum + and not attribute.attisdropped + ), '[]'::jsonb) as update_columns, + pg_catalog.pg_get_expr(trigger_row.tgqual, trigger_row.tgrelid, true) as when_expression, + pg_catalog.encode(trigger_row.tgargs, 'hex') as argument_hex + from pg_catalog.pg_trigger trigger_row + join pg_catalog.pg_class relation on relation.oid = trigger_row.tgrelid + join pg_catalog.pg_namespace table_namespace on table_namespace.oid = relation.relnamespace + join pg_catalog.pg_proc procedure on procedure.oid = trigger_row.tgfoid + join pg_catalog.pg_namespace function_namespace on function_namespace.oid = procedure.pronamespace + where pg_catalog.left(trigger_row.tgname, {len(V3_TRIGGER_PREFIX)}) = {sql_literal(V3_TRIGGER_PREFIX)} +), function_contract as ( + select + (select pg_catalog.count(*) from actual_v3_functions) = + (select pg_catalog.count(*) from expected_functions) + and not exists ( + select 1 + from expected_functions expected + where not exists ( + select 1 + from actual_v3_functions actual + where actual.function_schema = expected.function_schema + and actual.function_name = expected.function_name + and actual.function_identity_arguments = expected.function_identity_arguments + and actual.function_owner = expected.function_owner + and actual.function_security_definer = expected.function_security_definer + and actual.function_config = expected.function_config + and actual.function_source_sha256 = expected.function_source_sha256 + and ( + expected.function_owner_only_execute is null + or actual.function_owner_only_execute = expected.function_owner_only_execute + ) + and actual.function_language = 'plpgsql' + and actual.function_kind = 'f' + and actual.function_argument_count = 0 + and actual.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype + ) + ) as valid +), catalog_lock_function_contract as ( + select pg_catalog.count(*) = 1 + and pg_catalog.bool_and( + function_owner = {sql_literal(V3_GATE_OWNER)} + and function_security_definer + and function_config = '["search_path=pg_catalog, pg_temp"]'::jsonb + and function_source_sha256 = {sql_literal(V3_CATALOG_LOCK_FUNCTION_SOURCE_SHA256)} + and function_language = 'plpgsql' + and function_kind = 'f' + and function_argument_count = 0 + and function_identity_arguments = '' + and prorettype = 'pg_catalog.void'::pg_catalog.regtype + and apply_execute + and execute_acl_exact + ) as valid + from actual_v3_catalog_lock_functions +), trigger_contract as ( + select + (select pg_catalog.count(*) from actual_v3_triggers) = + (select pg_catalog.count(*) from expected_triggers) + and not exists ( + select 1 + from expected_triggers expected + where not exists ( + select 1 + from actual_v3_triggers actual + where actual.schema_name = expected.schema_name + and actual.table_name = expected.table_name + and actual.tgname = expected.trigger_name + and not actual.tgisinternal + and actual.tgenabled = 'O' + and actual.tgtype = expected.trigger_type + and actual.update_columns = expected.update_columns + and actual.when_expression is not distinct from expected.when_expression + and actual.tgnargs = expected.argument_count + and actual.argument_hex = expected.argument_hex + and actual.tgconstraint = 0::pg_catalog.oid + and not actual.tgdeferrable + and not actual.tginitdeferred + and actual.tgconstrrelid = 0::pg_catalog.oid + and actual.tgparentid = 0::pg_catalog.oid + and actual.tgoldtable is null + and actual.tgnewtable is null + and actual.function_schema = expected.function_schema + and actual.function_name = expected.function_name + and actual.function_identity_arguments = expected.function_identity_arguments + and exists ( + select 1 + from actual_v3_functions function_row + where function_row.oid = actual.tgfoid + and function_row.function_schema = expected.function_schema + and function_row.function_name = expected.function_name + and function_row.function_identity_arguments = expected.function_identity_arguments + ) + ) + ) as valid +), catalog_state as ( + select + exists (select 1 from actual_v3_functions) + or exists (select 1 from actual_v3_catalog_lock_functions) + or exists (select 1 from actual_v3_triggers) as artifacts_present, + (select valid from function_contract) as functions_valid, + (select valid from catalog_lock_function_contract) as catalog_lock_function_valid, + (select valid from trigger_contract) as triggers_valid, + exists ( + select 1 + from pg_catalog.pg_roles owner_role + where owner_role.rolname = {sql_literal(V3_GATE_OWNER)} + and not owner_role.rolcanlogin + and not owner_role.rolinherit + and not owner_role.rolsuper + and not owner_role.rolcreatedb + and not owner_role.rolcreaterole + and not owner_role.rolreplication + and not owner_role.rolbypassrls + ) as owner_role_valid, + not exists ( + select 1 + from pg_catalog.pg_auth_members membership + join pg_catalog.pg_roles member_role on member_role.oid = membership.member + join pg_catalog.pg_roles granted_role on granted_role.oid = membership.roleid + where member_role.rolname in ({protected_roles}) + or granted_role.rolname in ({protected_roles}) + ) as memberships_valid +) +select case + when not artifacts_present then 'V2' + when functions_valid and catalog_lock_function_valid and triggers_valid + and owner_role_valid and memberships_valid then 'V3' + else 'INVALID' +end{into_clause} +from catalog_state""" + + +def _v3_contract_lock_sql() -> str: + statements: list[str] = [] + for schema_name, table_name in V3_PROTECTED_RELATIONS: + if (schema_name, table_name) == ("kb_stage", "kb_proposals"): + statements.append(f"select kb_stage.{V3_CATALOG_LOCK_FUNCTION}();") + else: + statements.append(f"lock table {schema_name}.{table_name} in row exclusive mode;") + return "\n".join(statements) + + +def _v3_contract_transaction_guard_sql() -> str: + state_query = build_v3_contract_state_query(into="v_contract_state") + return f"""{_v3_contract_lock_sql()} +do $v3_contract_guard$ +declare + v_contract_state text; +begin +{state_query}; + if v_contract_state <> 'V3' then + raise exception using + errcode = '55000', + message = pg_catalog.format( + 'apply_proposal: V3 catalog contract is %s; refusing mutating transaction', + v_contract_state + ); + end if; +end +$v3_contract_guard$;""" + + +V3_CLAIM_TYPES = {"empirical", "predictive", "causal", "normative", "conceptual"} +V3_ACCESS_SCOPES = {"agent_private", "collective_internal", "public"} +V3_SOURCE_ORIGINS = {"operator_upload", "agent_fetch", "contributor", "system_import", "unknown"} +V3_PROVENANCE_STATES = {"verified", "unverified", "disputed"} +V3_SOURCE_CLASSES = { + "primary_record", + "peer_reviewed", + "institutional", + "journalistic", + "self_published", + "unknown", +} +V3_AUTO_PROMOTION_POLICIES = {"eligible", "human_required", "blocked"} +V3_EVIDENCE_POLARITIES = {"supports", "challenges", "illustrates"} +V3_RELATION_TYPES = {"supports", "challenges", "depends_on", "contradicts"} +V3_SUPPORT_TIERS = {"none", "weak", "moderate", "strong"} +V3_ASSESSMENT_STATES = {"insufficient", "support_dominant", "challenge_dominant", "mixed"} + + # --------------------------------------------------------------------------- # # SQL helpers # # --------------------------------------------------------------------------- # @@ -182,6 +816,16 @@ def _uuid(value: Any, path: str, *, nullable: bool = False) -> str | None: raise ValueError(f"{path} must be a UUID{' or null' if nullable else ''}") from exc +def _v3_uuid(value: Any, path: str, *, nullable: bool = False) -> str | None: + """Require the reviewed V3 spelling to equal PostgreSQL's UUID text form.""" + canonical = _uuid(value, path, nullable=nullable) + if canonical is None: + return None + if not isinstance(value, str) or value != canonical: + raise ValueError(f"{path} must be a canonical lowercase hyphenated UUID") + return canonical + + def _weight(value: Any, path: str) -> Any: if value is None: return None @@ -192,6 +836,41 @@ def _weight(value: Any, path: str) -> Any: return value +def _required_text(value: Any, path: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{path} must be a non-empty string") + return value + + +def _optional_text(value: Any, path: str) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError(f"{path} must be a string or null") + return value + + +def _sha256(value: Any, path: str) -> str: + text = _required_text(value, path) + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{path} must be a lowercase SHA-256 hex digest") + return text + + +def _utc_timestamp(value: Any, path: str) -> str: + try: + return replay_receipt.canonical_utc_timestamp(value, path=path) + except ValueError as exc: + raise ValueError(f"{path} must be a timezone-aware ISO-8601 timestamp") from exc + + +def _json_object(value: Any, path: str, *, nonempty: bool = False) -> dict[str, Any]: + if not isinstance(value, dict) or (nonempty and not value): + qualifier = "non-empty " if nonempty else "" + raise ValueError(f"{path} must be a {qualifier}object") + return value + + def _rows(payload: dict[str, Any], key: str, *, required: bool = False) -> list[dict[str, Any]]: value = payload.get(key) if value is None: @@ -220,6 +899,365 @@ def _reject_unknown(mapping: dict[str, Any], allowed: set[str], path: str) -> No raise ValueError(f"{path} contains unknown fields: {unknown}") +def _v3_evidence_set_hash(claim_id: str, rows: list[dict[str, Any]]) -> str: + evidence = [ + { + "source_id": row["source_id"], + "source_content_hash": row["source_content_hash"], + "polarity": row["polarity"], + "locator": row["locator_json"], + } + for row in rows + if row["claim_id"] == claim_id + ] + evidence.sort(key=lambda row: json.dumps(row, separators=(",", ":"), sort_keys=True)) + payload = json.dumps( + {"claim_id": claim_id, "evidence": evidence}, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _normalize_v3_approve_claim(apply_payload: dict[str, Any]) -> dict[str, Any]: + _reject_unknown( + apply_payload, + { + "contract_version", + "source_proposal_id", + "agent_id", + "claims", + "sources", + "evidence", + "edges", + "assessments", + "reasoning_tools", + "normalization_manifest", + }, + "approve_claim apply_payload", + ) + if apply_payload.get("contract_version") != 3: + raise ValueError("V3 approve_claim apply_payload.contract_version must equal 3") + for key in ("claims", "sources", "evidence", "edges", "assessments", "reasoning_tools"): + if key not in apply_payload: + raise ValueError(f"V3 approve_claim apply_payload requires explicit '{key}' collection") + if apply_payload.get("reasoning_tools") != []: + raise ValueError("V3 approve_claim does not admit reasoning_tools; use a separately reviewed capability") + if apply_payload.get("source_proposal_id") is not None: + _v3_uuid(apply_payload["source_proposal_id"], "approve_claim.source_proposal_id") + manifest = apply_payload.get("normalization_manifest") + if manifest is not None: + _json_object(manifest, "approve_claim.normalization_manifest") + + agent_id = _v3_uuid(apply_payload.get("agent_id"), "approve_claim.agent_id", nullable=True) + raw_claims = _rows(apply_payload, "claims", required=True) + raw_sources = _rows(apply_payload, "sources", required=True) + raw_evidence = _rows(apply_payload, "evidence", required=True) + raw_edges = _rows(apply_payload, "edges") + raw_assessments = _rows(apply_payload, "assessments", required=True) + + claims: list[dict[str, Any]] = [] + for index, row in enumerate(raw_claims): + path = f"approve_claim.claims[{index}]" + _reject_unknown( + row, + { + "id", + "owner_agent_id", + "type", + "proposition", + "body_markdown", + "scope", + "access_scope", + "status", + "revision_criteria", + "revision_kind", + "supersedes_id", + "tags", + "created_by", + }, + path, + ) + claim_type = row.get("type") + if claim_type not in V3_CLAIM_TYPES: + raise ValueError(f"{path}.type must be one of {sorted(V3_CLAIM_TYPES)}") + status = row.get("status", "active") + if status not in {"active", "contested"}: + raise ValueError(f"{path}.status must be active or contested for an original admission") + revision_kind = row.get("revision_kind", "original") + if revision_kind != "original" or row.get("supersedes_id") is not None: + raise ValueError("V3 minimum writer admits original claims only; revision transition is not yet proven") + access_scope = row.get("access_scope") + if access_scope not in V3_ACCESS_SCOPES: + raise ValueError(f"{path}.access_scope must be one of {sorted(V3_ACCESS_SCOPES)}") + owner_agent_id = _v3_uuid(row.get("owner_agent_id"), f"{path}.owner_agent_id", nullable=True) + if access_scope == "agent_private" and owner_agent_id is None: + raise ValueError(f"{path}.owner_agent_id is required for agent_private claims") + created_by = _v3_uuid(row.get("created_by", agent_id), f"{path}.created_by", nullable=True) + if created_by is None: + raise ValueError(f"{path}.created_by or apply_payload.agent_id is required") + tags = row.get("tags") or [] + if not isinstance(tags, list) or any(not isinstance(tag, str) or not tag.strip() for tag in tags): + raise ValueError(f"{path}.tags must be a list of non-empty strings") + claims.append( + { + "id": _v3_uuid(row.get("id"), f"{path}.id"), + "owner_agent_id": owner_agent_id, + "type": claim_type, + "proposition": _required_text(row.get("proposition"), f"{path}.proposition"), + "body_markdown": _required_text(row.get("body_markdown"), f"{path}.body_markdown"), + "scope": _required_text(row.get("scope"), f"{path}.scope"), + "access_scope": access_scope, + "status": status, + "revision_criteria": _required_text(row.get("revision_criteria"), f"{path}.revision_criteria"), + "revision_kind": revision_kind, + "supersedes_id": None, + "tags": tags, + "created_by": created_by, + } + ) + + sources: list[dict[str, Any]] = [] + for index, row in enumerate(raw_sources): + path = f"approve_claim.sources[{index}]" + _reject_unknown( + row, + { + "id", + "source_type", + "canonical_url", + "storage_uri", + "excerpt", + "content_hash", + "access_scope", + "ingestion_origin", + "provenance_status", + "source_class", + "auto_promotion_policy", + "captured_at", + "created_by", + }, + path, + ) + source_type = row.get("source_type") + if source_type not in SOURCE_TYPES: + raise ValueError(f"{path}.source_type must be one of {sorted(SOURCE_TYPES)}") + canonical_url = _optional_text(row.get("canonical_url"), f"{path}.canonical_url") + storage_uri = _optional_text(row.get("storage_uri"), f"{path}.storage_uri") + if not (canonical_url and canonical_url.strip()) and not (storage_uri and storage_uri.strip()): + raise ValueError(f"{path} requires canonical_url or storage_uri") + access_scope = row.get("access_scope") + if access_scope not in V3_ACCESS_SCOPES: + raise ValueError(f"{path}.access_scope must be one of {sorted(V3_ACCESS_SCOPES)}") + ingestion_origin = row.get("ingestion_origin") + if ingestion_origin not in V3_SOURCE_ORIGINS: + raise ValueError(f"{path}.ingestion_origin must be one of {sorted(V3_SOURCE_ORIGINS)}") + provenance_status = row.get("provenance_status") + if provenance_status not in V3_PROVENANCE_STATES: + raise ValueError(f"{path}.provenance_status must be one of {sorted(V3_PROVENANCE_STATES)}") + source_class = row.get("source_class") + if source_class not in V3_SOURCE_CLASSES: + raise ValueError(f"{path}.source_class must be one of {sorted(V3_SOURCE_CLASSES)}") + auto_promotion_policy = row.get("auto_promotion_policy") + if auto_promotion_policy not in V3_AUTO_PROMOTION_POLICIES: + raise ValueError(f"{path}.auto_promotion_policy must be one of {sorted(V3_AUTO_PROMOTION_POLICIES)}") + created_by = _v3_uuid(row.get("created_by", agent_id), f"{path}.created_by", nullable=True) + if created_by is None: + raise ValueError(f"{path}.created_by or apply_payload.agent_id is required") + sources.append( + { + "id": _v3_uuid(row.get("id"), f"{path}.id"), + "source_type": source_type, + "canonical_url": canonical_url, + "storage_uri": storage_uri, + "excerpt": _optional_text(row.get("excerpt"), f"{path}.excerpt"), + "content_hash": _sha256(row.get("content_hash"), f"{path}.content_hash"), + "access_scope": access_scope, + "ingestion_origin": ingestion_origin, + "provenance_status": provenance_status, + "source_class": source_class, + "auto_promotion_policy": auto_promotion_policy, + "captured_at": _utc_timestamp(row.get("captured_at"), f"{path}.captured_at"), + "created_by": created_by, + } + ) + + claim_ids = {row["id"] for row in claims} + source_by_id = {row["id"]: row for row in sources} + evidence: list[dict[str, Any]] = [] + polarity_to_role = {"supports": "grounds", "challenges": "contradicts", "illustrates": "illustrates"} + for index, row in enumerate(raw_evidence): + path = f"approve_claim.evidence[{index}]" + _reject_unknown( + row, + { + "id", + "claim_id", + "source_id", + "polarity", + "excerpt", + "locator_json", + "source_content_hash", + "rationale", + "created_by", + }, + path, + ) + claim_id = _v3_uuid(row.get("claim_id"), f"{path}.claim_id") + source_id = _v3_uuid(row.get("source_id"), f"{path}.source_id") + if claim_id not in claim_ids or source_id not in source_by_id: + raise ValueError(f"{path} must link payload-owned claim and source rows") + polarity = row.get("polarity") + if polarity not in V3_EVIDENCE_POLARITIES: + raise ValueError(f"{path}.polarity must be one of {sorted(V3_EVIDENCE_POLARITIES)}") + source_content_hash = _sha256(row.get("source_content_hash"), f"{path}.source_content_hash") + if source_content_hash != source_by_id[source_id]["content_hash"]: + raise ValueError(f"{path}.source_content_hash does not match the payload source") + created_by = _v3_uuid(row.get("created_by", agent_id), f"{path}.created_by", nullable=True) + if created_by is None: + raise ValueError(f"{path}.created_by or apply_payload.agent_id is required") + evidence.append( + { + "id": _v3_uuid(row.get("id"), f"{path}.id"), + "claim_id": claim_id, + "source_id": source_id, + "role": polarity_to_role[polarity], + "polarity": polarity, + "excerpt": _optional_text(row.get("excerpt"), f"{path}.excerpt"), + "locator_json": _json_object(row.get("locator_json"), f"{path}.locator_json", nonempty=True), + "source_content_hash": source_content_hash, + "rationale": _required_text(row.get("rationale"), f"{path}.rationale"), + "created_by": created_by, + } + ) + + edges: list[dict[str, Any]] = [] + for index, row in enumerate(raw_edges): + path = f"approve_claim.edges[{index}]" + _reject_unknown( + row, + { + "id", + "from_claim", + "to_claim", + "relation_type", + "rationale", + "scope_or_conditions", + "created_by", + }, + path, + ) + from_claim = _v3_uuid(row.get("from_claim"), f"{path}.from_claim") + to_claim = _v3_uuid(row.get("to_claim"), f"{path}.to_claim") + relation_type = row.get("relation_type") + if relation_type not in V3_RELATION_TYPES: + raise ValueError(f"{path}.relation_type must be one of {sorted(V3_RELATION_TYPES)}") + if from_claim == to_claim: + raise ValueError(f"{path} cannot be a self-edge") + if not ({from_claim, to_claim} & claim_ids): + raise ValueError(f"{path} must involve at least one payload-owned claim") + if relation_type == "contradicts" and from_claim > to_claim: + raise ValueError(f"{path} contradicts endpoints must use canonical UUID order") + created_by = _v3_uuid(row.get("created_by", agent_id), f"{path}.created_by", nullable=True) + if created_by is None: + raise ValueError(f"{path}.created_by or apply_payload.agent_id is required") + edges.append( + { + "id": _v3_uuid(row.get("id"), f"{path}.id"), + "from_claim": from_claim, + "to_claim": to_claim, + "relation_type": relation_type, + "edge_type": "requires" if relation_type == "depends_on" else relation_type, + "rationale": _required_text(row.get("rationale"), f"{path}.rationale"), + "scope_or_conditions": _optional_text(row.get("scope_or_conditions"), f"{path}.scope_or_conditions"), + "created_by": created_by, + } + ) + + assessments: list[dict[str, Any]] = [] + claim_type_by_id = {row["id"]: row["type"] for row in claims} + for index, row in enumerate(raw_assessments): + path = f"approve_claim.assessments[{index}]" + _reject_unknown( + row, + { + "id", + "claim_id", + "support_tier", + "challenge_tier", + "overall_state", + "rationale", + "evidence_set_hash", + "supersedes_assessment_id", + }, + path, + ) + claim_id = _v3_uuid(row.get("claim_id"), f"{path}.claim_id") + if claim_id not in claim_ids: + raise ValueError(f"{path}.claim_id must identify a payload-owned claim") + support_tier = row.get("support_tier") + challenge_tier = row.get("challenge_tier") + if support_tier is not None and support_tier not in V3_SUPPORT_TIERS: + raise ValueError(f"{path}.support_tier must be null or one of {sorted(V3_SUPPORT_TIERS)}") + if challenge_tier is not None and challenge_tier not in V3_SUPPORT_TIERS: + raise ValueError(f"{path}.challenge_tier must be null or one of {sorted(V3_SUPPORT_TIERS)}") + if claim_type_by_id[claim_id] in {"empirical", "predictive", "causal"} and ( + support_tier is None or challenge_tier is None + ): + raise ValueError(f"{path} requires support_tier and challenge_tier for a truth-apt claim") + overall_state = row.get("overall_state") + if overall_state not in V3_ASSESSMENT_STATES: + raise ValueError(f"{path}.overall_state must be one of {sorted(V3_ASSESSMENT_STATES)}") + evidence_set_hash = _sha256(row.get("evidence_set_hash"), f"{path}.evidence_set_hash") + if evidence_set_hash != _v3_evidence_set_hash(claim_id, evidence): + raise ValueError(f"{path}.evidence_set_hash does not match the accepted evidence set") + supersedes_assessment_id = _v3_uuid( + row.get("supersedes_assessment_id"), f"{path}.supersedes_assessment_id", nullable=True + ) + if supersedes_assessment_id is not None: + raise ValueError("V3 minimum writer admits first assessments only; assessment supersession is not proven") + assessments.append( + { + "id": _v3_uuid(row.get("id"), f"{path}.id"), + "claim_id": claim_id, + "support_tier": support_tier, + "challenge_tier": challenge_tier, + "overall_state": overall_state, + "rationale": _required_text(row.get("rationale"), f"{path}.rationale"), + "evidence_set_hash": evidence_set_hash, + "supersedes_assessment_id": None, + } + ) + + _dedupe([row["id"] for row in claims], "V3 approve_claim claims ids") + _dedupe([row["id"] for row in sources], "V3 approve_claim sources ids") + _dedupe([row["content_hash"] for row in sources], "V3 approve_claim source hashes") + _dedupe([row["id"] for row in evidence], "V3 approve_claim evidence ids") + _dedupe( + [[row["claim_id"], row["source_id"], row["polarity"]] for row in evidence], + "V3 approve_claim evidence keys", + ) + _dedupe([row["id"] for row in edges], "V3 approve_claim edge ids") + _dedupe( + [[row["from_claim"], row["to_claim"], row["relation_type"]] for row in edges], + "V3 approve_claim edge keys", + ) + _dedupe([row["id"] for row in assessments], "V3 approve_claim assessment ids") + _dedupe([row["claim_id"] for row in assessments], "V3 approve_claim assessment claim ids") + assessed_claims = {row["claim_id"] for row in assessments} + if assessed_claims != claim_ids: + raise ValueError("V3 approve_claim requires exactly one evidence assessment for every admitted claim") + + return { + "agent_id": agent_id, + "claims": claims, + "sources": sources, + "evidence": evidence, + "edges": edges, + "assessments": assessments, + } + + def _validate_approval_meta(approval: dict[str, Any], proposal_type: str, apply_payload: dict[str, Any]) -> None: if not isinstance(approval, dict): raise ValueError("approved proposal metadata is required") @@ -232,6 +1270,7 @@ def _validate_approval_meta(approval: dict[str, Any], proposal_type: str, apply_ raise ValueError("approved proposal requires reviewed_by_handle") if not str(approval.get("reviewed_at") or "").strip(): raise ValueError("approved proposal requires reviewed_at") + _utc_timestamp(approval["reviewed_at"], "approved proposal reviewed_at") if not str(approval.get("review_note") or "").strip(): raise ValueError("approved proposal requires review_note") _uuid( @@ -242,13 +1281,14 @@ def _validate_approval_meta(approval: dict[str, Any], proposal_type: str, apply_ def _approval_guard_sql(proposal_id: str, approval: dict[str, Any]) -> str: + reviewed_at = _utc_timestamp(approval["reviewed_at"], "approved proposal reviewed_at") return f"""select kb_stage.assert_approved_proposal( {sql_literal(proposal_id)}::uuid, {sql_literal(approval["proposal_type"])}, {_jsonb(approval["payload"])}, {sql_literal(approval["reviewed_by_handle"])}, {sql_literal(approval.get("reviewed_by_agent_id"))}::uuid, - {sql_literal(approval["reviewed_at"])}::timestamptz, + {sql_literal(reviewed_at)}::timestamptz, {sql_literal(approval["review_note"])} );""" @@ -260,6 +1300,7 @@ def _ledger_and_verify( approval: dict[str, Any], ) -> str: """Verify canonical rows, then finish the locked, payload-bound apply.""" + reviewed_at = _utc_timestamp(approval["reviewed_at"], "approved proposal reviewed_at") checks_sql = f"""do $verify$ begin {extra_checks} @@ -271,7 +1312,7 @@ $verify$;""" {_jsonb(approval["payload"])}, {sql_literal(approval["reviewed_by_handle"])}, {sql_literal(approval.get("reviewed_by_agent_id"))}::uuid, - {sql_literal(approval["reviewed_at"])}::timestamptz, + {sql_literal(reviewed_at)}::timestamptz, {sql_literal(approval["review_note"])}, {sql_literal(applied_by)} );""" @@ -474,6 +1515,231 @@ on conflict (claim_id, source_id, role) do nothing;""" return _wrap_txn(canonical, ledger, _approval_guard_sql(proposal_id, approval)) +def build_approve_claim_v3_sql( + apply_payload: dict[str, Any], + proposal_id: str, + applied_by: str | None, + approval: dict[str, Any], + *, + fresh_owned: bool = False, + fresh_preflight_sha256: str | None = None, +) -> str: + """Build an exact-row idempotent V3 claim/source/evidence transaction.""" + _validate_approval_meta(approval, "approve_claim", apply_payload) + bundle = _normalize_v3_approve_claim(apply_payload) + if fresh_owned: + _sha256(fresh_preflight_sha256, "fresh-owned approve_claim preflight") + elif fresh_preflight_sha256 is not None: + raise ValueError("fresh preflight SHA-256 is only valid for fresh-owned approve_claim") + + statements: list[str] = [] + checks: list[str] = [] + if fresh_owned: + statements.extend( + [ + f"-- fresh-owned preflight SHA-256: {fresh_preflight_sha256}", + ( + "select pg_advisory_xact_lock(hashtextextended(" + f"'proposal-fresh-apply:' || {sql_literal(proposal_id)}, 0));" + ), + ] + ) + + for row in bundle["sources"]: + statements.append( + f"""do $source_conflict$ +begin + if exists (select 1 from public.sources + where hash = {sql_literal(row["content_hash"])} + and id <> {sql_literal(row["id"])}::uuid) then + raise exception 'V3 approve_claim: source hash collision for source %', {sql_literal(row["id"])}; + end if; +end +$source_conflict$; +insert into public.sources ( + id, source_type, url, storage_path, excerpt, hash, created_by, captured_at, + canonical_url, storage_uri, content_hash, access_scope, ingestion_origin, + provenance_status, source_class, auto_promotion_policy, accepted_by_proposal_id +) values ( + {sql_literal(row["id"])}::uuid, {sql_literal(row["source_type"])}, + {sql_literal(row["canonical_url"])}, {sql_literal(row["storage_uri"])}, + {sql_literal(row["excerpt"])}, {sql_literal(row["content_hash"])}, + {sql_literal(row["created_by"])}::uuid, {sql_literal(row["captured_at"])}::timestamptz, + {sql_literal(row["canonical_url"])}, {sql_literal(row["storage_uri"])}, + {sql_literal(row["content_hash"])}, {sql_literal(row["access_scope"])}, + {sql_literal(row["ingestion_origin"])}, {sql_literal(row["provenance_status"])}, + {sql_literal(row["source_class"])}, {sql_literal(row["auto_promotion_policy"])}, + {sql_literal(proposal_id)}::uuid +) +on conflict do nothing;""" + ) + checks.append( + f""" if not exists (select 1 from public.sources + where id = {sql_literal(row["id"])}::uuid + and source_type = {sql_literal(row["source_type"])} + and url is not distinct from {sql_literal(row["canonical_url"])} + and storage_path is not distinct from {sql_literal(row["storage_uri"])} + and excerpt is not distinct from {sql_literal(row["excerpt"])} + and hash = {sql_literal(row["content_hash"])} + and created_by is not distinct from {sql_literal(row["created_by"])}::uuid + and captured_at = {sql_literal(row["captured_at"])}::timestamptz + and canonical_url is not distinct from {sql_literal(row["canonical_url"])} + and storage_uri is not distinct from {sql_literal(row["storage_uri"])} + and content_hash = {sql_literal(row["content_hash"])} + and access_scope = {sql_literal(row["access_scope"])} + and ingestion_origin = {sql_literal(row["ingestion_origin"])} + and provenance_status = {sql_literal(row["provenance_status"])} + and source_class = {sql_literal(row["source_class"])} + and auto_promotion_policy = {sql_literal(row["auto_promotion_policy"])} + and accepted_by_proposal_id = {sql_literal(proposal_id)}::uuid) then + raise exception 'V3 approve_claim: source row does not match strict payload: %', {sql_literal(row["id"])}; + end if;""" + ) + + for row in bundle["claims"]: + tags = _text_array(row["tags"]) + statements.append( + f"""insert into public.claims ( + id, owner_agent_id, type, text, proposition, body_markdown, scope, access_scope, + status, admission_state, revision_criteria, revision_kind, supersedes_id, + confidence, tags, created_by, superseded_by, accepted_by_proposal_id +) values ( + {sql_literal(row["id"])}::uuid, {sql_literal(row["owner_agent_id"])}::uuid, + {sql_literal(row["type"])}, {sql_literal(row["proposition"])}, {sql_literal(row["proposition"])}, + {sql_literal(row["body_markdown"])}, {sql_literal(row["scope"])}, {sql_literal(row["access_scope"])}, + {sql_literal(row["status"])}, 'admitted', {sql_literal(row["revision_criteria"])}, + {sql_literal(row["revision_kind"])}, null, null, {tags}, + {sql_literal(row["created_by"])}::uuid, null, {sql_literal(proposal_id)}::uuid +) +on conflict do nothing;""" + ) + checks.append( + f""" if not exists (select 1 from public.claims + where id = {sql_literal(row["id"])}::uuid + and owner_agent_id is not distinct from {sql_literal(row["owner_agent_id"])}::uuid + and type = {sql_literal(row["type"])} + and text = {sql_literal(row["proposition"])} + and proposition = {sql_literal(row["proposition"])} + and body_markdown = {sql_literal(row["body_markdown"])} + and scope = {sql_literal(row["scope"])} + and access_scope = {sql_literal(row["access_scope"])} + and status = {sql_literal(row["status"])} + and admission_state = 'admitted' + and revision_criteria = {sql_literal(row["revision_criteria"])} + and revision_kind = 'original' + and supersedes_id is null + and confidence is null + and tags = {tags} + and created_by is not distinct from {sql_literal(row["created_by"])}::uuid + and superseded_by is null + and accepted_by_proposal_id = {sql_literal(proposal_id)}::uuid) then + raise exception 'V3 approve_claim: claim row does not match strict payload: %', {sql_literal(row["id"])}; + end if;""" + ) + + for row in bundle["evidence"]: + statements.append( + f"""insert into public.claim_evidence ( + id, claim_id, source_id, role, weight, created_by, excerpt, locator_json, + source_content_hash, rationale, polarity, accepted_from_proposal_id +) values ( + {sql_literal(row["id"])}::uuid, {sql_literal(row["claim_id"])}::uuid, + {sql_literal(row["source_id"])}::uuid, {sql_literal(row["role"])}::evidence_role, + null, {sql_literal(row["created_by"])}::uuid, {sql_literal(row["excerpt"])}, + {_jsonb(row["locator_json"])}, {sql_literal(row["source_content_hash"])}, + {sql_literal(row["rationale"])}, {sql_literal(row["polarity"])}, {sql_literal(proposal_id)}::uuid +) +on conflict do nothing;""" + ) + checks.append( + f""" if not exists (select 1 from public.claim_evidence + where id = {sql_literal(row["id"])}::uuid + and claim_id = {sql_literal(row["claim_id"])}::uuid + and source_id = {sql_literal(row["source_id"])}::uuid + and role = {sql_literal(row["role"])}::evidence_role + and weight is null + and created_by is not distinct from {sql_literal(row["created_by"])}::uuid + and excerpt is not distinct from {sql_literal(row["excerpt"])} + and locator_json = {_jsonb(row["locator_json"])} + and source_content_hash = {sql_literal(row["source_content_hash"])} + and rationale = {sql_literal(row["rationale"])} + and polarity = {sql_literal(row["polarity"])} + and accepted_from_proposal_id = {sql_literal(proposal_id)}::uuid) then + raise exception 'V3 approve_claim: evidence row does not match strict payload: %', {sql_literal(row["id"])}; + end if;""" + ) + + for row in bundle["edges"]: + statements.append( + f"""insert into public.claim_edges ( + id, from_claim, to_claim, edge_type, weight, created_by, relation_type, + rationale, scope_or_conditions, accepted_by_proposal_id +) values ( + {sql_literal(row["id"])}::uuid, {sql_literal(row["from_claim"])}::uuid, + {sql_literal(row["to_claim"])}::uuid, {sql_literal(row["edge_type"])}::edge_type, + null, {sql_literal(row["created_by"])}::uuid, {sql_literal(row["relation_type"])}, + {sql_literal(row["rationale"])}, {sql_literal(row["scope_or_conditions"])}, + {sql_literal(proposal_id)}::uuid +) +on conflict do nothing;""" + ) + checks.append( + f""" if not exists (select 1 from public.claim_edges + where id = {sql_literal(row["id"])}::uuid + and from_claim = {sql_literal(row["from_claim"])}::uuid + and to_claim = {sql_literal(row["to_claim"])}::uuid + and edge_type = {sql_literal(row["edge_type"])}::edge_type + and weight is null + and created_by is not distinct from {sql_literal(row["created_by"])}::uuid + and relation_type = {sql_literal(row["relation_type"])} + and rationale = {sql_literal(row["rationale"])} + and scope_or_conditions is not distinct from {sql_literal(row["scope_or_conditions"])} + and accepted_by_proposal_id = {sql_literal(proposal_id)}::uuid) then + raise exception 'V3 approve_claim: edge row does not match strict payload: %', {sql_literal(row["id"])}; + end if;""" + ) + + for row in bundle["assessments"]: + statements.append( + f"""insert into public.claim_evidence_assessments ( + id, claim_id, support_tier, challenge_tier, overall_state, rationale, + evidence_set_hash, accepted_by_proposal_id, supersedes_assessment_id +) values ( + {sql_literal(row["id"])}::uuid, {sql_literal(row["claim_id"])}::uuid, + {sql_literal(row["support_tier"])}, {sql_literal(row["challenge_tier"])}, + {sql_literal(row["overall_state"])}, {sql_literal(row["rationale"])}, + {sql_literal(row["evidence_set_hash"])}, {sql_literal(proposal_id)}::uuid, + {sql_literal(row["supersedes_assessment_id"])}::uuid +) +on conflict do nothing;""" + ) + checks.append( + f""" if not exists (select 1 from public.claim_evidence_assessments + where id = {sql_literal(row["id"])}::uuid + and claim_id = {sql_literal(row["claim_id"])}::uuid + and support_tier is not distinct from {sql_literal(row["support_tier"])} + and challenge_tier is not distinct from {sql_literal(row["challenge_tier"])} + and overall_state = {sql_literal(row["overall_state"])} + and rationale = {sql_literal(row["rationale"])} + and evidence_set_hash = {sql_literal(row["evidence_set_hash"])} + and accepted_by_proposal_id = {sql_literal(proposal_id)}::uuid + and supersedes_assessment_id is not distinct from {sql_literal(row["supersedes_assessment_id"])}::uuid) then + raise exception 'V3 approve_claim: assessment row does not match strict payload: %', {sql_literal(row["id"])}; + end if;""" + ) + + canonical = "\n\n".join(statements) + ledger = _ledger_and_verify(proposal_id, applied_by or SERVICE_AGENT_HANDLE, "\n".join(checks), approval) + return _wrap_txn( + canonical, + ledger, + _approval_guard_sql(proposal_id, approval), + serializable=True, + contract_guard_sql=_v3_contract_transaction_guard_sql(), + session_advisory_lock_key=V3_CATALOG_DDL_ADVISORY_LOCK_KEY, + ) + + def build_approve_claim_sql( apply_payload: dict[str, Any], proposal_id: str, @@ -485,6 +1751,15 @@ def build_approve_claim_sql( ) -> str: """Build one conflict-safe transaction for a reviewed canonical graph bundle.""" _validate_approval_meta(approval, "approve_claim", apply_payload) + if apply_payload.get("contract_version") == 3: + return build_approve_claim_v3_sql( + apply_payload, + proposal_id, + applied_by, + approval, + fresh_owned=fresh_owned, + fresh_preflight_sha256=fresh_preflight_sha256, + ) _reject_unknown( apply_payload, { @@ -864,15 +2139,26 @@ def _wrap_txn( approval_guard_sql: str, *, serializable: bool = False, + contract_guard_sql: str = "", + session_advisory_lock_key: int | None = None, ) -> str: + session_lock_sql = "" + session_unlock_sql = "" + if session_advisory_lock_key is not None: + session_lock_sql = f"select pg_catalog.pg_advisory_lock_shared({session_advisory_lock_key});\n" + session_unlock_sql = f"select pg_catalog.pg_advisory_unlock_shared({session_advisory_lock_key});\n" return ( - "begin;\n" + session_lock_sql + + "begin;\n" + ("set transaction isolation level serializable;\n" if serializable else "") + + "set local timezone = 'UTC';\n" + "set local standard_conforming_strings = on;\n" + + (f"{contract_guard_sql}\n" if contract_guard_sql else "") + f"{approval_guard_sql}\n" + f"{canonical_sql}\n" + f"{ledger_sql}\n" + "commit;\n" + + session_unlock_sql ) @@ -884,6 +2170,35 @@ BUILDERS = { } +def proposal_contract_version(proposal: dict[str, Any]) -> int: + """Return the exact typed apply contract or reject malformed/hybrid shapes.""" + proposal_type = proposal.get("proposal_type") + payload = proposal.get("payload") + if not isinstance(payload, dict): + raise ValueError("proposal payload must be a JSON object") + apply_payload = payload.get("apply_payload") + if not isinstance(apply_payload, dict): + raise ValueError("proposal payload.apply_payload must be a JSON object") + + raw_version = apply_payload.get("contract_version") + if proposal_type == "approve_claim": + if type(raw_version) is not int or raw_version not in {2, 3}: + raise ValueError("approve_claim apply_payload.contract_version must be integer 2 or 3") + return raw_version + if "contract_version" in apply_payload: + raise ValueError(f"proposal_type {proposal_type!r} is a V2 legacy contract and must omit contract_version") + return 2 + + +def assert_expected_contract_version(proposal: dict[str, Any], expected: int | None) -> int: + actual = proposal_contract_version(proposal) + if expected is not None and actual != expected: + raise ValueError(f"proposal contract version {actual} does not match expected version {expected}") + if actual == 3 and proposal.get("proposal_type") != "approve_claim": + raise ValueError("V3 apply contract supports only proposal_type 'approve_claim'") + return actual + + def build_apply_sql( proposal: dict[str, Any], applied_by: str | None, @@ -891,10 +2206,11 @@ def build_apply_sql( fresh_owned: bool = False, fresh_preflight_sha256: str | None = None, ) -> str: + assert_expected_contract_version(proposal, None) ptype = proposal["proposal_type"] if ptype not in BUILDERS: raise ValueError(f"apply_proposal cannot apply proposal_type {ptype!r}") - payload = proposal.get("payload") or {} + payload = proposal["payload"] apply_payload = payload.get("apply_payload") if apply_payload is None: raise ValueError( @@ -981,9 +2297,10 @@ def run_psql( "-At", "-q", ] + session_sql = "set timezone = 'UTC';\n" + sql result = subprocess.run( command, - input=sql, + input=session_sql, text=True, capture_output=True, env={"PGPASSWORD": password, "PATH": "/usr/bin:/bin:/usr/local/bin"}, @@ -1004,11 +2321,13 @@ def load_proposal(args: argparse.Namespace, password: str) -> dict[str, Any]: 'payload', payload, 'reviewed_by_handle', reviewed_by_handle, 'reviewed_by_agent_id', reviewed_by_agent_id::text, - 'reviewed_at', reviewed_at::text, + 'reviewed_at', case when reviewed_at is null then null else + to_char(reviewed_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') end, 'review_note', review_note, 'applied_by_handle', applied_by_handle, 'applied_by_agent_id', applied_by_agent_id::text, - 'applied_at', applied_at::text)::text + 'applied_at', case when applied_at is null then null else + to_char(applied_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') end)::text from kb_stage.kb_proposals where id = {sql_literal(args.proposal_id)}::uuid;""" out = run_psql(args, sql, password).strip() @@ -1071,6 +2390,13 @@ def parse_args(argv: list[str]) -> argparse.Namespace: p.add_argument("--db", default=DEFAULT_DB) p.add_argument("--host", default=DEFAULT_HOST) p.add_argument("--role", default=DEFAULT_ROLE) + p.add_argument( + "--expected-contract-version", + type=int, + choices=(2, 3), + default=None, + help="reassert one exact proposal contract before any mutating SQL or receipt recovery", + ) p.add_argument( "--fresh-preflight", type=Path, @@ -1128,6 +2454,7 @@ def main(argv: list[str] | None = None) -> int: if args.receipt_only: password = load_password(args.secrets_file) proposal = load_proposal(args, password) + assert_expected_contract_version(proposal, args.expected_contract_version) assert_receiptable(proposal) path, receipt = capture_replay_receipt(args, proposal, password) print( @@ -1147,12 +2474,14 @@ def main(argv: list[str] | None = None) -> int: # Dry-run still needs the proposal to build SQL; read it as kb_apply. password = load_password(args.secrets_file) proposal = load_proposal(args, password) + assert_expected_contract_version(proposal, args.expected_contract_version) assert_applyable(proposal) print(build_apply_sql_for_args(proposal, args, args.applied_by)) return 0 password = load_password(args.secrets_file) proposal = load_proposal(args, password) + assert_expected_contract_version(proposal, args.expected_contract_version) assert_applyable(proposal) sql = build_apply_sql_for_args(proposal, args, args.applied_by) run_psql(args, sql, password) diff --git a/scripts/apply_worker.py b/scripts/apply_worker.py index 6d3dc13..fc3c060 100644 --- a/scripts/apply_worker.py +++ b/scripts/apply_worker.py @@ -24,6 +24,9 @@ Safety posture * Gated OFF by default. Without ``--enable`` (or ``KB_APPLY_WORKER_ENABLED=1``) the worker only REPORTS what it would apply -- it performs no writes. Flip it on only after the first apply has been proven by hand. +* Pinned to one proposal contract version. V2 remains the default; a V3 worker + must be selected explicitly with ``--contract-version 3`` or + ``KB_APPLY_WORKER_CONTRACT_VERSION=3`` after the V3 schema is installed. * The optional render step is a configurable hook (``--render-cmd`` / ``KB_APPLY_RENDER_CMD``). Only ``revise_strategy`` currently has a proven agent-owned render contract. Other apply types update canonical state without @@ -38,6 +41,7 @@ import os import stat import subprocess import sys +import uuid from pathlib import Path from typing import Any @@ -56,6 +60,9 @@ RENDER_TYPES = ("revise_strategy",) POSTCOMMIT_RECEIPT_FAILURE_MARKER = "committed, but private replay receipt capture failed" POSTCOMMIT_RECEIPT_RECOVERY_MARKER = ". Recover without reapplying via: " +CONTRACT_STATE_V2 = "V2" +CONTRACT_STATE_V3 = "V3" +CONTRACT_STATE_INVALID = "INVALID" class PostCommitReceiptError(RuntimeError): @@ -69,28 +76,71 @@ def build_candidate_query( types: tuple = WORKER_TYPES, limit: int = 20, excluded_ids: tuple = (), + contract_version: int = 2, + proposal_id: str | None = None, ) -> str: - """SELECT approved, applyable proposals that carry an apply_payload. + """SELECT approved proposals for one exact apply contract version. Filters on status='approved' at the DB level -- this is the structural guarantee that the worker cannot act on anything a human has not approved. IDs at the failure ceiling are excluded before LIMIT so exhausted rows cannot occupy every fetched slot and starve later eligible proposals. """ - type_list = ", ".join(ap.sql_literal(t) for t in types) + if contract_version not in {2, 3}: + raise ValueError("worker contract_version must be 2 or 3") + requested_types = tuple(dict.fromkeys(types)) + if contract_version == 3: + eligible_types = tuple(proposal_type for proposal_type in requested_types if proposal_type == "approve_claim") + if not eligible_types: + raise ValueError("V3 worker requires proposal_type 'approve_claim'") + type_contract_clause = ( + "proposal_type = 'approve_claim'\n" + " and pg_catalog.jsonb_typeof(payload->'apply_payload'->'contract_version') = 'number'\n" + " and payload->'apply_payload'->>'contract_version' = '3'" + ) + else: + eligible_types = tuple(proposal_type for proposal_type in requested_types if proposal_type in WORKER_TYPES) + if not eligible_types: + raise ValueError("V2 worker has no supported proposal types") + legacy_types = tuple(proposal_type for proposal_type in eligible_types if proposal_type != "approve_claim") + clauses = [] + if legacy_types: + legacy_type_list = ", ".join(ap.sql_literal(proposal_type) for proposal_type in legacy_types) + clauses.append( + f"(proposal_type in ({legacy_type_list}) and not (payload->'apply_payload' ? 'contract_version'))" + ) + if "approve_claim" in eligible_types: + clauses.append( + "(proposal_type = 'approve_claim' " + "and pg_catalog.jsonb_typeof(payload->'apply_payload'->'contract_version') = 'number' " + "and payload->'apply_payload'->>'contract_version' = '2')" + ) + type_contract_clause = "(\n " + "\n or ".join(clauses) + "\n )" excluded = tuple(str(proposal_id) for proposal_id in excluded_ids) exclusion_clause = "" if excluded: excluded_list = ", ".join(ap.sql_literal(proposal_id) for proposal_id in excluded) exclusion_clause = f"\n and id::text not in ({excluded_list})" + exact_clause = "" + if proposal_id is not None: + try: + canonical_proposal_id = str(uuid.UUID(str(proposal_id))) + except ValueError as exc: + raise ValueError("proposal_id must be a full canonical UUID") from exc + if str(proposal_id) != canonical_proposal_id: + raise ValueError("proposal_id must be a canonical lowercase UUID") + exact_clause = f"\n and id = {ap.sql_literal(canonical_proposal_id)}::uuid" return f"""select jsonb_build_object( 'id', id::text, 'proposal_type', proposal_type, - 'agent_id', payload->'apply_payload'->>'agent_id')::text + 'agent_id', payload->'apply_payload'->>'agent_id', + 'contract_version', coalesce(payload->'apply_payload'->>'contract_version', '2'))::text from kb_stage.kb_proposals where status = 'approved' - and proposal_type in ({type_list}) - and payload ? 'apply_payload'{exclusion_clause} + and pg_catalog.jsonb_typeof(payload) = 'object' + and payload ? 'apply_payload' + and pg_catalog.jsonb_typeof(payload->'apply_payload') = 'object' + and {type_contract_clause}{exact_clause}{exclusion_clause} order by created_at asc limit {int(limit)};""" @@ -238,9 +288,29 @@ def fetch_candidates( password: str, excluded_ids: tuple, ) -> list[dict[str, Any]]: - sql = build_candidate_query(limit=args.limit, excluded_ids=excluded_ids) + sql = build_candidate_query( + limit=args.limit, + excluded_ids=excluded_ids, + contract_version=args.contract_version, + proposal_id=getattr(args, "proposal_id", None), + ) out = ap.run_psql(_psql_args(args), sql, password) - return parse_candidates(out) + candidates = parse_candidates(out) + expected_version = str(args.contract_version) + allowed_types = {"approve_claim"} if args.contract_version == 3 else set(WORKER_TYPES) + for candidate in candidates: + if candidate.get("proposal_type") not in allowed_types or candidate.get("contract_version") != expected_version: + raise RuntimeError("candidate query returned a proposal outside the exact worker contract") + return candidates + + +def detect_installed_contract(args: argparse.Namespace, password: str) -> str: + """Classify the exact database apply contract before candidate selection.""" + sql = ap.build_v3_contract_state_query() + ";" + raw = ap.run_psql(_psql_args(args), sql, password).strip() + if raw not in {CONTRACT_STATE_V2, CONTRACT_STATE_V3, CONTRACT_STATE_INVALID}: + raise RuntimeError("database apply contract readback was not V2, V3, or INVALID") + return raw def apply_one(args: argparse.Namespace, proposal_id: str) -> None: @@ -250,6 +320,8 @@ def apply_one(args: argparse.Namespace, proposal_id: str) -> None: sys.executable, str(args.apply_script), proposal_id, + "--expected-contract-version", + str(args.contract_version), "--applied-by", args.applied_by, "--secrets-file", @@ -267,6 +339,9 @@ def apply_one(args: argparse.Namespace, proposal_id: str) -> None: "--receipt-out", str(receipt_path), ] + fresh_preflight = getattr(args, "fresh_preflight", None) + if fresh_preflight is not None: + cmd.extend(("--fresh-preflight", str(fresh_preflight))) result = subprocess.run(cmd, text=True, capture_output=True, check=False) committed = result.returncode == 0 or has_exact_postcommit_receipt_failure( result.stderr, @@ -327,10 +402,56 @@ def render_one(args: argparse.Namespace, agent_id: str | None) -> str: # --------------------------------------------------------------------------- # def run(args: argparse.Namespace) -> int: password = ap.load_password(args.secrets_file) + installed_contract = detect_installed_contract(args, password) + if installed_contract == CONTRACT_STATE_INVALID: + print( + "REFUSE execution: database apply contract integrity is INVALID; " + "no candidate selection or writes attempted.", + file=sys.stderr, + ) + return 1 + if args.contract_version == 3 and installed_contract == CONTRACT_STATE_V2: + print( + "REFUSE contract v3 execution: the database V3 cutover is not installed; " + "no candidate selection or writes attempted.", + file=sys.stderr, + ) + return 1 + failure_counts = load_failure_state(args.failure_state_file) exhausted_ids = tuple(proposal_id for proposal_id, count in failure_counts.items() if count >= args.max_attempts) - candidates = fetch_candidates(args, password, exhausted_ids) + fetch_exclusions = () if args.contract_version == 2 and installed_contract == CONTRACT_STATE_V3 else exhausted_ids + candidates = fetch_candidates(args, password, fetch_exclusions) + exact_proposal_id = getattr(args, "proposal_id", None) + if exact_proposal_id is not None and (len(candidates) != 1 or candidates[0].get("id") != exact_proposal_id): + if not candidates and getattr(args, "require_candidate", False): + print( + f"required approved+applyable contract v{args.contract_version} proposal " + f"{exact_proposal_id} was not selected; no writes attempted.", + file=sys.stderr, + ) + return 2 + print( + f"REFUSE exact proposal execution: approved+applyable contract v{args.contract_version} " + f"proposal {exact_proposal_id} was not selected exactly once; no writes attempted.", + file=sys.stderr, + ) + return 1 + + if args.contract_version == 2 and installed_contract == CONTRACT_STATE_V3: + print( + "SKIP contract v2 execution: the database V3 cutover is installed; " + f"{len(candidates)} approved V2 proposal(s) in this fetch remain review-visible. " + "Migrate each packet to a newly reviewed V3 proposal or explicitly reject/cancel it; no writes attempted.", + file=sys.stderr, + ) + for candidate in candidates: + print( + f" retained V2 proposal {candidate['id']} ({candidate['proposal_type']})", + file=sys.stderr, + ) + return 0 for proposal_id in exhausted_ids: print( f"EXCLUDE failure-ceiling ID {proposal_id}: failed " @@ -340,7 +461,7 @@ def run(args: argparse.Namespace) -> int: ) if not candidates: - print("no approved+applyable proposals; nothing to do") + print(f"no approved+applyable contract v{args.contract_version} proposals; nothing to do") return 0 split = partition_candidates(candidates, failure_counts, args.max_attempts, args.max_per_tick) @@ -356,7 +477,8 @@ def run(args: argparse.Namespace) -> int: enabled = args.enable or os.environ.get("KB_APPLY_WORKER_ENABLED") == "1" if not enabled: print( - f"[report-only] worker disabled; {len(split['to_apply'])} proposal(s) would apply " + f"[report-only] contract v{args.contract_version} worker disabled; " + f"{len(split['to_apply'])} proposal(s) would apply " f"this tick (cap {args.max_per_tick}):" ) for c in split["to_apply"]: @@ -405,7 +527,23 @@ def parse_args(argv: list[str]) -> argparse.Namespace: action="store_true", help="actually apply (default: report-only). Also honored via KB_APPLY_WORKER_ENABLED=1.", ) + p.add_argument( + "--contract-version", + type=int, + choices=(2, 3), + default=os.environ.get("KB_APPLY_WORKER_CONTRACT_VERSION", "2"), + help="apply only proposals for one exact contract version (default 2)", + ) p.add_argument("--limit", type=int, default=20, help="max candidates fetched per tick") + p.add_argument( + "--proposal-id", + help="select exactly one canonical UUID; refuse instead of falling through to another approved proposal", + ) + p.add_argument( + "--require-candidate", + action="store_true", + help="fail when --proposal-id is not selected instead of treating an empty tick as success", + ) p.add_argument( "--max-per-tick", type=int, @@ -435,6 +573,11 @@ def parse_args(argv: list[str]) -> argparse.Namespace: default=str(HERE / "apply_proposal.py"), help="path to the apply_proposal.py engine (the sole apply path)", ) + p.add_argument( + "--fresh-preflight", + type=Path, + help="exact fresh-owned preflight forwarded to the apply engine; requires --proposal-id", + ) p.add_argument( "--receipt-dir", default=( @@ -456,7 +599,19 @@ def parse_args(argv: list[str]) -> argparse.Namespace: p.add_argument("--db", default=ap.DEFAULT_DB) p.add_argument("--host", default=ap.DEFAULT_HOST) p.add_argument("--role", default=ap.DEFAULT_ROLE) - return p.parse_args(argv) + args = p.parse_args(argv) + if args.proposal_id is not None: + try: + canonical_proposal_id = str(uuid.UUID(args.proposal_id)) + except (AttributeError, TypeError, ValueError) as exc: + p.error(f"--proposal-id must be a UUID: {exc}") + if args.proposal_id != canonical_proposal_id: + p.error("--proposal-id must be a canonical lowercase UUID") + if args.require_candidate and args.proposal_id is None: + p.error("--require-candidate requires --proposal-id") + if args.fresh_preflight is not None and args.proposal_id is None: + p.error("--fresh-preflight requires --proposal-id") + return args def main(argv: list[str] | None = None) -> int: diff --git a/scripts/compile_kb_source_packet.py b/scripts/compile_kb_source_packet.py index 103634c..f497bb7 100644 --- a/scripts/compile_kb_source_packet.py +++ b/scripts/compile_kb_source_packet.py @@ -2,10 +2,10 @@ """Compile one hash-bound source document into a pending-review KB proposal. The command is deliberately build-only. It validates a raw artifact, its UTF-8 -text extraction, and an extraction manifest; then it reuses Leo's rich proposal -normalizer and strict staging preflight to produce a schema-valid -``approve_claim`` child. It never connects to Postgres and never executes the -generated staging SQL. +text extraction, and an extraction manifest. V1/V2 reuse Leo's rich proposal +normalizer; V3 additionally requires the exact extractor specification and +builds the explicit epistemic payload. The command never connects to Postgres +or executes the generated staging SQL. """ from __future__ import annotations @@ -18,6 +18,7 @@ import re import sys import tempfile import uuid +from datetime import datetime, timezone from pathlib import Path from typing import Any from urllib.parse import urlsplit @@ -31,11 +32,14 @@ import stage_normalized_proposal as stage # noqa: E402 MANIFEST_SCHEMA = "livingip.kbSourceExtractionManifest.v1" MANIFEST_SCHEMA_V2 = "livingip.kbSourceExtractionManifest.v2" MANIFEST_SCHEMAS = {MANIFEST_SCHEMA, MANIFEST_SCHEMA_V2} +MANIFEST_SCHEMA_V3 = "livingip.kbSourceExtractionManifest.v3" BUNDLE_SCHEMA = "livingip.kbSourceProposalBundle.v1" +BUNDLE_SCHEMA_V3 = "livingip.kbSourceProposalBundle.v3" STATUS_SCHEMA = "livingip.kbSourceCompilerStatus.v1" SHA256_RE = re.compile(r"^[0-9a-f]{64}$") KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._:-]{0,127}$") URI_SCHEME_RE = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE) +ISO_TIMESTAMP_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$") UNSAFE_URI_SCHEMES = {"data", "javascript", "vbscript"} TOP_LEVEL_KEYS = { @@ -55,6 +59,57 @@ DEDUPE_KEYS = {"database_search_query", "candidate_claims", "duplicates"} DEDUPE_CANDIDATE_KEYS = {"id", "text", "score", "evidence_count", "edge_count"} DEDUPE_DUPLICATE_KEYS = {"claim_id", "source_quote", "reason"} +TOP_LEVEL_KEYS_V3 = { + "schema", + "artifact_sha256", + "extracted_text_sha256", + "extractor", + "agent_id", + "source", + "claims", +} +EXTRACTOR_KEYS_V3 = {"name", "version", "spec_sha256"} +SOURCE_KEYS_V3 = { + "identity", + "source_key", + "source_type", + "title", + "locator", + "access_scope", + "ingestion_origin", + "provenance_status", + "source_class", + "auto_promotion_policy", + "captured_at", +} +CLAIM_KEYS_V3 = { + "claim_key", + "type", + "proposition", + "body_markdown", + "scope", + "access_scope", + "owner_agent_id", + "status", + "revision_criteria", + "revision_kind", + "supersedes_id", + "tags", + "evidence", + "assessment", +} +EVIDENCE_KEYS_V3 = {"evidence_key", "quote", "polarity", "locator", "rationale"} +LOCATOR_KEYS_V3 = {"unit", "start", "end"} +ASSESSMENT_KEYS_V3 = { + "support_tier", + "challenge_tier", + "overall_state", + "rationale", + "supersedes_assessment_id", +} +TRUTH_APT_V3_CLAIM_TYPES = {"empirical", "predictive", "causal"} +ASSESSMENT_RELEVANT_POLARITIES = {"supports", "challenges"} + class CompilerError(RuntimeError): """Raised when an input cannot safely become a reviewable proposal.""" @@ -72,10 +127,37 @@ def canonical_sha256(value: Any) -> str: return sha256_bytes(canonical_json_bytes(value)) +def _load_duplicate_free_json(value: bytes, label: str) -> Any: + try: + text = value.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise CompilerError(f"{label} must be valid UTF-8 JSON: {exc}") from exc + + def reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, item in pairs: + if key in result: + raise CompilerError(f"{label} contains duplicate JSON object key: {key!r}") + result[key] = item + return result + + try: + return json.loads(text, object_pairs_hook=reject_duplicate_pairs) + except json.JSONDecodeError as exc: + raise CompilerError(f"{label} must be valid UTF-8 JSON: {exc}") from exc + + def stable_uuid(label: str) -> str: return str(uuid.uuid5(uuid.NAMESPACE_URL, f"livingip:kb-source-compiler:v1:{label}")) +def stable_uuid_v3(row_kind: str, packet_identity_sha256: str, semantic_key: Any) -> str: + material = canonical_json_bytes( + ["livingip", "kb-source-compiler", 3, packet_identity_sha256, row_kind, semantic_key] + ).decode("ascii") + return str(uuid.uuid5(uuid.NAMESPACE_URL, material)) + + def _require_object(value: Any, label: str) -> dict[str, Any]: if not isinstance(value, dict): raise CompilerError(f"{label} must be an object") @@ -88,6 +170,13 @@ def _require_exact_keys(value: dict[str, Any], allowed: set[str], label: str) -> raise CompilerError(f"{label} contains unsupported keys: {', '.join(unknown)}") +def _require_exact_shape(value: dict[str, Any], required: set[str], label: str) -> None: + _require_exact_keys(value, required, label) + missing = sorted(required - set(value)) + if missing: + raise CompilerError(f"{label} is missing required keys: {', '.join(missing)}") + + def _require_nonempty_string(value: Any, label: str) -> str: if not isinstance(value, str) or not value.strip(): raise CompilerError(f"{label} must be a non-empty string") @@ -110,6 +199,51 @@ def _require_key(value: Any, label: str) -> str: return text +def _require_canonical_uuid(value: Any, label: str, *, nullable: bool = False) -> str | None: + if value is None and nullable: + return None + if not isinstance(value, str): + raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID") + try: + canonical = str(uuid.UUID(value)) + except ValueError as exc: + raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID") from exc + if value != canonical: + raise CompilerError(f"{label} must be a canonical lowercase hyphenated UUID") + return canonical + + +def _require_canonical_utc_timestamp(value: Any, label: str) -> str: + text = _require_nonempty_string(value, label) + if not ISO_TIMESTAMP_RE.fullmatch(text): + raise CompilerError(f"{label} must be a timezone-aware ISO-8601 timestamp with at most 6 fractional digits") + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as exc: + raise CompilerError(f"{label} must be a valid ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise CompilerError(f"{label} must include a timezone") + return parsed.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def _require_nullable_enum(value: Any, allowed: set[str], label: str) -> str | None: + if value is None: + return None + if not isinstance(value, str) or value not in allowed: + raise CompilerError(f"{label} must be null or one of {sorted(allowed)}") + return value + + +def _require_string_list(value: Any, label: str) -> list[str]: + if not isinstance(value, list) or any( + not isinstance(item, str) or not item.strip() or item != item.strip() for item in value + ): + raise CompilerError(f"{label} must be a list of non-empty strings without surrounding whitespace") + if len(value) != len(set(value)): + raise CompilerError(f"{label} contains duplicates") + return sorted(value) + + def _require_stable_reference(value: Any, label: str) -> str: text = _require_nonempty_string(value, label) if any(character.isspace() for character in text) or not URI_SCHEME_RE.match(text): @@ -162,6 +296,258 @@ def _require_nonnegative_integer(value: Any, label: str) -> int: return value +def _v3_quote_binding(text_bytes: bytes, quote: Any, locator_value: Any, label: str) -> dict[str, Any]: + exact = _require_nonempty_string(quote, f"{label}.quote") + locator = _require_object(locator_value, f"{label}.locator") + _require_exact_shape(locator, LOCATOR_KEYS_V3, f"{label}.locator") + if locator.get("unit") != "utf8_bytes": + raise CompilerError(f"{label}.locator.unit must equal 'utf8_bytes'") + start = _require_nonnegative_integer(locator.get("start"), f"{label}.locator.start") + end = _require_nonnegative_integer(locator.get("end"), f"{label}.locator.end") + quote_bytes = exact.encode("utf-8") + if end <= start or end > len(text_bytes) or text_bytes[start:end] != quote_bytes: + raise CompilerError(f"{label}.locator does not bind the exact quote bytes in the extracted text") + return { + "quote": exact, + "quote_sha256": sha256_bytes(quote_bytes), + "unit": "utf8_bytes", + "start": start, + "end": end, + } + + +def _validate_v3_assessment(value: Any, claim_type: str, evidence: list[dict[str, Any]], label: str) -> dict[str, Any]: + assessment = _require_object(value, label) + _require_exact_shape(assessment, ASSESSMENT_KEYS_V3, label) + support_tier = _require_nullable_enum(assessment.get("support_tier"), ap.V3_SUPPORT_TIERS, f"{label}.support_tier") + challenge_tier = _require_nullable_enum( + assessment.get("challenge_tier"), ap.V3_SUPPORT_TIERS, f"{label}.challenge_tier" + ) + if claim_type in TRUTH_APT_V3_CLAIM_TYPES and (support_tier is None or challenge_tier is None): + raise CompilerError(f"{label} requires support_tier and challenge_tier for a truth-apt claim") + overall_state = assessment.get("overall_state") + if overall_state not in ap.V3_ASSESSMENT_STATES: + raise CompilerError(f"{label}.overall_state must be one of {sorted(ap.V3_ASSESSMENT_STATES)}") + + polarities = {row["polarity"] for row in evidence} + if not polarities & ASSESSMENT_RELEVANT_POLARITIES: + raise CompilerError(f"{label} requires at least one supports or challenges evidence row") + if support_tier not in {None, "none"} and "supports" not in polarities: + raise CompilerError(f"{label}.support_tier requires supports evidence") + if challenge_tier not in {None, "none"} and "challenges" not in polarities: + raise CompilerError(f"{label}.challenge_tier requires challenges evidence") + if overall_state == "support_dominant" and "supports" not in polarities: + raise CompilerError(f"{label}.overall_state=support_dominant requires supports evidence") + if overall_state == "challenge_dominant" and "challenges" not in polarities: + raise CompilerError(f"{label}.overall_state=challenge_dominant requires challenges evidence") + if overall_state == "mixed" and not {"supports", "challenges"} <= polarities: + raise CompilerError(f"{label}.overall_state=mixed requires supports and challenges evidence") + if assessment.get("supersedes_assessment_id") is not None: + raise CompilerError(f"{label}.supersedes_assessment_id must be null for a first assessment") + return { + "support_tier": support_tier, + "challenge_tier": challenge_tier, + "overall_state": overall_state, + "rationale": _require_nonempty_string(assessment.get("rationale"), f"{label}.rationale"), + "supersedes_assessment_id": None, + } + + +def _validate_manifest_v3( + manifest: dict[str, Any], + artifact_sha256: str, + text_sha256: str, + extractor_spec_sha256: str, + text_bytes: bytes, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + _require_exact_shape(manifest, TOP_LEVEL_KEYS_V3, "manifest") + if manifest.get("schema") != MANIFEST_SCHEMA_V3: + raise CompilerError(f"manifest.schema must equal {MANIFEST_SCHEMA_V3!r}") + + declared_artifact_sha256 = _require_sha256(manifest.get("artifact_sha256"), "manifest.artifact_sha256") + declared_text_sha256 = _require_sha256(manifest.get("extracted_text_sha256"), "manifest.extracted_text_sha256") + if declared_artifact_sha256 != artifact_sha256: + raise CompilerError("artifact SHA-256 does not match manifest.artifact_sha256") + if declared_text_sha256 != text_sha256: + raise CompilerError("extracted text SHA-256 does not match manifest.extracted_text_sha256") + + extractor = _require_object(manifest.get("extractor"), "manifest.extractor") + _require_exact_shape(extractor, EXTRACTOR_KEYS_V3, "manifest.extractor") + declared_spec_sha256 = _require_sha256(extractor.get("spec_sha256"), "manifest.extractor.spec_sha256") + if declared_spec_sha256 != extractor_spec_sha256: + raise CompilerError("extractor specification SHA-256 does not match manifest.extractor.spec_sha256") + validated_extractor = { + "name": _require_nonempty_string(extractor.get("name"), "manifest.extractor.name"), + "version": _require_nonempty_string(extractor.get("version"), "manifest.extractor.version"), + "spec_sha256": extractor_spec_sha256, + } + agent_id = _require_canonical_uuid(manifest.get("agent_id"), "manifest.agent_id") + + source = _require_object(manifest.get("source"), "manifest.source") + _require_exact_shape(source, SOURCE_KEYS_V3, "manifest.source") + source_type = source.get("source_type") + if source_type not in ap.SOURCE_TYPES: + raise CompilerError(f"manifest.source.source_type must be one of {sorted(ap.SOURCE_TYPES)}") + source_access_scope = source.get("access_scope") + if source_access_scope not in ap.V3_ACCESS_SCOPES: + raise CompilerError(f"manifest.source.access_scope must be one of {sorted(ap.V3_ACCESS_SCOPES)}") + ingestion_origin = source.get("ingestion_origin") + if ingestion_origin not in ap.V3_SOURCE_ORIGINS: + raise CompilerError(f"manifest.source.ingestion_origin must be one of {sorted(ap.V3_SOURCE_ORIGINS)}") + provenance_status = source.get("provenance_status") + if provenance_status not in ap.V3_PROVENANCE_STATES: + raise CompilerError(f"manifest.source.provenance_status must be one of {sorted(ap.V3_PROVENANCE_STATES)}") + source_class = source.get("source_class") + if source_class not in ap.V3_SOURCE_CLASSES: + raise CompilerError(f"manifest.source.source_class must be one of {sorted(ap.V3_SOURCE_CLASSES)}") + auto_promotion_policy = source.get("auto_promotion_policy") + if auto_promotion_policy not in ap.V3_AUTO_PROMOTION_POLICIES: + raise CompilerError( + f"manifest.source.auto_promotion_policy must be one of {sorted(ap.V3_AUTO_PROMOTION_POLICIES)}" + ) + source_identity = _require_stable_reference(source.get("identity"), "manifest.source.identity") + source_locator = _require_stable_reference(source.get("locator"), "manifest.source.locator") + expected_source_identity = f"document:sha256:{artifact_sha256}" + expected_source_locator = f"artifact://sha256/{artifact_sha256}" + if source_identity != expected_source_identity: + raise CompilerError( + "manifest.source.identity must equal the content-addressed identity for the actual artifact bytes" + ) + if source_locator != expected_source_locator: + raise CompilerError("manifest.source.locator must equal artifact://sha256/") + validated_source = { + "identity": source_identity, + "source_key": _require_key(source.get("source_key"), "manifest.source.source_key"), + "source_type": source_type, + "title": _require_nonempty_string(source.get("title"), "manifest.source.title"), + "locator": source_locator, + "access_scope": source_access_scope, + "ingestion_origin": ingestion_origin, + "provenance_status": provenance_status, + "source_class": source_class, + "auto_promotion_policy": auto_promotion_policy, + "captured_at": _require_canonical_utc_timestamp(source.get("captured_at"), "manifest.source.captured_at"), + } + + raw_claims = manifest.get("claims") + if not isinstance(raw_claims, list) or not raw_claims: + raise CompilerError("manifest.claims must be a non-empty list of objects") + if len(raw_claims) > ap.MAX_BUNDLE_ROWS["claims"]: + raise CompilerError(f"manifest.claims exceeds {ap.MAX_BUNDLE_ROWS['claims']} rows") + + validated_claims: list[dict[str, Any]] = [] + quote_bindings: list[dict[str, Any]] = [] + seen_claim_keys: set[str] = set() + evidence_count = 0 + for claim_index, raw_claim in enumerate(raw_claims): + label = f"manifest.claims[{claim_index}]" + claim = _require_object(raw_claim, label) + _require_exact_shape(claim, CLAIM_KEYS_V3, label) + claim_key = _require_key(claim.get("claim_key"), f"{label}.claim_key") + if claim_key in seen_claim_keys: + raise CompilerError(f"duplicate claim_key: {claim_key}") + seen_claim_keys.add(claim_key) + claim_type = claim.get("type") + if claim_type not in ap.V3_CLAIM_TYPES: + raise CompilerError(f"{label}.type must be one of {sorted(ap.V3_CLAIM_TYPES)}") + access_scope = claim.get("access_scope") + if access_scope not in ap.V3_ACCESS_SCOPES: + raise CompilerError(f"{label}.access_scope must be one of {sorted(ap.V3_ACCESS_SCOPES)}") + owner_agent_id = _require_canonical_uuid(claim.get("owner_agent_id"), f"{label}.owner_agent_id", nullable=True) + if access_scope == "agent_private" and owner_agent_id is None: + raise CompilerError(f"{label}.owner_agent_id is required for agent_private claims") + status = claim.get("status") + if status not in {"active", "contested"}: + raise CompilerError(f"{label}.status must be active or contested") + if claim.get("revision_kind") != "original" or claim.get("supersedes_id") is not None: + raise CompilerError(f"{label} must describe an original claim with supersedes_id=null") + + raw_evidence = claim.get("evidence") + if not isinstance(raw_evidence, list) or not raw_evidence: + raise CompilerError(f"{label}.evidence must be a non-empty list of exact locator-bound rows") + evidence_count += len(raw_evidence) + if evidence_count > ap.MAX_BUNDLE_ROWS["evidence"]: + raise CompilerError(f"manifest evidence exceeds {ap.MAX_BUNDLE_ROWS['evidence']} rows") + validated_evidence: list[dict[str, Any]] = [] + seen_evidence_keys: set[str] = set() + seen_polarities: set[str] = set() + for evidence_index, raw_row in enumerate(raw_evidence): + evidence_label = f"{label}.evidence[{evidence_index}]" + row = _require_object(raw_row, evidence_label) + _require_exact_shape(row, EVIDENCE_KEYS_V3, evidence_label) + evidence_key = _require_key(row.get("evidence_key"), f"{evidence_label}.evidence_key") + if evidence_key in seen_evidence_keys: + raise CompilerError(f"{evidence_label}.evidence_key duplicates an earlier evidence row") + seen_evidence_keys.add(evidence_key) + polarity = row.get("polarity") + if polarity not in ap.V3_EVIDENCE_POLARITIES: + raise CompilerError(f"{evidence_label}.polarity must be one of {sorted(ap.V3_EVIDENCE_POLARITIES)}") + if polarity in seen_polarities: + raise CompilerError(f"{evidence_label}.polarity duplicates a claim-local source/polarity key") + seen_polarities.add(polarity) + binding = _v3_quote_binding(text_bytes, row.get("quote"), row.get("locator"), evidence_label) + validated_evidence.append( + { + "evidence_key": evidence_key, + "quote": binding["quote"], + "polarity": polarity, + "locator": { + "unit": binding["unit"], + "start": binding["start"], + "end": binding["end"], + }, + "rationale": _require_nonempty_string(row.get("rationale"), f"{evidence_label}.rationale"), + } + ) + quote_bindings.append( + { + "claim_key": claim_key, + "evidence_key": evidence_key, + "kind": "evidence", + **binding, + } + ) + validated_evidence.sort(key=lambda row: row["evidence_key"]) + + validated_claims.append( + { + "claim_key": claim_key, + "type": claim_type, + "proposition": _require_nonempty_string(claim.get("proposition"), f"{label}.proposition"), + "body_markdown": _require_nonempty_string(claim.get("body_markdown"), f"{label}.body_markdown"), + "scope": _require_nonempty_string(claim.get("scope"), f"{label}.scope"), + "access_scope": access_scope, + "owner_agent_id": owner_agent_id, + "status": status, + "revision_criteria": _require_nonempty_string( + claim.get("revision_criteria"), f"{label}.revision_criteria" + ), + "revision_kind": "original", + "supersedes_id": None, + "tags": _require_string_list(claim.get("tags"), f"{label}.tags"), + "evidence": validated_evidence, + "assessment": _validate_v3_assessment( + claim.get("assessment"), claim_type, validated_evidence, f"{label}.assessment" + ), + } + ) + + validated_claims.sort(key=lambda row: row["claim_key"]) + quote_bindings.sort(key=lambda row: (row["claim_key"], row["evidence_key"])) + return ( + { + "schema": MANIFEST_SCHEMA_V3, + "artifact_sha256": artifact_sha256, + "extracted_text_sha256": text_sha256, + "extractor": validated_extractor, + "agent_id": agent_id, + "source": validated_source, + "claims": validated_claims, + }, + quote_bindings, + ) + + def _validate_dedupe(value: Any, extracted_text: str) -> dict[str, Any]: dedupe = _require_object(value, "manifest.dedupe") _require_exact_keys(dedupe, DEDUPE_KEYS, "manifest.dedupe") @@ -460,6 +846,408 @@ def build_parent_proposal( } +def _v3_identity_material(manifest: dict[str, Any], manifest_file_sha256: str) -> tuple[dict[str, Any], str, str]: + candidate_content = { + "agent_id": manifest["agent_id"], + "source": manifest["source"], + "claims": manifest["claims"], + } + candidate_content_sha256 = canonical_sha256(candidate_content) + identity_material = { + "artifact_sha256": manifest["artifact_sha256"], + "extracted_text_sha256": manifest["extracted_text_sha256"], + "extractor": manifest["extractor"], + "manifest_file_sha256": manifest_file_sha256, + "source_identity": manifest["source"]["identity"], + "source_locator": manifest["source"]["locator"], + "normalized_candidate_content_sha256": candidate_content_sha256, + } + return identity_material, candidate_content_sha256, canonical_sha256(identity_material) + + +def _build_v3_parent_proposal( + manifest: dict[str, Any], + manifest_canonical_sha256: str, + identity_material: dict[str, Any], + candidate_content_sha256: str, + packet_identity_sha256: str, +) -> dict[str, Any]: + source = manifest["source"] + parent_id = stable_uuid_v3("source_proposal", packet_identity_sha256, candidate_content_sha256) + return { + "id": parent_id, + "proposal_type": "approve_claim", + "status": "pending_review", + "proposed_by_handle": "leo", + "proposed_by_agent_id": manifest["agent_id"], + "channel": "source_document_compiler_v3", + "source_ref": f"source-compiler-v3:{packet_identity_sha256[:32]}", + "rationale": "Compile one retained semantic extraction into a hash-bound V3 claim proposal for review.", + "payload": { + "source_document": { + "manifest_schema": MANIFEST_SCHEMA_V3, + "manifest_file_sha256": identity_material["manifest_file_sha256"], + "manifest_canonical_sha256": manifest_canonical_sha256, + "artifact_sha256": manifest["artifact_sha256"], + "extracted_text_sha256": manifest["extracted_text_sha256"], + "source_identity": source["identity"], + "source_locator": source["locator"], + "extractor": manifest["extractor"], + }, + "identity_material": identity_material, + "normalized_candidate_content_sha256": candidate_content_sha256, + "packet_identity_sha256": packet_identity_sha256, + "semantic_extraction_is_retained_input": True, + "model_training_claim": False, + }, + } + + +def _v3_source_excerpt(manifest: dict[str, Any]) -> str: + excerpts: list[str] = [] + seen: set[str] = set() + for claim in manifest["claims"]: + for evidence in claim["evidence"]: + quote = evidence["quote"] + if quote not in seen: + seen.add(quote) + excerpts.append(quote) + return "\n\n".join(excerpts) + + +def _v3_locator_json(manifest: dict[str, Any], binding: dict[str, Any]) -> dict[str, Any]: + return { + "schema": "livingip.extractedTextQuoteLocator.v1", + "source_identity": manifest["source"]["identity"], + "source_locator": manifest["source"]["locator"], + "artifact_sha256": manifest["artifact_sha256"], + "extracted_text_sha256": manifest["extracted_text_sha256"], + "extractor": manifest["extractor"], + "unit": binding["unit"], + "start": binding["start"], + "end": binding["end"], + "quote_sha256": binding["quote_sha256"], + } + + +def _ensure_distinct_v3_ids(values: list[str]) -> None: + if len(values) != len(set(values)): + raise CompilerError("compiled V3 packet contains duplicate deterministic ids") + + +def _build_v3_strict_child( + manifest: dict[str, Any], + manifest_canonical_sha256: str, + quote_bindings: list[dict[str, Any]], + parent: dict[str, Any], + identity_material: dict[str, Any], + candidate_content_sha256: str, + packet_identity_sha256: str, +) -> dict[str, Any]: + source = manifest["source"] + source_id = stable_uuid_v3("source", packet_identity_sha256, source["source_key"]) + claim_ids = { + claim["claim_key"]: stable_uuid_v3("claim", packet_identity_sha256, claim["claim_key"]) + for claim in manifest["claims"] + } + binding_by_key = {(binding["claim_key"], binding["evidence_key"]): binding for binding in quote_bindings} + canonical_url, storage_uri = _locator_fields(source["locator"]) + source_row = { + "id": source_id, + "source_type": source["source_type"], + "canonical_url": canonical_url, + "storage_uri": storage_uri, + "excerpt": _v3_source_excerpt(manifest), + "content_hash": manifest["artifact_sha256"], + "access_scope": source["access_scope"], + "ingestion_origin": source["ingestion_origin"], + "provenance_status": source["provenance_status"], + "source_class": source["source_class"], + "auto_promotion_policy": source["auto_promotion_policy"], + "captured_at": source["captured_at"], + "created_by": manifest["agent_id"], + } + + claim_rows: list[dict[str, Any]] = [] + evidence_rows: list[dict[str, Any]] = [] + assessment_rows: list[dict[str, Any]] = [] + for claim in manifest["claims"]: + claim_id = claim_ids[claim["claim_key"]] + claim_rows.append( + { + "id": claim_id, + "owner_agent_id": claim["owner_agent_id"], + "type": claim["type"], + "proposition": claim["proposition"], + "body_markdown": claim["body_markdown"], + "scope": claim["scope"], + "access_scope": claim["access_scope"], + "status": claim["status"], + "revision_criteria": claim["revision_criteria"], + "revision_kind": "original", + "supersedes_id": None, + "tags": claim["tags"], + "created_by": manifest["agent_id"], + } + ) + for evidence in claim["evidence"]: + binding = binding_by_key[(claim["claim_key"], evidence["evidence_key"])] + evidence_rows.append( + { + "id": stable_uuid_v3( + "evidence", + packet_identity_sha256, + [claim["claim_key"], evidence["evidence_key"]], + ), + "claim_id": claim_id, + "source_id": source_id, + "polarity": evidence["polarity"], + "excerpt": evidence["quote"], + "locator_json": _v3_locator_json(manifest, binding), + "source_content_hash": manifest["artifact_sha256"], + "rationale": evidence["rationale"], + "created_by": manifest["agent_id"], + } + ) + + for claim in manifest["claims"]: + claim_id = claim_ids[claim["claim_key"]] + assessment = claim["assessment"] + evidence_set_hash = ap._v3_evidence_set_hash(claim_id, evidence_rows) + assessment_rows.append( + { + "id": stable_uuid_v3("assessment", packet_identity_sha256, claim["claim_key"]), + "claim_id": claim_id, + "support_tier": assessment["support_tier"], + "challenge_tier": assessment["challenge_tier"], + "overall_state": assessment["overall_state"], + "rationale": assessment["rationale"], + "evidence_set_hash": evidence_set_hash, + "supersedes_assessment_id": None, + } + ) + + parent_packet_sha256 = canonical_sha256(parent) + apply_payload = { + "contract_version": 3, + "source_proposal_id": parent["id"], + "agent_id": manifest["agent_id"], + "claims": claim_rows, + "sources": [source_row], + "evidence": evidence_rows, + "edges": [], + "assessments": assessment_rows, + "reasoning_tools": [], + "normalization_manifest": { + "schema": "livingip.kbSourceProposalNormalization.v3", + "semantic_extraction_statement": ( + "The semantic extraction is a retained input; this is a reproducible learning input, " + "not a model-training claim." + ), + "semantic_extraction_is_retained_input": True, + "model_training_claim": False, + "manifest_schema": MANIFEST_SCHEMA_V3, + "manifest_file_sha256": identity_material["manifest_file_sha256"], + "manifest_identity_binding": "sha256(exact duplicate-free UTF-8 JSON file bytes)", + "manifest_canonical_sha256": manifest_canonical_sha256, + "artifact_sha256": manifest["artifact_sha256"], + "extracted_text_sha256": manifest["extracted_text_sha256"], + "extractor": manifest["extractor"], + "source_identity": source["identity"], + "source_locator": source["locator"], + "normalized_candidate_content_sha256": candidate_content_sha256, + "identity_material": identity_material, + "packet_identity_sha256": packet_identity_sha256, + "parent_packet_sha256": parent_packet_sha256, + "quote_bindings": quote_bindings, + }, + } + payload = {"apply_payload": apply_payload} + payload_sha256 = canonical_sha256(payload) + child_id = stable_uuid_v3("pending_review_proposal", packet_identity_sha256, payload_sha256) + _ensure_distinct_v3_ids( + [ + parent["id"], + child_id, + source_id, + *[row["id"] for row in claim_rows], + *[row["id"] for row in evidence_rows], + *[row["id"] for row in assessment_rows], + ] + ) + return { + "id": child_id, + "proposal_type": "approve_claim", + "status": "pending_review", + "proposed_by_handle": parent["proposed_by_handle"], + "proposed_by_agent_id": manifest["agent_id"], + "channel": parent["channel"], + "source_ref": f"source-compiler-v3:{packet_identity_sha256[:32]}:{payload_sha256[:16]}", + "rationale": f"Strict V3 source proposal compiled from retained packet {parent['id']}.", + "payload": payload, + "parent_packet_sha256": parent_packet_sha256, + "payload_sha256": payload_sha256, + } + + +def _validate_v3_compiled_child( + child: dict[str, Any], manifest: dict[str, Any], quote_bindings: list[dict[str, Any]] +) -> None: + if child.get("status") != "pending_review" or child.get("proposal_type") != "approve_claim": + raise CompilerError("V3 compilation did not produce one pending_review approve_claim proposal") + payload = child.get("payload") + apply_payload = payload.get("apply_payload") if isinstance(payload, dict) else None + if not isinstance(apply_payload, dict) or apply_payload.get("contract_version") != 3: + raise CompilerError("strict child has no schema-valid approve_claim V3 apply_payload") + if apply_payload.get("reasoning_tools") != [] or apply_payload.get("edges") != []: + raise CompilerError("V3 source compilation admits neither reasoning_tools nor inferred edges") + if child.get("payload_sha256") != canonical_sha256(payload): + raise CompilerError("strict child payload_sha256 does not bind the exact V3 payload") + + claims = apply_payload.get("claims") or [] + sources = apply_payload.get("sources") or [] + evidence = apply_payload.get("evidence") or [] + assessments = apply_payload.get("assessments") or [] + if len(claims) != len(manifest["claims"]) or len(sources) != 1: + raise CompilerError("V3 strict child row counts differ from the extraction manifest") + if len(evidence) != sum(len(claim["evidence"]) for claim in manifest["claims"]): + raise CompilerError("V3 strict child evidence count differs from the extraction manifest") + if len(assessments) != len(claims): + raise CompilerError("V3 strict child requires exactly one assessment per claim") + _ensure_distinct_v3_ids( + [ + child["id"], + *[row.get("id") for row in sources], + *[row.get("id") for row in claims], + *[row.get("id") for row in evidence], + *[row.get("id") for row in assessments], + ] + ) + + source_row = sources[0] + if source_row.get("content_hash") != manifest["artifact_sha256"]: + raise CompilerError("V3 source content_hash does not bind the original artifact") + expected_url, expected_storage_uri = _locator_fields(manifest["source"]["locator"]) + if source_row.get("canonical_url") != expected_url or source_row.get("storage_uri") != expected_storage_uri: + raise CompilerError("V3 source row does not retain the validated content-addressed locator") + if source_row.get("captured_at") != _require_canonical_utc_timestamp( + source_row.get("captured_at"), "V3 source captured_at" + ): + raise CompilerError("V3 source captured_at is not in writer-canonical fixed-microsecond UTC Z form") + binding_by_locator = {canonical_sha256(_v3_locator_json(manifest, binding)): binding for binding in quote_bindings} + for row in evidence: + binding = binding_by_locator.get(canonical_sha256(row.get("locator_json"))) + if binding is None or row.get("excerpt") != binding["quote"]: + raise CompilerError("V3 evidence does not retain an exact locator-bound quote") + if row.get("source_content_hash") != manifest["artifact_sha256"]: + raise CompilerError("V3 evidence source_content_hash does not bind the original artifact") + assessment_by_claim = {row.get("claim_id"): row for row in assessments} + if len(assessment_by_claim) != len(claims): + raise CompilerError("V3 strict child contains duplicate assessment claim ids") + for claim in claims: + assessment = assessment_by_claim.get(claim.get("id")) + if assessment is None: + raise CompilerError("V3 strict child is missing a claim-local evidence assessment") + if assessment.get("evidence_set_hash") != ap._v3_evidence_set_hash(claim["id"], evidence): + raise CompilerError("V3 assessment evidence_set_hash does not match claim-local evidence") + + +def _compile_v3_source_packet( + artifact_bytes: bytes, + text_bytes: bytes, + manifest_bytes: bytes, + extractor_spec_bytes: bytes, + manifest_object: dict[str, Any], +) -> dict[str, Any]: + artifact_sha256 = sha256_bytes(artifact_bytes) + text_sha256 = sha256_bytes(text_bytes) + extractor_spec_sha256 = sha256_bytes(extractor_spec_bytes) + manifest_file_sha256 = sha256_bytes(manifest_bytes) + manifest, quote_bindings = _validate_manifest_v3( + manifest_object, + artifact_sha256, + text_sha256, + extractor_spec_sha256, + text_bytes, + ) + manifest_canonical_sha256 = canonical_sha256(manifest) + identity_material, candidate_content_sha256, packet_identity_sha256 = _v3_identity_material( + manifest, manifest_file_sha256 + ) + parent = _build_v3_parent_proposal( + manifest, + manifest_canonical_sha256, + identity_material, + candidate_content_sha256, + packet_identity_sha256, + ) + child = _build_v3_strict_child( + manifest, + manifest_canonical_sha256, + quote_bindings, + parent, + identity_material, + candidate_content_sha256, + packet_identity_sha256, + ) + _validate_v3_compiled_child(child, manifest, quote_bindings) + stage_sql = stage.build_stage_sql(child) + stable_bundle = { + "schema": BUNDLE_SCHEMA_V3, + "status": "pending_review", + "build_only": True, + "database_write_performed": False, + "canonical_apply_performed": False, + "inputs": { + "artifact_bytes": len(artifact_bytes), + "extracted_text_bytes": len(text_bytes), + "manifest_bytes": len(manifest_bytes), + "extractor_spec_bytes": len(extractor_spec_bytes), + }, + "validated_manifest": manifest, + "quote_bindings": quote_bindings, + "parent_proposal": parent, + "strict_child_proposal": child, + "stage_preview": { + "target_table": "kb_stage.kb_proposals", + "target_status": "pending_review", + "executed": False, + "stage_sql_sha256": sha256_bytes(stage_sql.encode("utf-8")), + }, + "compiler_contract": { + "manifest_contract": MANIFEST_SCHEMA_V3, + "output_contract": BUNDLE_SCHEMA_V3, + "approve_claim_contract_version": 3, + "strict_preflight": "scripts/compile_kb_source_packet.py::_validate_v3_compiled_child", + "writer_apply_invoked": False, + "writer_timestamp_contract": "fixed-microsecond UTC Z", + "manifest_identity_binding": "sha256(exact duplicate-free UTF-8 JSON file bytes)", + "extractor_spec_binding": "sha256(exact independently supplied artifact bytes)", + "semantic_extraction_is_retained_input": True, + "semantic_extraction_statement": ( + "The semantic extraction is a retained input; this is the smallest reproducible learning input, " + "not a model-training claim." + ), + "model_training_claim": False, + "source_from_chat_labels": False, + "database_stage_executed": False, + }, + "hashes": { + "artifact_sha256": artifact_sha256, + "extracted_text_sha256": text_sha256, + "extractor_spec_sha256": extractor_spec_sha256, + "manifest_file_sha256": manifest_file_sha256, + "manifest_canonical_sha256": manifest_canonical_sha256, + "normalized_candidate_content_sha256": candidate_content_sha256, + "packet_identity_sha256": packet_identity_sha256, + "parent_packet_sha256": child["parent_packet_sha256"], + "strict_child_payload_sha256": child["payload_sha256"], + }, + } + stable_bundle["hashes"]["bundle_content_sha256"] = canonical_sha256(stable_bundle) + return stable_bundle + + def _validate_compiled_child( child: dict[str, Any], manifest: dict[str, Any], quote_bindings: list[dict[str, Any]] ) -> None: @@ -503,7 +1291,12 @@ def _validate_compiled_child( raise CompilerError("one or more exact quotes exceed the current canonical source excerpt capacity") -def compile_source_packet(artifact_path: Path, text_path: Path, manifest_path: Path) -> dict[str, Any]: +def compile_source_packet( + artifact_path: Path, + text_path: Path, + manifest_path: Path, + extractor_spec_path: Path | None = None, +) -> dict[str, Any]: artifact_bytes = _read_required_file(artifact_path, "artifact") text_bytes = _read_required_file(text_path, "extracted text") manifest_bytes = _read_required_file(manifest_path, "extraction manifest") @@ -511,14 +1304,24 @@ def compile_source_packet(artifact_path: Path, text_path: Path, manifest_path: P extracted_text = text_bytes.decode("utf-8", errors="strict") except UnicodeDecodeError as exc: raise CompilerError(f"extracted text must be valid UTF-8: {exc}") from exc - try: - manifest_raw = json.loads(manifest_bytes.decode("utf-8", errors="strict")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise CompilerError(f"extraction manifest must be valid UTF-8 JSON: {exc}") from exc + manifest_raw = _load_duplicate_free_json(manifest_bytes, "extraction manifest") manifest_object = _require_object(manifest_raw, "manifest") artifact_sha256 = sha256_bytes(artifact_bytes) text_sha256 = sha256_bytes(text_bytes) + if manifest_object.get("schema") == MANIFEST_SCHEMA_V3: + if extractor_spec_path is None: + raise CompilerError("V3 requires an independently supplied extractor specification artifact") + extractor_spec_bytes = _read_required_file(extractor_spec_path, "extractor specification") + return _compile_v3_source_packet( + artifact_bytes, + text_bytes, + manifest_bytes, + extractor_spec_bytes, + manifest_object, + ) + if extractor_spec_path is not None: + raise CompilerError("extractor specification input is only valid for a V3 manifest") manifest, quote_bindings = _validate_manifest(manifest_object, artifact_sha256, text_sha256, extracted_text) manifest_canonical_sha256 = canonical_sha256(manifest) parent = build_parent_proposal(manifest, manifest_canonical_sha256, quote_bindings) @@ -573,6 +1376,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--artifact", required=True, type=Path, help="raw source artifact; any binary format") parser.add_argument("--text", required=True, type=Path, help="strict UTF-8 extraction of the artifact") parser.add_argument("--manifest", required=True, type=Path, help="extraction manifest JSON") + parser.add_argument( + "--extractor-spec", + type=Path, + help="exact extractor specification artifact; required for V3 and rejected for V1/V2", + ) parser.add_argument("--output", required=True, type=Path, help="private path for the compiled proposal bundle") return parser.parse_args(argv) @@ -612,9 +1420,11 @@ def main(argv: list[str] | None = None) -> int: try: output_path = args.output.resolve() input_paths = {args.artifact.resolve(), args.text.resolve(), args.manifest.resolve()} + if args.extractor_spec is not None: + input_paths.add(args.extractor_spec.resolve()) if output_path in input_paths: raise CompilerError("--output must not overwrite an input file") - bundle = compile_source_packet(args.artifact, args.text, args.manifest) + bundle = compile_source_packet(args.artifact, args.text, args.manifest, args.extractor_spec) except (CompilerError, OSError, ValueError): _print_status("rejected", reason="input_validation_failed") return 2 diff --git a/scripts/export_kb_transition_replay_material.py b/scripts/export_kb_transition_replay_material.py new file mode 100644 index 0000000..1fd95a8 --- /dev/null +++ b/scripts/export_kb_transition_replay_material.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Export exact private replay material for one guarded applied proposal. + +The exporter is intentionally read-only. It validates an existing worker replay +receipt, reads the approved and applied proposal rows through one gate-owned +function, and atomically publishes deterministic genesis-ledger input. +""" + +from __future__ import annotations + +import argparse +import json +import os +import stat +import sys +import tempfile +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import apply_proposal as apply_engine # noqa: E402 +import kb_apply_replay_receipt as replay_receipt # noqa: E402 + +MATERIAL_ARTIFACT = "teleo_strict_proposal_replay_material" +MATERIAL_CONTRACT_VERSION = 1 +MATERIAL_FIELDS = frozenset( + { + "artifact", + "contract_version", + "sequence", + "approved_proposal", + "approval_snapshot", + "applied_proposal", + "replay_receipt", + } +) +PROPOSAL_FIELDS = frozenset( + { + "id", + "proposal_type", + "status", + "proposed_by_handle", + "proposed_by_agent_id", + "channel", + "source_ref", + "rationale", + "payload", + "reviewed_by_handle", + "reviewed_by_agent_id", + "reviewed_at", + "review_note", + "applied_by_handle", + "applied_by_agent_id", + "applied_at", + "created_at", + "updated_at", + } +) +APPROVAL_FIELDS = frozenset( + { + "proposal_id", + "proposal_type", + "payload", + "reviewed_by_handle", + "reviewed_by_agent_id", + "reviewed_by_db_role", + "reviewed_at", + "review_note", + } +) +PROPOSAL_TRANSITION_FIELDS = frozenset( + {"status", "applied_by_handle", "applied_by_agent_id", "applied_at", "updated_at"} +) +PROPOSAL_UUID_FIELDS = frozenset({"id", "proposed_by_agent_id", "reviewed_by_agent_id", "applied_by_agent_id"}) +PROPOSAL_TIMESTAMP_FIELDS = frozenset({"reviewed_at", "applied_at", "created_at", "updated_at"}) +APPROVAL_UUID_FIELDS = frozenset({"proposal_id", "reviewed_by_agent_id"}) +APPROVAL_TIMESTAMP_FIELDS = frozenset({"reviewed_at"}) +RECEIPT_PROPOSAL_FIELDS = ( + "id", + "proposal_type", + "status", + "payload", + "reviewed_by_handle", + "reviewed_by_agent_id", + "reviewed_at", + "review_note", + "applied_by_handle", + "applied_by_agent_id", + "applied_at", +) + + +class ExportError(RuntimeError): + """A fail-closed error that contains no proposal payload material.""" + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _same_json(left: Any, right: Any) -> bool: + return _canonical_json(left) == _canonical_json(right) + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ExportError("JSON input contains duplicate object fields") + result[key] = value + return result + + +def _reject_nonfinite_number(value: str) -> None: + raise ExportError(f"JSON input contains invalid numeric constant {value}") + + +def _parse_json_object(raw: str, *, label: str) -> dict[str, Any]: + try: + value = json.loads( + raw, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_nonfinite_number, + ) + except (json.JSONDecodeError, UnicodeError) as exc: + raise ExportError(f"{label} is not valid UTF-8 JSON") from exc + if not isinstance(value, dict): + raise ExportError(f"{label} must contain one JSON object") + return value + + +def _absolute_path(value: str | Path) -> Path: + path = Path(value).expanduser() + if not path.is_absolute(): + path = Path.cwd() / path + return path + + +def _load_private_receipt(path: Path) -> dict[str, Any]: + path = _absolute_path(path) + try: + metadata = path.lstat() + except OSError as exc: + raise ExportError("worker replay receipt is unavailable") from exc + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise ExportError("worker replay receipt must be a regular non-symlink file") + if stat.S_IMODE(metadata.st_mode) != 0o600: + raise ExportError("worker replay receipt permissions must be exactly 0600") + try: + raw = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise ExportError("worker replay receipt is unreadable") from exc + return _parse_json_object(raw, label="worker replay receipt") + + +def _require_exact_fields(value: Any, expected: frozenset[str], *, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ExportError(f"{label} must be an object") + observed = frozenset(value) + if observed != expected: + raise ExportError( + f"{label} has invalid fields; missing_count={len(expected - observed)}, " + f"extra_count={len(observed - expected)}" + ) + return value + + +def _canonical_uuid(value: Any, *, label: str, nullable: bool = False) -> str | None: + if value is None and nullable: + return None + try: + return str(uuid.UUID(str(value))) + except (AttributeError, TypeError, ValueError) as exc: + raise ExportError(f"{label} must be a UUID") from exc + + +def _canonical_timestamp(value: Any, *, label: str, nullable: bool = False) -> str | None: + if value is None and nullable: + return None + try: + return replay_receipt.canonical_utc_timestamp(value, path=label) + except ValueError as exc: + raise ExportError(f"{label} must be a timezone-aware timestamp") from exc + + +def _normalize_proposal(value: Any, *, label: str) -> dict[str, Any]: + row = dict(_require_exact_fields(value, PROPOSAL_FIELDS, label=label)) + for field in PROPOSAL_UUID_FIELDS: + row[field] = _canonical_uuid( + row[field], + label=f"{label}.{field}", + nullable=field != "id", + ) + for field in PROPOSAL_TIMESTAMP_FIELDS: + row[field] = _canonical_timestamp( + row[field], + label=f"{label}.{field}", + nullable=field in {"reviewed_at", "applied_at"}, + ) + if not isinstance(row["proposal_type"], str) or not row["proposal_type"]: + raise ExportError(f"{label}.proposal_type must be a non-empty string") + if not isinstance(row["status"], str) or not row["status"]: + raise ExportError(f"{label}.status must be a non-empty string") + if not isinstance(row["payload"], dict): + raise ExportError(f"{label}.payload must be an object") + for field in ( + PROPOSAL_FIELDS + - PROPOSAL_UUID_FIELDS + - PROPOSAL_TIMESTAMP_FIELDS + - { + "payload", + } + ): + if field in {"proposal_type", "status"}: + continue + if row[field] is not None and not isinstance(row[field], str): + raise ExportError(f"{label}.{field} must be text or null") + return row + + +def _normalize_approval(value: Any, *, label: str) -> dict[str, Any]: + row = dict(_require_exact_fields(value, APPROVAL_FIELDS, label=label)) + for field in APPROVAL_UUID_FIELDS: + row[field] = _canonical_uuid(row[field], label=f"{label}.{field}") + for field in APPROVAL_TIMESTAMP_FIELDS: + row[field] = _canonical_timestamp(row[field], label=f"{label}.{field}") + if not isinstance(row["payload"], dict): + raise ExportError(f"{label}.payload must be an object") + for field in APPROVAL_FIELDS - APPROVAL_UUID_FIELDS - APPROVAL_TIMESTAMP_FIELDS - {"payload"}: + if not isinstance(row[field], str) or not row[field].strip(): + raise ExportError(f"{label}.{field} must be non-empty text") + return row + + +def _parse_timestamp(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def validate_material( + material: dict[str, Any], + *, + expected_proposal_id: str | None = None, + expected_sequence: int | None = None, +) -> dict[str, Any]: + """Validate and canonicalize the exact seven-field replay material.""" + _require_exact_fields(material, MATERIAL_FIELDS, label="replay material") + if material["artifact"] != MATERIAL_ARTIFACT: + raise ExportError("replay material artifact is unsupported") + if type(material["contract_version"]) is not int or material["contract_version"] != MATERIAL_CONTRACT_VERSION: + raise ExportError("replay material contract version is unsupported") + sequence = material["sequence"] + if type(sequence) is not int or sequence <= 0: + raise ExportError("replay material sequence must be a positive integer") + if expected_sequence is not None and sequence != expected_sequence: + raise ExportError("replay material sequence does not match the requested sequence") + + approved = _normalize_proposal(material["approved_proposal"], label="approved_proposal") + approval = _normalize_approval(material["approval_snapshot"], label="approval_snapshot") + applied = _normalize_proposal(material["applied_proposal"], label="applied_proposal") + proposal_id = _canonical_uuid(applied["id"], label="applied_proposal.id") + if expected_proposal_id is not None and proposal_id != _canonical_uuid( + expected_proposal_id, label="expected_proposal_id" + ): + raise ExportError("database proposal id does not match the requested proposal") + if approved["id"] != proposal_id or approval["proposal_id"] != proposal_id: + raise ExportError("approved, approval, and applied proposal ids do not match") + if approved["status"] != "approved" or applied["status"] != "applied": + raise ExportError("replay material requires exact approved and applied proposal states") + if any(approved[field] is not None for field in ("applied_by_handle", "applied_by_agent_id", "applied_at")): + raise ExportError("approved proposal snapshot already contains apply metadata") + if not applied["applied_by_handle"] or not applied["applied_by_agent_id"] or not applied["applied_at"]: + raise ExportError("applied proposal row lacks exact apply metadata") + if not approved["reviewed_by_handle"] or not approved["reviewed_by_agent_id"] or not approved["reviewed_at"]: + raise ExportError("approved proposal row lacks exact review metadata") + + changed_fields = {field for field in PROPOSAL_FIELDS if not _same_json(approved[field], applied[field])} + if changed_fields != PROPOSAL_TRANSITION_FIELDS: + raise ExportError("proposal rows differ outside or omit the exact guarded apply transition") + + approval_projection = { + "proposal_id": approved["id"], + "proposal_type": approved["proposal_type"], + "payload": approved["payload"], + "reviewed_by_handle": approved["reviewed_by_handle"], + "reviewed_by_agent_id": approved["reviewed_by_agent_id"], + "reviewed_at": approved["reviewed_at"], + "review_note": approved["review_note"], + } + for field, expected in approval_projection.items(): + if not _same_json(approval[field], expected): + raise ExportError(f"approval snapshot does not match approved proposal field {field}") + + receipt = material["replay_receipt"] + if not isinstance(receipt, dict): + raise ExportError("replay_receipt must be an object") + try: + replay_receipt.validate_replay_receipt(receipt, expected_proposal_id=proposal_id) + except ValueError as exc: + raise ExportError("worker replay receipt failed its exact row-level contract") from exc + receipt_proposal = receipt.get("proposal") + if not isinstance(receipt_proposal, dict): + raise ExportError("worker replay receipt has no proposal envelope") + applied_envelope = {field: applied[field] for field in RECEIPT_PROPOSAL_FIELDS} + if not _same_json(receipt_proposal, applied_envelope): + raise ExportError("worker replay receipt does not match the exact applied proposal row") + + created_at = _parse_timestamp(approved["created_at"]) + reviewed_at = _parse_timestamp(approved["reviewed_at"]) + approved_updated_at = _parse_timestamp(approved["updated_at"]) + applied_at = _parse_timestamp(applied["applied_at"]) + applied_updated_at = _parse_timestamp(applied["updated_at"]) + if not created_at <= reviewed_at <= approved_updated_at < applied_at <= applied_updated_at: + raise ExportError("proposal transition timestamps are not in exact lifecycle order") + + return { + "artifact": MATERIAL_ARTIFACT, + "contract_version": MATERIAL_CONTRACT_VERSION, + "sequence": sequence, + "approved_proposal": approved, + "approval_snapshot": approval, + "applied_proposal": applied, + "replay_receipt": receipt, + } + + +def fetch_transition_rows(args: argparse.Namespace, password: str, proposal_id: str) -> dict[str, Any]: + sql = f"""begin transaction isolation level repeatable read read only; +select kb_stage.export_applied_proposal_replay_rows( + {apply_engine.sql_literal(proposal_id)}::uuid +)::text; +rollback; +""" + raw = apply_engine.run_psql(args, sql, password, redact_output_on_error=True).strip() + if not raw or "\n" in raw: + raise ExportError("applied transition export returned an invalid row count") + return _parse_json_object(raw, label="applied transition export") + + +def _resolved_output(path: Path) -> Path: + absolute = _absolute_path(path) + if absolute.name in {"", ".", ".."}: + raise ExportError("output path must name a file") + return absolute.parent.resolve(strict=False) / absolute.name + + +def _paths_alias(left: Path, right: Path) -> bool: + if left.resolve(strict=False) == right.resolve(strict=False): + return True + try: + return left.exists() and right.exists() and os.path.samefile(left, right) + except OSError: + return False + + +def _assert_output_safe(path: Path, protected_paths: tuple[Path, ...]) -> None: + for protected in protected_paths: + if _paths_alias(path, protected): + raise ExportError("output path aliases a protected input or engine file") + try: + metadata = path.lstat() + except FileNotFoundError: + return + if path.is_symlink() or not stat.S_ISREG(metadata.st_mode): + raise ExportError("output path must be a regular non-symlink file") + if metadata.st_nlink != 1: + raise ExportError("output path has unsafe hard-link aliases") + if stat.S_IMODE(metadata.st_mode) != 0o600: + raise ExportError("existing output permissions must be exactly 0600") + + +def _sync_directory(path: Path) -> None: + directory_fd = os.open(path, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + + +def write_private_material( + path: Path, + material: dict[str, Any], + *, + protected_paths: tuple[Path, ...], +) -> tuple[Path, str]: + """Publish once, or accept an exact 0600 rerun without overwriting drift.""" + output = _resolved_output(path) + output.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + _assert_output_safe(output, protected_paths) + payload = (json.dumps(material, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8") + if output.exists(): + if output.read_bytes() != payload: + raise ExportError("existing output contains different replay material") + return output, "unchanged" + + fd, temporary_name = tempfile.mkstemp(prefix=f".{output.name}.", dir=str(output.parent)) + temporary = Path(temporary_name) + published = False + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, output) + published = True + except FileExistsError: + _assert_output_safe(output, protected_paths) + if output.read_bytes() != payload: + raise ExportError("output changed concurrently during replay material export") from None + _sync_directory(output.parent) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + _assert_output_safe(output, protected_paths) + if output.read_bytes() != payload: + raise ExportError("published replay material failed exact readback") + return output, "created" if published else "unchanged" + + +def _positive_sequence(value: str) -> int: + try: + sequence = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("sequence must be a positive integer") from exc + if sequence <= 0: + raise argparse.ArgumentTypeError("sequence must be a positive integer") + return sequence + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("proposal_id", help="UUID of the applied proposal") + parser.add_argument("--replay-receipt", required=True, help="private 0600 worker replay receipt") + parser.add_argument("--sequence", required=True, type=_positive_sequence, help="positive ledger sequence") + parser.add_argument("--output", required=True, help="private replay material output path") + parser.add_argument("--secrets-file", default=apply_engine.DEFAULT_SECRETS_FILE) + parser.add_argument("--container", default=apply_engine.DEFAULT_CONTAINER) + parser.add_argument("--db", default=apply_engine.DEFAULT_DB) + parser.add_argument("--host", default=apply_engine.DEFAULT_HOST) + parser.add_argument("--role", default=apply_engine.DEFAULT_ROLE) + return parser.parse_args(argv) + + +def run(args: argparse.Namespace) -> int: + proposal_id = _canonical_uuid(args.proposal_id, label="proposal_id") + receipt_path = _absolute_path(args.replay_receipt) + receipt = _load_private_receipt(receipt_path) + try: + replay_receipt.validate_replay_receipt(receipt, expected_proposal_id=proposal_id) + except ValueError as exc: + raise ExportError("worker replay receipt failed its exact row-level contract") from exc + + password = apply_engine.load_password(args.secrets_file) + rows = fetch_transition_rows(args, password, proposal_id) + material = validate_material( + { + "artifact": MATERIAL_ARTIFACT, + "contract_version": MATERIAL_CONTRACT_VERSION, + "sequence": args.sequence, + "approved_proposal": rows.get("approved_proposal"), + "approval_snapshot": rows.get("approval_snapshot"), + "applied_proposal": rows.get("applied_proposal"), + "replay_receipt": receipt, + }, + expected_proposal_id=proposal_id, + expected_sequence=args.sequence, + ) + protected_paths = tuple( + _absolute_path(path) + for path in ( + receipt_path, + args.secrets_file, + Path(__file__), + Path(apply_engine.__file__), + Path(replay_receipt.__file__), + ) + ) + _, disposition = write_private_material( + Path(args.output), + material, + protected_paths=protected_paths, + ) + print(f"replay material {disposition}: proposal={proposal_id} sequence={args.sequence}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + try: + return run(parse_args(sys.argv[1:] if argv is None else argv)) + except ExportError as exc: + print(f"replay material export refused: {exc}", file=sys.stderr) + return 1 + except OSError: + print("replay material export refused: private file operation failed", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/kb_apply_prereqs.sql b/scripts/kb_apply_prereqs.sql index 839f6b5..7998133 100644 --- a/scripts/kb_apply_prereqs.sql +++ b/scripts/kb_apply_prereqs.sql @@ -10,6 +10,12 @@ -- -- The deployer provisions passwords separately in kb-review.env/kb-apply.env. -- This file contains no credentials and is safe to reuse in disposable clones. +-- Before the first V3 contract migration, run this file, stop the V2 worker, +-- then ALTER ROLE kb_apply NOLOGIN and end its sessions. After V3 is installed, +-- this file may re-enable the login: a V2-configured worker detects the cutover +-- and leaves historical V2 approvals review-visible until migrate/reject. + +set timezone = 'UTC'; begin; @@ -74,10 +80,54 @@ insert into public.agents (id, handle, kind) values ('44444444-4444-4444-4444-444444444444', 'kb-apply', 'service') on conflict (handle) do nothing; +-- Canonicalize the existing human reviewer identity without changing its UUID. +-- Historical proposal/approval reviewer text is deliberately left untouched. +do $reviewer_identity$ +declare + v_legacy_id uuid; + v_canonical_id uuid; +begin + select id into v_legacy_id + from public.agents + where handle = 'm3ta' + for update; + + select id into v_canonical_id + from public.agents + where handle = 'm3taversal' + for update; + + if v_legacy_id is not null + and v_canonical_id is not null + and v_legacy_id <> v_canonical_id then + raise exception + 'kb_apply_prereqs: both reviewer handles m3ta (%) and m3taversal (%) exist with different agent UUIDs; resolve the identity conflict before rerunning', + v_legacy_id, v_canonical_id; + end if; + + if v_canonical_id is null and v_legacy_id is not null then + update public.agents + set handle = 'm3taversal' + where id = v_legacy_id + and handle = 'm3ta'; + if not found then + raise exception + 'kb_apply_prereqs: legacy reviewer m3ta changed during canonical identity migration'; + end if; + v_canonical_id := v_legacy_id; + end if; + + if v_canonical_id is null then + raise exception + 'kb_apply_prereqs: required human reviewer agent is missing; restore exactly one public.agents row with handle m3ta or m3taversal before rerunning'; + end if; +end +$reviewer_identity$; + -- DB-login to human-reviewer mapping. Only the gate owner/admin can change it; -- the approval function verifies --reviewed-by against this mapping rather than --- trusting caller-supplied identity. The current dedicated reviewer credential --- maps to the existing canonical m3ta human agent when that row is present. +-- trusting caller-supplied identity. The dedicated reviewer credential always +-- maps to the canonical m3taversal human agent. create table if not exists kb_stage.kb_review_principals ( db_role name primary key, reviewed_by_handle text not null, @@ -96,7 +146,8 @@ create table if not exists kb_stage.kb_proposal_approvals ( reviewed_by_agent_id uuid not null references public.agents(id), reviewed_by_db_role name not null, reviewed_at timestamptz not null, - review_note text not null + review_note text not null, + approved_proposal_snapshot jsonb ); -- Existing canonical/proposal tables must remain owned by the deployer. Gate @@ -119,22 +170,28 @@ begin ('public', 'strategies'), ('public', 'strategy_nodes'), ('public', 'claim_evidence'), + ('public', 'claim_evidence_assessments'), ('public', 'claim_edges'), ('public', 'claims'), ('public', 'sources'), ('public', 'reasoning_tools'), ('kb_stage', 'kb_proposals'), ('kb_stage', 'kb_review_principals'), - ('kb_stage', 'kb_proposal_approvals') + ('kb_stage', 'kb_proposal_approvals'), + ('kb_stage', 'teleo_v3_installation_ownership') ) and relation.relkind in ('r', 'p') and not ( (namespace.nspname = 'kb_stage' - and relation.relname in ('kb_review_principals', 'kb_proposal_approvals') + and relation.relname in ( + 'kb_review_principals', 'kb_proposal_approvals', 'teleo_v3_installation_ownership' + ) and owner_role.rolname in (current_user, 'kb_gate_owner')) or (not ( namespace.nspname = 'kb_stage' - and relation.relname in ('kb_review_principals', 'kb_proposal_approvals') + and relation.relname in ( + 'kb_review_principals', 'kb_proposal_approvals', 'teleo_v3_installation_ownership' + ) ) and owner_role.rolname = current_user) ); @@ -161,7 +218,8 @@ begin and procedure.proname in ( 'approve_strict_proposal', 'assert_approved_proposal', - 'finish_approved_proposal' + 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ) and owner_role.rolname not in (current_user, 'kb_gate_owner'); @@ -185,7 +243,8 @@ begin and procedure.proname in ( 'approve_strict_proposal', 'assert_approved_proposal', - 'finish_approved_proposal' + 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ) and not ( (procedure.proname = 'approve_strict_proposal' @@ -199,6 +258,8 @@ begin or (procedure.proname = 'finish_approved_proposal' and pg_catalog.oidvectortypes(procedure.proargtypes) = 'uuid, text, jsonb, text, uuid, timestamp with time zone, text, text') + or (procedure.proname = 'export_applied_proposal_replay_rows' + and pg_catalog.oidvectortypes(procedure.proargtypes) = 'uuid') ); if v_drift is not null then @@ -224,8 +285,19 @@ begin coalesce(relation.relacl, pg_catalog.acldefault('r', relation.relowner)) ) acl where (namespace.nspname, relation.relname) in ( + ('public', 'agents'), + ('public', 'strategies'), + ('public', 'strategy_nodes'), + ('public', 'claim_evidence'), + ('public', 'claim_evidence_assessments'), + ('public', 'claim_edges'), + ('public', 'claims'), + ('public', 'sources'), + ('public', 'reasoning_tools'), + ('kb_stage', 'kb_proposals'), ('kb_stage', 'kb_review_principals'), - ('kb_stage', 'kb_proposal_approvals') + ('kb_stage', 'kb_proposal_approvals'), + ('kb_stage', 'teleo_v3_installation_ownership') ) and acl.grantee <> 0 and acl.grantee <> relation.relowner @@ -256,8 +328,19 @@ begin join pg_catalog.pg_namespace namespace on namespace.oid = relation.relnamespace cross join lateral pg_catalog.aclexplode(attribute.attacl) acl where (namespace.nspname, relation.relname) in ( + ('public', 'agents'), + ('public', 'strategies'), + ('public', 'strategy_nodes'), + ('public', 'claim_evidence'), + ('public', 'claim_evidence_assessments'), + ('public', 'claim_edges'), + ('public', 'claims'), + ('public', 'sources'), + ('public', 'reasoning_tools'), + ('kb_stage', 'kb_proposals'), ('kb_stage', 'kb_review_principals'), - ('kb_stage', 'kb_proposal_approvals') + ('kb_stage', 'kb_proposal_approvals'), + ('kb_stage', 'teleo_v3_installation_ownership') ) and not attribute.attisdropped and acl.grantee <> 0 @@ -293,7 +376,8 @@ begin and procedure.proname in ( 'approve_strict_proposal', 'assert_approved_proposal', - 'finish_approved_proposal' + 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ) and acl.grantee <> 0 and acl.grantee <> procedure.proowner @@ -314,13 +398,41 @@ drop function if exists kb_stage.approve_strict_proposal(uuid, text, text); alter table kb_stage.kb_review_principals owner to kb_gate_owner; alter table kb_stage.kb_proposal_approvals owner to kb_gate_owner; +alter table kb_stage.kb_proposal_approvals + add column if not exists approved_proposal_snapshot jsonb; + +comment on column kb_stage.kb_proposal_approvals.approved_proposal_snapshot is + 'Exact post-approval kb_proposals row. NULL means exact replay material is unavailable.'; + +-- A still-approved legacy row is the only state whose complete approved shape +-- remains recoverable without guessing. Already-applied rows stay NULL because +-- their pre-apply fields, especially updated_at, cannot be reconstructed. +update kb_stage.kb_proposal_approvals approval + set approved_proposal_snapshot = pg_catalog.to_jsonb(proposal) + from kb_stage.kb_proposals proposal + where approval.proposal_id = proposal.id + and approval.approved_proposal_snapshot is null + and proposal.status = 'approved' + and proposal.applied_by_handle is null + and proposal.applied_by_agent_id is null + and proposal.applied_at is null + and approval.proposal_type = proposal.proposal_type + and approval.payload = proposal.payload + and approval.reviewed_by_handle = proposal.reviewed_by_handle + and approval.reviewed_by_agent_id is not distinct from proposal.reviewed_by_agent_id + and approval.reviewed_at = proposal.reviewed_at + and approval.review_note = proposal.review_note; + insert into kb_stage.kb_review_principals (db_role, reviewed_by_handle, reviewed_by_agent_id, active, created_at) -select 'kb_review'::name, a.handle, a.id, true, +select 'kb_review'::name, 'm3taversal', a.id, true, '1970-01-01 00:00:00+00'::timestamptz from public.agents a - where a.handle = 'm3ta' -on conflict (db_role) do nothing; + where a.handle = 'm3taversal' +on conflict (db_role) do update + set reviewed_by_handle = excluded.reviewed_by_handle, + reviewed_by_agent_id = excluded.reviewed_by_agent_id, + active = excluded.active; -- Reset every explicit privilege held by the protected roles before granting -- the documented matrix. This removes stale DELETE/TRUNCATE/REFERENCES/TRIGGER @@ -372,13 +484,15 @@ begin ('public', 'strategies'), ('public', 'strategy_nodes'), ('public', 'claim_evidence'), + ('public', 'claim_evidence_assessments'), ('public', 'claim_edges'), ('public', 'claims'), ('public', 'sources'), ('public', 'reasoning_tools'), ('kb_stage', 'kb_proposals'), ('kb_stage', 'kb_review_principals'), - ('kb_stage', 'kb_proposal_approvals') + ('kb_stage', 'kb_proposal_approvals'), + ('kb_stage', 'teleo_v3_installation_ownership') ) and attribute.attnum > 0 and not attribute.attisdropped @@ -392,7 +506,8 @@ begin ); if protected_table.schema_name = 'kb_stage' and protected_table.table_name in ( - 'kb_proposals', 'kb_review_principals', 'kb_proposal_approvals' + 'kb_proposals', 'kb_review_principals', 'kb_proposal_approvals', + 'teleo_v3_installation_ownership' ) then execute pg_catalog.format( 'revoke all privileges (%s) on table %I.%I from public', @@ -417,6 +532,45 @@ grant select, insert on public.claims to kb_apply; grant select, insert on public.sources to kb_apply; grant select, insert on public.reasoning_tools to kb_apply; +-- The V3 assessment table is installed by a separate additive migration. Keep +-- this prerequisite runnable before that migration, then normalize its ACLs on +-- the first post-migration rerun. +do $v3_assessment_acl$ +declare + v_columns text; +begin + if pg_catalog.to_regclass('public.claim_evidence_assessments') is not null then + execute 'revoke all privileges on table public.claim_evidence_assessments ' + 'from public, kb_gate_owner, kb_apply, kb_review'; + select pg_catalog.string_agg(pg_catalog.format('%I', attribute.attname), ', ' order by attribute.attnum) + into v_columns + from pg_catalog.pg_attribute attribute + where attribute.attrelid = 'public.claim_evidence_assessments'::regclass + and attribute.attnum > 0 + and not attribute.attisdropped; + if v_columns is not null then + execute pg_catalog.format( + 'revoke all privileges (%s) on table public.claim_evidence_assessments ' + 'from public, kb_gate_owner, kb_apply, kb_review', + v_columns + ); + end if; + execute 'grant select, insert on public.claim_evidence_assessments to kb_apply'; + execute 'grant select on public.claims, public.sources, public.claim_evidence, ' + 'public.claim_edges, public.claim_evidence_assessments to kb_gate_owner'; + end if; +end +$v3_assessment_acl$; + +do $v3_installation_ownership_acl$ +begin + if pg_catalog.to_regclass('kb_stage.teleo_v3_installation_ownership') is not null then + execute 'revoke all privileges on table kb_stage.teleo_v3_installation_ownership ' + 'from public, kb_gate_owner, kb_apply, kb_review'; + end if; +end +$v3_installation_ownership_acl$; + -- Keep the primary ledger denial explicit in addition to the full reset above. revoke update on kb_stage.kb_proposals from kb_apply; grant select on kb_stage.kb_proposals to kb_apply; @@ -527,14 +681,6 @@ begin end if; v_reviewed_at := pg_catalog.clock_timestamp(); - insert into kb_stage.kb_proposal_approvals - (proposal_id, proposal_type, payload, reviewed_by_handle, - reviewed_by_agent_id, reviewed_by_db_role, reviewed_at, review_note) - values - (v_row.id, v_row.proposal_type, v_row.payload, v_principal.reviewed_by_handle, - v_principal.reviewed_by_agent_id, session_user::name, v_reviewed_at, - pg_catalog.btrim(p_review_note)); - update kb_stage.kb_proposals set status = 'approved', reviewed_by_handle = v_principal.reviewed_by_handle, @@ -552,6 +698,15 @@ begin p_proposal_id; end if; + insert into kb_stage.kb_proposal_approvals + (proposal_id, proposal_type, payload, reviewed_by_handle, + reviewed_by_agent_id, reviewed_by_db_role, reviewed_at, review_note, + approved_proposal_snapshot) + values + (v_row.id, v_row.proposal_type, v_row.payload, v_principal.reviewed_by_handle, + v_principal.reviewed_by_agent_id, session_user::name, v_reviewed_at, + pg_catalog.btrim(p_review_note), pg_catalog.to_jsonb(v_row)); + return pg_catalog.jsonb_build_object( 'id', v_row.id::text, 'proposal_type', v_row.proposal_type, @@ -598,6 +753,7 @@ begin and a.reviewed_by_agent_id is not distinct from p_reviewed_by_agent_id and a.reviewed_at = p_reviewed_at and a.review_note = p_review_note + and a.approved_proposal_snapshot = pg_catalog.to_jsonb(p) for update of p; if not found then raise exception 'assert_approved_proposal: proposal % is not the exact immutable approved snapshot', @@ -649,6 +805,7 @@ begin and a.reviewed_by_agent_id is not distinct from p_reviewed_by_agent_id and a.reviewed_at = p_reviewed_at and a.review_note = p_review_note + and a.approved_proposal_snapshot = pg_catalog.to_jsonb(p) for update of p; if not found then raise exception 'finish_approved_proposal: proposal % is not the locked immutable approved snapshot', @@ -694,12 +851,88 @@ begin end $function$; +-- Read-only transition export. kb_apply receives only EXECUTE on this function, +-- never SELECT on the approval table. The stored approved row must be a complete +-- kb_proposals record and differ from the applied row only by the guarded apply +-- transition before any private material leaves PostgreSQL. +create or replace function kb_stage.export_applied_proposal_replay_rows( + p_proposal_id uuid +) returns jsonb +language plpgsql +stable +security definer +set search_path = pg_catalog, pg_temp +as $function$ +declare + v_rows jsonb; +begin + select pg_catalog.jsonb_build_object( + 'approved_proposal', approval.approved_proposal_snapshot, + 'approval_snapshot', + pg_catalog.to_jsonb(approval) - 'approved_proposal_snapshot', + 'applied_proposal', pg_catalog.to_jsonb(proposal) + ) + into v_rows + from kb_stage.kb_proposals proposal + join kb_stage.kb_proposal_approvals approval + on approval.proposal_id = proposal.id + where proposal.id = p_proposal_id + and proposal.status = 'applied' + and proposal.applied_by_handle is not null + and proposal.applied_by_agent_id is not null + and proposal.applied_at is not null + and proposal.reviewed_by_handle is not null + and proposal.reviewed_by_agent_id is not null + and proposal.reviewed_at is not null + and proposal.review_note is not null + and nullif(pg_catalog.btrim(approval.reviewed_by_db_role::text), '') is not null + and pg_catalog.jsonb_typeof(approval.approved_proposal_snapshot) = 'object' + and approval.approved_proposal_snapshot = pg_catalog.to_jsonb( + pg_catalog.jsonb_populate_record( + null::kb_stage.kb_proposals, + approval.approved_proposal_snapshot + ) + ) + and approval.approved_proposal_snapshot->>'status' = 'approved' + and approval.approved_proposal_snapshot->'applied_by_handle' = 'null'::jsonb + and approval.approved_proposal_snapshot->'applied_by_agent_id' = 'null'::jsonb + and approval.approved_proposal_snapshot->'applied_at' = 'null'::jsonb + and approval.proposal_id = (approval.approved_proposal_snapshot->>'id')::uuid + and approval.proposal_type = approval.approved_proposal_snapshot->>'proposal_type' + and approval.payload = approval.approved_proposal_snapshot->'payload' + and approval.reviewed_by_handle = + approval.approved_proposal_snapshot->>'reviewed_by_handle' + and approval.reviewed_by_agent_id = + (approval.approved_proposal_snapshot->>'reviewed_by_agent_id')::uuid + and approval.reviewed_at = + (approval.approved_proposal_snapshot->>'reviewed_at')::timestamptz + and approval.review_note = approval.approved_proposal_snapshot->>'review_note' + and ( + approval.approved_proposal_snapshot || pg_catalog.jsonb_build_object( + 'status', proposal.status, + 'applied_by_handle', proposal.applied_by_handle, + 'applied_by_agent_id', proposal.applied_by_agent_id, + 'applied_at', proposal.applied_at, + 'updated_at', proposal.updated_at + ) + ) = pg_catalog.to_jsonb(proposal); + + if v_rows is null then + raise exception 'export_applied_proposal_replay_rows: exact applied transition unavailable for proposal %', + p_proposal_id; + end if; + return v_rows; +end +$function$; + alter function kb_stage.approve_strict_proposal(uuid, text, jsonb, text, text) owner to kb_gate_owner; alter function kb_stage.assert_approved_proposal(uuid, text, jsonb, text, uuid, timestamptz, text) owner to kb_gate_owner; alter function kb_stage.finish_approved_proposal(uuid, text, jsonb, text, uuid, timestamptz, text, text) owner to kb_gate_owner; +alter function kb_stage.export_applied_proposal_replay_rows(uuid) + owner to kb_gate_owner; revoke all on function kb_stage.approve_strict_proposal(uuid, text, jsonb, text, text) from public, kb_gate_owner, kb_apply, kb_review; @@ -707,6 +940,8 @@ revoke all on function kb_stage.assert_approved_proposal(uuid, text, jsonb, text from public, kb_gate_owner, kb_apply, kb_review; revoke all on function kb_stage.finish_approved_proposal(uuid, text, jsonb, text, uuid, timestamptz, text, text) from public, kb_gate_owner, kb_apply, kb_review; +revoke all on function kb_stage.export_applied_proposal_replay_rows(uuid) + from public, kb_gate_owner, kb_apply, kb_review; grant execute on function kb_stage.approve_strict_proposal(uuid, text, jsonb, text, text) to kb_review; @@ -714,6 +949,8 @@ grant execute on function kb_stage.assert_approved_proposal(uuid, text, jsonb, t to kb_apply; grant execute on function kb_stage.finish_approved_proposal(uuid, text, jsonb, text, uuid, timestamptz, text, text) to kb_apply; +grant execute on function kb_stage.export_applied_proposal_replay_rows(uuid) + to kb_apply; -- Assert the complete protected catalog contract before commit. These checks -- make reruns an upgrade proof rather than a best-effort sequence of grants. @@ -755,6 +992,20 @@ begin raise exception 'kb_apply_prereqs: protected role membership appeared during migration: %', v_drift; end if; + if exists (select 1 from public.agents where handle = 'm3ta') + or not exists ( + select 1 + from kb_stage.kb_review_principals principal + join public.agents reviewer + on reviewer.id = principal.reviewed_by_agent_id + where principal.db_role = 'kb_review'::name + and principal.reviewed_by_handle = 'm3taversal' + and principal.active + and reviewer.handle = 'm3taversal' + ) then + raise exception 'kb_apply_prereqs: kb_review principal did not normalize to active m3taversal'; + end if; + select pg_catalog.string_agg( pg_catalog.format( '%I on %I usage=%s create=%s', @@ -792,21 +1043,27 @@ begin ('public', 'strategies'), ('public', 'strategy_nodes'), ('public', 'claim_evidence'), + ('public', 'claim_evidence_assessments'), ('public', 'claim_edges'), ('public', 'claims'), ('public', 'sources'), ('public', 'reasoning_tools'), ('kb_stage', 'kb_proposals'), ('kb_stage', 'kb_review_principals'), - ('kb_stage', 'kb_proposal_approvals') + ('kb_stage', 'kb_proposal_approvals'), + ('kb_stage', 'teleo_v3_installation_ownership') ) and not ( (namespace.nspname = 'kb_stage' - and relation.relname in ('kb_review_principals', 'kb_proposal_approvals') + and relation.relname in ( + 'kb_review_principals', 'kb_proposal_approvals', 'teleo_v3_installation_ownership' + ) and owner_role.rolname = 'kb_gate_owner') or (not ( namespace.nspname = 'kb_stage' - and relation.relname in ('kb_review_principals', 'kb_proposal_approvals') + and relation.relname in ( + 'kb_review_principals', 'kb_proposal_approvals', 'teleo_v3_installation_ownership' + ) ) and owner_role.rolname = current_user) ); @@ -815,11 +1072,24 @@ begin raise exception 'kb_apply_prereqs: protected table ownership mismatch: %', v_drift; end if; + if not exists ( + select 1 + from pg_catalog.pg_attribute attribute + where attribute.attrelid = 'kb_stage.kb_proposal_approvals'::regclass + and attribute.attname = 'approved_proposal_snapshot' + and attribute.atttypid = 'pg_catalog.jsonb'::regtype + and not attribute.attnotnull + and not attribute.attisdropped + ) then + raise exception 'kb_apply_prereqs: approved proposal snapshot column is not nullable JSONB'; + end if; + with expected(signature) as ( values ('approve_strict_proposal(uuid, text, jsonb, text, text)'), ('assert_approved_proposal(uuid, text, jsonb, text, uuid, timestamp with time zone, text)'), - ('finish_approved_proposal(uuid, text, jsonb, text, uuid, timestamp with time zone, text, text)') + ('finish_approved_proposal(uuid, text, jsonb, text, uuid, timestamp with time zone, text, text)'), + ('export_applied_proposal_replay_rows(uuid)') ), actual(signature) as ( select procedure.proname || '(' || pg_catalog.oidvectortypes(procedure.proargtypes) || ')' from pg_catalog.pg_proc procedure @@ -828,7 +1098,8 @@ begin and procedure.proname in ( 'approve_strict_proposal', 'assert_approved_proposal', - 'finish_approved_proposal' + 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ) ), delta(kind, signature) as ( select 'unexpected', actual.signature @@ -867,13 +1138,22 @@ begin and procedure.proname in ( 'approve_strict_proposal', 'assert_approved_proposal', - 'finish_approved_proposal' + 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ) and ( owner_role.rolname <> 'kb_gate_owner' or not procedure.prosecdef or pg_catalog.array_to_string(procedure.proconfig, ',') is distinct from 'search_path=pg_catalog, pg_temp' + or ( + procedure.proname = 'export_applied_proposal_replay_rows' + and ( + procedure.provolatile <> 's' + or procedure.prokind <> 'f' + or procedure.prorettype <> 'pg_catalog.jsonb'::regtype + ) + ) ); if v_drift is not null then @@ -886,13 +1166,15 @@ begin ('public', 'strategies'), ('public', 'strategy_nodes'), ('public', 'claim_evidence'), + ('public', 'claim_evidence_assessments'), ('public', 'claim_edges'), ('public', 'claims'), ('public', 'sources'), ('public', 'reasoning_tools'), ('kb_stage', 'kb_proposals'), ('kb_stage', 'kb_review_principals'), - ('kb_stage', 'kb_proposal_approvals') + ('kb_stage', 'kb_proposal_approvals'), + ('kb_stage', 'teleo_v3_installation_ownership') ) select pg_catalog.string_agg( pg_catalog.format( @@ -920,8 +1202,7 @@ begin left join pg_catalog.pg_roles grantee_role on grantee_role.oid = acl.grantee where attribute.attnum > 0 and not attribute.attisdropped - and (acl.grantee = 0 - or grantee_role.rolname in ('kb_gate_owner', 'kb_review', 'kb_apply')); + and acl.grantee <> relation.relowner; if v_drift is not null then raise exception 'kb_apply_prereqs: protected column ACL mismatch: %', v_drift; @@ -952,19 +1233,42 @@ begin ('kb_stage', 'kb_proposals', 'kb_gate_owner', 'UPDATE', false), ('kb_stage', 'kb_proposals', 'kb_apply', 'SELECT', false), ('kb_stage', 'kb_proposals', 'kb_review', 'SELECT', false) + union all + select 'public', 'claim_evidence_assessments', 'kb_apply', 'SELECT', false + where pg_catalog.to_regclass('public.claim_evidence_assessments') is not null + union all + select 'public', 'claim_evidence_assessments', 'kb_apply', 'INSERT', false + where pg_catalog.to_regclass('public.claim_evidence_assessments') is not null + union all + select 'public', 'claims', 'kb_gate_owner', 'SELECT', false + where pg_catalog.to_regclass('public.claim_evidence_assessments') is not null + union all + select 'public', 'sources', 'kb_gate_owner', 'SELECT', false + where pg_catalog.to_regclass('public.claim_evidence_assessments') is not null + union all + select 'public', 'claim_evidence', 'kb_gate_owner', 'SELECT', false + where pg_catalog.to_regclass('public.claim_evidence_assessments') is not null + union all + select 'public', 'claim_edges', 'kb_gate_owner', 'SELECT', false + where pg_catalog.to_regclass('public.claim_evidence_assessments') is not null + union all + select 'public', 'claim_evidence_assessments', 'kb_gate_owner', 'SELECT', false + where pg_catalog.to_regclass('public.claim_evidence_assessments') is not null ), tracked(schema_name, object_name) as ( values ('public', 'agents'), ('public', 'strategies'), ('public', 'strategy_nodes'), ('public', 'claim_evidence'), + ('public', 'claim_evidence_assessments'), ('public', 'claim_edges'), ('public', 'claims'), ('public', 'sources'), ('public', 'reasoning_tools'), ('kb_stage', 'kb_proposals'), ('kb_stage', 'kb_review_principals'), - ('kb_stage', 'kb_proposal_approvals') + ('kb_stage', 'kb_proposal_approvals'), + ('kb_stage', 'teleo_v3_installation_ownership') ), actual(schema_name, object_name, grantee, privilege_type, is_grantable) as ( select distinct namespace.nspname::text, @@ -985,8 +1289,6 @@ begin ) acl left join pg_catalog.pg_roles grantee_role on grantee_role.oid = acl.grantee where acl.grantee <> relation.relowner - and (acl.grantee = 0 - or grantee_role.rolname in ('kb_gate_owner', 'kb_review', 'kb_apply')) ), delta(kind, schema_name, object_name, grantee, privilege_type, is_grantable) as ( select 'unexpected', actual.* from actual @@ -1020,6 +1322,8 @@ begin ('assert_approved_proposal(uuid, text, jsonb, text, uuid, timestamp with time zone, text)', 'kb_apply', 'EXECUTE', false), ('finish_approved_proposal(uuid, text, jsonb, text, uuid, timestamp with time zone, text, text)', + 'kb_apply', 'EXECUTE', false), + ('export_applied_proposal_replay_rows(uuid)', 'kb_apply', 'EXECUTE', false) ), actual(signature, grantee, privilege_type, is_grantable) as ( select distinct @@ -1039,7 +1343,8 @@ begin and procedure.proname in ( 'approve_strict_proposal', 'assert_approved_proposal', - 'finish_approved_proposal' + 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ) and acl.grantee <> procedure.proowner ), delta(kind, signature, grantee, privilege_type, is_grantable) as ( diff --git a/scripts/kb_apply_replay_receipt.py b/scripts/kb_apply_replay_receipt.py index 2e89345..6c7bf76 100644 --- a/scripts/kb_apply_replay_receipt.py +++ b/scripts/kb_apply_replay_receipt.py @@ -13,13 +13,79 @@ from __future__ import annotations import hashlib import json import os +import re import tempfile +import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any RECEIPT_CONTRACT_VERSION = 1 DEFAULT_RECEIPT_DIR = "/opt/teleo-eval/logs/kb-apply-receipts" +UTC_SESSION_SQL = "set timezone = 'UTC';" +_ISO_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$") +_UUID_FIELDS = { + "id", + "agent_id", + "owner_agent_id", + "created_by", + "superseded_by", + "supersedes_id", + "accepted_by_proposal_id", + "claim_id", + "source_id", + "accepted_from_proposal_id", + "from_claim", + "to_claim", + "supersedes_assessment_id", + "reviewed_by_agent_id", + "applied_by_agent_id", +} +_TIMESTAMP_FIELDS = {"captured_at", "created_at", "updated_at", "reviewed_at", "applied_at"} + + +def canonical_utc_timestamp(value: Any, *, path: str = "timestamp") -> str: + """Normalize a typed timestamp to fixed-microsecond UTC receipt text.""" + if not isinstance(value, str) or not _ISO_TIMESTAMP.fullmatch(value): + raise ValueError(f"{path} must be a timezone-aware ISO-8601 timestamp with at most 6 fractional digits") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError(f"{path} must be a valid ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise ValueError(f"{path} must include a timezone") + return parsed.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def _canonical_uuid(value: Any, *, path: str) -> str: + try: + return str(uuid.UUID(str(value))) + except (AttributeError, TypeError, ValueError) as exc: + raise ValueError(f"{path} must be a UUID") from exc + + +def _normalize_typed_value(field: str, value: Any, *, path: str) -> Any: + if value is None: + return None + if field in _UUID_FIELDS: + return _canonical_uuid(value, path=path) + if field in _TIMESTAMP_FIELDS: + return canonical_utc_timestamp(value, path=path) + return value + + +def _normalize_typed_row(row: dict[str, Any], *, path: str) -> dict[str, Any]: + return {field: _normalize_typed_value(field, value, path=f"{path}.{field}") for field, value in row.items()} + + +def normalize_typed_value(field: str, value: Any, *, path: str) -> Any: + """Canonicalize a value according to its PostgreSQL column type.""" + return _normalize_typed_value(field, value, path=path) + + +def normalize_typed_row(row: dict[str, Any], *, path: str) -> dict[str, Any]: + """Canonicalize UUID and timestamp columns without weakening text equality.""" + return _normalize_typed_row(row, path=path) def _canonical_json(value: Any) -> str: @@ -56,6 +122,12 @@ def _require_applied_strict_proposal(proposal: dict[str, Any]) -> dict[str, Any] apply_payload = payload.get("apply_payload") if isinstance(payload, dict) else None if not isinstance(apply_payload, dict): raise ValueError("replay receipt requires a strict payload.apply_payload object") + raw_version = apply_payload.get("contract_version") + if proposal.get("proposal_type") == "approve_claim": + if type(raw_version) is not int or raw_version not in {2, 3}: + raise ValueError("approve_claim replay receipt requires integer contract_version 2 or 3") + elif "contract_version" in apply_payload: + raise ValueError("legacy replay receipt proposal types must omit contract_version") return apply_payload @@ -108,6 +180,8 @@ def _build_attach_evidence_postflight_sql(apply_payload: dict[str, Any]) -> str: def _build_approve_claim_postflight_sql(apply_payload: dict[str, Any]) -> str: + if apply_payload.get("contract_version") == 3: + return _build_approve_claim_v3_postflight_sql(apply_payload) claims = apply_payload.get("claims") or [] sources = apply_payload.get("sources") or [] evidence = apply_payload.get("evidence") or [] @@ -154,6 +228,44 @@ def _build_approve_claim_postflight_sql(apply_payload: dict[str, Any]) -> str: return f"select jsonb_build_object({arguments})::text;" +def _build_approve_claim_v3_postflight_sql(apply_payload: dict[str, Any]) -> str: + fields = { + "claims": _coalesced_json_agg( + "public.claims", + "c", + _id_filter("c", [row.get("id") for row in apply_payload.get("claims") or []]), + "c.id::text", + ), + "sources": _coalesced_json_agg( + "public.sources", + "s", + _id_filter("s", [row.get("id") for row in apply_payload.get("sources") or []]), + "s.id::text", + ), + "claim_evidence": _coalesced_json_agg( + "public.claim_evidence", + "ce", + _id_filter("ce", [row.get("id") for row in apply_payload.get("evidence") or []]), + "ce.id::text", + ), + "claim_edges": _coalesced_json_agg( + "public.claim_edges", + "e", + _id_filter("e", [row.get("id") for row in apply_payload.get("edges") or []]), + "e.id::text", + ), + "claim_evidence_assessments": _coalesced_json_agg( + "public.claim_evidence_assessments", + "a", + _id_filter("a", [row.get("id") for row in apply_payload.get("assessments") or []]), + "a.id::text", + ), + "reasoning_tools": "'[]'::jsonb", + } + arguments = ", ".join(f"{_sql_literal(name)}, {expression}" for name, expression in fields.items()) + return f"select jsonb_build_object({arguments})::text;" + + def _build_revise_strategy_postflight_sql(proposal: dict[str, Any], apply_payload: dict[str, Any]) -> str: strategy = apply_payload.get("strategy") or {} agent_id = apply_payload.get("agent_id") @@ -193,14 +305,16 @@ def build_postflight_sql(proposal: dict[str, Any]) -> str: apply_payload = _require_applied_strict_proposal(proposal) proposal_type = proposal.get("proposal_type") if proposal_type == "add_edge": - return _build_add_edge_postflight_sql(apply_payload) - if proposal_type == "attach_evidence": - return _build_attach_evidence_postflight_sql(apply_payload) - if proposal_type == "approve_claim": - return _build_approve_claim_postflight_sql(apply_payload) - if proposal_type == "revise_strategy": - return _build_revise_strategy_postflight_sql(proposal, apply_payload) - raise ValueError(f"replay receipt does not support proposal_type {proposal_type!r}") + query = _build_add_edge_postflight_sql(apply_payload) + elif proposal_type == "attach_evidence": + query = _build_attach_evidence_postflight_sql(apply_payload) + elif proposal_type == "approve_claim": + query = _build_approve_claim_postflight_sql(apply_payload) + elif proposal_type == "revise_strategy": + query = _build_revise_strategy_postflight_sql(proposal, apply_payload) + else: + raise ValueError(f"replay receipt does not support proposal_type {proposal_type!r}") + return UTC_SESSION_SQL + "\n" + query def expected_row_counts(proposal: dict[str, Any]) -> dict[str, int]: @@ -211,6 +325,15 @@ def expected_row_counts(proposal: dict[str, Any]) -> dict[str, int]: if proposal_type == "attach_evidence": return {"claim_evidence": len(apply_payload.get("evidence") or [])} if proposal_type == "approve_claim": + if apply_payload.get("contract_version") == 3: + return { + "claims": len(apply_payload.get("claims") or []), + "sources": len(apply_payload.get("sources") or []), + "claim_evidence": len(apply_payload.get("evidence") or []), + "claim_edges": len(apply_payload.get("edges") or []), + "claim_evidence_assessments": len(apply_payload.get("assessments") or []), + "reasoning_tools": 0, + } return { "claims": len(apply_payload.get("claims") or []), "sources": len(apply_payload.get("sources") or []), @@ -232,12 +355,13 @@ def _normalize_rows(canonical_rows: dict[str, Any], expected_counts: dict[str, i rows = canonical_rows.get(table) if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows): raise ValueError(f"postflight {table} must be a list of row objects") - normalized[table] = sorted(rows, key=_canonical_json) + typed_rows = [_normalize_typed_row(row, path=f"postflight.{table}") for row in rows] + normalized[table] = sorted(typed_rows, key=_canonical_json) return normalized def _semantic_projection(row: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]: - return {field: row.get(field) for field in fields} + return {field: _normalize_typed_value(field, row.get(field), path=f"semantic.{field}") for field in fields} def _expected_semantic_rows(proposal: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: @@ -269,6 +393,8 @@ def _expected_semantic_rows(proposal: dict[str, Any]) -> dict[str, list[dict[str ] } if proposal_type == "approve_claim": + if apply_payload.get("contract_version") == 3: + return _expected_v3_semantic_rows(proposal, apply_payload) agent_id = apply_payload.get("agent_id") return { "claims": [ @@ -359,12 +485,115 @@ def _expected_semantic_rows(proposal: dict[str, Any]) -> dict[str, list[dict[str raise ValueError(f"replay receipt does not support proposal_type {proposal_type!r}") +def _expected_v3_semantic_rows( + proposal: dict[str, Any], apply_payload: dict[str, Any] +) -> dict[str, list[dict[str, Any]]]: + proposal_id = _canonical_uuid(proposal.get("id"), path="proposal.id") + agent_id = apply_payload.get("agent_id") + polarity_to_role = {"supports": "grounds", "challenges": "contradicts", "illustrates": "illustrates"} + return { + "claims": [ + { + "id": row.get("id"), + "owner_agent_id": row.get("owner_agent_id"), + "type": row.get("type"), + "text": row.get("proposition"), + "proposition": row.get("proposition"), + "body_markdown": row.get("body_markdown"), + "scope": row.get("scope"), + "access_scope": row.get("access_scope"), + "status": row.get("status", "active"), + "admission_state": "admitted", + "revision_criteria": row.get("revision_criteria"), + "revision_kind": "original", + "supersedes_id": None, + "confidence": None, + "tags": row.get("tags") or [], + "created_by": row.get("created_by", agent_id), + "superseded_by": None, + "accepted_by_proposal_id": proposal_id, + } + for row in apply_payload.get("claims") or [] + ], + "sources": [ + { + "id": row.get("id"), + "source_type": row.get("source_type"), + "url": row.get("canonical_url"), + "storage_path": row.get("storage_uri"), + "excerpt": row.get("excerpt"), + "hash": row.get("content_hash"), + "created_by": row.get("created_by", agent_id), + "captured_at": row.get("captured_at"), + "canonical_url": row.get("canonical_url"), + "storage_uri": row.get("storage_uri"), + "content_hash": row.get("content_hash"), + "access_scope": row.get("access_scope"), + "ingestion_origin": row.get("ingestion_origin"), + "provenance_status": row.get("provenance_status"), + "source_class": row.get("source_class"), + "auto_promotion_policy": row.get("auto_promotion_policy"), + "accepted_by_proposal_id": proposal_id, + } + for row in apply_payload.get("sources") or [] + ], + "claim_evidence": [ + { + "id": row.get("id"), + "claim_id": row.get("claim_id"), + "source_id": row.get("source_id"), + "role": polarity_to_role.get(row.get("polarity")), + "weight": None, + "created_by": row.get("created_by", agent_id), + "excerpt": row.get("excerpt"), + "locator_json": row.get("locator_json"), + "source_content_hash": row.get("source_content_hash"), + "rationale": row.get("rationale"), + "polarity": row.get("polarity"), + "accepted_from_proposal_id": proposal_id, + } + for row in apply_payload.get("evidence") or [] + ], + "claim_edges": [ + { + "id": row.get("id"), + "from_claim": row.get("from_claim"), + "to_claim": row.get("to_claim"), + "edge_type": "requires" if row.get("relation_type") == "depends_on" else row.get("relation_type"), + "weight": None, + "created_by": row.get("created_by", agent_id), + "relation_type": row.get("relation_type"), + "rationale": row.get("rationale"), + "scope_or_conditions": row.get("scope_or_conditions"), + "accepted_by_proposal_id": proposal_id, + } + for row in apply_payload.get("edges") or [] + ], + "claim_evidence_assessments": [ + { + "id": row.get("id"), + "claim_id": row.get("claim_id"), + "support_tier": row.get("support_tier"), + "challenge_tier": row.get("challenge_tier"), + "overall_state": row.get("overall_state"), + "rationale": row.get("rationale"), + "evidence_set_hash": row.get("evidence_set_hash"), + "accepted_by_proposal_id": proposal_id, + "supersedes_assessment_id": row.get("supersedes_assessment_id"), + } + for row in apply_payload.get("assessments") or [] + ], + "reasoning_tools": [], + } + + def _assert_semantic_rows_match(proposal: dict[str, Any], rows: dict[str, list[dict[str, Any]]]) -> None: expected = _expected_semantic_rows(proposal) for table, expected_rows in expected.items(): fields = tuple(expected_rows[0]) if expected_rows else () + expected_projected = [_semantic_projection(row, fields) for row in expected_rows] actual_projected = [_semantic_projection(row, fields) for row in rows.get(table, [])] - if sorted(expected_rows, key=_canonical_json) != sorted(actual_projected, key=_canonical_json): + if sorted(expected_projected, key=_canonical_json) != sorted(actual_projected, key=_canonical_json): raise ValueError( f"row-level postflight semantic mismatch for {table}; " "canonical values do not match the immutable strict payload" @@ -397,17 +626,21 @@ def build_replay_receipt( _assert_semantic_rows_match(proposal, rows) proposal_envelope = { - "id": str(proposal.get("id")), + "id": _canonical_uuid(proposal.get("id"), path="proposal.id"), "proposal_type": proposal.get("proposal_type"), "status": proposal.get("status"), "payload": proposal.get("payload"), "reviewed_by_handle": proposal.get("reviewed_by_handle"), - "reviewed_by_agent_id": proposal.get("reviewed_by_agent_id"), - "reviewed_at": proposal.get("reviewed_at"), + "reviewed_by_agent_id": _normalize_typed_value( + "reviewed_by_agent_id", proposal.get("reviewed_by_agent_id"), path="proposal.reviewed_by_agent_id" + ), + "reviewed_at": _normalize_typed_value("reviewed_at", proposal.get("reviewed_at"), path="proposal.reviewed_at"), "review_note": proposal.get("review_note"), "applied_by_handle": proposal.get("applied_by_handle"), - "applied_by_agent_id": proposal.get("applied_by_agent_id"), - "applied_at": proposal.get("applied_at"), + "applied_by_agent_id": _normalize_typed_value( + "applied_by_agent_id", proposal.get("applied_by_agent_id"), path="proposal.applied_by_agent_id" + ), + "applied_at": _normalize_typed_value("applied_at", proposal.get("applied_at"), path="proposal.applied_at"), } row_hashes = {table: [_sha256_json(row) for row in table_rows] for table, table_rows in rows.items()} table_hashes = {table: _sha256_json(table_rows) for table, table_rows in rows.items()} @@ -423,7 +656,9 @@ def build_replay_receipt( return { **replay_material, "artifact": "kb_apply_replay_receipt", - "exported_at_utc": exported_at_utc or datetime.now(timezone.utc).isoformat(), + "exported_at_utc": canonical_utc_timestamp( + exported_at_utc or datetime.now(timezone.utc).isoformat(), path="exported_at_utc" + ), "expected_row_counts": expected, "actual_row_counts": actual, "checks": { @@ -464,7 +699,9 @@ def validate_replay_receipt( apply_engine = receipt.get("apply_engine") if not isinstance(proposal, dict) or not isinstance(rows, dict) or not isinstance(apply_engine, dict): raise ValueError("replay receipt is missing proposal, canonical rows, or apply-engine metadata") - if expected_proposal_id is not None and proposal.get("id") != expected_proposal_id: + if expected_proposal_id is not None and proposal.get("id") != _canonical_uuid( + expected_proposal_id, path="expected_proposal_id" + ): raise ValueError("replay receipt proposal id does not match") try: rebuilt = build_replay_receipt( diff --git a/scripts/kb_proposal_review_packet.py b/scripts/kb_proposal_review_packet.py index 4faaab5..c55819b 100755 --- a/scripts/kb_proposal_review_packet.py +++ b/scripts/kb_proposal_review_packet.py @@ -18,6 +18,7 @@ a file for tests and UI development. from __future__ import annotations import argparse +import hashlib import json import sys from pathlib import Path @@ -30,6 +31,11 @@ import apply_proposal as ap # noqa: E402 import apply_worker as aw # noqa: E402 +def canonical_sha256(value: Any) -> str: + encoded = json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def _payload_list(payload: dict[str, Any], key: str) -> list[Any]: value = payload.get(key) return value if isinstance(value, list) else [] @@ -47,7 +53,9 @@ def classify_proposal(proposal: dict[str, Any]) -> dict[str, Any]: status = proposal.get("status") proposal_type = proposal.get("proposal_type") - has_apply_payload = "apply_payload" in payload + apply_payload = _payload_dict(payload, "apply_payload") + has_apply_payload = bool(apply_payload) + contract_version = apply_payload.get("contract_version", 2) if has_apply_payload else None worker_supported_type = proposal_type in aw.WORKER_TYPES worker_applyable = ( status == "approved" and worker_supported_type and has_apply_payload @@ -73,6 +81,11 @@ def classify_proposal(proposal: dict[str, Any]) -> dict[str, Any]: governance_standard = _payload_dict(payload, "proposed_governance_standard") approval_request = _payload_dict(payload, "approval_request") review_request = _payload_dict(payload, "review_request") + strict_rows = { + key: _payload_list(apply_payload, key) + for key in ("claims", "sources", "evidence", "edges", "assessments", "reasoning_tools") + } + strict_normalization_manifest = _payload_dict(apply_payload, "normalization_manifest") missing_contract = [] if not has_apply_payload: @@ -125,6 +138,7 @@ def classify_proposal(proposal: dict[str, Any]) -> dict[str, Any]: }, "payload_summary": { "proposal_kind": payload.get("proposal_kind"), + "contract_version": contract_version, "old_claim_id": _payload_dict(payload, "old_claim").get("id"), "claim_candidate_count": len(claim_candidates), "source_candidate_count": len(source_candidates), @@ -136,6 +150,13 @@ def classify_proposal(proposal: dict[str, Any]) -> dict[str, Any]: "do_not_approve_yet": approval_request.get("do_not_approve_yet", []), "requested_decision": review_request.get("requested_decision"), "not_requested": review_request.get("not_requested"), + "strict_row_counts": {key: len(rows) for key, rows in strict_rows.items()}, + }, + "strict_review_material": { + "apply_payload_sha256": canonical_sha256(apply_payload) if has_apply_payload else None, + "proposal_payload_sha256": canonical_sha256(payload), + "normalization_manifest": strict_normalization_manifest, + "rows": strict_rows, }, "applyability": { "missing_contract": missing_contract, @@ -225,8 +246,10 @@ def render_markdown(packets: list[dict[str, Any]]) -> str: f"- Worker applyable: {p['worker_applyable']}", f"- Reviewed by: {auth.get('reviewed_by_handle') or 'unreviewed'}", f"- Proposal kind: {summary.get('proposal_kind') or 'unknown'}", + f"- Contract version: {summary.get('contract_version') or 'none'}", f"- Claim candidates: {summary['claim_candidate_count']}", f"- Source candidates: {summary['source_candidate_count']}", + f"- Strict rows: {json.dumps(summary['strict_row_counts'], sort_keys=True)}", f"- Missing contract: {', '.join(applyability['missing_contract']) or 'none'}", f"- Next admin action: {p['next_admin_action']}", ] diff --git a/scripts/proposal_apply_lifecycle.py b/scripts/proposal_apply_lifecycle.py index 90cf91d..be31a86 100644 --- a/scripts/proposal_apply_lifecycle.py +++ b/scripts/proposal_apply_lifecycle.py @@ -119,6 +119,11 @@ def _require_approved_bundle(proposal: dict[str, Any]) -> dict[str, Any]: apply_payload = payload.get("apply_payload") if isinstance(payload, dict) else None if not isinstance(apply_payload, dict): raise ValueError("approved proposal requires payload.apply_payload") + if apply_payload.get("contract_version") != 2: + raise ValueError( + "compensating rollback supports approve_claim contract_version 2 only; " + "V3 admitted rows are immutable and require an append-only correction" + ) # Reuse the audited builder as the strict contract validator. ap.build_apply_sql(proposal, ap.SERVICE_AGENT_HANDLE) return apply_payload diff --git a/scripts/run_approve_claim_clone_canary.py b/scripts/run_approve_claim_clone_canary.py index c2e6d55..010db29 100644 --- a/scripts/run_approve_claim_clone_canary.py +++ b/scripts/run_approve_claim_clone_canary.py @@ -32,7 +32,7 @@ import apply_proposal as ap # noqa: E402 import proposal_apply_lifecycle as lifecycle # noqa: E402 SAFE_DB_RE = re.compile(r"^[a-z][a-z0-9_]{1,62}$") -REVIEWER_HANDLE = "m3ta" +REVIEWER_HANDLE = "m3taversal" REVIEW_NOTE = "Reviewed exact strict payload inside the disposable clone." DEFAULT_REVIEW_SECRETS_FILE = "/home/teleo/.hermes/profiles/leoclean/secrets/kb-review.env" SECURITY_DEFINER_DEPENDENCIES = { diff --git a/scripts/run_leo_integrated_learning_lifecycle.py b/scripts/run_leo_integrated_learning_lifecycle.py index 13e401d..9b4960b 100644 --- a/scripts/run_leo_integrated_learning_lifecycle.py +++ b/scripts/run_leo_integrated_learning_lifecycle.py @@ -17,6 +17,14 @@ The extracted text is used as both the hash-bound source artifact and its strict UTF-8 extraction. ``proposal-packet.json`` is an independently retained compiler expectation: the harness refuses to continue unless a fresh compile matches it byte-for-byte under canonical JSON. + +When every ``--genesis-*`` input is supplied, the harness copies every input +and execution engine into one private read-only staging directory. Before +Docker it validates the genesis against a repository-pinned trusted exporter +receipt/source identity and binds the executing commit and engine hashes. It +then restores and manifest-proves the real genesis without synthetic canonical +bootstrap or preseed and runs the gated apply worker lifecycle twice. The +detailed receipt is private mode 0600; stdout is sanitized. """ from __future__ import annotations @@ -25,41 +33,155 @@ import argparse import hashlib import json import os +import re import secrets import shutil +import stat import subprocess import sys import tempfile import time import uuid +from dataclasses import dataclass from pathlib import Path from typing import Any HERE = Path(__file__).resolve().parent REPO_ROOT = HERE.parent +OPS_DIR = REPO_ROOT / "ops" sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(OPS_DIR)) import apply_proposal as ap # noqa: E402 +import capture_vps_canonical_postgres_snapshot as snapshot_capture # noqa: E402 import compile_kb_source_packet as compiler # noqa: E402 import kb_proposal_review_packet as review_packets # noqa: E402 import proposal_apply_lifecycle as apply_lifecycle # noqa: E402 +import run_local_canonical_postgres_rebuild as canonical_rebuild # noqa: E402 import stage_normalized_proposal as stage # noqa: E402 +from private_receipt_io import print_private_receipt_summary, write_private_json # noqa: E402 +from verify_postgres_parity_manifest import load_manifest # noqa: E402 RECEIPT_SCHEMA = "livingip.leoIntegratedLearningLifecycleReceipt.v1" +GENESIS_RECEIPT_SCHEMA = "livingip.leoGenesisIntegratedLearningLifecycleReceipt.v1" +GENESIS_RECEIPT_ARTIFACT = "leo_genesis_integrated_learning_lifecycle" CONTAINER_LABEL = "livingip.canary=leo-integrated-learning-lifecycle" DATABASE = "teleo" -REVIEWER_HANDLE = "m3ta" +REVIEWER_HANDLE = "m3taversal" REVIEWER_ID = "11111111-1111-4111-8111-111111111111" DEFAULT_REVIEW_NOTE = ( "Approved the exact hash-bound candidate rows for this disposable integrated-learning lifecycle only." ) -POSTGRES_IMAGE = "postgres:16-alpine" +POSTGRES_IMAGE = os.environ.get("LEOCLEAN_RUNTIME_POSTGRES_IMAGE", "postgres:16-alpine") +DEFAULT_GENESIS_IMAGE = ( + "docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777" +) +SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") +IMAGE_DIGEST_RE = re.compile(r"\S+@sha256:[0-9a-f]{64}\Z") +PARITY_SQL_PATH = OPS_DIR / "postgres_parity_manifest.sql" +GUARD_PREREQUISITES_PATH = HERE / "kb_apply_prereqs.sql" +GUARD_PREREQUISITES_DESTINATION = "/tmp/kb-apply-prereqs.sql" +GENESIS_REPEAT_COUNT = 2 +TRUSTED_EXPORTER_ARTIFACT = "vps_canonical_postgres_exported_snapshot" +TRUSTED_EXPORTER_RECEIPTS: dict[str, dict[str, Any]] = { + "5e0030179d1ae9568e5619588ec1f10f573893eb036a15ce6a98e232a30de92b": { + "run_id": "canonical-20260715t204305z-final", + "capture_authorization_ref": "codex-delegation:019f47af-f2cc-7c10-b928-a9283afa2621", + "source": { + "ssh_target": "root@77.42.65.182", + "container": "teleo-pg", + "database": "teleo", + "service": "leoclean-gateway.service", + }, + "dump_sha256": "64eea334a3515a83d405e27b08c65891080afc7f2d051bbd198f07f2ccfabf3b", + "manifest_sha256": "ef04591b295d1a49e6baf6ee71a6b98f854916add9a2d3869c6ddbed1c8cef9b", + "manifest_sql_sha256": "67c287169394758df58fcab15103f5a9f9b185d266af82c35a3c2bc87aa79095", + "provenance_binding_sha256": "2442ea51b4048aedd5b370d840ee04745c21742525a6b2b950e58468d1d5bd08", + } +} +ENGINE_INPUT_PATHS = { + "engine.runner": HERE / "run_leo_integrated_learning_lifecycle.py", + "engine.apply_worker": HERE / "apply_worker.py", + "engine.apply_proposal": HERE / "apply_proposal.py", + "engine.approve_proposal": HERE / "approve_proposal.py", + "engine.apply_lifecycle": HERE / "proposal_apply_lifecycle.py", + "engine.apply_receipt": HERE / "kb_apply_replay_receipt.py", + "engine.compiler": HERE / "compile_kb_source_packet.py", + "engine.stage": HERE / "stage_normalized_proposal.py", + "engine.proposal_normalizer": HERE / "kb_proposal_normalize.py", + "engine.rich_proposal_plan": HERE / "kb_rich_proposal_creation_plan.py", + "engine.clone_checkpoint": HERE / "run_leo_clone_bound_handler_checkpoint.py", + "engine.review_packet": HERE / "kb_proposal_review_packet.py", + "engine.guard_sql": GUARD_PREREQUISITES_PATH, + "engine.canonical_rebuild": OPS_DIR / "run_local_canonical_postgres_rebuild.py", + "engine.parity_verifier": OPS_DIR / "verify_postgres_parity_manifest.py", + "engine.parity_sql": PARITY_SQL_PATH, + "engine.private_receipt": OPS_DIR / "private_receipt_io.py", + "engine.snapshot_exporter": OPS_DIR / "capture_vps_canonical_postgres_snapshot.py", +} +STAGED_ENGINE_DESTINATIONS = { + "engine.apply_worker": Path("engine/scripts/apply_worker.py"), + "engine.apply_proposal": Path("engine/scripts/apply_proposal.py"), + "engine.approve_proposal": Path("engine/scripts/approve_proposal.py"), + "engine.apply_lifecycle": Path("engine/scripts/proposal_apply_lifecycle.py"), + "engine.apply_receipt": Path("engine/scripts/kb_apply_replay_receipt.py"), + "engine.guard_sql": Path("engine/scripts/kb_apply_prereqs.sql"), + "engine.parity_sql": Path("engine/ops/postgres_parity_manifest.sql"), +} +PAYLOAD_CONTROLLED_FIELDS: dict[str, tuple[str, ...]] = { + "claims": ("id", "type", "text", "status", "confidence", "tags", "created_by", "superseded_by"), + "sources": ("id", "source_type", "url", "storage_path", "excerpt", "hash", "created_by"), + "claim_evidence": ("claim_id", "source_id", "role", "weight", "created_by"), + "claim_edges": ("from_claim", "to_claim", "edge_type", "weight", "created_by"), + "reasoning_tools": ("id", "agent_id", "name", "description", "category"), +} class LifecycleError(RuntimeError): """Raised when the isolated lifecycle cannot prove an exact transition.""" +@dataclass(frozen=True) +class GenesisInputs: + """Fully validated, immutable inputs for one direct genesis restore.""" + + dump: Path + dump_sha256: str + dump_bytes: int + manifest_path: Path + manifest_sha256: str + manifest_bytes: int + manifest: dict[str, Any] + image: str + exporter_receipt_path: Path + exporter_receipt_sha256: str + exporter_receipt: dict[str, Any] + source_identity: dict[str, str] + capture_authorization_ref: str + provenance_binding_sha256: str + trust_root_receipt_sha256: str + role_settings_capture_complete: bool + parity_sql_path: Path + guard_prerequisites_path: Path + approve_script_path: Path + apply_worker_script_path: Path + apply_script_path: Path + + +@dataclass(frozen=True) +class StagedLifecycleInputs: + """Private read-only byte set consumed by both genesis-backed runs.""" + + root: Path + source_packet: Path + genesis_dump: Path + genesis_manifest: Path + exporter_receipt: Path + files: dict[str, dict[str, Any]] + manifest_sha256: str + execution: dict[str, Any] + + def canonical_json_bytes(value: Any) -> bytes: return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") @@ -80,6 +202,457 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def absolute_path_value_pointers(value: Any, pointer: str = "$") -> list[str]: + """Return receipt locations that expose host-specific absolute paths.""" + if isinstance(value, dict): + paths: list[str] = [] + for key, child in value.items(): + paths.extend(absolute_path_value_pointers(child, f"{pointer}.{key}")) + return paths + if isinstance(value, list): + paths = [] + for index, child in enumerate(value): + paths.extend(absolute_path_value_pointers(child, f"{pointer}[{index}]")) + return paths + if isinstance(value, str) and value.startswith("/"): + return [pointer] + return [] + + +def manifest_for_role_replay(manifest: dict[str, Any]) -> dict[str, Any]: + """Project legacy role fields for replay without upgrading their parity claim.""" + normalized = json.loads(json.dumps(manifest)) + try: + role_items = normalized["singleton"]["application_roles"]["items"] + except (KeyError, TypeError) as exc: + raise LifecycleError("genesis manifest is missing application role inventory") from exc + if not isinstance(role_items, list): + raise LifecycleError("genesis manifest application role inventory is malformed") + for item in role_items: + if not isinstance(item, dict): + raise LifecycleError("genesis manifest application role entry is malformed") + if "settings" not in item: + read_only = item.get("default_transaction_read_only") + item["settings"] = ( + [{"name": "default_transaction_read_only", "value": read_only}] if read_only is not None else [] + ) + item.setdefault("database_settings", []) + return normalized + + +def role_settings_capture_complete(manifest: dict[str, Any]) -> bool: + try: + items = manifest["singleton"]["application_roles"]["items"] + except (KeyError, TypeError): + return False + return isinstance(items, list) and all( + isinstance(item, dict) + and isinstance(item.get("settings"), list) + and isinstance(item.get("database_settings"), list) + for item in items + ) + + +def _regular_non_symlink(path: Path, label: str) -> os.stat_result: + try: + result = path.lstat() + except OSError as exc: + raise LifecycleError(f"{label} is unavailable: {exc}") from exc + if not stat.S_ISREG(result.st_mode) or path.is_symlink(): + raise LifecycleError(f"{label} must be a regular non-symlink file") + return result + + +def _copy_private_input(source: Path, destination: Path, label: str) -> dict[str, Any]: + """Copy one regular file through no-follow descriptors and hash copied bytes.""" + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + source_fd = os.open(source, flags) + except OSError as exc: + raise LifecycleError(f"{label} could not be opened safely: {exc}") from exc + digest = hashlib.sha256() + byte_count = 0 + try: + source_stat = os.fstat(source_fd) + if not stat.S_ISREG(source_stat.st_mode): + raise LifecycleError(f"{label} must remain a regular file while staging") + destination_fd = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + while True: + chunk = os.read(source_fd, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + byte_count += len(chunk) + view = memoryview(chunk) + while view: + written = os.write(destination_fd, view) + view = view[written:] + os.fsync(destination_fd) + finally: + os.close(destination_fd) + finally: + os.close(source_fd) + destination.chmod(0o400) + return { + "relative_path": str(destination), + "bytes": byte_count, + "sha256": digest.hexdigest(), + "mode": "0400", + } + + +def execution_identity() -> dict[str, Any]: + engine_relpaths = { + logical_name: str(path.relative_to(REPO_ROOT)) for logical_name, path in sorted(ENGINE_INPUT_PATHS.items()) + } + head = run_tool( + ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"], + action="executing Git commit readback", + ).stdout.strip() + if not re.fullmatch(r"[0-9a-f]{40}", head): + raise LifecycleError("executing Git commit readback was malformed") + + head_sha256: dict[str, str] = {} + for logical_name, relative_path in engine_relpaths.items(): + source = ENGINE_INPUT_PATHS[logical_name] + _regular_non_symlink(source, logical_name) + committed = subprocess.run( + ["git", "-C", str(REPO_ROOT), "show", f"{head}:{relative_path}"], + capture_output=True, + check=False, + ) + if committed.returncode != 0: + raise LifecycleError(f"executing engine dependency is not tracked at {head}: {relative_path}") + current_bytes = source.read_bytes() + if current_bytes != committed.stdout: + raise LifecycleError(f"executing engine dependency does not match {head}: {relative_path}") + head_sha256[logical_name] = hashlib.sha256(committed.stdout).hexdigest() + + return { + "git_head": head, + "engine_paths_match_head": True, + "engine_relpaths": sorted(engine_relpaths.values()), + "engine_head_sha256": head_sha256, + } + + +def stage_lifecycle_inputs( + source_packet: Path, + genesis_dump: Path, + genesis_manifest: Path, + genesis_exporter_receipt: Path, +) -> StagedLifecycleInputs: + """Copy every external input and subprocess engine into one private byte set.""" + source_packet = source_packet.expanduser().resolve() + sources: dict[str, tuple[Path, Path]] = { + "genesis.dump": (genesis_dump.expanduser().resolve(), Path("inputs/genesis/teleo-canonical.dump")), + "genesis.manifest": ( + genesis_manifest.expanduser().resolve(), + Path("inputs/genesis/source-manifest.jsonl"), + ), + "genesis.exporter_receipt": ( + genesis_exporter_receipt.expanduser().resolve(), + Path("inputs/genesis/exporter-receipt.json"), + ), + "source.extracted_text": ( + source_packet / "prepared" / "extracted-text.txt", + Path("inputs/source-packet/prepared/extracted-text.txt"), + ), + "source.manifest": ( + source_packet / "prepared" / "source-manifest.json", + Path("inputs/source-packet/prepared/source-manifest.json"), + ), + "source.proposal_packet": ( + source_packet / "proposal-packet.json", + Path("inputs/source-packet/proposal-packet.json"), + ), + } + for logical_name, source in ENGINE_INPUT_PATHS.items(): + destination = STAGED_ENGINE_DESTINATIONS.get( + logical_name, + Path("engine/bound") / f"{logical_name.replace('.', '__')}--{source.name}", + ) + sources[logical_name] = (source, destination) + + for logical_name, (source, _destination) in sources.items(): + _regular_non_symlink(source, logical_name) + + root = Path(tempfile.mkdtemp(prefix="leo-genesis-immutable-stage-")) + root.chmod(0o700) + files: dict[str, dict[str, Any]] = {} + try: + for logical_name, (source, relative_destination) in sorted(sources.items()): + destination = root / relative_destination + evidence = _copy_private_input(source, destination, logical_name) + evidence["relative_path"] = str(relative_destination) + files[logical_name] = evidence + execution = execution_identity() + staged_engine_sha256 = { + logical_name: files[logical_name]["sha256"] for logical_name in sorted(ENGINE_INPUT_PATHS) + } + if staged_engine_sha256 != execution["engine_head_sha256"]: + raise LifecycleError("staged engine dependency bytes do not match the executing Git commit") + manifest_payload = { + "schema": "livingip.leoImmutableLifecycleInputs.v1", + "execution": execution, + "files": files, + } + manifest_sha256 = canonical_sha256(manifest_payload) + for directory in sorted((path for path in root.rglob("*") if path.is_dir()), reverse=True): + directory.chmod(0o500) + root.chmod(0o500) + staged = StagedLifecycleInputs( + root=root, + source_packet=root / "inputs/source-packet", + genesis_dump=root / "inputs/genesis/teleo-canonical.dump", + genesis_manifest=root / "inputs/genesis/source-manifest.jsonl", + exporter_receipt=root / "inputs/genesis/exporter-receipt.json", + files=files, + manifest_sha256=manifest_sha256, + execution=execution, + ) + verify_staged_inputs(staged) + return staged + except Exception: + cleanup_staged_inputs(root) + raise + + +def verify_staged_inputs(staged: StagedLifecycleInputs) -> dict[str, Any]: + expected_paths = {str(item["relative_path"]) for item in staged.files.values()} + actual_paths = { + str(path.relative_to(staged.root)) for path in staged.root.rglob("*") if path.is_file() or path.is_symlink() + } + if actual_paths != expected_paths: + raise LifecycleError("private immutable staging file set drifted") + observed: dict[str, dict[str, Any]] = {} + for logical_name, expected in sorted(staged.files.items()): + path = staged.root / str(expected["relative_path"]) + path_stat = _regular_non_symlink(path, logical_name) + if stat.S_IMODE(path_stat.st_mode) != 0o400: + raise LifecycleError(f"private immutable staging mode drifted for {logical_name}") + observed[logical_name] = { + "relative_path": str(expected["relative_path"]), + "bytes": path_stat.st_size, + "sha256": sha256_file(path), + "mode": "0400", + } + manifest_payload = { + "schema": "livingip.leoImmutableLifecycleInputs.v1", + "execution": staged.execution, + "files": observed, + } + observed_manifest_sha256 = canonical_sha256(manifest_payload) + if observed != staged.files or observed_manifest_sha256 != staged.manifest_sha256: + raise LifecycleError("private immutable staging bytes drifted from preflight") + return { + "manifest_sha256": observed_manifest_sha256, + "files_sha256": canonical_sha256({name: row["sha256"] for name, row in observed.items()}), + "file_count": len(observed), + "all_files_match_preflight": True, + } + + +def cleanup_staged_inputs(root: Path | None) -> dict[str, Any]: + if root is None: + return {"attempted": False, "root_absent": True} + if root.exists(): + for path in root.rglob("*"): + try: + path.chmod(0o700 if path.is_dir() else 0o600) + except OSError: + pass + try: + root.chmod(0o700) + except OSError: + pass + shutil.rmtree(root, ignore_errors=True) + return {"attempted": True, "root_absent": not root.exists()} + + +def _service_state_healthy(value: Any) -> bool: + if not isinstance(value, dict): + return False + try: + main_pid = int(value.get("MainPID") or 0) + restarts = int(value.get("NRestarts") or 0) + except (TypeError, ValueError): + return False + return ( + value.get("ActiveState") == "active" and value.get("SubState") == "running" and main_pid > 0 and restarts >= 0 + ) + + +def validate_genesis_inputs( + genesis_dump: Path, + genesis_manifest: Path, + genesis_exporter_receipt: Path, + image: str, + *, + parity_sql_path: Path = PARITY_SQL_PATH, + guard_prerequisites_path: Path = GUARD_PREREQUISITES_PATH, + approve_script_path: Path = HERE / "approve_proposal.py", + apply_worker_script_path: Path = HERE / "apply_worker.py", + apply_script_path: Path = HERE / "apply_proposal.py", +) -> GenesisInputs: + """Validate staged genesis bytes against the exporter receipt, without Docker.""" + if not IMAGE_DIGEST_RE.fullmatch(image): + raise LifecycleError("--image must be pinned by a lowercase sha256 digest") + dump = genesis_dump.expanduser().resolve() + manifest_path = genesis_manifest.expanduser().resolve() + receipt_path = genesis_exporter_receipt.expanduser().resolve() + if len({dump, manifest_path, receipt_path}) != 3: + raise LifecycleError("genesis dump, manifest, and exporter receipt must be distinct files") + _regular_non_symlink(receipt_path, "genesis exporter receipt") + if receipt_path.stat().st_mode & 0o077: + raise LifecycleError("genesis exporter receipt permissions must be private") + try: + canonical_rebuild.validate_custom_dump(dump) + except (OSError, ValueError) as exc: + raise LifecycleError(f"invalid genesis dump: {exc}") from exc + try: + exporter_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + if not isinstance(exporter_receipt, dict): + raise LifecycleError("genesis exporter receipt must contain one JSON object") + manifest = load_manifest(manifest_path) + role_manifest = manifest_for_role_replay(manifest) + canonical_rebuild.application_role_sql(role_manifest, DATABASE) + except (OSError, UnicodeError, ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise LifecycleError("genesis exporter receipt or manifest is invalid") from exc + + dump_sha256 = sha256_file(dump) + manifest_sha256 = sha256_file(manifest_path) + exporter_receipt_sha256 = sha256_file(receipt_path) + trust_root = TRUSTED_EXPORTER_RECEIPTS.get(exporter_receipt_sha256) + receipt_dump = exporter_receipt.get("dump") or {} + receipt_manifest = exporter_receipt.get("manifest") or {} + source = exporter_receipt.get("source") or {} + snapshot = exporter_receipt.get("snapshot") or {} + source_context = exporter_receipt.get("source_context") or {} + source_service = exporter_receipt.get("source_service") or {} + binding = exporter_receipt.get("provenance_binding") or {} + authorization_ref = str(exporter_receipt.get("capture_authorization_ref") or "") + manifest_sql_sha256 = str(receipt_manifest.get("manifest_sql_sha256") or "") + source_context_sha256 = str(source_context.get("sha256") or "") + source_identity_valid = ( + isinstance(source, dict) + and set(source) == {"ssh_target", "container", "database", "service"} + and all( + isinstance(value, str) and value and not any(char in value for char in "\r\n\x00") + for value in source.values() + ) + ) + expected_binding = snapshot_capture.build_provenance_binding( + run_id=str(exporter_receipt.get("run_id") or ""), + authorization_ref=authorization_ref, + source=source, + snapshot=snapshot, + dump_sha256=dump_sha256, + manifest_sql_sha256=manifest_sql_sha256, + source_manifest_sha256=manifest_sha256, + source_context_sha256=source_context_sha256, + ) + application_roles = manifest["singleton"]["application_roles"].get("items", []) + table_count = len(manifest["tables"]) + total_rows = sum(int(row["row_count"]) for row in manifest["tables"].values()) + before = source_service.get("before") + after = source_service.get("after") + checks = { + "trusted_exporter_receipt_hash": trust_root is not None, + "trusted_exporter_run_id": trust_root is not None and exporter_receipt.get("run_id") == trust_root["run_id"], + "trusted_capture_authorization": trust_root is not None + and authorization_ref == trust_root["capture_authorization_ref"], + "trusted_source_identity": trust_root is not None and source == trust_root["source"], + "trusted_dump_identity": trust_root is not None and dump_sha256 == trust_root["dump_sha256"], + "trusted_manifest_identity": trust_root is not None and manifest_sha256 == trust_root["manifest_sha256"], + "trusted_manifest_engine": trust_root is not None and manifest_sql_sha256 == trust_root["manifest_sql_sha256"], + "trusted_provenance_binding": trust_root is not None + and binding.get("sha256") == trust_root["provenance_binding_sha256"], + "artifact": exporter_receipt.get("artifact") == TRUSTED_EXPORTER_ARTIFACT, + "schema": exporter_receipt.get("schema") == snapshot_capture.SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "receipt_version": exporter_receipt.get("receipt_version") == 2, + "status": exporter_receipt.get("status") == "pass", + "run_id": bool(exporter_receipt.get("run_id")), + "capture_authorization_bound": bool(authorization_ref), + "source_identity_bound": source_identity_valid, + "snapshot_exported": exporter_receipt.get("snapshot_exported") is True, + "snapshot_identity_bound": all( + bool(snapshot.get(field)) + for field in ("captured_at_utc", "exported_snapshot_id", "txid_snapshot", "wal_lsn", "system_identifier") + ), + "source_service_unchanged": exporter_receipt.get("service_unchanged") is True + and source_service.get("unchanged") is True + and before == after + and _service_state_healthy(before), + "source_not_mutated": exporter_receipt.get("production_db_mutated") is False, + "telegram_not_sent": exporter_receipt.get("telegram_send_attempted") is False, + "dump_hash_from_exporter_receipt": receipt_dump.get("sha256") == dump_sha256, + "dump_bytes_from_exporter_receipt": receipt_dump.get("bytes") == dump.stat().st_size, + "manifest_hash_from_exporter_receipt": receipt_manifest.get("sha256") == manifest_sha256, + "manifest_sql_hash_bound": bool(SHA256_RE.fullmatch(manifest_sql_sha256)), + "manifest_table_count_bound": receipt_manifest.get("table_count") == table_count, + "manifest_total_rows_bound": receipt_manifest.get("total_rows") == total_rows, + "manifest_application_roles_bound": receipt_manifest.get("application_roles") == application_roles, + "manifest_database_bound": source_identity_valid + and source.get("database") == manifest["singleton"]["identity"].get("database"), + "manifest_captured_read_only": manifest["singleton"]["identity"].get("transaction_read_only") == "on", + "source_context_hash_bound": bool(SHA256_RE.fullmatch(source_context_sha256)), + "provenance_binding_payload": binding.get("payload") == expected_binding["payload"], + "provenance_binding_sha256": binding.get("sha256") == expected_binding["sha256"], + "provenance_binding_algorithm": binding.get("algorithm") == "sha256", + } + failed = sorted(name for name, passed in checks.items() if not passed) + if failed: + raise LifecycleError("genesis exporter receipt failed validation: " + ", ".join(failed)) + return GenesisInputs( + dump=dump, + dump_sha256=dump_sha256, + dump_bytes=dump.stat().st_size, + manifest_path=manifest_path, + manifest_sha256=manifest_sha256, + manifest_bytes=manifest_path.stat().st_size, + manifest=manifest, + image=image, + exporter_receipt_path=receipt_path, + exporter_receipt_sha256=exporter_receipt_sha256, + exporter_receipt=exporter_receipt, + source_identity={key: str(value) for key, value in source.items()}, + capture_authorization_ref=authorization_ref, + provenance_binding_sha256=str(binding["sha256"]), + trust_root_receipt_sha256=exporter_receipt_sha256, + role_settings_capture_complete=role_settings_capture_complete(manifest), + parity_sql_path=parity_sql_path, + guard_prerequisites_path=guard_prerequisites_path, + approve_script_path=approve_script_path, + apply_worker_script_path=apply_worker_script_path, + apply_script_path=apply_script_path, + ) + + +def payload_controlled_fingerprint(rows: dict[str, list[dict[str, Any]]]) -> dict[str, Any]: + """Hash only immutable payload-controlled fields, excluding generated IDs/times.""" + if set(rows) != set(PAYLOAD_CONTROLLED_FIELDS): + raise LifecycleError("canonical target rows do not contain the exact approve-claim table set") + projected: dict[str, list[dict[str, Any]]] = {} + for family, fields in PAYLOAD_CONTROLLED_FIELDS.items(): + family_rows = rows[family] + if not isinstance(family_rows, list) or any(not isinstance(row, dict) for row in family_rows): + raise LifecycleError(f"canonical target rows for {family} are malformed") + projected[family] = sorted( + ({field: row.get(field) for field in fields} for row in family_rows), + key=canonical_json_bytes, + ) + return { + "schema": "livingip.leoPayloadControlledRowsFingerprint.v1", + "sha256": canonical_sha256(projected), + "table_sha256": {family: canonical_sha256(family_rows) for family, family_rows in projected.items()}, + "row_counts": _row_counts(projected), + "rows": projected, + } + + def parse_last_json_line(output: str) -> Any: for line in reversed(output.splitlines()): line = line.strip() @@ -120,9 +693,20 @@ def run_tool(arguments: list[str], *, action: str) -> subprocess.CompletedProces class DisposablePostgres: - def __init__(self, name: str, admin_password: str) -> None: + def __init__( + self, + name: str, + admin_password: str, + *, + image: str = POSTGRES_IMAGE, + genesis_backed: bool = False, + run_group: str | None = None, + ) -> None: self.name = name self.admin_password = admin_password + self.image = image + self.genesis_backed = genesis_backed + self.run_group = run_group or uuid.uuid4().hex self.docker = shutil.which("docker") if not self.docker: raise LifecycleError("docker executable is unavailable") @@ -149,6 +733,7 @@ class DisposablePostgres: return result def start(self) -> dict[str, Any]: + auth_environment = ["-e", f"POSTGRES_PASSWORD={self.admin_password}"] result = self._run( [ "run", @@ -162,13 +747,14 @@ class DisposablePostgres: CONTAINER_LABEL, "--label", f"livingip.canary.instance={self.name}", + "--label", + f"livingip.canary.run_group={self.run_group}", "--tmpfs", "/var/lib/postgresql/data:rw,nosuid,size=384m", "-e", - f"POSTGRES_PASSWORD={self.admin_password}", - "-e", f"POSTGRES_DB={DATABASE}", - POSTGRES_IMAGE, + *auth_environment, + self.image, ] ) self.started = True @@ -203,11 +789,15 @@ class DisposablePostgres: "container_id": result.stdout.strip(), "container_name": self.name, "image": inspect.get("Config", {}).get("Image"), + "image_reference_exact": inspect.get("Config", {}).get("Image") == self.image, + "image_digest_pinned": bool(IMAGE_DIGEST_RE.fullmatch(self.image)) if self.genesis_backed else None, + "host_auth_trust_requested": False, "network_mode": inspect.get("HostConfig", {}).get("NetworkMode"), "tmpfs_data_dir": "/var/lib/postgresql/data" in inspect.get("HostConfig", {}).get("Tmpfs", {}), "mounts": mounts, "canary_label": labels.get("livingip.canary"), "instance_label": labels.get("livingip.canary.instance"), + "run_group_label": labels.get("livingip.canary.run_group"), } if self.environment["network_mode"] != "none": raise LifecycleError("disposable Postgres is not networkless") @@ -217,6 +807,12 @@ class DisposablePostgres: raise LifecycleError("disposable Postgres canary label is missing") if self.environment["instance_label"] != self.name: raise LifecycleError("disposable Postgres instance label is missing") + if self.environment["run_group_label"] != self.run_group: + raise LifecycleError("disposable Postgres run-group label is missing") + if not self.environment["image_reference_exact"]: + raise LifecycleError("disposable Postgres did not use the exact requested image reference") + if self.genesis_backed and not self.environment["image_digest_pinned"]: + raise LifecycleError("genesis-backed Postgres image is not digest pinned") return self.environment def psql(self, sql: str) -> str: @@ -235,7 +831,7 @@ class DisposablePostgres: "ON_ERROR_STOP=1", "-Atq", ], - input_text=sql, + input_text="set timezone = 'UTC';\n" + sql, ) return result.stdout.strip() @@ -246,6 +842,185 @@ class DisposablePostgres: raise LifecycleError("Postgres readback did not contain JSON") return parsed + def restore_genesis(self, inputs: GenesisInputs) -> dict[str, Any]: + """Restore one validated genesis directly and prove its parity manifest.""" + if not self.genesis_backed: + raise LifecycleError("genesis restore requires a genesis-backed disposable database") + role_manifest = manifest_for_role_replay(inputs.manifest) + role_capture_complete = inputs.role_settings_capture_complete + try: + application_roles = canonical_rebuild.execute_application_role_bootstrap( + self.docker, + self.name, + DATABASE, + role_manifest, + timeout=30.0, + ) + role_memberships = canonical_rebuild.execute_role_membership_replay( + self.docker, + self.name, + DATABASE, + role_manifest, + timeout=30.0, + ) + except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as exc: + raise LifecycleError("manifest-declared application role bootstrap failed") from exc + + copy_dump = self._run( + ["cp", str(inputs.dump), f"{self.name}:{canonical_rebuild.DUMP_DESTINATION}"], + ) + restore = self._run( + [ + "exec", + self.name, + "pg_restore", + "-U", + "postgres", + "-d", + DATABASE, + "--no-owner", + "--exit-on-error", + canonical_rebuild.DUMP_DESTINATION, + ] + ) + try: + database_acl_statements = canonical_rebuild.execute_database_acl_replay( + self.docker, + self.name, + DATABASE, + role_manifest, + timeout=30.0, + ) + parity = canonical_rebuild.run_parity_check( + self.docker, + self.name, + DATABASE, + source_manifest_path=inputs.manifest_path, + source_manifest=role_manifest, + manifest_sql=inputs.parity_sql_path, + command_timeout=180.0, + max_target_query_ms=2000.0, + max_target_source_ratio=20.0, + ) + key_counts, schema_table_count = canonical_rebuild.collect_key_counts( + self.docker, + self.name, + DATABASE, + timeout=30.0, + ) + except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired, json.JSONDecodeError) as exc: + raise LifecycleError("restored genesis manifest proof could not complete") from exc + if parity.get("status") != "pass": + problems = _safe_error_text(json.dumps(parity.get("problems", []), sort_keys=True)) + raise LifecycleError(f"restored genesis does not match the supplied parity manifest: {problems}") + copy_guards = self._run( + ["cp", str(inputs.guard_prerequisites_path), f"{self.name}:{GUARD_PREREQUISITES_DESTINATION}"], + ) + install_guards = self._run( + [ + "exec", + self.name, + "psql", + "-X", + "-U", + "postgres", + "-d", + DATABASE, + "-Atq", + "-v", + "ON_ERROR_STOP=1", + "-f", + GUARD_PREREQUISITES_DESTINATION, + ] + ) + source_manifest = dict(parity["source_manifest"]) + source_manifest.pop("path", None) + return { + "dump": { + "bytes": inputs.dump_bytes, + "sha256": inputs.dump_sha256, + "exporter_receipt_binding_verified_before_docker": True, + "copy_command_stdout_sha256": sha256_text(copy_dump.stdout), + }, + "manifest": { + "bytes": inputs.manifest_bytes, + "sha256": inputs.manifest_sha256, + "exporter_receipt_binding_verified_before_docker": True, + "compatibility_projection": { + "raw_bytes_unchanged": True, + "role_settings_capture_complete": role_capture_complete, + "derived_fields": ( + [] + if role_capture_complete + else [ + "application_roles[].settings", + "application_roles[].database_settings", + ] + ), + "derivation": ( + None + if role_capture_complete + else "settings mirrors captured default_transaction_read_only; database_settings defaults empty" + ), + }, + }, + "exporter_receipt": { + "sha256": inputs.exporter_receipt_sha256, + "trust_root_receipt_sha256": inputs.trust_root_receipt_sha256, + "schema": inputs.exporter_receipt["schema"], + "artifact": inputs.exporter_receipt["artifact"], + "run_id": inputs.exporter_receipt["run_id"], + "capture_authorization_ref": inputs.capture_authorization_ref, + "source_identity": inputs.source_identity, + "provenance_binding_sha256": inputs.provenance_binding_sha256, + "trusted_contract_validated_before_docker": True, + }, + "image": { + "reference": inputs.image, + "digest_pinned": True, + "runtime_reference_exact": self.environment.get("image_reference_exact") is True, + }, + "restore": { + "direct_custom_format_pg_restore": True, + "snapshot_repackaged": False, + "synthetic_schema_bootstrap_skipped": True, + "canonical_payload_preseed_performed": False, + "application_roles_from_manifest": application_roles, + "role_membership_statement_count": role_memberships, + "database_acl_statement_count": database_acl_statements, + "pg_restore_stdout_sha256": sha256_text(restore.stdout), + "pg_restore_stderr_sha256": sha256_text(restore.stderr), + "guard_prerequisites_sha256": sha256_file(inputs.guard_prerequisites_path), + "manifest_parity_proven_before_guard_install": True, + "guard_copy_stdout_sha256": sha256_text(copy_guards.stdout), + "guard_install_stdout_sha256": sha256_text(install_guards.stdout), + }, + "parity": { + "status": parity["status"], + "verifier": parity["verifier"], + "source_manifest": source_manifest, + "target_manifest": parity["target_manifest"], + "problems": parity["problems"], + "claim_scope": ( + "full_captured_manifest_including_role_and_database_settings" + if role_capture_complete + else "data_schema_and_captured_role_attributes_only" + ), + "role_global_parity": { + "status": "proven" if role_capture_complete else "not_proven", + "captured_source_settings_present": role_capture_complete, + "reason": ( + None + if role_capture_complete + else "source manifest omitted role settings/database settings; replay used a compatibility projection" + ), + }, + "details": parity["details"], + }, + "key_counts": key_counts, + "canonical_schema_table_count": schema_table_count, + } + def cleanup(self) -> dict[str, Any]: remove = self._run(["rm", "-f", self.name], check=False) if self.started else None inspect = self._run(["inspect", self.name], check=False) @@ -267,12 +1042,27 @@ class DisposablePostgres: ], check=False, ) + run_group = self._run( + [ + "container", + "ls", + "-a", + "--filter", + f"label=livingip.canary.run_group={self.run_group}", + "--filter", + f"label=livingip.canary.instance={self.name}", + "--format", + "{{.Names}}", + ], + check=False, + ) return { "remove_attempted": self.started, "remove_returncode": remove.returncode if remove else None, "container_absent": inspect.returncode != 0, "name_readback_clear": names.returncode == 0 and not names.stdout.strip(), "instance_label_readback_clear": labels.returncode == 0 and not labels.stdout.strip(), + "run_group_readback_clear": run_group.returncode == 0 and not run_group.stdout.strip(), "network_mode_was_none": self.environment.get("network_mode") == "none", "tmpfs_data_dir_was_present": self.environment.get("tmpfs_data_dir") is True, "docker_volume_mounts_observed": self.environment.get("mounts", []), @@ -396,6 +1186,48 @@ values ({ap.sql_literal(REVIEWER_ID)}::uuid, {ap.sql_literal(REVIEWER_HANDLE)}, """ +def initialize_database( + postgres: DisposablePostgres, + *, + genesis: GenesisInputs | None, + review_password: str, + apply_password: str, +) -> dict[str, Any]: + """Initialize exactly one allowed database mode.""" + if genesis is not None: + restored = postgres.restore_genesis(genesis) + postgres.psql( + f"alter role kb_review password {ap.sql_literal(review_password)};\n" + f"alter role kb_apply password {ap.sql_literal(apply_password)};" + ) + return { + "mode": "exporter_receipt_bound_genesis_restore", + "synthetic_schema_bootstrap_skipped": True, + "canonical_payload_preseed_performed": False, + "worker_credentials_configured_for_password_auth": True, + "genesis": restored, + } + postgres.psql(bootstrap_sql(review_password, apply_password)) + postgres.psql(GUARD_PREREQUISITES_PATH.read_text(encoding="utf-8")) + return { + "mode": "legacy_synthetic_fixture_schema", + "synthetic_schema_bootstrap_skipped": False, + "canonical_payload_preseed_performed": False, + "worker_credentials_configured_for_password_auth": True, + } + + +def source_packet_input_paths(source_packet: Path) -> frozenset[Path]: + packet = source_packet.expanduser().resolve() + return frozenset( + { + packet / "prepared" / "extracted-text.txt", + packet / "prepared" / "source-manifest.json", + packet / "proposal-packet.json", + } + ) + + def compile_source_packet(source_packet: Path) -> tuple[dict[str, Any], dict[str, Any], dict[str, Path]]: paths = { "artifact": source_packet / "prepared" / "extracted-text.txt", @@ -428,13 +1260,15 @@ def proposal_readback(postgres: DisposablePostgres, proposal_id: str) -> dict[st 'payload', payload, 'reviewed_by_handle', reviewed_by_handle, 'reviewed_by_agent_id', reviewed_by_agent_id::text, - 'reviewed_at', reviewed_at::text, + 'reviewed_at', case when reviewed_at is null then null else + to_char(reviewed_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') end, 'review_note', review_note, 'applied_by_handle', applied_by_handle, 'applied_by_agent_id', applied_by_agent_id::text, - 'applied_at', applied_at::text, - 'created_at', created_at::text, - 'updated_at', updated_at::text + 'applied_at', case when applied_at is null then null else + to_char(applied_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') end, + 'created_at', to_char(created_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), + 'updated_at', to_char(updated_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') )::text from kb_stage.kb_proposals where id = {ap.sql_literal(proposal_id)}::uuid;""" @@ -452,7 +1286,7 @@ def approval_readback(postgres: DisposablePostgres, proposal_id: str) -> dict[st 'reviewed_by_handle', reviewed_by_handle, 'reviewed_by_agent_id', reviewed_by_agent_id::text, 'reviewed_by_db_role', reviewed_by_db_role::text, - 'reviewed_at', reviewed_at::text, + 'reviewed_at', to_char(reviewed_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), 'review_note', review_note )::text from kb_stage.kb_proposal_approvals @@ -565,12 +1399,36 @@ def _all_target_rows_empty(rows: dict[str, list[dict[str, Any]]]) -> bool: return set(rows) == set(apply_lifecycle.ROW_TABLES) and all(not table_rows for table_rows in rows.values()) +def require_fresh_initial_state(initial: dict[str, Any]) -> None: + """Reject proposal or canonical target preseeding before any staging action.""" + state = initial.get("state") + if not isinstance(state, dict): + raise LifecycleError("initial database state is malformed") + if state.get("proposal") is not None or state.get("immutable_approval") is not None: + raise LifecycleError("proposal or approval preseed is forbidden before staging") + rows = state.get("canonical_rows") + if not isinstance(rows, dict) or not _all_target_rows_empty(rows): + raise LifecycleError("canonical payload target preseed is forbidden before staging") + + def _row_counts(rows: dict[str, list[dict[str, Any]]]) -> dict[str, int]: return {family: len(table_rows) for family, table_rows in rows.items()} def normalized_canonical_rows(rows: dict[str, list[dict[str, Any]]]) -> dict[str, list[dict[str, Any]]]: - return {family: sorted(table_rows, key=canonical_json_bytes) for family, table_rows in sorted(rows.items())} + return { + family: sorted( + ( + ap.replay_receipt.normalize_typed_row( + row, + path=f"integrated_lifecycle.{family}", + ) + for row in table_rows + ), + key=canonical_json_bytes, + ) + for family, table_rows in sorted(rows.items()) + } def run_lifecycle( @@ -580,6 +1438,10 @@ def run_lifecycle( review_action: str, reviewed_by: str = REVIEWER_HANDLE, review_note: str = DEFAULT_REVIEW_NOTE, + genesis: GenesisInputs | None = None, + persist_receipt: bool = True, + run_group: str | None = None, + run_number: int | None = None, ) -> dict[str, Any]: if review_action != "approve": raise LifecycleError("the only supported explicit review action is 'approve'") @@ -590,22 +1452,36 @@ def run_lifecycle( source_packet = source_packet.resolve() output = output.resolve() + if output in source_packet_input_paths(source_packet): + raise LifecycleError("--output must not overwrite a retained source-packet input") temp_root = Path(tempfile.mkdtemp(prefix="leo-integrated-learning-")) - private_apply_receipt_path = temp_root / "apply-receipt.json" - container_name = f"leo-integrated-learning-{os.getpid()}-{uuid.uuid4().hex[:10]}" - postgres = DisposablePostgres(container_name, secrets.token_urlsafe(32)) + worker_receipt_dir = temp_root / "worker-receipts" + private_apply_receipt_path: Path | None = None + mode_slug = "genesis" if genesis is not None else "fixture" + container_name = f"leo-integrated-learning-{mode_slug}-{os.getpid()}-{uuid.uuid4().hex[:10]}" + run_group = run_group or uuid.uuid4().hex + postgres: DisposablePostgres | None = None receipt: dict[str, Any] = { + "artifact": "leo_integrated_learning_lifecycle_run", "schema": RECEIPT_SCHEMA, "status": "fail", - "required_tier": "realistic_isolated_local_container_end_to_end", + "mode": "exporter_receipt_bound_genesis" if genesis is not None else "legacy_synthetic_fixture", + "run_number": run_number, + "required_tier": ( + "T2_disposable_genesis_backed_runtime" + if genesis is not None + else "realistic_isolated_local_container_end_to_end" + ), "current_tier": "not_proven", - "source_packet": str(source_packet), + "source_packet": "source-packet", "review_action": review_action, "safety_boundary": { "self_created_container_only": True, "existing_database_target_argument_supported": False, "vps_or_gcp_target_supported": False, "telegram_send_performed": False, + "model_calls_performed": False, + "genesis_restore_requested": genesis is not None, }, "checks": {}, } @@ -616,17 +1492,23 @@ def run_lifecycle( parent = compiled["parent_proposal"] proposal_id = str(child["id"]) apply_payload = child["payload"]["apply_payload"] + worker_contract_version = apply_payload.get("contract_version", 2) + if worker_contract_version != 2: + raise LifecycleError("this integrated genesis lifecycle is pinned to the V2 apply contract") manifest = json.loads(paths["manifest"].read_text(encoding="utf-8")) receipt["source"] = { "source_identity": manifest["source"]["identity"], "source_locator": manifest["source"]["locator"], "artifact_sha256": sha256_file(paths["artifact"]), "manifest_canonical_sha256": canonical_sha256(manifest), + "retained_proposal_packet_file_sha256": sha256_file(paths["expected"]), "retained_proposal_packet_canonical_sha256": canonical_sha256(expected), } receipt["compiled"] = { "status": compiled["status"], "bundle_sha256": canonical_sha256(compiled), + "canonical_bytes_sha256": hashlib.sha256(canonical_json_bytes(compiled)).hexdigest(), + "retained_packet_canonical_bytes_sha256": hashlib.sha256(canonical_json_bytes(expected)).hexdigest(), "parent_proposal_id": parent["id"], "strict_proposal_id": proposal_id, "candidate_counts": { @@ -637,6 +1519,13 @@ def run_lifecycle( compiled_identities = exact_identities(child) receipt["identities"] = compiled_identities + postgres = DisposablePostgres( + container_name, + secrets.token_urlsafe(32), + image=genesis.image if genesis is not None else POSTGRES_IMAGE, + genesis_backed=genesis is not None, + run_group=run_group, + ) environment = postgres.start() receipt["environment"] = environment review_password = secrets.token_urlsafe(32) @@ -647,10 +1536,15 @@ def run_lifecycle( write_secret_file(review_secrets, "KB_REVIEW_PASSWORD", review_password) write_secret_file(apply_secrets, "KB_APPLY_PASSWORD", apply_password) - postgres.psql(bootstrap_sql(review_password, apply_password)) - postgres.psql((HERE / "kb_apply_prereqs.sql").read_text(encoding="utf-8")) + receipt["database_initialization"] = initialize_database( + postgres, + genesis=genesis, + review_password=review_password, + apply_password=apply_password, + ) initial = database_state(postgres, child) + require_fresh_initial_state(initial) stage_result = postgres.psql_json(stage.build_stage_sql(child)) pending = proposal_readback(postgres, proposal_id) if pending is None: @@ -668,9 +1562,10 @@ def run_lifecycle( }, } + approve_script = genesis.approve_script_path if genesis is not None else HERE / "approve_proposal.py" approve_base = [ *_tool_base( - HERE / "approve_proposal.py", + approve_script, proposal_id, review_secrets, "kb_review", @@ -710,30 +1605,96 @@ def run_lifecycle( preflight = apply_lifecycle.build_fresh_preflight(approved, target_rows_before) preflight_path.write_text(json.dumps(preflight, indent=2, sort_keys=True) + "\n", encoding="utf-8") preflight_path.chmod(0o600) - apply_base = [ - *_tool_base( - HERE / "apply_proposal.py", - proposal_id, - apply_secrets, - "kb_apply", - container_name, - ), + failure_state_path = temp_root / "worker-failures.json" + private_apply_receipt_path = worker_receipt_dir / f"{proposal_id}.json" + worker_script = genesis.apply_worker_script_path if genesis is not None else HERE / "apply_worker.py" + apply_script = genesis.apply_script_path if genesis is not None else HERE / "apply_proposal.py" + worker_base = [ + sys.executable, + str(worker_script), + "--proposal-id", + proposal_id, + "--contract-version", + str(worker_contract_version), + "--limit", + "1", + "--max-per-tick", + "1", + "--max-attempts", + "3", + "--failure-state-file", + str(failure_state_path), + "--apply-script", + str(apply_script), + "--receipt-dir", + str(worker_receipt_dir), + "--secrets-file", + str(apply_secrets), + "--container", + container_name, + "--db", + DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_apply", "--fresh-preflight", str(preflight_path), - "--receipt-out", - str(private_apply_receipt_path), ] - apply_dry_run = run_tool([*apply_base, "--dry-run"], action="guarded apply dry-run") - apply_markers = ( - "kb_stage.assert_approved_proposal", - "kb_stage.finish_approved_proposal", + worker_report_only = run_tool( + [*worker_base, "--require-candidate"], + action="disabled apply worker candidate preflight", + ) + report_only_state = database_state(postgres, child) + report_marker = f"would apply {proposal_id} (approve_claim)" + if report_marker not in worker_report_only.stdout: + raise LifecycleError("disabled apply worker did not select the exact approved proposal") + + expected_retry_failure = subprocess.run( + [ + *worker_base, + "--require-candidate", + "--enable", + "--applied-by", + "missing-disposable-worker-agent", + ], + text=True, + capture_output=True, + check=False, + ) + if expected_retry_failure.returncode != 1: + raise LifecycleError("apply worker retry probe did not fail exactly once before commit") + retry_state = json.loads(failure_state_path.read_text(encoding="utf-8")) + retry_probe_state = database_state(postgres, child) + if retry_state != {proposal_id: 1}: + raise LifecycleError("apply worker retry probe did not persist the exact failure count") + if private_apply_receipt_path.exists(): + raise LifecycleError("failed apply worker retry unexpectedly wrote a replay receipt") + + worker_execute = run_tool( + [*worker_base, "--require-candidate", "--enable"], + action="enabled apply worker retry", ) - if any(marker not in apply_dry_run.stdout for marker in apply_markers): - raise LifecycleError("apply dry-run did not bind both approval guards") - apply_execute = run_tool(apply_base, action="guarded canonical apply") if not private_apply_receipt_path.is_file(): - raise LifecycleError("guarded apply did not retain its private replay receipt") + raise LifecycleError("apply worker did not retain its private replay receipt") + if stat.S_IMODE(private_apply_receipt_path.stat().st_mode) != 0o600: + raise LifecycleError("apply worker replay receipt is not mode 0600") apply_receipt = json.loads(private_apply_receipt_path.read_text(encoding="utf-8")) + failure_state_after_success = json.loads(failure_state_path.read_text(encoding="utf-8")) + post_apply_before = database_state(postgres, child) + worker_post_apply_refusal = subprocess.run( + [*worker_base, "--enable"], + text=True, + capture_output=True, + check=False, + ) + refusal_marker = ( + f"REFUSE exact proposal execution: approved+applyable contract v{worker_contract_version} " + f"proposal {proposal_id} " + "was not selected exactly once; no writes attempted." + ) + if worker_post_apply_refusal.returncode != 1 or refusal_marker not in worker_post_apply_refusal.stderr: + raise LifecycleError("post-apply exact worker did not refuse the already-applied proposal") applied = proposal_readback(postgres, proposal_id) if applied is None: raise LifecycleError("applied proposal readback is missing") @@ -743,18 +1704,34 @@ def run_lifecycle( recompiled, _expected_again, _paths_again = compile_source_packet(source_packet) recompiled_child = recompiled["strict_child_proposal"] final_identities = exact_identities(recompiled_child, final_rows) + controlled_fingerprint = payload_controlled_fingerprint(final_rows) receipt["identities"] = final_identities receipt["phases"]["applied"] = { "database_fingerprint": applied_state["database_fingerprint"], "canonical_fingerprint": applied_state["canonical_fingerprint"], "proposal": applied, "canonical_rows": final_rows, - "apply_action": { - "dry_run": { - **command_receipt(apply_dry_run), - "approval_guards_present": all(marker in apply_dry_run.stdout for marker in apply_markers), + "payload_controlled_fingerprint": controlled_fingerprint, + "worker_action": { + "exact_candidate_id": proposal_id, + "credential_role": "kb_apply", + "fresh_preflight_sha256": sha256_file(preflight_path), + "report_only": command_receipt(worker_report_only, expected_marker=report_marker), + "report_only_database_fingerprint": report_only_state["database_fingerprint"], + "expected_retry_failure": command_receipt(expected_retry_failure), + "retry_failure_count": retry_state[proposal_id], + "retry_probe_database_fingerprint": retry_probe_state["database_fingerprint"], + "enabled_retry": command_receipt( + worker_execute, + expected_marker=f"applied {proposal_id} (approve_claim)", + ), + "failure_state_after_success": failure_state_after_success, + "post_apply_exact_refusal": { + **command_receipt(worker_post_apply_refusal), + "expected_refusal_marker": refusal_marker, + "expected_refusal_marker_present": refusal_marker in worker_post_apply_refusal.stderr, }, - "execute": command_receipt(apply_execute), + "private_receipt_mode": "0600", }, "apply_receipt": apply_receipt, } @@ -774,8 +1751,12 @@ def run_lifecycle( actual_counts = _row_counts(final_rows) checks = { "fresh_compile_matches_retained_packet": canonical_json_bytes(compiled) == canonical_json_bytes(expected), + "fresh_compile_matches_retained_packet_byte_identically": canonical_json_bytes(compiled) + == canonical_json_bytes(expected), "compiler_emits_pending_review": compiled.get("status") == "pending_review", "initial_target_rows_absent": _all_target_rows_empty(initial["state"]["canonical_rows"]), + "initial_proposal_and_approval_absent": initial["state"]["proposal"] is None + and initial["state"]["immutable_approval"] is None, "staged_proposal_exact": pending.get("id") == proposal_id and pending.get("payload") == child["payload"], "staged_status_pending_review": pending.get("status") == "pending_review", "review_packet_requires_human_review": packet.get("review_state") == "needs_human_review", @@ -786,6 +1767,19 @@ def run_lifecycle( "immutable_approval_matches_reviewer": immutable_approval.get("reviewed_by_handle") == reviewed_by, "review_did_not_change_canonical_rows": approved_state["canonical_fingerprint"] == initial["canonical_fingerprint"], + "worker_report_only_selected_exact_candidate": report_marker in worker_report_only.stdout, + "worker_enable_gate_prevented_apply": report_only_state["database_fingerprint"] + == approved_state["database_fingerprint"], + "worker_retry_failure_was_precommit": retry_probe_state["database_fingerprint"] + == approved_state["database_fingerprint"], + "worker_retry_count_persisted_once": retry_state == {proposal_id: 1}, + "worker_retry_state_cleared_after_success": failure_state_after_success == {}, + "worker_enabled_retry_applied_exact_candidate": f"applied {proposal_id} (approve_claim)" + in worker_execute.stdout, + "worker_post_apply_exact_refusal_did_not_retry": worker_post_apply_refusal.returncode == 1 + and refusal_marker in worker_post_apply_refusal.stderr + and post_apply_before["database_fingerprint"] == applied_state["database_fingerprint"], + "worker_receipt_private_mode_0600": stat.S_IMODE(private_apply_receipt_path.stat().st_mode) == 0o600, "apply_status_applied": applied.get("status") == "applied" and bool(applied.get("applied_at")), "apply_changed_canonical_fingerprint": applied_state["canonical_fingerprint"] != initial["canonical_fingerprint"], @@ -793,6 +1787,7 @@ def run_lifecycle( "apply_receipt_replay_ready": apply_receipt.get("replay_ready") is True, "apply_receipt_rows_exact": normalized_canonical_rows(apply_receipt.get("canonical_rows")) == normalized_canonical_rows(final_rows), + "payload_controlled_fields_fingerprinted": controlled_fingerprint["row_counts"] == actual_counts, "recompile_matches_initial": canonical_json_bytes(recompiled) == canonical_json_bytes(compiled), "recompile_proposal_id_exact": recompiled_child["id"] == proposal_id, "database_fingerprints_distinguish_all_transitions": len( @@ -805,6 +1800,40 @@ def run_lifecycle( ) == 4, } + if genesis is not None: + genesis_restore = receipt["database_initialization"]["genesis"] + checks.update( + { + "genesis_dump_exporter_receipt_bound_before_docker": genesis_restore["dump"][ + "exporter_receipt_binding_verified_before_docker" + ] + is True, + "genesis_manifest_exporter_receipt_bound_before_docker": genesis_restore["manifest"][ + "exporter_receipt_binding_verified_before_docker" + ] + is True, + "genesis_exporter_receipt_trusted_contract_valid": genesis_restore["exporter_receipt"][ + "trusted_contract_validated_before_docker" + ] + is True, + "genesis_data_schema_parity_proven": genesis_restore["parity"]["status"] == "pass" + and not genesis_restore["parity"]["problems"], + "genesis_role_global_parity_claim_honest": genesis_restore["parity"]["role_global_parity"]["status"] + == ("proven" if genesis.role_settings_capture_complete else "not_proven"), + "genesis_image_digest_pinned": genesis_restore["image"]["digest_pinned"] is True + and genesis_restore["image"]["runtime_reference_exact"] is True, + "worker_password_auth_not_trust": receipt["environment"]["host_auth_trust_requested"] is False + and receipt["database_initialization"]["worker_credentials_configured_for_password_auth"] is True, + "synthetic_bootstrap_skipped": receipt["database_initialization"][ + "synthetic_schema_bootstrap_skipped" + ] + is True, + "canonical_payload_preseed_forbidden": receipt["database_initialization"][ + "canonical_payload_preseed_performed" + ] + is False, + } + ) receipt["checks"] = checks if not all(checks.values()): failed = [name for name, passed in checks.items() if not passed] @@ -812,15 +1841,44 @@ def run_lifecycle( except (LifecycleError, OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: receipt["error"] = {"type": type(exc).__name__, "message": str(exc)} finally: - cleanup = postgres.cleanup() + try: + cleanup = ( + postgres.cleanup() + if postgres is not None + else { + "remove_attempted": False, + "container_absent": True, + "name_readback_clear": True, + "instance_label_readback_clear": True, + "run_group_readback_clear": True, + "network_mode_was_none": False, + "tmpfs_data_dir_was_present": False, + "docker_volume_mounts_observed": [], + } + ) + except (LifecycleError, OSError) as exc: + cleanup = { + "remove_attempted": bool(postgres is not None and postgres.started), + "container_absent": False, + "name_readback_clear": False, + "instance_label_readback_clear": False, + "run_group_readback_clear": False, + "network_mode_was_none": False, + "tmpfs_data_dir_was_present": False, + "docker_volume_mounts_observed": [], + "cleanup_error": {"type": type(exc).__name__, "message": str(exc)}, + } shutil.rmtree(temp_root, ignore_errors=True) cleanup["temporary_secret_root_absent"] = not temp_root.exists() - cleanup["private_apply_receipt_removed_with_secret_root"] = not private_apply_receipt_path.exists() + cleanup["private_apply_receipt_removed_with_secret_root"] = ( + private_apply_receipt_path is None or not private_apply_receipt_path.exists() + ) cleanup["all"] = all( ( cleanup.get("container_absent") is True, cleanup.get("name_readback_clear") is True, cleanup.get("instance_label_readback_clear") is True, + cleanup.get("run_group_readback_clear") is True, cleanup.get("network_mode_was_none") is True, cleanup.get("tmpfs_data_dir_was_present") is True, cleanup.get("docker_volume_mounts_observed") == [], @@ -834,14 +1892,26 @@ def run_lifecycle( "pass" if "error" not in receipt and receipt["checks"] and all(receipt["checks"].values()) else "fail" ) receipt["current_tier"] = ( - "realistic_isolated_local_container_end_to_end" + ( + "T2_disposable_genesis_backed_runtime" + if genesis is not None + else "realistic_isolated_local_container_end_to_end" + ) if receipt["status"] == "pass" else "runtime_verification_failed" ) receipt["strongest_claim"] = ( - "A retained real source packet was freshly compiled, staged, explicitly reviewed, guarded-applied " - "into networkless disposable Postgres, deterministically recompiled, and read back with exact rows; " - "the container and private temporary files were then independently absent." + ( + "An exporter-receipt-bound real genesis dump was directly restored and manifest-proven before a retained " + "source packet was freshly compiled, staged, explicitly reviewed, selected and fresh-owned applied " + "through the gated worker, deterministically recompiled, and read back with payload-controlled " + "fingerprints in networkless disposable Postgres; the container and private temporary files were " + "then independently absent." + if genesis is not None + else "A retained real source packet was freshly compiled, staged, explicitly reviewed, " + "guarded-applied into networkless disposable Postgres, deterministically recompiled, and read " + "back with exact rows; the container and private temporary files were then independently absent." + ) if receipt["status"] == "pass" else "The integrated-learning lifecycle did not pass all fail-closed checks." ) @@ -850,10 +1920,418 @@ def run_lifecycle( "live Telegram send or reply", "human acceptance of these candidate claims for production", ] - output.parent.mkdir(parents=True, exist_ok=True) - temporary_output = output.with_name(f".{output.name}.{uuid.uuid4().hex}.tmp") - temporary_output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") - os.replace(temporary_output, output) + if persist_receipt: + write_private_json(output, receipt) + return receipt + + +def semantic_run_summary(receipt: dict[str, Any]) -> dict[str, Any]: + """Project one run onto deterministic source, payload, and restore semantics.""" + try: + identities = receipt["identities"] + applied = receipt["phases"]["applied"] + recompiled = receipt["phases"]["recompiled_readback"] + genesis = receipt["database_initialization"]["genesis"] + apply_receipt = applied["apply_receipt"] + return { + "consumed_inputs_manifest_sha256": receipt["consumed_inputs"]["preflight_manifest_sha256"], + "source": receipt["source"], + "compiled_bundle_sha256": receipt["compiled"]["bundle_sha256"], + "proposal_id": identities["proposal_id"], + "claim_ids": identities["claim_ids"], + "source_ids": identities["source_ids"], + "planned_evidence_keys": identities["planned_evidence_keys"], + "phase_statuses": { + "staged": receipt["phases"]["staged_pending_review"]["proposal"]["status"], + "approved": receipt["phases"]["approved_unapplied"]["proposal"]["status"], + "applied": applied["proposal"]["status"], + }, + "payload_controlled_fingerprint": applied["payload_controlled_fingerprint"], + "apply_payload_sha256": apply_receipt["hashes"]["apply_payload_sha256"], + "expected_row_counts": apply_receipt["expected_row_counts"], + "actual_row_counts": apply_receipt["actual_row_counts"], + "recompiled_bundle_sha256": recompiled["bundle_sha256"], + "recompile_matches_initial": recompiled["matches_initial_compile"], + "apply_worker": { + "exact_candidate_id": applied["worker_action"]["exact_candidate_id"], + "credential_role": applied["worker_action"]["credential_role"], + "retry_failure_count": applied["worker_action"]["retry_failure_count"], + "failure_state_after_success": applied["worker_action"]["failure_state_after_success"], + "private_receipt_mode": applied["worker_action"]["private_receipt_mode"], + }, + "genesis": { + "dump_sha256": genesis["dump"]["sha256"], + "manifest_sha256": genesis["manifest"]["sha256"], + "exporter_receipt_sha256": genesis["exporter_receipt"]["sha256"], + "provenance_binding_sha256": genesis["exporter_receipt"]["provenance_binding_sha256"], + "image_reference": genesis["image"]["reference"], + "parity_status": genesis["parity"]["status"], + "parity_problems": genesis["parity"]["problems"], + "parity_claim_scope": genesis["parity"]["claim_scope"], + "role_global_parity": genesis["parity"]["role_global_parity"], + "key_counts": genesis["key_counts"], + "canonical_schema_table_count": genesis["canonical_schema_table_count"], + }, + } + except (KeyError, TypeError) as exc: + raise LifecycleError("completed genesis run is missing repeatability evidence") from exc + + +def verify_run_group_cleanup(run_group: str) -> dict[str, Any]: + """Prove that no container created by this two-run invocation remains.""" + docker = shutil.which("docker") + if not docker: + return { + "checked": False, + "run_group": run_group, + "container_names": [], + "all_absent": False, + "error": "docker executable is unavailable", + } + try: + result = subprocess.run( + [ + docker, + "container", + "ls", + "-a", + "--filter", + f"label=livingip.canary.run_group={run_group}", + "--format", + "{{.Names}}", + ], + text=True, + capture_output=True, + check=False, + ) + except OSError as exc: + return { + "checked": False, + "run_group": run_group, + "container_names": [], + "all_absent": False, + "error": type(exc).__name__, + } + names = sorted(line.strip() for line in result.stdout.splitlines() if line.strip()) + return { + "checked": True, + "run_group": run_group, + "container_names": names, + "docker_readback_returncode": result.returncode, + "stdout_sha256": sha256_text(result.stdout), + "stderr_sha256": sha256_text(result.stderr), + "all_absent": result.returncode == 0 and not names, + } + + +def run_genesis_lifecycle( + source_packet: Path, + output: Path, + *, + review_action: str, + genesis_dump: Path, + genesis_manifest: Path, + genesis_exporter_receipt: Path, + image: str, + reviewed_by: str = REVIEWER_HANDLE, + review_note: str = DEFAULT_REVIEW_NOTE, +) -> dict[str, Any]: + """Run the same genesis-backed lifecycle twice and compare deterministic semantics.""" + output = output.expanduser().resolve() + source_packet = source_packet.expanduser().resolve() + protected_path_candidates = { + genesis_dump.expanduser().resolve(), + genesis_manifest.expanduser().resolve(), + genesis_exporter_receipt.expanduser().resolve(), + *source_packet_input_paths(source_packet), + } + if output in protected_path_candidates: + raise LifecycleError("--output must not overwrite a genesis or retained source-packet input") + run_group = f"leo-genesis-{uuid.uuid4().hex}" + receipt: dict[str, Any] = { + "artifact": GENESIS_RECEIPT_ARTIFACT, + "schema": GENESIS_RECEIPT_SCHEMA, + "status": "fail", + "required_tier": "T2_disposable_genesis_backed_runtime", + "current_tier": "not_proven", + "run_group": run_group, + "repeat_count_required": GENESIS_REPEAT_COUNT, + "source_packet": "source-packet", + "safety_boundary": { + "disposable_networkless_postgres_only": True, + "existing_database_target_supported": False, + "vps_or_gcp_mutation_supported": False, + "model_calls_performed": False, + "synthetic_canonical_bootstrap_supported": False, + "canonical_payload_preseed_supported": False, + "snapshot_repackaging_performed": False, + }, + "pre_docker_validation": {"status": "pending", "docker_commands_attempted": 0}, + "runs": [], + "repeatability": {"status": "not_run"}, + "checks": {}, + } + staged: StagedLifecycleInputs | None = None + genesis: GenesisInputs | None = None + final_staging_verification: dict[str, Any] | None = None + + try: + if review_action != "approve": + raise LifecycleError("the only supported explicit review action is 'approve'") + if reviewed_by != REVIEWER_HANDLE: + raise LifecycleError(f"disposable reviewer must be {REVIEWER_HANDLE!r}") + if not review_note.strip(): + raise LifecycleError("review note must be non-empty") + staged = stage_lifecycle_inputs( + source_packet, + genesis_dump, + genesis_manifest, + genesis_exporter_receipt, + ) + genesis = validate_genesis_inputs( + staged.genesis_dump, + staged.genesis_manifest, + staged.exporter_receipt, + image, + parity_sql_path=staged.root / STAGED_ENGINE_DESTINATIONS["engine.parity_sql"], + guard_prerequisites_path=staged.root / STAGED_ENGINE_DESTINATIONS["engine.guard_sql"], + approve_script_path=staged.root / STAGED_ENGINE_DESTINATIONS["engine.approve_proposal"], + apply_worker_script_path=staged.root / STAGED_ENGINE_DESTINATIONS["engine.apply_worker"], + apply_script_path=staged.root / STAGED_ENGINE_DESTINATIONS["engine.apply_proposal"], + ) + compiled, retained, source_paths = compile_source_packet(staged.source_packet) + receipt["execution"] = { + **staged.execution, + "engine_sha256": {name: row["sha256"] for name, row in staged.files.items() if name.startswith("engine.")}, + "engine_set_sha256": canonical_sha256( + {name: row["sha256"] for name, row in staged.files.items() if name.startswith("engine.")} + ), + } + receipt["immutable_staging"] = { + "schema": "livingip.leoImmutableLifecycleInputs.v1", + "private_directory_mode": "0500", + "file_mode": "0400", + "preflight_manifest_sha256": staged.manifest_sha256, + "files": staged.files, + "original_paths_reopened_after_staging": False, + } + receipt["inputs"] = { + "genesis_dump": { + "bytes": genesis.dump_bytes, + "sha256": genesis.dump_sha256, + }, + "genesis_manifest": { + "bytes": genesis.manifest_bytes, + "sha256": genesis.manifest_sha256, + "role_settings_capture_complete": genesis.role_settings_capture_complete, + }, + "genesis_exporter_receipt": { + "bytes": staged.files["genesis.exporter_receipt"]["bytes"], + "sha256": genesis.exporter_receipt_sha256, + "trust_root_receipt_sha256": genesis.trust_root_receipt_sha256, + "schema": genesis.exporter_receipt["schema"], + "artifact": genesis.exporter_receipt["artifact"], + "run_id": genesis.exporter_receipt["run_id"], + "capture_authorization_ref": genesis.capture_authorization_ref, + "source_identity": genesis.source_identity, + "provenance_binding_sha256": genesis.provenance_binding_sha256, + }, + "image": genesis.image, + "source_packet": { + "compiled_bundle_sha256": canonical_sha256(compiled), + "retained_packet_file_sha256": sha256_file(source_paths["expected"]), + "retained_packet_canonical_sha256": canonical_sha256(retained), + }, + } + receipt["pre_docker_validation"] = { + "status": "pass", + "docker_commands_attempted": 0, + "dump_custom_format_signature_valid": True, + "trusted_exporter_receipt_validated": True, + "trusted_exporter_receipt_sha256": genesis.trust_root_receipt_sha256, + "dump_hash_bound_by_exporter_receipt": True, + "manifest_hash_bound_by_exporter_receipt": True, + "source_identity_and_authorization_bound": True, + "provenance_binding_verified": True, + "manifest_contract_loaded": True, + "retained_packet_recompiled_byte_identically": canonical_json_bytes(compiled) + == canonical_json_bytes(retained), + "image_digest_pinned": True, + "immutable_staging_manifest_sha256": staged.manifest_sha256, + "executing_git_head": staged.execution["git_head"], + "engine_set_sha256": receipt["execution"]["engine_set_sha256"], + "committed_engine_dependency_closure_verified": staged.execution["engine_paths_match_head"] is True + and receipt["execution"]["engine_sha256"] == staged.execution["engine_head_sha256"], + } + + for run_number in range(1, GENESIS_REPEAT_COUNT + 1): + consumed_before = verify_staged_inputs(staged) + run_receipt = run_lifecycle( + staged.source_packet, + output, + review_action=review_action, + reviewed_by=reviewed_by, + review_note=review_note, + genesis=genesis, + persist_receipt=False, + run_group=run_group, + run_number=run_number, + ) + consumed_after = verify_staged_inputs(staged) + run_receipt["consumed_inputs"] = { + "preflight_manifest_sha256": staged.manifest_sha256, + "executing_git_head": staged.execution["git_head"], + "engine_set_sha256": receipt["execution"]["engine_set_sha256"], + "immutable_files_sha256": consumed_before["files_sha256"], + "verified_before_run": consumed_before, + "verified_after_run": consumed_after, + "original_paths_reopened": False, + } + receipt["runs"].append(run_receipt) + + run_summaries = [semantic_run_summary(run) for run in receipt["runs"] if run.get("status") == "pass"] + repeatability_sha256 = [canonical_sha256(summary) for summary in run_summaries] + post_apply_fingerprints = [ + run["phases"]["applied"]["payload_controlled_fingerprint"]["sha256"] + for run in receipt["runs"] + if run.get("status") == "pass" + ] + identical_post_apply_fingerprints = ( + len(post_apply_fingerprints) == GENESIS_REPEAT_COUNT and len(set(post_apply_fingerprints)) == 1 + ) + semantic_match = ( + len(run_summaries) == GENESIS_REPEAT_COUNT + and len(set(repeatability_sha256)) == 1 + and identical_post_apply_fingerprints + and all(run["cleanup"].get("all") is True for run in receipt["runs"]) + ) + receipt["repeatability"] = { + "status": "pass" if semantic_match else "fail", + "semantic_fingerprints_sha256": repeatability_sha256, + "semantic_match": semantic_match, + "post_apply_fingerprints_sha256": post_apply_fingerprints, + "identical_post_apply_fingerprints": identical_post_apply_fingerprints, + "generated_fields_excluded": [ + "claim_evidence.id", + "claim_edges.id", + "created_at", + "updated_at", + "reviewed_at", + "applied_at", + "container identity", + "Postgres system identifier", + ], + "summaries": run_summaries, + } + except (LifecycleError, OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: + if receipt["pre_docker_validation"].get("status") == "pending": + receipt["pre_docker_validation"]["status"] = "fail" + receipt["error"] = {"type": type(exc).__name__, "message": str(exc)} + finally: + if staged is not None: + try: + final_staging_verification = verify_staged_inputs(staged) + except (LifecycleError, OSError) as exc: + final_staging_verification = { + "all_files_match_preflight": False, + "error": {"type": type(exc).__name__, "message": str(exc)}, + } + receipt.setdefault("error", {"type": type(exc).__name__, "message": str(exc)}) + staging_cleanup = cleanup_staged_inputs(staged.root if staged is not None else None) + if receipt["pre_docker_validation"].get("status") == "pass": + group_cleanup = verify_run_group_cleanup(run_group) + else: + group_cleanup = { + "checked": False, + "run_group": run_group, + "container_names": [], + "all_absent": True, + "reason": "input validation failed before any Docker command", + } + per_run_cleanup = [run.get("cleanup", {}).get("all") is True for run in receipt["runs"]] + receipt["cleanup"] = { + "per_run": per_run_cleanup, + "per_run_all": len(per_run_cleanup) == GENESIS_REPEAT_COUNT and all(per_run_cleanup), + "run_group_readback": group_cleanup, + "zero_orphan_containers": group_cleanup.get("all_absent") is True, + "immutable_staging_final_verification": final_staging_verification, + "immutable_staging_cleanup": staging_cleanup, + "zero_staging_leftovers": staging_cleanup["root_absent"] is True, + } + expected_run_binding = ( + { + "preflight_manifest_sha256": staged.manifest_sha256, + "executing_git_head": staged.execution["git_head"], + "engine_set_sha256": receipt.get("execution", {}).get("engine_set_sha256"), + "immutable_files_sha256": final_staging_verification.get("files_sha256") + if isinstance(final_staging_verification, dict) + else None, + } + if staged is not None + else None + ) + run_input_bindings = [ + { + key: run.get("consumed_inputs", {}).get(key) + for key in ( + "preflight_manifest_sha256", + "executing_git_head", + "engine_set_sha256", + "immutable_files_sha256", + ) + } + for run in receipt["runs"] + ] + absolute_path_values = absolute_path_value_pointers(receipt) + receipt["checks"] = { + "all_inputs_validated_before_docker": receipt["pre_docker_validation"].get("status") == "pass", + "all_inputs_copied_to_private_immutable_staging": staged is not None + and receipt.get("immutable_staging", {}).get("preflight_manifest_sha256") == staged.manifest_sha256, + "both_runs_consumed_preflight_bound_bytes": len(run_input_bindings) == GENESIS_REPEAT_COUNT + and expected_run_binding is not None + and all(binding == expected_run_binding for binding in run_input_bindings), + "exactly_two_runs_completed": len(receipt["runs"]) == GENESIS_REPEAT_COUNT, + "both_runs_passed": len(receipt["runs"]) == GENESIS_REPEAT_COUNT + and all(run.get("status") == "pass" for run in receipt["runs"]), + "both_runs_match_semantically": receipt["repeatability"].get("semantic_match") is True, + "identical_post_apply_fingerprints": receipt["repeatability"].get("identical_post_apply_fingerprints") + is True, + "both_runs_cleaned_up": receipt["cleanup"]["per_run_all"], + "zero_orphan_containers": receipt["cleanup"]["zero_orphan_containers"], + "zero_staging_leftovers": receipt["cleanup"]["zero_staging_leftovers"], + "committed_engine_dependency_closure_verified": staged is not None + and staged.execution.get("engine_paths_match_head") is True + and receipt.get("execution", {}).get("engine_sha256") == staged.execution.get("engine_head_sha256"), + "portable_receipt_has_no_absolute_paths": not absolute_path_values, + } + receipt["status"] = "pass" if receipt["checks"] and all(receipt["checks"].values()) else "fail" + if receipt["status"] == "fail" and "error" not in receipt: + failed_runs = [run.get("run_number") for run in receipt["runs"] if run.get("status") != "pass"] + receipt["error"] = { + "type": "GenesisRunFailure", + "message": f"genesis lifecycle runs did not pass: {failed_runs}", + } + receipt["current_tier"] = ( + "T2_disposable_genesis_backed_runtime" if receipt["status"] == "pass" else "runtime_verification_failed" + ) + receipt["strongest_claim"] = ( + "The same immutable staged bytes were consumed twice from one trusted exporter-receipt-bound genesis " + "in independent networkless disposable PostgreSQL containers; both exact apply-worker lifecycles, " + "post-apply fingerprints, recompiles, and zero-leftover readbacks matched." + if receipt["status"] == "pass" + else "The two-run genesis-backed lifecycle did not pass every fail-closed T2 check." + ) + receipt["not_proven"] = [ + "VPS or GCP canonical mutation", + "production access or production readiness", + "live Telegram send or reply", + "blank-schema reconstruction of historical canonical knowledge", + ] + if genesis is not None and not genesis.role_settings_capture_complete: + receipt["not_proven"].append( + "full role/global parity because the retained source manifest omitted role and database settings" + ) + write_private_json(output, receipt) return receipt @@ -861,6 +2339,14 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source-packet", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--genesis-dump", type=Path) + parser.add_argument("--genesis-manifest", type=Path) + parser.add_argument("--genesis-exporter-receipt", type=Path) + parser.add_argument( + "--image", + default=DEFAULT_GENESIS_IMAGE, + help="digest-pinned PostgreSQL image used only by genesis-backed mode", + ) parser.add_argument( "--review-action", required=True, @@ -869,30 +2355,67 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument("--reviewed-by", default=REVIEWER_HANDLE) parser.add_argument("--review-note", default=DEFAULT_REVIEW_NOTE) - return parser.parse_args(argv) + args = parser.parse_args(argv) + genesis_values = ( + args.genesis_dump, + args.genesis_manifest, + args.genesis_exporter_receipt, + ) + if any(value is not None for value in genesis_values) and not all(value is not None for value in genesis_values): + parser.error( + "genesis-backed mode requires --genesis-dump, --genesis-manifest, and --genesis-exporter-receipt together" + ) + if args.genesis_dump is not None and not IMAGE_DIGEST_RE.fullmatch(args.image): + parser.error("--image must be pinned by a lowercase sha256 digest") + protected_inputs = set(source_packet_input_paths(args.source_packet)) + if args.genesis_dump is not None: + protected_inputs.update( + { + args.genesis_dump.expanduser().resolve(), + args.genesis_manifest.expanduser().resolve(), + args.genesis_exporter_receipt.expanduser().resolve(), + } + ) + if args.output.expanduser().resolve() in protected_inputs: + parser.error("--output must not overwrite a genesis or retained source-packet input") + return args def main(argv: list[str] | None = None) -> int: args = parse_args(argv) - receipt = run_lifecycle( - args.source_packet, - args.output, - review_action=args.review_action, - reviewed_by=args.reviewed_by, - review_note=args.review_note, - ) - print( - json.dumps( - { - "status": receipt["status"], - "receipt": str(args.output.resolve()), - "proposal_id": receipt.get("identities", {}).get("proposal_id"), - "current_tier": receipt["current_tier"], - "cleanup": receipt["cleanup"].get("all"), - }, - sort_keys=True, + if args.genesis_dump is not None: + receipt = run_genesis_lifecycle( + args.source_packet, + args.output, + review_action=args.review_action, + genesis_dump=args.genesis_dump, + genesis_manifest=args.genesis_manifest, + genesis_exporter_receipt=args.genesis_exporter_receipt, + image=args.image, + reviewed_by=args.reviewed_by, + review_note=args.review_note, + ) + print_private_receipt_summary(args.output.resolve(), receipt) + else: + receipt = run_lifecycle( + args.source_packet, + args.output, + review_action=args.review_action, + reviewed_by=args.reviewed_by, + review_note=args.review_note, + ) + print( + json.dumps( + { + "status": receipt["status"], + "receipt": str(args.output.resolve()), + "proposal_id": receipt.get("identities", {}).get("proposal_id"), + "current_tier": receipt["current_tier"], + "cleanup": receipt["cleanup"].get("all"), + }, + sort_keys=True, + ) ) - ) return 0 if receipt["status"] == "pass" else 1 diff --git a/scripts/run_leo_negative_permissions_canary.py b/scripts/run_leo_negative_permissions_canary.py index 76403b3..2bce3e2 100644 --- a/scripts/run_leo_negative_permissions_canary.py +++ b/scripts/run_leo_negative_permissions_canary.py @@ -30,7 +30,7 @@ READ_ROLE_SQL = ROOT / "ops" / "observatory_read_role.sql" DATABASE = "teleo_canonical" IAM_USER = "sa-observatory-read-adapter@teleo-501523.iam" PARTICIPANT_HANDLE = "m3taversal" -DEFAULT_IMAGE = "postgres:16-alpine" +DEFAULT_IMAGE = os.environ.get("LEOCLEAN_RUNTIME_POSTGRES_IMAGE", "postgres:16-alpine") CANARY_LABEL = "leo-negative-permissions" BOOTSTRAP_SQL = r""" diff --git a/scripts/stage_normalized_proposal.py b/scripts/stage_normalized_proposal.py index 9fe9a56..d6a078e 100644 --- a/scripts/stage_normalized_proposal.py +++ b/scripts/stage_normalized_proposal.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import copy import hashlib import json import sys @@ -18,6 +19,9 @@ import apply_proposal as ap # noqa: E402 import kb_proposal_normalize as normalize # noqa: E402 import run_leo_clone_bound_handler_checkpoint as bound # noqa: E402 +V3_BUNDLE_SCHEMA = "livingip.kbSourceProposalBundle.v3" +V3_MANIFEST_SCHEMA = "livingip.kbSourceExtractionManifest.v3" + def canonical_sha256(value: Any) -> str: encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() @@ -175,24 +179,116 @@ commit; """ -def stage_normalized_child(container: str, database: str, parent: dict[str, Any]) -> dict[str, Any]: +def prepare_compiled_v3_child(bundle: dict[str, Any]) -> dict[str, Any]: + """Revalidate one build-only V3 compiler bundle before disposable staging.""" + + if bundle.get("schema") != V3_BUNDLE_SCHEMA: + raise bound.CheckpointError(f"compiled bundle schema must equal {V3_BUNDLE_SCHEMA!r}") + if ( + bundle.get("status") != "pending_review" + or bundle.get("build_only") is not True + or bundle.get("database_write_performed") is not False + or bundle.get("canonical_apply_performed") is not False + ): + raise bound.CheckpointError("compiled V3 bundle is not build-only pending-review material") + + hashes = bundle.get("hashes") + if not isinstance(hashes, dict): + raise bound.CheckpointError("compiled V3 bundle hashes must be an object") + expected_bundle_hash = hashes.get("bundle_content_sha256") + preimage = copy.deepcopy(bundle) + preimage_hashes = preimage.get("hashes") + if not isinstance(expected_bundle_hash, str) or not isinstance(preimage_hashes, dict): + raise bound.CheckpointError("compiled V3 bundle content hash is missing") + preimage_hashes.pop("bundle_content_sha256", None) + if canonical_sha256(preimage) != expected_bundle_hash: + raise bound.CheckpointError("compiled V3 bundle content hash does not match") + + manifest = bundle.get("validated_manifest") + child = bundle.get("strict_child_proposal") + if not isinstance(manifest, dict) or manifest.get("schema") != V3_MANIFEST_SCHEMA: + raise bound.CheckpointError("compiled V3 bundle has no validated V3 manifest") + if not isinstance(child, dict): + raise bound.CheckpointError("compiled V3 bundle has no strict child proposal") + payload = child.get("payload") + apply_payload = payload.get("apply_payload") if isinstance(payload, dict) else None + if ( + child.get("status") != "pending_review" + or child.get("proposal_type") != "approve_claim" + or not isinstance(apply_payload, dict) + or apply_payload.get("contract_version") != 3 + ): + raise bound.CheckpointError("compiled V3 child is not one pending-review approve_claim v3 proposal") + + payload_hash = canonical_sha256(payload) + if child.get("payload_sha256") != payload_hash or hashes.get("strict_child_payload_sha256") != payload_hash: + raise bound.CheckpointError("compiled V3 child payload hash does not match") + if child.get("parent_packet_sha256") != hashes.get("parent_packet_sha256"): + raise bound.CheckpointError("compiled V3 parent packet hash does not match") + + stage_preview = bundle.get("stage_preview") + stage_sql_hash = hashlib.sha256(build_stage_sql(child).encode("utf-8")).hexdigest() + if ( + not isinstance(stage_preview, dict) + or stage_preview.get("target_table") != "kb_stage.kb_proposals" + or stage_preview.get("target_status") != "pending_review" + or stage_preview.get("executed") is not False + or stage_preview.get("stage_sql_sha256") != stage_sql_hash + ): + raise bound.CheckpointError("compiled V3 stage preview does not match the exact child") + + validation_snapshot = copy.deepcopy(child) + validation_snapshot.update( + { + "status": "approved", + "reviewed_by_handle": "v3-compiler-preflight", + "reviewed_by_agent_id": None, + "reviewed_at": "2000-01-01T00:00:00+00:00", + "review_note": "Build-only validation of exact compiled V3 material.", + } + ) + try: + ap.build_apply_sql(validation_snapshot, ap.SERVICE_AGENT_HANDLE) + except (TypeError, ValueError) as exc: + raise bound.CheckpointError(f"compiled V3 child failed strict writer preflight: {exc}") from exc + return copy.deepcopy(child) + + +def _stage_child(container: str, database: str, child: dict[str, Any]) -> dict[str, Any]: target = bound.container_identity(container) - production = bound.container_identity(bound.PRODUCTION_CONTAINER) + try: + production = bound.container_identity(bound.PRODUCTION_CONTAINER) + except bound.CheckpointError as exc: + detail = str(exc).lower() + if "no such object" not in detail and "no such container" not in detail: + raise + production = {"id": None, "mount_sources": []} bound.validate_disposable_target_identity(target, production) if target["id"] == production["id"]: raise bound.CheckpointError("normalized staging target resolves to production") - child = prepare_normalized_child(parent) result = bound._psql_json(str(target["id"]), database, build_stage_sql(child)) if result.get("proposal_id") != child["id"] or result.get("status") != "pending_review": raise bound.CheckpointError("normalized staging did not return the exact pending child") return {"child": child, "database_receipt": result, "bound_container_id": str(target["id"])} +def stage_normalized_child(container: str, database: str, parent: dict[str, Any]) -> dict[str, Any]: + return _stage_child(container, database, prepare_normalized_child(parent)) + + +def stage_compiled_v3_bundle(container: str, database: str, bundle: dict[str, Any]) -> dict[str, Any]: + return _stage_child(container, database, prepare_compiled_v3_child(bundle)) + + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--container", required=True) parser.add_argument("--db", required=True) - parser.add_argument("--proposal-file", required=True, type=Path) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--proposal-file", type=Path, help="rich V1/V2 proposal to normalize then stage") + source.add_argument( + "--compiled-v3-bundle", type=Path, help="build-only V3 compiler bundle to revalidate then stage" + ) return parser.parse_args(argv) @@ -201,10 +297,15 @@ def main(argv: list[str] | None = None) -> int: try: if args.container == bound.PRODUCTION_CONTAINER: raise bound.CheckpointError("--container must be a disposable non-production target") - parent = json.loads(args.proposal_file.read_text(encoding="utf-8")) - if not isinstance(parent, dict): - raise bound.CheckpointError("--proposal-file must contain one JSON object") - result = stage_normalized_child(args.container, args.db, parent) + input_path = args.compiled_v3_bundle or args.proposal_file + packet = json.loads(input_path.read_text(encoding="utf-8")) + if not isinstance(packet, dict): + raise bound.CheckpointError("staging input must contain one JSON object") + result = ( + stage_compiled_v3_bundle(args.container, args.db, packet) + if args.compiled_v3_bundle + else stage_normalized_child(args.container, args.db, packet) + ) except (OSError, json.JSONDecodeError, bound.CheckpointError, SystemExit, ValueError) as exc: print(json.dumps({"status": "rejected", "error": str(exc)}, sort_keys=True)) return 2 @@ -216,6 +317,9 @@ def main(argv: list[str] | None = None) -> int: "payload_sha256": result["child"]["payload_sha256"], "created": result["database_receipt"].get("created"), "bound_container_id": result["bound_container_id"], + "contract_version": (result["child"].get("payload") or {}) + .get("apply_payload", {}) + .get("contract_version", 2), }, sort_keys=True, ) diff --git a/systemd/teleo-kb-apply-worker.service b/systemd/teleo-kb-apply-worker.service index d9cabde..b63347d 100644 --- a/systemd/teleo-kb-apply-worker.service +++ b/systemd/teleo-kb-apply-worker.service @@ -14,7 +14,11 @@ ExecStart=/opt/teleo-eval/pipeline/.venv/bin/python3 \ /opt/teleo-eval/workspaces/deploy-infra/scripts/apply_worker.py # Ships INERT: no --enable here, so each tick is report-only until an operator opts # in. Flip on (after the first manual apply is proven) by dropping an EnvironmentFile -# with KB_APPLY_WORKER_ENABLED=1 and KB_APPLY_RENDER_CMD=... — no unit edit needed. +# with KB_APPLY_WORKER_ENABLED=1, KB_APPLY_WORKER_CONTRACT_VERSION=2|3, +# and KB_APPLY_RENDER_CMD=... -- no unit edit needed. Once the V3 database cutover +# is installed, a V2-configured tick skips retained historical V2 approvals without +# applying them; set version 3 before enabling new execution. Migrate or reject each +# retained V2 packet explicitly through review instead of deleting it from the queue. EnvironmentFile=-/opt/teleo-eval/secrets/kb-apply-worker.env StandardOutput=journal StandardError=journal diff --git a/tests/fixtures/teleo_corpus_rebuild/forgejo/project-record.md b/tests/fixtures/teleo_corpus_rebuild/forgejo/project-record.md new file mode 100644 index 0000000..cc863d4 --- /dev/null +++ b/tests/fixtures/teleo_corpus_rebuild/forgejo/project-record.md @@ -0,0 +1,11 @@ +# Forgejo Project Record + +The retained project record names the repository and keeps source provenance reviewable. + +Repository: living-ip/teleo-infrastructure + +## Review Boundary + +Every extracted section remains a pending candidate until a separate reviewer approves it. + +No local corpus rebuild may stage or apply canonical database rows. diff --git a/tests/fixtures/teleo_corpus_rebuild/operator-notes.txt b/tests/fixtures/teleo_corpus_rebuild/operator-notes.txt new file mode 100644 index 0000000..ffbfbfc --- /dev/null +++ b/tests/fixtures/teleo_corpus_rebuild/operator-notes.txt @@ -0,0 +1,2 @@ +The local rebuild inventories exact UTF-8 bytes and source-root-relative paths. +Its receipt binds the source manifest, candidate index, extractor specification, and V3 proposal files. diff --git a/tests/test_apply_proposal.py b/tests/test_apply_proposal.py index 65dad6f..dd17043 100644 --- a/tests/test_apply_proposal.py +++ b/tests/test_apply_proposal.py @@ -4,6 +4,8 @@ Pure tests: exercise the SQL builders, payload validation, and dispatch without a live database. Runnable under pytest or standalone (`python3 tests/test_apply_proposal.py`). """ +import json +import re import sys from pathlib import Path @@ -100,6 +102,26 @@ def test_revise_strategy_sql_shape(): assert "::jsonb" in sql +def test_guarded_sql_canonicalizes_reviewed_timestamp_to_stable_utc_text(): + payload = _revise_payload()["apply_payload"] + approval = _approval_meta("revise_strategy", payload) + approval["reviewed_at"] = "2026-07-10T02:00:00.1+02:00" + + sql = ap.build_revise_strategy_sql(payload, "pid-1", "kb-apply", approval) + + assert sql.count("2026-07-10T00:00:00.100000Z") == 2 + assert "2026-07-10T02:00:00.1+02:00" not in sql + + +def test_guarded_sql_rejects_reviewed_timestamp_without_timezone(): + payload = _revise_payload()["apply_payload"] + approval = _approval_meta("revise_strategy", payload) + approval["reviewed_at"] = "2026-07-10T00:00:00" + + with pytest.raises(ValueError, match="timezone-aware ISO-8601"): + ap.build_revise_strategy_sql(payload, "pid-1", "kb-apply", approval) + + def test_revise_strategy_requires_agent_id(): payload = _revise_payload()["apply_payload"] del payload["agent_id"] @@ -240,6 +262,135 @@ def _approve_claim_bundle(): } +def _approve_claim_v3_bundle(): + agent_id = "11111111-1111-4111-8111-111111111111" + source_b = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" + evidence_a = { + "id": "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", + "claim_id": CLAIM_A, + "source_id": SOURCE_A, + "polarity": "supports", + "excerpt": "The primary record supports claim A.", + "locator_json": {"section": "A", "paragraph": 1}, + "source_content_hash": "a" * 64, + "rationale": "The cited paragraph directly supports the bounded proposition.", + "created_by": agent_id, + } + evidence_b = { + "id": "ffffffff-ffff-4fff-8fff-ffffffffffff", + "claim_id": CLAIM_B, + "source_id": source_b, + "polarity": "challenges", + "excerpt": "The primary record challenges claim B.", + "locator_json": {"section": "B", "paragraph": 2}, + "source_content_hash": "b" * 64, + "rationale": "The cited paragraph directly challenges the bounded proposition.", + "created_by": agent_id, + } + evidence = [evidence_a, evidence_b] + return { + "contract_version": 3, + "agent_id": agent_id, + "claims": [ + { + "id": CLAIM_A, + "type": "empirical", + "proposition": "Claim A has a reproducible evidence chain.", + "body_markdown": "Claim A is bounded to the cited primary record and exact locator.", + "scope": "V3 apply-writer disposable verification.", + "access_scope": "collective_internal", + "status": "active", + "revision_criteria": "Revise if the cited source or locator cannot be reproduced.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["v3", "apply-writer"], + "created_by": agent_id, + }, + { + "id": CLAIM_B, + "type": "conceptual", + "proposition": "Claim B remains explicitly challengeable.", + "body_markdown": "Claim B records a conceptual object together with its challenge evidence.", + "scope": "V3 apply-writer disposable verification.", + "access_scope": "collective_internal", + "status": "contested", + "revision_criteria": "Revise when the challenge is resolved or materially changes.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["v3", "challenge"], + "created_by": agent_id, + }, + ], + "sources": [ + { + "id": SOURCE_A, + "source_type": "paper", + "canonical_url": "https://example.test/v3/a", + "storage_uri": None, + "excerpt": "Primary source A.", + "content_hash": "a" * 64, + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-18T10:00:00+00:00", + "created_by": agent_id, + }, + { + "id": source_b, + "source_type": "observation", + "canonical_url": None, + "storage_uri": "gs://livingip-test/v3/source-b", + "excerpt": "Primary source B.", + "content_hash": "b" * 64, + "access_scope": "collective_internal", + "ingestion_origin": "system_import", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-18T10:01:00+00:00", + "created_by": agent_id, + }, + ], + "evidence": evidence, + "edges": [ + { + "id": "12121212-1212-4121-8121-121212121212", + "from_claim": CLAIM_A, + "to_claim": CLAIM_B, + "relation_type": "challenges", + "rationale": "Claim A supplies a bounded reason to challenge claim B.", + "scope_or_conditions": "Only inside the disposable V3 verification scope.", + "created_by": agent_id, + } + ], + "assessments": [ + { + "id": "13131313-1313-4131-8131-131313131313", + "claim_id": CLAIM_A, + "support_tier": "strong", + "challenge_tier": "none", + "overall_state": "support_dominant", + "rationale": "One exact primary record supports the bounded empirical claim.", + "evidence_set_hash": ap._v3_evidence_set_hash(CLAIM_A, evidence), + "supersedes_assessment_id": None, + }, + { + "id": "14141414-1414-4141-8141-141414141414", + "claim_id": CLAIM_B, + "support_tier": None, + "challenge_tier": None, + "overall_state": "challenge_dominant", + "rationale": "The conceptual claim has an explicit challenge without numeric evidence tiers.", + "evidence_set_hash": ap._v3_evidence_set_hash(CLAIM_B, evidence), + "supersedes_assessment_id": None, + }, + ], + "reasoning_tools": [], + } + + def test_approve_claim_bundle_sql_is_atomic_and_row_verified(): payload = _approve_claim_bundle() sql = ap.build_approve_claim_sql( @@ -264,6 +415,233 @@ def test_approve_claim_bundle_sql_is_atomic_and_row_verified(): assert "kb_stage.finish_approved_proposal" in sql +def test_v3_approve_claim_sql_writes_complete_epistemic_bundle() -> None: + payload = _approve_claim_v3_bundle() + proposal_id = "99999999-9999-4999-8999-999999999999" + sql = ap.build_approve_claim_sql( + payload, + proposal_id, + None, + _approval_meta("approve_claim", payload), + ) + + shared_lock = f"select pg_catalog.pg_advisory_lock_shared({ap.V3_CATALOG_DDL_ADVISORY_LOCK_KEY});" + shared_unlock = f"select pg_catalog.pg_advisory_unlock_shared({ap.V3_CATALOG_DDL_ADVISORY_LOCK_KEY});" + assert sql.startswith(f"{shared_lock}\nbegin;") + assert "set transaction isolation level serializable" in sql + assert sql.rstrip().endswith(shared_unlock) + assert sql.count(shared_lock) == 1 + assert sql.count(shared_unlock) == 1 + assert "insert into public.claims" in sql + assert "body_markdown" in sql + assert "accepted_by_proposal_id" in sql + assert "insert into public.sources" in sql + assert "content_hash" in sql + assert "insert into public.claim_evidence (" in sql + assert "locator_json" in sql + assert "insert into public.claim_edges (" in sql + assert "relation_type" in sql + assert "insert into public.claim_evidence_assessments" in sql + assert "evidence_set_hash" in sql + assert "insert into public.reasoning_tools" not in sql + expected_inserts = sum(len(payload[key]) for key in ("sources", "claims", "evidence", "edges", "assessments")) + assert sql.lower().count("on conflict do nothing") == expected_inserts + assert sql.index("do $v3_contract_guard$") < sql.index("kb_stage.assert_approved_proposal") + relation_locks = [ + f"select kb_stage.{ap.V3_CATALOG_LOCK_FUNCTION}();", + "lock table public.claim_edges in row exclusive mode;", + "lock table public.claim_evidence in row exclusive mode;", + "lock table public.claim_evidence_assessments in row exclusive mode;", + "lock table public.claims in row exclusive mode;", + "lock table public.sources in row exclusive mode;", + ] + ordered_catalog_guard = [ + shared_lock, + "begin;", + "set transaction isolation level serializable;", + *relation_locks, + "do $v3_contract_guard$", + "commit;", + shared_unlock, + ] + assert [sql.index(statement) for statement in ordered_catalog_guard] == sorted( + sql.index(statement) for statement in ordered_catalog_guard + ) + assert sql.index("kb_stage.assert_approved_proposal") < sql.index("insert into public.sources") + assert sql.index("insert into public.claim_evidence_assessments") < sql.index("kb_stage.finish_approved_proposal") + assert sql.index("on conflict do nothing") < sql.index("do $verify$") + assert "set local timezone = 'UTC'" in sql + assert "apply_proposal: V3 catalog contract is %s; refusing mutating transaction" in sql + assert "and superseded_by is null" in sql + expected_creator_checks = sum(len(payload[key]) for key in ("sources", "claims", "evidence", "edges")) + assert sql.count("created_by is not distinct from") == expected_creator_checks + + +def test_v3_catalog_guard_requires_the_full_hardened_contract() -> None: + sql = ap.build_v3_contract_state_query() + + assert ap.V3_CATALOG_DDL_ADVISORY_LOCK_KEY == 6072343533146485505 + assert ap.V3_PROTECTED_RELATIONS == ( + ("kb_stage", "kb_proposals"), + ("public", "claim_edges"), + ("public", "claim_evidence"), + ("public", "claim_evidence_assessments"), + ("public", "claims"), + ("public", "sources"), + ) + assert ( + tuple(sorted({(row["schema_name"], row["table_name"]) for row in ap.V3_EXPECTED_TRIGGER_CONTRACT})) + == ap.V3_PROTECTED_RELATIONS + ) + assert len(ap.V3_EXPECTED_TRIGGER_CONTRACT) == 24 + assert len({row["trigger_name"] for row in ap.V3_EXPECTED_TRIGGER_CONTRACT}) == 24 + assert all(row["when_expression"] is None for row in ap.V3_EXPECTED_TRIGGER_CONTRACT) + assert "when not artifacts_present then 'V2'" in sql + assert "when functions_valid and catalog_lock_function_valid and triggers_valid" in sql + assert "and owner_role_valid and memberships_valid then 'V3'" in sql + assert "else 'INVALID'" in sql + assert "actual.tgenabled = 'O'" in sql + assert "actual.tgtype = expected.trigger_type" in sql + assert "actual.update_columns = expected.update_columns" in sql + assert "actual.when_expression is not distinct from expected.when_expression" in sql + assert "actual.argument_hex = expected.argument_hex" in sql + assert "actual.tgconstraint = 0::pg_catalog.oid" in sql + assert "actual.tgoldtable is null" in sql + assert "actual.tgnewtable is null" in sql + assert "actual.tgfoid" in sql + assert "owner_role.rolname = E'kb_gate_owner'" in sql + assert "actual.function_security_definer = expected.function_security_definer" in sql + assert "actual.function_source_sha256 = expected.function_source_sha256" in sql + assert f"procedure.proname = E'{ap.V3_CATALOG_LOCK_FUNCTION}'" in sql + assert ap.V3_CATALOG_LOCK_FUNCTION_SOURCE_SHA256 in sql + assert "and prorettype = 'pg_catalog.void'::pg_catalog.regtype" in sql + assert "and grantee.rolname = 'kb_apply'" in sql + assert "and execute_acl_exact" in sql + assert "acl.grantee <> procedure.proowner" in sql + assert "from pg_catalog.pg_auth_members membership" in sql + + +def test_migration_trigger_contract_and_catalog_queries_match_worker_builder() -> None: + migration = (REPO_ROOT / "ops" / "teleo_v3_epistemic_contract.sql").read_text(encoding="utf-8") + contract_match = re.search(r"\$trigger_contract\$\s*(\[.*?\])\s*\$trigger_contract\$", migration, re.DOTALL) + + assert contract_match is not None + assert json.loads(contract_match.group(1)) == list(ap.V3_EXPECTED_TRIGGER_CONTRACT) + exclusive_lock = f"select pg_catalog.pg_advisory_xact_lock({ap.V3_CATALOG_DDL_ADVISORY_LOCK_KEY});" + assert migration.count(exclusive_lock) == 1 + assert migration.index(exclusive_lock) < migration.index( + f"create or replace function kb_stage.{ap.V3_CATALOG_LOCK_FUNCTION}()" + ) + assert migration.index(exclusive_lock) < migration.index( + "create or replace function kb_stage.teleo_v3_enforce_apply_contract_cutover()" + ) + assert migration.index(exclusive_lock) < migration.index("create or replace trigger") + + expected_query = ap.build_v3_contract_state_query( + into="v_contract_state", + expected_contract_expression="pg_catalog.current_setting('teleo.v3_expected_trigger_contract')", + ) + for marker in ("STATE", "POSTFLIGHT"): + query_match = re.search( + rf"TELEO_V3_CONTRACT_{marker}_QUERY_BEGIN\n(.*?)\n\s*-- TELEO_V3_CONTRACT_{marker}_QUERY_END", + migration, + re.DOTALL, + ) + assert query_match is not None + assert query_match.group(1).strip() == expected_query + ";" + + +@pytest.mark.parametrize( + ("mutation", "error"), + [ + (lambda payload: payload["claims"][0].update(body_markdown=""), "body_markdown"), + (lambda payload: payload["claims"][0].update(revision_kind="semantic"), "original claims only"), + (lambda payload: payload["sources"][0].update(content_hash="not-a-hash"), "SHA-256"), + (lambda payload: payload["evidence"][0].update(source_content_hash="b" * 64), "does not match"), + (lambda payload: payload["assessments"][0].update(evidence_set_hash="c" * 64), "evidence_set_hash"), + ( + lambda payload: payload["assessments"][0].update( + supersedes_assessment_id="15151515-1515-4151-8151-151515151515" + ), + "assessment supersession", + ), + (lambda payload: payload.update(reasoning_tools=[{"id": CLAIM_A}]), "reasoning_tools"), + ], +) +def test_v3_approve_claim_rejects_incomplete_or_unbound_payloads(mutation, error) -> None: + payload = _approve_claim_v3_bundle() + mutation(payload) + with pytest.raises(ValueError, match=error): + ap.build_approve_claim_sql( + payload, + "99999999-9999-4999-8999-999999999999", + None, + _approval_meta("approve_claim", payload), + ) + + +def test_v3_approve_claim_preserves_exact_reviewed_text_for_replay_receipts() -> None: + payload = _approve_claim_v3_bundle() + payload["claims"][0]["body_markdown"] = " Exact reviewed body with intentional spacing. " + + normalized = ap._normalize_v3_approve_claim(payload) + + assert normalized["claims"][0]["body_markdown"] == payload["claims"][0]["body_markdown"] + + +@pytest.mark.parametrize( + ("captured_at", "expected"), + [ + ("2026-07-18T10:00:00Z", "2026-07-18T10:00:00.000000Z"), + ("2026-07-18T12:00:00+02:00", "2026-07-18T10:00:00.000000Z"), + ("2026-07-18T12:00:00.123400+02:00", "2026-07-18T10:00:00.123400Z"), + ("2026-07-18T10:00:00.1Z", "2026-07-18T10:00:00.100000Z"), + ], +) +def test_v3_approve_claim_normalizes_capture_timestamp_to_fixed_utc( + captured_at: str, + expected: str, +) -> None: + payload = _approve_claim_v3_bundle() + payload["sources"][0]["captured_at"] = captured_at + + normalized = ap._normalize_v3_approve_claim(payload) + + assert normalized["sources"][0]["captured_at"] == expected + + +@pytest.mark.parametrize( + "captured_at", + [ + "2026-07-18 10:00:00+00:00", + "2026-07-18T10:00:00", + "2026-07-18T10:00:00.1234567Z", + "not-a-timestamp", + ], +) +def test_v3_approve_claim_rejects_untyped_capture_timestamp(captured_at: str) -> None: + payload = _approve_claim_v3_bundle() + payload["sources"][0]["captured_at"] = captured_at + with pytest.raises(ValueError, match="timezone-aware ISO-8601"): + ap._normalize_v3_approve_claim(payload) + + +@pytest.mark.parametrize( + "spelling", + [ + CLAIM_A.upper(), + "{" + CLAIM_A + "}", + CLAIM_A.replace("-", ""), + ], +) +def test_v3_approve_claim_rejects_noncanonical_uuid_spellings(spelling: str) -> None: + payload = _approve_claim_v3_bundle() + payload["claims"][0]["id"] = spelling + + with pytest.raises(ValueError, match="canonical lowercase hyphenated UUID"): + ap._normalize_v3_approve_claim(payload) + + def test_approve_claim_semantic_rows_accept_matches_and_reject_payload_mismatches(): payload = _approve_claim_bundle() sql = ap.build_approve_claim_sql( @@ -445,6 +823,72 @@ def test_build_apply_sql_requires_apply_payload(): ap.build_apply_sql(proposal, None) +@pytest.mark.parametrize( + ("proposal", "error"), + [ + ({"proposal_type": "approve_claim", "payload": []}, "payload must be a JSON object"), + ( + {"proposal_type": "approve_claim", "payload": {"apply_payload": []}}, + "payload.apply_payload must be a JSON object", + ), + ( + { + "proposal_type": "approve_claim", + "payload": {"apply_payload": {"contract_version": "3"}}, + }, + "must be integer 2 or 3", + ), + ( + { + "proposal_type": "approve_claim", + "payload": {"apply_payload": {"contract_version": 3.0}}, + }, + "must be integer 2 or 3", + ), + ( + { + "proposal_type": "approve_claim", + "payload": {"apply_payload": {"contract_version": True}}, + }, + "must be integer 2 or 3", + ), + ( + { + "proposal_type": "add_edge", + "payload": {"apply_payload": {"contract_version": 3}}, + }, + "must omit contract_version", + ), + ( + { + "proposal_type": "add_edge", + "payload": {"apply_payload": {"contract_version": None}}, + }, + "must omit contract_version", + ), + ], +) +def test_exact_contract_gate_rejects_malformed_and_hybrid_proposals( + proposal: dict, + error: str, +) -> None: + with pytest.raises(ValueError, match=error): + ap.proposal_contract_version(proposal) + + +def test_mutating_subprocess_contract_expectation_is_reasserted() -> None: + payload = _approve_claim_v3_bundle() + proposal = { + "id": "99999999-9999-4999-8999-999999999999", + "proposal_type": "approve_claim", + "payload": {"apply_payload": payload}, + } + + assert ap.assert_expected_contract_version(proposal, 3) == 3 + with pytest.raises(ValueError, match="does not match expected version 2"): + ap.assert_expected_contract_version(proposal, 2) + + def test_build_apply_sql_rejects_unsupported_type(): proposal = {"id": "pid", "proposal_type": "reject_claim", "payload": {"apply_payload": {}}} with pytest.raises(ValueError): diff --git a/tests/test_apply_worker.py b/tests/test_apply_worker.py index cd04d1d..2960a7d 100644 --- a/tests/test_apply_worker.py +++ b/tests/test_apply_worker.py @@ -42,18 +42,27 @@ sys.path.insert(0, str(REPO_ROOT / "scripts")) import apply_worker as w # noqa: E402 +REAL_DETECT_INSTALLED_CONTRACT = w.detect_installed_contract + def _dependency_restore(): original_load_password = w.ap.load_password + original_run_psql = w.ap.run_psql original_load_failure_state = w.load_failure_state original_fetch_candidates = w.fetch_candidates + original_detect_installed_contract = w.detect_installed_contract original_apply_one = w.apply_one original_render_one = w.render_one original_subprocess_run = w.subprocess.run + w.detect_installed_contract = lambda args, _password: ( + w.CONTRACT_STATE_V3 if args.contract_version == 3 else w.CONTRACT_STATE_V2 + ) yield w.ap.load_password = original_load_password + w.ap.run_psql = original_run_psql w.load_failure_state = original_load_failure_state w.fetch_candidates = original_fetch_candidates + w.detect_installed_contract = original_detect_installed_contract w.apply_one = original_apply_one w.render_one = original_render_one w.subprocess.run = original_subprocess_run @@ -86,6 +95,80 @@ def test_candidate_query_requires_apply_payload(): assert "payload ? 'apply_payload'" in sql +def test_candidate_query_is_pinned_to_v2_by_default() -> None: + sql = w.build_candidate_query() + assert "contract_version' = '2'" in sql + assert "contract_version' = '3'" not in sql + + +def test_candidate_query_can_be_pinned_to_v3_without_selecting_v2() -> None: + sql = w.build_candidate_query(contract_version=3) + assert "contract_version' = '3'" in sql + assert "contract_version' = '2'" not in sql + assert "proposal_type = 'approve_claim'" in sql + assert "proposal_type in ('revise_strategy'" not in sql + assert "jsonb_typeof(payload) = 'object'" in sql + assert "jsonb_typeof(payload->'apply_payload') = 'object'" in sql + assert "jsonb_typeof(payload->'apply_payload'->'contract_version') = 'number'" in sql + + +def test_candidate_query_rejects_unknown_contract_version() -> None: + with pytest.raises(ValueError, match="contract_version"): + w.build_candidate_query(contract_version=4) + + +def test_cli_can_select_the_v3_worker_contract_explicitly() -> None: + assert w.parse_args(["--contract-version", "3"]).contract_version == 3 + + +def test_cli_rejects_a_noncanonical_exact_proposal_uuid() -> None: + with pytest.raises(SystemExit): + w.parse_args(["--proposal-id", "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA"]) + + +def test_candidate_query_can_select_one_exact_proposal_without_queue_fallback() -> None: + proposal_id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + + sql = w.build_candidate_query(contract_version=3, proposal_id=proposal_id) + + exact_clause = f"id = {w.ap.sql_literal(proposal_id)}::uuid" + assert exact_clause in sql + assert sql.index(exact_clause) < sql.index("limit 20") + + +def test_candidate_query_rejects_noncanonical_exact_proposal_id() -> None: + with pytest.raises(ValueError, match="canonical"): + w.build_candidate_query(contract_version=3, proposal_id="AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA") + + +def test_installed_contract_detection_is_exact_and_tri_state() -> None: + seen: list[str] = [] + w.ap.run_psql = lambda _args, sql, _password: seen.append(sql) or w.CONTRACT_STATE_V3 + + assert REAL_DETECT_INSTALLED_CONTRACT(_args(contract_version=3), "pw") == w.CONTRACT_STATE_V3 + assert "then 'V2'" in seen[0] + assert "else 'INVALID'" in seen[0] + assert "actual.tgenabled = 'O'" in seen[0] + assert "actual.update_columns = expected.update_columns" in seen[0] + assert "actual.when_expression is not distinct from expected.when_expression" in seen[0] + assert "actual.argument_hex = expected.argument_hex" in seen[0] + assert "actual.tgfoid" in seen[0] + assert "owner_role.rolname = E'kb_gate_owner'" in seen[0] + assert "actual.function_security_definer = expected.function_security_definer" in seen[0] + assert "actual.function_source_sha256 = expected.function_source_sha256" in seen[0] + assert "search_path=pg_catalog, pg_temp" in seen[0] + assert "acl.grantee <> procedure.proowner" in seen[0] + assert "not owner_role.rolcanlogin" in seen[0] + assert "from pg_catalog.pg_auth_members membership" in seen[0] + + +def test_installed_contract_detection_rejects_unknown_readback() -> None: + w.ap.run_psql = lambda *_args, **_kwargs: "drifted" + + with pytest.raises(RuntimeError, match="V2, V3, or INVALID"): + REAL_DETECT_INSTALLED_CONTRACT(_args(contract_version=3), "pw") + + def test_candidate_query_lists_worker_types_only(): sql = w.build_candidate_query() for t in w.WORKER_TYPES: @@ -116,6 +199,20 @@ def test_parse_candidates_skips_blank_lines(): assert rows[0]["id"] == "a" +def test_fetch_candidates_fails_closed_on_hybrid_query_result() -> None: + w.ap.run_psql = lambda *_args, **_kwargs: json.dumps( + { + "id": "hybrid", + "proposal_type": "add_edge", + "agent_id": None, + "contract_version": "3", + } + ) + + with pytest.raises(RuntimeError, match="outside the exact worker contract"): + w.fetch_candidates(_args(contract_version=3), "pw", ()) + + # --- render hook ---------------------------------------------------------- # def test_render_command_none_when_unset(): assert w.build_render_command("", "agent-1") is None @@ -134,6 +231,8 @@ def test_render_command_expands_agent_id(): def _args(**over): base = dict( enable=False, + contract_version=2, + proposal_id=None, limit=20, max_per_tick=1, max_attempts=3, @@ -269,9 +368,28 @@ def test_apply_one_supplies_deterministic_private_receipt_path(): assert len(calls) == 1 assert calls[0][calls[0].index("--receipt-dir") + 1] == str(expected.parent) assert calls[0][calls[0].index("--receipt-out") + 1] == str(expected) + assert calls[0][calls[0].index("--expected-contract-version") + 1] == "2" assert stat.S_IMODE(expected.stat().st_mode) == 0o600 +def test_apply_one_forwards_the_exact_fresh_preflight() -> None: + proposal_id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + receipt_dir = Path(tempfile.mkdtemp()) / "receipts" + preflight = Path(tempfile.mkdtemp()) / "fresh-preflight.json" + calls = [] + + def _run(cmd, **_kwargs): + calls.append(cmd) + receipt_path = Path(cmd[cmd.index("--receipt-out") + 1]) + _write_valid_private_receipt(receipt_path, proposal_id) + return _completed(0, stdout='{"applied": true}') + + w.subprocess.run = _run + w.apply_one(_args(receipt_dir=str(receipt_dir), fresh_preflight=preflight), proposal_id) + + assert calls[0][calls[0].index("--fresh-preflight") + 1] == str(preflight) + + def test_apply_one_recovers_postcommit_receipt_failure_without_reapplying(): proposal_id = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" receipt_dir = Path(tempfile.mkdtemp()) / "receipts" @@ -406,6 +524,61 @@ def test_postcommit_receipt_error_does_not_increment_apply_failure_count(): assert w.load_failure_state(path) == {} +def test_installed_v3_contract_retains_and_skips_visible_v2_queue(capsys) -> None: + applied: list[str] = [] + w.ap.load_password = lambda _f: "pw" + w.detect_installed_contract = lambda _args, _password: w.CONTRACT_STATE_V3 + w.fetch_candidates = lambda _args, _password, excluded: [ + { + "id": "legacy-v2", + "proposal_type": "add_edge", + "agent_id": None, + "contract_version": "2", + } + ] + w.apply_one = lambda _args, proposal_id: applied.append(proposal_id) + + rc = w.run(_args(enable=True, contract_version=2)) + + captured = capsys.readouterr() + assert rc == 0 + assert applied == [] + assert "SKIP contract v2 execution" in captured.err + assert "remain review-visible" in captured.err + assert "migrate" in captured.err.lower() + assert "reject/cancel" in captured.err + + +def test_v3_worker_refuses_before_database_contract_install(capsys) -> None: + applied: list[str] = [] + w.ap.load_password = lambda _f: "pw" + w.detect_installed_contract = lambda _args, _password: w.CONTRACT_STATE_V2 + w.fetch_candidates = lambda *_args: (_ for _ in ()).throw(AssertionError("candidate selection must not run")) + w.apply_one = lambda _args, proposal_id: applied.append(proposal_id) + + rc = w.run(_args(enable=True, contract_version=3)) + + assert rc == 1 + assert applied == [] + assert "V3 cutover is not installed" in capsys.readouterr().err + + +def test_invalid_contract_aborts_both_workers_before_candidate_selection(capsys) -> None: + selected: list[int] = [] + applied: list[str] = [] + w.ap.load_password = lambda _f: "pw" + w.detect_installed_contract = lambda _args, _password: w.CONTRACT_STATE_INVALID + w.fetch_candidates = lambda *_args: selected.append(1) or [] + w.apply_one = lambda _args, proposal_id: applied.append(proposal_id) + + for version in (2, 3): + assert w.run(_args(enable=True, contract_version=version)) == 1 + + assert selected == [] + assert applied == [] + assert capsys.readouterr().err.count("no candidate selection or writes attempted") == 2 + + def test_report_only_gate_does_not_apply(monkeypatch=None): calls = [] w.ap.load_password = lambda _f: "pw" @@ -417,6 +590,45 @@ def test_report_only_gate_does_not_apply(monkeypatch=None): assert calls == [] # disabled worker performs NO writes +def test_exact_proposal_selector_refuses_missing_target_without_applying(capsys) -> None: + proposal_id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + applied: list[str] = [] + w.ap.load_password = lambda _f: "pw" + w.fetch_candidates = lambda *_args: [] + w.apply_one = lambda _args, selected: applied.append(selected) + + rc = w.run(_args(enable=True, contract_version=3, proposal_id=proposal_id)) + + captured = capsys.readouterr() + assert rc == 1 + assert applied == [] + assert proposal_id in captured.err + assert "no writes attempted" in captured.err + + +def test_required_exact_proposal_returns_distinct_missing_candidate_status(capsys) -> None: + proposal_id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + applied: list[str] = [] + w.ap.load_password = lambda _f: "pw" + w.fetch_candidates = lambda *_args: [] + w.apply_one = lambda _args, selected: applied.append(selected) + + rc = w.run( + _args( + enable=True, + contract_version=3, + proposal_id=proposal_id, + require_candidate=True, + ) + ) + + captured = capsys.readouterr() + assert rc == 2 + assert applied == [] + assert proposal_id in captured.err + assert "no writes attempted" in captured.err + + def test_enabled_worker_applies_and_renders(): applied, rendered = [], [] w.ap.load_password = lambda _f: "pw" diff --git a/tests/test_build_v3_genesis_ledger_bundle.py b/tests/test_build_v3_genesis_ledger_bundle.py new file mode 100644 index 0000000..c10974f --- /dev/null +++ b/tests/test_build_v3_genesis_ledger_bundle.py @@ -0,0 +1,547 @@ +from __future__ import annotations + +import hashlib +import json +import stat +from pathlib import Path + +import pytest + +from ops import build_v3_genesis_ledger_bundle as builder +from ops import run_local_genesis_ledger_rebuild as genesis_rebuild +from scripts import compile_kb_source_packet as compiler + +REVIEWER_ID = "11111111-1111-4111-8111-111111111111" +SERVICE_ID = "44444444-4444-4444-8444-444444444444" +PRIVATE_MARKER = "PRIVATE-V3-MATERIAL-MUST-NOT-BE-PRINTED" + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _write_manifest(path: Path) -> Path: + singleton_rows = [ + { + "kind": "identity", + "database": "teleo_fixture", + "server_version_num": 160010, + "transaction_read_only": "on", + "server_address": None, + "server_port": None, + "ssl": False, + "ssl_version": None, + "database_collation": "C", + "database_ctype": "C", + }, + {"kind": "schemas", "items": ["kb_stage", "public"]}, + ] + singleton_rows.extend( + {"kind": kind, "items": []} + for kind in sorted(genesis_rebuild.load_manifest.__globals__["SINGLETON_KINDS"] - {"identity", "schemas"}) + ) + rows = [ + *singleton_rows, + { + "kind": "table", + "schema": "public", + "table": "claims", + "row_count": 1, + "rowset_md5": "f" * 32, + }, + ] + path.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), encoding="utf-8") + return path + + +def _compiled_child(tmp_path: Path, *, salt: str) -> dict: + artifact_bytes = f"retained artifact {salt}\n".encode() + quote = f"The exact retained source {salt} requires review before canonical apply." + extracted_text = f"Header {salt}\n{quote}\n" + text_bytes = extracted_text.encode() + quote_bytes = quote.encode() + start = text_bytes.index(quote_bytes) + extractor_spec = f"fixture-extractor={salt}\n".encode() + artifact_sha256 = _sha256(artifact_bytes) + + manifest = { + "schema": compiler.MANIFEST_SCHEMA_V3, + "artifact_sha256": artifact_sha256, + "extracted_text_sha256": _sha256(text_bytes), + "extractor": { + "name": "fixture-semantic-extractor", + "version": "3.1.0", + "spec_sha256": _sha256(extractor_spec), + }, + "agent_id": REVIEWER_ID, + "source": { + "identity": f"document:sha256:{artifact_sha256}", + "source_key": f"source_{salt}", + "source_type": "article", + "title": f"Source {salt}", + "locator": f"artifact://sha256/{artifact_sha256}", + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-18T10:00:00+00:00", + }, + "claims": [ + { + "claim_key": f"review_gate_{salt}", + "type": "empirical", + "proposition": f"Source {salt} remains non-canonical until guarded review and apply succeed.", + "body_markdown": f"The retained source {salt} explicitly preserves the review boundary.", + "scope": f"The exact retained source fixture {salt}.", + "access_scope": "collective_internal", + "owner_agent_id": None, + "status": "active", + "revision_criteria": "Revise if the exact source bytes or review contract changes.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["genesis", "provenance", salt], + "evidence": [ + { + "evidence_key": f"exact_quote_{salt}", + "quote": quote, + "polarity": "supports", + "locator": { + "unit": "utf8_bytes", + "start": start, + "end": start + len(quote_bytes), + }, + "rationale": "The exact byte-bound sentence directly supports the bounded proposition.", + } + ], + "assessment": { + "support_tier": "strong", + "challenge_tier": "none", + "overall_state": "support_dominant", + "rationale": "One exact verified primary-record sentence supports the claim.", + "supersedes_assessment_id": None, + }, + } + ], + } + + input_dir = tmp_path / f"compiler-{salt}" + input_dir.mkdir() + artifact = input_dir / "artifact.bin" + text = input_dir / "extracted.txt" + manifest_path = input_dir / "manifest.json" + extractor_path = input_dir / "extractor.spec" + artifact.write_bytes(artifact_bytes) + text.write_text(extracted_text, encoding="utf-8") + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + extractor_path.write_bytes(extractor_spec) + return compiler.compile_source_packet(artifact, text, manifest_path, extractor_path)["strict_child_proposal"] + + +def _applied_envelope(proposal: dict) -> dict: + fields = ( + "id", + "proposal_type", + "status", + "payload", + "reviewed_by_handle", + "reviewed_by_agent_id", + "reviewed_at", + "review_note", + "applied_by_handle", + "applied_by_agent_id", + "applied_at", + ) + return {field: proposal[field] for field in fields} + + +def _v3_material(tmp_path: Path, *, sequence: int, salt: str) -> dict: + child = _compiled_child(tmp_path, salt=salt) + approved = { + "id": child["id"], + "proposal_type": child["proposal_type"], + "status": "approved", + "proposed_by_handle": child["proposed_by_handle"], + "proposed_by_agent_id": child["proposed_by_agent_id"], + "channel": child["channel"], + "source_ref": child["source_ref"], + "rationale": PRIVATE_MARKER, + "payload": child["payload"], + "reviewed_by_handle": "m3taversal", + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_at": "2026-07-18T10:01:00.000000Z", + "review_note": "Reviewed the exact source-normalized V3 payload.", + "applied_by_handle": None, + "applied_by_agent_id": None, + "applied_at": None, + "created_at": "2026-07-18T10:00:30.000000Z", + "updated_at": "2026-07-18T10:01:00.000000Z", + } + applied = { + **approved, + "status": "applied", + "applied_by_handle": "kb-apply", + "applied_by_agent_id": SERVICE_ID, + "applied_at": "2026-07-18T10:02:00.000000Z", + "updated_at": "2026-07-18T10:02:00.000001Z", + } + approval = { + "proposal_id": approved["id"], + "proposal_type": approved["proposal_type"], + "payload": approved["payload"], + "reviewed_by_handle": approved["reviewed_by_handle"], + "reviewed_by_agent_id": approved["reviewed_by_agent_id"], + "reviewed_by_db_role": "kb_review", + "reviewed_at": approved["reviewed_at"], + "review_note": approved["review_note"], + } + + proposal = _applied_envelope(applied) + apply_payload = proposal["payload"]["apply_payload"] + rows = genesis_rebuild.replay_receipt._expected_v3_semantic_rows(proposal, apply_payload) + created_at = "2026-07-18T10:01:30.000000Z" + for table, table_rows in rows.items(): + for row in table_rows: + if table == "claims": + row.update(created_at=created_at, updated_at=created_at) + elif table in { + "sources", + "claim_evidence", + "claim_edges", + "claim_evidence_assessments", + }: + row["created_at"] = created_at + apply_sql = genesis_rebuild.apply_engine.build_apply_sql(proposal, applied["applied_by_handle"]) + receipt = genesis_rebuild.replay_receipt.build_replay_receipt( + proposal, + rows, + apply_sql_sha256=genesis_rebuild.replay_receipt.sha256_text(apply_sql), + apply_sql_source="exact_executed_sql", + exported_at_utc="2026-07-18T10:03:00.000000Z", + ) + return { + "artifact": genesis_rebuild.MATERIAL_ARTIFACT, + "contract_version": genesis_rebuild.MATERIAL_CONTRACT_VERSION, + "sequence": sequence, + "approved_proposal": approved, + "approval_snapshot": approval, + "applied_proposal": applied, + "replay_receipt": receipt, + } + + +def _v2_material() -> dict: + proposal_id = "70000000-0000-4000-8000-000000000001" + claim_a = "70000000-0000-4000-8000-000000000002" + claim_b = "70000000-0000-4000-8000-000000000003" + payload = {"apply_payload": {"from_claim": claim_a, "to_claim": claim_b, "edge_type": "supports", "weight": 0.8}} + approved = { + "id": proposal_id, + "proposal_type": "add_edge", + "status": "approved", + "proposed_by_handle": "fixture", + "proposed_by_agent_id": REVIEWER_ID, + "channel": "fixture", + "source_ref": "private://fixture", + "rationale": PRIVATE_MARKER, + "payload": payload, + "reviewed_by_handle": "m3taversal", + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_at": "2026-07-18T10:01:00.000000Z", + "review_note": "Reviewed exact V2 fixture.", + "applied_by_handle": None, + "applied_by_agent_id": None, + "applied_at": None, + "created_at": "2026-07-18T10:00:30.000000Z", + "updated_at": "2026-07-18T10:01:00.000000Z", + } + applied = { + **approved, + "status": "applied", + "applied_by_handle": "kb-apply", + "applied_by_agent_id": SERVICE_ID, + "applied_at": "2026-07-18T10:02:00.000000Z", + "updated_at": "2026-07-18T10:02:00.000001Z", + } + approval = { + "proposal_id": proposal_id, + "proposal_type": "add_edge", + "payload": payload, + "reviewed_by_handle": "m3taversal", + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_by_db_role": "kb_review", + "reviewed_at": approved["reviewed_at"], + "review_note": approved["review_note"], + } + receipt = genesis_rebuild.replay_receipt.build_replay_receipt( + _applied_envelope(applied), + { + "claim_edges": [ + { + "id": "70000000-0000-4000-8000-000000000004", + "from_claim": claim_a, + "to_claim": claim_b, + "edge_type": "supports", + "weight": 0.8, + "created_by": None, + "created_at": "2026-07-18T10:01:30.000000Z", + } + ] + }, + apply_sql_sha256=genesis_rebuild.replay_receipt.sha256_text("exact V2 fixture SQL"), + apply_sql_source="exact_executed_sql", + exported_at_utc="2026-07-18T10:03:00.000000Z", + ) + return { + "artifact": genesis_rebuild.MATERIAL_ARTIFACT, + "contract_version": genesis_rebuild.MATERIAL_CONTRACT_VERSION, + "sequence": 1, + "approved_proposal": approved, + "approval_snapshot": approval, + "applied_proposal": applied, + "replay_receipt": receipt, + } + + +def _write_json(path: Path, value: object, *, private: bool = False) -> Path: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if private: + path.chmod(0o600) + return path + + +@pytest.fixture +def bundle_inputs(tmp_path: Path) -> dict[str, object]: + dump = tmp_path / "genesis.dump" + dump.write_bytes(b"PGDMPfixture") + genesis_manifest = _write_manifest(tmp_path / "genesis-manifest.jsonl") + final_manifest = _write_manifest(tmp_path / "final-manifest.jsonl") + materials = [ + _write_json(tmp_path / "material-0001.json", _v3_material(tmp_path, sequence=1, salt="alpha"), private=True), + _write_json(tmp_path / "material-0002.json", _v3_material(tmp_path, sequence=2, salt="beta"), private=True), + ] + note = tmp_path / "review-note.txt" + note.write_text(f"Explicitly reviewed all source-backed entries. {PRIVATE_MARKER}\n", encoding="utf-8") + return { + "dump": dump, + "genesis_manifest": genesis_manifest, + "final_manifest": final_manifest, + "materials": materials, + "note": note, + } + + +def _build_args(inputs: dict[str, object], ledger: Path, *, admission: Path | None = None) -> list[str]: + args = [ + "--genesis-dump", + str(inputs["dump"]), + "--genesis-manifest", + str(inputs["genesis_manifest"]), + "--final-manifest", + str(inputs["final_manifest"]), + ] + for material in inputs["materials"]: + args.extend(("--material", str(material))) + args.extend(("--output-ledger", str(ledger))) + if admission is not None: + args.extend( + ( + "--emit-admission-manifest", + "--admission-output", + str(admission), + "--reviewed-by", + "m3taversal", + "--reviewed-by-agent-id", + REVIEWER_ID, + "--reviewed-at", + "2026-07-18T14:00:00+02:00", + "--review-note-file", + str(inputs["note"]), + "--admission-scope", + "collective_core", + "--admission-basis", + "forgejo_primary_source", + ) + ) + return args + + +def _verify_args(inputs: dict[str, object], ledger: Path, admission: Path | None = None) -> list[str]: + args = [ + "--genesis-dump", + str(inputs["dump"]), + "--genesis-manifest", + str(inputs["genesis_manifest"]), + "--verify-ledger", + str(ledger), + "--ledger-sha256", + genesis_rebuild.canonical_rebuild.sha256_file(ledger), + ] + if admission is not None: + args.extend( + ( + "--admission-manifest", + str(admission), + "--admission-manifest-sha256", + genesis_rebuild.canonical_rebuild.sha256_file(admission), + ) + ) + return args + + +def test_deterministic_build_and_verify_every_pin_with_no_writes( + bundle_inputs: dict[str, object], tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + first_ledger = tmp_path / "ledger-first.json" + second_ledger = tmp_path / "ledger-second.json" + first_admission = tmp_path / "admission-first.json" + second_admission = tmp_path / "admission-second.json" + + assert builder.main(_build_args(bundle_inputs, first_ledger, admission=first_admission)) == 0 + first_output = capsys.readouterr() + assert builder.main(_build_args(bundle_inputs, second_ledger, admission=second_admission)) == 0 + second_output = capsys.readouterr() + + assert first_ledger.read_bytes() == second_ledger.read_bytes() + assert first_admission.read_bytes() == second_admission.read_bytes() + assert PRIVATE_MARKER not in first_output.out + first_output.err + second_output.out + second_output.err + before = {path: (path.read_bytes(), path.stat().st_mtime_ns) for path in (first_ledger, first_admission)} + + assert builder.main(_verify_args(bundle_inputs, first_ledger, first_admission)) == 0 + verification = json.loads(capsys.readouterr().out) + + assert verification["action"] == "verify" + assert verification["entry_count"] == 2 + assert verification["admission"]["review_metadata_provenance"] == builder.REVIEW_METADATA_PROVENANCE + assert verification["admission"]["cryptographically_authenticated"] is False + assert before == {path: (path.read_bytes(), path.stat().st_mtime_ns) for path in (first_ledger, first_admission)} + + +def test_reordered_materials_and_v2_material_are_refused( + bundle_inputs: dict[str, object], tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + reversed_inputs = {**bundle_inputs, "materials": list(reversed(bundle_inputs["materials"]))} + assert builder.main(_build_args(reversed_inputs, tmp_path / "reordered.json")) == 1 + assert not (tmp_path / "reordered.json").exists() + assert json.loads(capsys.readouterr().err)["code"] == "invalid_exported_material" + + v2_path = _write_json(tmp_path / "v2-material.json", _v2_material(), private=True) + v2_inputs = {**bundle_inputs, "materials": [v2_path]} + assert builder.main(_build_args(v2_inputs, tmp_path / "v2-ledger.json")) == 1 + assert not (tmp_path / "v2-ledger.json").exists() + assert json.loads(capsys.readouterr().err)["code"] == "legacy_material_forbidden" + + +def test_hash_drift_engine_tamper_and_symlink_material_fail_closed( + bundle_inputs: dict[str, object], tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + ledger = tmp_path / "ledger.json" + assert builder.main(_build_args(bundle_inputs, ledger)) == 0 + capsys.readouterr() + + first_material = bundle_inputs["materials"][0] + first_material.write_bytes(first_material.read_bytes() + b" ") + assert builder.main(_verify_args(bundle_inputs, ledger)) == 1 + assert json.loads(capsys.readouterr().err)["code"] == "genesis_preflight_refused" + + first_material.write_bytes(first_material.read_bytes()[:-1]) + tampered = json.loads(ledger.read_text(encoding="utf-8")) + tampered["engine"]["apply_engine_sha256"] = "0" * 64 + _write_json(ledger, tampered, private=True) + assert builder.main(_verify_args(bundle_inputs, ledger)) == 1 + assert json.loads(capsys.readouterr().err)["code"] == "genesis_preflight_refused" + + symlink = tmp_path / "material-symlink.json" + symlink.symlink_to(bundle_inputs["materials"][0]) + symlink_inputs = {**bundle_inputs, "materials": [symlink]} + assert builder.main(_build_args(symlink_inputs, tmp_path / "symlink-ledger.json")) == 1 + assert json.loads(capsys.readouterr().err)["code"] == "unsafe_input" + + +@pytest.mark.parametrize( + ("mutation", "expected_code"), + [ + ("missing_explicit_flag", "implicit_admission_forbidden"), + ("missing_reviewer_id", "incomplete_admission_review"), + ("alternate_reviewer", "invalid_reviewer_handle"), + ("noncanonical_reviewer_id", "invalid_reviewer_identity"), + ], +) +def test_admission_requires_complete_exact_operator_review( + bundle_inputs: dict[str, object], + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + mutation: str, + expected_code: str, +) -> None: + ledger = tmp_path / f"{mutation}-ledger.json" + admission = tmp_path / f"{mutation}-admission.json" + args = _build_args(bundle_inputs, ledger, admission=admission) + if mutation == "missing_explicit_flag": + args.remove("--emit-admission-manifest") + elif mutation == "missing_reviewer_id": + index = args.index("--reviewed-by-agent-id") + del args[index : index + 2] + elif mutation == "alternate_reviewer": + args[args.index("--reviewed-by") + 1] = "m3ta" + else: + args[args.index("--reviewed-by-agent-id") + 1] = "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA" + + assert builder.main(args) == 1 + output = capsys.readouterr() + assert json.loads(output.err)["code"] == expected_code + assert PRIVATE_MARKER not in output.out + output.err + assert not ledger.exists() + assert not admission.exists() + + +def test_admission_always_selects_all_entries_and_partial_manifest_is_rejected( + bundle_inputs: dict[str, object], tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + ledger = tmp_path / "ledger.json" + admission = tmp_path / "admission.json" + assert builder.main(_build_args(bundle_inputs, ledger, admission=admission)) == 0 + capsys.readouterr() + manifest = json.loads(admission.read_text(encoding="utf-8")) + assert [entry["sequence"] for entry in manifest["entries"]] == [1, 2] + + manifest["entries"].pop() + _write_json(admission, manifest, private=True) + assert builder.main(_verify_args(bundle_inputs, ledger, admission)) == 1 + assert json.loads(capsys.readouterr().err)["code"] == "genesis_preflight_refused" + + +def test_existing_outputs_are_never_overwritten_and_outputs_are_private( + bundle_inputs: dict[str, object], tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + existing = tmp_path / "existing-ledger.json" + existing.write_text("operator-owned\n", encoding="utf-8") + assert builder.main(_build_args(bundle_inputs, existing)) == 1 + assert existing.read_text(encoding="utf-8") == "operator-owned\n" + assert json.loads(capsys.readouterr().err)["code"] == "existing_output" + + ledger = tmp_path / "private-ledger.json" + admission = tmp_path / "private-admission.json" + assert builder.main(_build_args(bundle_inputs, ledger, admission=admission)) == 0 + output = capsys.readouterr() + assert stat.S_IMODE(ledger.stat().st_mode) == 0o600 + assert stat.S_IMODE(admission.stat().st_mode) == 0o600 + assert ledger.stat().st_nlink == admission.stat().st_nlink == 1 + assert PRIVATE_MARKER not in output.out + output.err + + +def test_material_tamper_is_redacted_and_publishes_nothing( + bundle_inputs: dict[str, object], tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + material_path = bundle_inputs["materials"][0] + material = json.loads(material_path.read_text(encoding="utf-8")) + material["approved_proposal"]["payload"]["private_secret"] = PRIVATE_MARKER + _write_json(material_path, material, private=True) + tampered_inputs = {**bundle_inputs, "materials": [material_path]} + ledger = tmp_path / "tampered-ledger.json" + + assert builder.main(_build_args(tampered_inputs, ledger)) == 1 + output = capsys.readouterr() + assert not ledger.exists() + assert PRIVATE_MARKER not in output.out + output.err + assert json.loads(output.err)["status"] == "refused" diff --git a/tests/test_compile_kb_source_packet.py b/tests/test_compile_kb_source_packet.py index 6978699..834d8a1 100644 --- a/tests/test_compile_kb_source_packet.py +++ b/tests/test_compile_kb_source_packet.py @@ -3,9 +3,12 @@ from __future__ import annotations import copy import hashlib import json +import os import stat import subprocess import sys +import tarfile +import uuid from pathlib import Path import pytest @@ -24,6 +27,123 @@ EXTRACTED_TEXT = ( ) CLAIM_QUOTE = "A staged proposal remains non-canonical until a human review and guarded apply both succeed." EVIDENCE_QUOTE = "Every proposed claim must retain an exact quote from the extracted source text." +EXTRACTOR_SPEC_BYTES = ( + b"livingip semantic extractor specification\n" + b"name=fixture-semantic-extractor\n" + b"version=3.1.0\n" + b"quotes=exact-utf8-byte-locators\n" +) +REPAIRED_WRITER_HEAD = "6fdaf19b96d027bc614e8ade0d475ba0954740b9" +V1_RENDERED_BUNDLE_SHA256 = "64795816f6b28e61df34cd00a62ffae12c7cc40bb26c475a56d47b4507d1bf73" +V2_RENDERED_BUNDLE_SHA256 = "e0a240371885a2c7fc714dd1b08099ef122d472e76ae1065542c1f305a27d816" +EXACT_WRITER_FILES = ( + "scripts/apply_proposal.py", + "scripts/kb_apply_replay_receipt.py", +) +EXACT_WRITER_PROBE = r""" +import copy +import hashlib +import json +import sys +from pathlib import Path + +scripts_dir = Path(sys.argv[1]).resolve() +child_path = Path(sys.argv[2]).resolve() +resolved_commit = sys.argv[3] +forbidden_checkout = Path(sys.argv[4]).resolve() + + +def canonical_bytes(value): + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def sha256_bytes(value): + return hashlib.sha256(value).hexdigest() + + +sys.path.insert(0, str(scripts_dir)) +import apply_proposal as writer +import kb_apply_replay_receipt as replay_receipt + +module_origins = { + "apply_proposal": str(Path(writer.__file__).resolve()), + "kb_apply_replay_receipt": str(Path(replay_receipt.__file__).resolve()), +} +expected_origins = { + "apply_proposal": str(scripts_dir / "apply_proposal.py"), + "kb_apply_replay_receipt": str(scripts_dir / "kb_apply_replay_receipt.py"), +} +assert module_origins == expected_origins +for origin in module_origins.values(): + origin_path = Path(origin) + assert origin_path.is_relative_to(scripts_dir) + assert not origin_path.is_relative_to(forbidden_checkout) + + +def forbidden_process(*_args, **_kwargs): + raise AssertionError("exact-writer compatibility probe attempted process or database execution") + + +writer.subprocess.run = forbidden_process +raw_child = child_path.read_bytes() +child = json.loads(raw_child.decode("utf-8")) +payload_before = copy.deepcopy(child["payload"]) +apply_payload = child["payload"]["apply_payload"] +apply_payload_before = canonical_bytes(apply_payload) +normalized = writer._normalize_v3_approve_claim(apply_payload) + +roles = {"supports": "grounds", "challenges": "contradicts", "illustrates": "illustrates"} +expected_evidence = [] +for row in apply_payload["evidence"]: + expected = copy.deepcopy(row) + expected["role"] = roles[row["polarity"]] + expected_evidence.append(expected) +assert normalized == { + "agent_id": apply_payload["agent_id"], + "claims": apply_payload["claims"], + "sources": apply_payload["sources"], + "evidence": expected_evidence, + "edges": apply_payload["edges"], + "assessments": apply_payload["assessments"], +} +assert normalized["sources"][0]["captured_at"] == apply_payload["sources"][0]["captured_at"] +assert apply_payload["sources"][0]["captured_at"].endswith(".000000Z") +assert canonical_bytes(apply_payload) == apply_payload_before + +approved = copy.deepcopy(child) +approved.update( + status="approved", + reviewed_by_handle="source-compiler-v3-exact-writer-test", + reviewed_by_agent_id=None, + reviewed_at="2000-01-01T00:00:00.000000Z", + review_note="Build-only exact-writer compatibility validation.", +) +sql = writer.build_apply_sql(approved, writer.SERVICE_AGENT_HANDLE) +assert "begin;" in sql +assert approved["payload"] == payload_before +assert child["payload"] == payload_before +assert child_path.read_bytes() == raw_child +assert sha256_bytes(canonical_bytes(child["payload"])) == child["payload_sha256"] + +receipt = { + "resolved_commit": resolved_commit, + "module_origins": module_origins, + "module_sha256": { + name: sha256_bytes(Path(origin).read_bytes()) for name, origin in module_origins.items() + }, + "input_child_file_sha256": sha256_bytes(raw_child), + "strict_child_payload_sha256": child["payload_sha256"], + "apply_payload_sha256_before": sha256_bytes(apply_payload_before), + "apply_payload_sha256_after": sha256_bytes(canonical_bytes(apply_payload)), + "payload_file_bytes_preserved": child_path.read_bytes() == raw_child, + "payload_object_preserved": child["payload"] == payload_before, + "canonical_timestamp_equal": normalized["sources"][0]["captured_at"] + == apply_payload["sources"][0]["captured_at"], + "build_apply_sql_sha256": sha256_bytes(sql.encode("utf-8")), + "sql_executed": False, +} +print(json.dumps(receipt, ensure_ascii=True, sort_keys=True)) +""" def _sha256(value: bytes) -> str: @@ -59,6 +179,78 @@ def _manifest() -> dict: } +def _byte_locator(quote: str, text: str = EXTRACTED_TEXT) -> dict: + text_bytes = text.encode("utf-8") + quote_bytes = quote.encode("utf-8") + start = text_bytes.index(quote_bytes) + return {"unit": "utf8_bytes", "start": start, "end": start + len(quote_bytes)} + + +def _manifest_v3( + *, + artifact_bytes: bytes = ARTIFACT_BYTES, + extracted_text: str = EXTRACTED_TEXT, + extractor_spec_bytes: bytes = EXTRACTOR_SPEC_BYTES, +) -> dict: + artifact_sha256 = _sha256(artifact_bytes) + return { + "schema": compiler.MANIFEST_SCHEMA_V3, + "artifact_sha256": artifact_sha256, + "extracted_text_sha256": _sha256(extracted_text.encode("utf-8")), + "extractor": { + "name": "fixture-semantic-extractor", + "version": "3.1.0", + "spec_sha256": _sha256(extractor_spec_bytes), + }, + "agent_id": "11111111-1111-4111-8111-111111111111", + "source": { + "identity": f"document:sha256:{artifact_sha256}", + "source_key": "source_compiler_note_v3", + "source_type": "article", + "title": "Source packet compiler note", + "locator": f"artifact://sha256/{artifact_sha256}", + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-18T10:00:00+00:00", + }, + "claims": [ + { + "claim_key": "review_gate_precedes_canonical_apply_v3", + "type": "empirical", + "proposition": "A staged proposal remains non-canonical until review and guarded apply succeed.", + "body_markdown": "The source explicitly separates staging from review and guarded canonical apply.", + "scope": "The retained source packet compiler note and its exact cited sentence.", + "access_scope": "collective_internal", + "owner_agent_id": None, + "status": "active", + "revision_criteria": "Revise if the bound source sentence changes or no longer supports the proposition.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["review", "v3", "provenance"], + "evidence": [ + { + "evidence_key": "review_gate_exact_sentence", + "quote": CLAIM_QUOTE, + "polarity": "supports", + "locator": _byte_locator(CLAIM_QUOTE, extracted_text), + "rationale": "The exact sentence directly supports the bounded proposition.", + } + ], + "assessment": { + "support_tier": "strong", + "challenge_tier": "none", + "overall_state": "support_dominant", + "rationale": "One exact verified primary-record sentence directly supports the claim.", + "supersedes_assessment_id": None, + }, + } + ], + } + + def _write_inputs(tmp_path: Path, manifest: dict | None = None, prefix: str = "input") -> tuple[Path, Path, Path]: tmp_path.mkdir(parents=True, exist_ok=True) artifact = tmp_path / f"{prefix}.bin" @@ -75,6 +267,173 @@ def _compile(tmp_path: Path, manifest: dict | None = None) -> dict: return compiler.compile_source_packet(*paths) +def _write_v3_inputs( + tmp_path: Path, + manifest: dict | None = None, + *, + artifact_bytes: bytes = ARTIFACT_BYTES, + extracted_text: str = EXTRACTED_TEXT, + extractor_spec_bytes: bytes = EXTRACTOR_SPEC_BYTES, + prefix: str = "v3-input", +) -> tuple[Path, Path, Path, Path]: + tmp_path.mkdir(parents=True, exist_ok=True) + artifact = tmp_path / f"{prefix}.bin" + text = tmp_path / f"{prefix}.txt" + manifest_path = tmp_path / f"{prefix}.json" + extractor_spec = tmp_path / f"{prefix}.extractor-spec" + artifact.write_bytes(artifact_bytes) + text.write_text(extracted_text, encoding="utf-8") + manifest_path.write_text( + json.dumps( + manifest + or _manifest_v3( + artifact_bytes=artifact_bytes, + extracted_text=extracted_text, + extractor_spec_bytes=extractor_spec_bytes, + ), + indent=2, + ), + encoding="utf-8", + ) + extractor_spec.write_bytes(extractor_spec_bytes) + return artifact, text, manifest_path, extractor_spec + + +def _compile_v3( + tmp_path: Path, + manifest: dict | None = None, + *, + artifact_bytes: bytes = ARTIFACT_BYTES, + extracted_text: str = EXTRACTED_TEXT, + extractor_spec_bytes: bytes = EXTRACTOR_SPEC_BYTES, +) -> dict: + return compiler.compile_source_packet( + *_write_v3_inputs( + tmp_path, + manifest, + artifact_bytes=artifact_bytes, + extracted_text=extracted_text, + extractor_spec_bytes=extractor_spec_bytes, + ) + ) + + +def _rendered_bundle_sha256(bundle: dict) -> str: + rendered = json.dumps(bundle, ensure_ascii=True, indent=2, sort_keys=True) + "\n" + return _sha256(rendered.encode("utf-8")) + + +def _resolve_exact_writer_commit(requested_head: str) -> str: + assert requested_head == REPAIRED_WRITER_HEAD, ( + "KB_SOURCE_COMPILER_WRITER_COMPAT_HEAD must name the exact reviewed writer commit " + f"{REPAIRED_WRITER_HEAD}, got {requested_head!r}" + ) + completed = subprocess.run( + ["git", "rev-parse", "--verify", f"{requested_head}^{{commit}}"], + cwd=REPO_ROOT, + capture_output=True, + check=False, + text=True, + ) + assert completed.returncode == 0, ( + f"could not resolve exact writer commit {requested_head}: {completed.stderr.strip()}" + ) + resolved = completed.stdout.strip() + assert resolved == requested_head, f"writer ref resolved to {resolved}, expected exact commit {requested_head}" + return resolved + + +def _git_file_sha256(commit: str, path: str) -> str: + completed = subprocess.run( + ["git", "show", f"{commit}:{path}"], + cwd=REPO_ROOT, + capture_output=True, + check=False, + ) + assert completed.returncode == 0, f"could not read {path} from exact writer commit {commit}" + return _sha256(completed.stdout) + + +def _run_exact_writer_compatibility(child: dict, tmp_path: Path) -> dict: + tmp_path.mkdir(parents=True, exist_ok=True) + requested_head = os.environ.get("KB_SOURCE_COMPILER_WRITER_COMPAT_HEAD", REPAIRED_WRITER_HEAD) + resolved = _resolve_exact_writer_commit(requested_head) + writer_root = tmp_path / "exact-writer" + writer_root.mkdir() + archive_path = tmp_path / "exact-writer.tar" + archived = subprocess.run( + [ + "git", + "archive", + "--format=tar", + f"--output={archive_path}", + resolved, + "--", + *EXACT_WRITER_FILES, + ], + cwd=REPO_ROOT, + capture_output=True, + check=False, + text=True, + ) + assert archived.returncode == 0, f"could not materialize exact writer commit {resolved}: {archived.stderr}" + with tarfile.open(archive_path, "r") as archive: + for member in archive.getmembers(): + member_path = Path(member.name) + assert not member_path.is_absolute() and ".." not in member_path.parts + archive.extractall(writer_root) + + writer_scripts = writer_root / "scripts" + expected_origins = { + "apply_proposal": str((writer_scripts / "apply_proposal.py").resolve()), + "kb_apply_replay_receipt": str((writer_scripts / "kb_apply_replay_receipt.py").resolve()), + } + expected_module_sha256 = { + "apply_proposal": _git_file_sha256(resolved, "scripts/apply_proposal.py"), + "kb_apply_replay_receipt": _git_file_sha256(resolved, "scripts/kb_apply_replay_receipt.py"), + } + child_path = tmp_path / "strict-child.json" + child_bytes = compiler.canonical_json_bytes(child) + child_path.write_bytes(child_bytes) + completed = subprocess.run( + [ + sys.executable, + "-I", + "-c", + EXACT_WRITER_PROBE, + str(writer_scripts), + str(child_path), + resolved, + str(REPO_ROOT), + ], + cwd=tmp_path, + capture_output=True, + check=False, + text=True, + env={"PYTHONHASHSEED": "0"}, + ) + assert completed.returncode == 0, ( + f"exact writer {resolved} compatibility execution failed\nstdout: {completed.stdout}\nstderr: {completed.stderr}" + ) + receipt = json.loads(completed.stdout) + assert receipt["resolved_commit"] == resolved + assert receipt["module_origins"] == expected_origins + assert receipt["module_sha256"] == expected_module_sha256 + assert receipt["input_child_file_sha256"] == _sha256(child_bytes) + assert receipt["strict_child_payload_sha256"] == child["payload_sha256"] + assert receipt["apply_payload_sha256_before"] == receipt["apply_payload_sha256_after"] + assert receipt["payload_file_bytes_preserved"] is True + assert receipt["payload_object_preserved"] is True + assert receipt["canonical_timestamp_equal"] is True + assert receipt["sql_executed"] is False + assert len(receipt["build_apply_sql_sha256"]) == 64 + receipt["requested_commit"] = requested_head + receipt_path = tmp_path / "exact-writer-compatibility-receipt.json" + receipt_path.write_text(json.dumps(receipt, ensure_ascii=True, indent=2, sort_keys=True) + "\n", encoding="utf-8") + receipt["receipt_path"] = str(receipt_path) + return receipt + + def test_compiles_deterministic_hash_bound_pending_review_bundle_without_db_write( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -162,6 +521,488 @@ def test_v2_manifest_carries_canonical_dedupe_candidates_into_review_packet(tmp_ assert staged_assessment == assessment +def test_v1_v2_rendered_bundle_bytes_match_pre_repair_baselines(tmp_path: Path) -> None: + artifact, text, manifest_path = _write_inputs(tmp_path / "v1") + v1 = compiler.compile_source_packet(artifact, text, manifest_path) + + v2_manifest = _manifest() + v2_manifest["schema"] = compiler.MANIFEST_SCHEMA_V2 + v2_manifest["dedupe"] = { + "database_search_query": "test query", + "candidate_claims": [], + "duplicates": [], + } + artifact, text, manifest_path = _write_inputs(tmp_path / "v2", v2_manifest) + v2 = compiler.compile_source_packet(artifact, text, manifest_path) + + assert _rendered_bundle_sha256(v1) == V1_RENDERED_BUNDLE_SHA256 + assert _rendered_bundle_sha256(v2) == V2_RENDERED_BUNDLE_SHA256 + + +def test_v3_compiles_exact_deterministic_pending_review_payload_without_db_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + first_paths = _write_v3_inputs(tmp_path / "first", prefix="original-source") + + def unexpected_db_call(*_args, **_kwargs): + raise AssertionError("V3 compiler attempted a database write") + + monkeypatch.setattr(compiler.stage, "stage_normalized_child", unexpected_db_call) + monkeypatch.setattr(compiler.stage.bound, "_psql_json", unexpected_db_call) + first = compiler.compile_source_packet(*first_paths) + second = compiler.compile_source_packet(*_write_v3_inputs(tmp_path / "copied", prefix="copied-source")) + + assert first == second + assert first["schema"] == compiler.BUNDLE_SCHEMA_V3 + assert first["status"] == "pending_review" + assert first["build_only"] is True + assert first["database_write_performed"] is False + assert first["canonical_apply_performed"] is False + assert first["stage_preview"] == { + "target_table": "kb_stage.kb_proposals", + "target_status": "pending_review", + "executed": False, + "stage_sql_sha256": first["stage_preview"]["stage_sql_sha256"], + } + assert len(first["stage_preview"]["stage_sql_sha256"]) == 64 + + parent = first["parent_proposal"] + child = first["strict_child_proposal"] + stage_sql = compiler.stage.build_stage_sql(child) + assert stage_sql.count("insert into kb_stage.kb_proposals") == 1 + assert "'pending_review'" in stage_sql + assert "public." not in stage_sql + assert first["stage_preview"]["stage_sql_sha256"] == _sha256(stage_sql.encode("utf-8")) + apply_payload = child["payload"]["apply_payload"] + assert parent["status"] == child["status"] == "pending_review" + assert child["proposal_type"] == "approve_claim" + assert child["payload_sha256"] == compiler.canonical_sha256(child["payload"]) + assert apply_payload["contract_version"] == 3 + assert apply_payload["source_proposal_id"] == parent["id"] + assert apply_payload["agent_id"] == _manifest_v3()["agent_id"] + assert apply_payload["edges"] == [] + assert apply_payload["reasoning_tools"] == [] + assert len(apply_payload["sources"]) == 1 + assert len(apply_payload["claims"]) == 1 + assert len(apply_payload["evidence"]) == 1 + assert len(apply_payload["assessments"]) == 1 + + source = apply_payload["sources"][0] + assert source == { + "id": source["id"], + "source_type": "article", + "canonical_url": None, + "storage_uri": _manifest_v3()["source"]["locator"], + "excerpt": CLAIM_QUOTE, + "content_hash": _sha256(ARTIFACT_BYTES), + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-18T10:00:00.000000Z", + "created_by": _manifest_v3()["agent_id"], + } + claim = apply_payload["claims"][0] + assert claim["proposition"] == _manifest_v3()["claims"][0]["proposition"] + assert claim["body_markdown"] == _manifest_v3()["claims"][0]["body_markdown"] + assert claim["scope"] == _manifest_v3()["claims"][0]["scope"] + assert claim["revision_criteria"] == _manifest_v3()["claims"][0]["revision_criteria"] + assert claim["revision_kind"] == "original" + assert claim["supersedes_id"] is None + assert claim["created_by"] == apply_payload["agent_id"] + + evidence = apply_payload["evidence"][0] + assert evidence["claim_id"] == claim["id"] + assert evidence["source_id"] == source["id"] + assert evidence["excerpt"] == CLAIM_QUOTE + assert evidence["polarity"] == "supports" + assert evidence["source_content_hash"] == source["content_hash"] + locator = evidence["locator_json"] + assert locator["unit"] == "utf8_bytes" + assert EXTRACTED_TEXT.encode("utf-8")[locator["start"] : locator["end"]] == CLAIM_QUOTE.encode("utf-8") + assert locator["quote_sha256"] == _sha256(CLAIM_QUOTE.encode("utf-8")) + assert locator["artifact_sha256"] == _sha256(ARTIFACT_BYTES) + assert locator["extracted_text_sha256"] == _sha256(EXTRACTED_TEXT.encode("utf-8")) + assert locator["extractor"] == _manifest_v3()["extractor"] + + assessment = apply_payload["assessments"][0] + assert assessment["claim_id"] == claim["id"] + assert assessment["evidence_set_hash"] == compiler.ap._v3_evidence_set_hash(claim["id"], [evidence]) + assert assessment["supersedes_assessment_id"] is None + + all_ids = [ + parent["id"], + child["id"], + source["id"], + claim["id"], + evidence["id"], + assessment["id"], + ] + assert len(all_ids) == len(set(all_ids)) + assert all(value == str(uuid.UUID(value)) for value in all_ids) + identity = apply_payload["normalization_manifest"]["identity_material"] + assert identity == { + "artifact_sha256": _sha256(ARTIFACT_BYTES), + "extracted_text_sha256": _sha256(EXTRACTED_TEXT.encode("utf-8")), + "extractor": _manifest_v3()["extractor"], + "manifest_file_sha256": first["hashes"]["manifest_file_sha256"], + "source_identity": _manifest_v3()["source"]["identity"], + "source_locator": _manifest_v3()["source"]["locator"], + "normalized_candidate_content_sha256": first["hashes"]["normalized_candidate_content_sha256"], + } + assert apply_payload["normalization_manifest"]["semantic_extraction_is_retained_input"] is True + assert apply_payload["normalization_manifest"]["model_training_claim"] is False + assert "retained input" in apply_payload["normalization_manifest"]["semantic_extraction_statement"] + assert first["hashes"]["extractor_spec_sha256"] == _sha256(EXTRACTOR_SPEC_BYTES) + assert first["compiler_contract"]["writer_apply_invoked"] is False + + without_bundle_hash = copy.deepcopy(first) + bundle_hash = without_bundle_hash["hashes"].pop("bundle_content_sha256") + assert bundle_hash == compiler.canonical_sha256(without_bundle_hash) + + +def test_v3_exact_writer_commit_preserves_payload_and_builds_apply_sql(tmp_path: Path) -> None: + bundle = _compile_v3(tmp_path) + apply_payload = bundle["strict_child_proposal"]["payload"]["apply_payload"] + assert apply_payload["sources"][0]["captured_at"] == "2026-07-18T10:00:00.000000Z" + receipt = _run_exact_writer_compatibility(bundle["strict_child_proposal"], tmp_path / "writer-proof") + + assert receipt["resolved_commit"] == REPAIRED_WRITER_HEAD + assert receipt["apply_payload_sha256_before"] == receipt["apply_payload_sha256_after"] + assert Path(receipt["receipt_path"]).is_file() + + +def test_v3_exact_writer_gate_cannot_fall_back_to_branch_local_apply_module( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bundle = _compile_v3(tmp_path / "payload") + branch_local_origin = str(Path(compiler.ap.__file__).resolve()) + + def forbidden_branch_local_writer(*_args, **_kwargs): + raise AssertionError("exact writer gate used branch-local compiler.ap") + + monkeypatch.setattr(compiler.ap, "_normalize_v3_approve_claim", forbidden_branch_local_writer) + monkeypatch.setattr(compiler.ap, "build_apply_sql", forbidden_branch_local_writer) + receipt = _run_exact_writer_compatibility(bundle["strict_child_proposal"], tmp_path / "writer-proof") + + assert branch_local_origin not in receipt["module_origins"].values() + assert all(Path(origin).is_relative_to(tmp_path) for origin in receipt["module_origins"].values()) + + +def test_v3_cli_rerun_is_byte_identical_and_emits_only_public_status(tmp_path: Path) -> None: + private_root = tmp_path / "private-v3-source" + artifact, text, manifest_path, extractor_spec = _write_v3_inputs(private_root) + outputs = [private_root / "compiled" / "first.json", private_root / "compiled" / "second.json"] + completed = [] + for output in outputs: + completed.append( + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "compile_kb_source_packet.py"), + "--artifact", + str(artifact), + "--text", + str(text), + "--manifest", + str(manifest_path), + "--extractor-spec", + str(extractor_spec), + "--output", + str(output), + ], + capture_output=True, + check=False, + ) + ) + + assert [item.returncode for item in completed] == [0, 0] + assert [item.stderr for item in completed] == [b"", b""] + assert completed[0].stdout == completed[1].stdout + assert outputs[0].read_bytes() == outputs[1].read_bytes() + assert json.loads(outputs[0].read_text(encoding="utf-8"))["schema"] == compiler.BUNDLE_SCHEMA_V3 + public_status = json.loads(completed[0].stdout.decode("utf-8")) + assert public_status["status"] == "pending_review" + assert public_status["database_write_performed"] is False + assert public_status["canonical_apply_performed"] is False + assert EXTRACTED_TEXT.encode("utf-8") not in completed[0].stdout + assert str(private_root).encode("utf-8") not in completed[0].stdout + assert all(stat.S_IMODE(output.stat().st_mode) == 0o600 for output in outputs) + + +def test_v3_stale_artifact_fails_closed_and_rehashed_artifact_changes_identity(tmp_path: Path) -> None: + original = _compile_v3(tmp_path / "original") + mutated_bytes = ARTIFACT_BYTES + b"mutated" + stale_manifest = _manifest_v3() + with pytest.raises(compiler.CompilerError, match="artifact SHA-256 does not match"): + _compile_v3(tmp_path / "stale", stale_manifest, artifact_bytes=mutated_bytes) + + rehashed_manifest = _manifest_v3(artifact_bytes=mutated_bytes) + stale_identity = copy.deepcopy(rehashed_manifest) + stale_identity["source"]["identity"] = _manifest_v3()["source"]["identity"] + with pytest.raises(compiler.CompilerError, match="content-addressed identity"): + _compile_v3(tmp_path / "stale-identity", stale_identity, artifact_bytes=mutated_bytes) + + stale_locator = copy.deepcopy(rehashed_manifest) + stale_locator["source"]["locator"] = _manifest_v3()["source"]["locator"] + with pytest.raises(compiler.CompilerError, match="actual artifact SHA-256"): + _compile_v3(tmp_path / "stale-locator", stale_locator, artifact_bytes=mutated_bytes) + + rehashed = _compile_v3(tmp_path / "rehashed", rehashed_manifest, artifact_bytes=mutated_bytes) + assert rehashed["strict_child_proposal"]["id"] != original["strict_child_proposal"]["id"] + assert rehashed["strict_child_proposal"]["payload_sha256"] != original["strict_child_proposal"]["payload_sha256"] + assert rehashed["hashes"]["packet_identity_sha256"] != original["hashes"]["packet_identity_sha256"] + + +@pytest.mark.parametrize("target", ["extractor_name", "extractor_version", "claim"]) +def test_v3_extraction_identity_and_normalized_candidate_mutations_change_proposal(tmp_path: Path, target: str) -> None: + original = _compile_v3(tmp_path / "original") + manifest = _manifest_v3() + if target == "extractor_name": + manifest["extractor"]["name"] = "fixture-semantic-extractor-revised" + elif target == "extractor_version": + manifest["extractor"]["version"] = "3.1.1" + else: + manifest["claims"][0]["proposition"] += " The proposal remains reviewable." + + mutated = _compile_v3(tmp_path / target, manifest) + assert mutated["strict_child_proposal"]["id"] != original["strict_child_proposal"]["id"] + assert mutated["strict_child_proposal"]["payload_sha256"] != original["strict_child_proposal"]["payload_sha256"] + assert mutated["hashes"]["packet_identity_sha256"] != original["hashes"]["packet_identity_sha256"] + + +def test_v3_requires_exact_independent_extractor_spec_and_binds_spec_mutations(tmp_path: Path) -> None: + artifact, text, manifest_path, _extractor_spec = _write_v3_inputs(tmp_path / "missing") + with pytest.raises(compiler.CompilerError, match="independently supplied extractor specification"): + compiler.compile_source_packet(artifact, text, manifest_path) + + mismatched_spec = EXTRACTOR_SPEC_BYTES + b"mismatch\n" + with pytest.raises(compiler.CompilerError, match="specification SHA-256 does not match"): + _compile_v3( + tmp_path / "mismatched", + _manifest_v3(), + extractor_spec_bytes=mismatched_spec, + ) + + original = _compile_v3(tmp_path / "original") + revised_spec = EXTRACTOR_SPEC_BYTES + b"normalization=revision-2\n" + revised_manifest = _manifest_v3(extractor_spec_bytes=revised_spec) + revised = _compile_v3( + tmp_path / "revised", + revised_manifest, + extractor_spec_bytes=revised_spec, + ) + assert revised["hashes"]["extractor_spec_sha256"] == _sha256(revised_spec) + assert revised["strict_child_proposal"]["id"] != original["strict_child_proposal"]["id"] + assert revised["strict_child_proposal"]["payload_sha256"] != original["strict_child_proposal"]["payload_sha256"] + assert revised["hashes"]["packet_identity_sha256"] != original["hashes"]["packet_identity_sha256"] + + +@pytest.mark.parametrize( + ("field", "value", "expected"), + [ + ("identity", "document:alternate:source", "content-addressed identity"), + ("locator", "artifact://mirror/source", "actual artifact SHA-256"), + ], +) +def test_v3_rejects_unbound_source_identity_or_locator(tmp_path: Path, field: str, value: str, expected: str) -> None: + manifest = _manifest_v3() + manifest["source"][field] = value + with pytest.raises(compiler.CompilerError, match=expected): + _compile_v3(tmp_path, manifest) + + +def test_v3_exact_manifest_bytes_are_identity_bound_but_paths_are_not(tmp_path: Path) -> None: + direct_paths = _write_v3_inputs(tmp_path / "direct", prefix="source") + copied_paths = _write_v3_inputs(tmp_path / "copied", prefix="renamed") + direct = compiler.compile_source_packet(*direct_paths) + copied = compiler.compile_source_packet(*copied_paths) + assert direct == copied + + compact_paths = _write_v3_inputs(tmp_path / "compact", prefix="source") + compact_manifest = compact_paths[2] + manifest_object = json.loads(compact_manifest.read_text(encoding="utf-8")) + compact_manifest.write_text( + json.dumps(manifest_object, separators=(",", ":"), sort_keys=True), + encoding="utf-8", + ) + compact = compiler.compile_source_packet(*compact_paths) + + assert compact["validated_manifest"] == direct["validated_manifest"] + assert compact["hashes"]["manifest_canonical_sha256"] == direct["hashes"]["manifest_canonical_sha256"] + assert compact["hashes"]["manifest_file_sha256"] != direct["hashes"]["manifest_file_sha256"] + assert compact["hashes"]["packet_identity_sha256"] != direct["hashes"]["packet_identity_sha256"] + assert compact["strict_child_proposal"]["id"] != direct["strict_child_proposal"]["id"] + assert compact["strict_child_proposal"]["payload_sha256"] != direct["strict_child_proposal"]["payload_sha256"] + + +def test_v3_valid_quote_binding_mutation_changes_identity_and_mismatches_fail_closed(tmp_path: Path) -> None: + original = _compile_v3(tmp_path / "original") + valid_mutation = _manifest_v3() + evidence = valid_mutation["claims"][0]["evidence"][0] + evidence["quote"] = EVIDENCE_QUOTE + evidence["locator"] = _byte_locator(EVIDENCE_QUOTE) + mutated = _compile_v3(tmp_path / "valid-mutation", valid_mutation) + assert mutated["strict_child_proposal"]["id"] != original["strict_child_proposal"]["id"] + assert mutated["strict_child_proposal"]["payload_sha256"] != original["strict_child_proposal"]["payload_sha256"] + + quote_mismatch = _manifest_v3() + quote_mismatch["claims"][0]["evidence"][0]["quote"] = "Invented source sentence." + with pytest.raises(compiler.CompilerError, match="does not bind the exact quote bytes"): + _compile_v3(tmp_path / "quote-mismatch", quote_mismatch) + + locator_mismatch = _manifest_v3() + locator_mismatch["claims"][0]["evidence"][0]["locator"]["start"] += 1 + with pytest.raises(compiler.CompilerError, match="does not bind the exact quote bytes"): + _compile_v3(tmp_path / "locator-mismatch", locator_mismatch) + + +def test_v3_rejects_missing_assessment_relevant_evidence_and_tampered_set_hash(tmp_path: Path) -> None: + manifest = _manifest_v3() + manifest["claims"][0]["evidence"][0]["polarity"] = "illustrates" + manifest["claims"][0]["assessment"].update( + support_tier="none", + challenge_tier="none", + overall_state="insufficient", + ) + with pytest.raises(compiler.CompilerError, match="requires at least one supports or challenges evidence"): + _compile_v3(tmp_path / "irrelevant", manifest) + + bundle = _compile_v3(tmp_path / "valid") + child = copy.deepcopy(bundle["strict_child_proposal"]) + child["payload"]["apply_payload"]["assessments"][0]["evidence_set_hash"] = "f" * 64 + child["payload_sha256"] = compiler.canonical_sha256(child["payload"]) + with pytest.raises(compiler.CompilerError, match="evidence_set_hash does not match claim-local evidence"): + compiler._validate_v3_compiled_child(child, bundle["validated_manifest"], bundle["quote_bindings"]) + + +@pytest.mark.parametrize( + ("mutation", "expected"), + [ + ( + lambda manifest: manifest.__setitem__("agent_id", manifest["agent_id"].replace("-", "")), + "canonical lowercase", + ), + ( + lambda manifest: manifest["source"].__setitem__("captured_at", "2026-07-18T10:00:00.0000000Z"), + "at most 6 fractional digits", + ), + (lambda manifest: manifest["claims"][0].__setitem__("type", "structural"), "type must be one of"), + ( + lambda manifest: manifest["claims"][0]["evidence"][0]["locator"].__setitem__("start", True), + "non-negative integer", + ), + ( + lambda manifest: manifest["claims"][0]["assessment"].__setitem__("support_tier", 1), + "support_tier must be null or one of", + ), + ], +) +def test_v3_rejects_noncanonical_typed_manifest_fields(tmp_path: Path, mutation, expected: str) -> None: + manifest = _manifest_v3() + mutation(manifest) + with pytest.raises(compiler.CompilerError, match=expected): + _compile_v3(tmp_path, manifest) + + +@pytest.mark.parametrize( + ("needle", "replacement"), + [ + ( + f'"schema": "{compiler.MANIFEST_SCHEMA_V3}",', + (f'"schema": "{compiler.MANIFEST_SCHEMA_V3}",\n "schema": "{compiler.MANIFEST_SCHEMA_V3}",'), + ), + ( + f'"schema": "{compiler.MANIFEST_SCHEMA_V3}",', + f'"schema": "{compiler.MANIFEST_SCHEMA_V3}",\n "schema": "{compiler.MANIFEST_SCHEMA}",', + ), + ( + f'"spec_sha256": "{_sha256(EXTRACTOR_SPEC_BYTES)}"', + ( + f'"spec_sha256": "{_sha256(EXTRACTOR_SPEC_BYTES)}",\n' + f' "spec_sha256": "{_sha256(EXTRACTOR_SPEC_BYTES)}"' + ), + ), + ( + '"overall_state": "support_dominant",', + '"overall_state": "support_dominant",\n "overall_state": "insufficient",', + ), + ], +) +def test_v3_rejects_identical_or_conflicting_duplicate_json_keys_at_any_depth( + tmp_path: Path, needle: str, replacement: str +) -> None: + paths = _write_v3_inputs(tmp_path) + manifest_path = paths[2] + raw_manifest = manifest_path.read_text(encoding="utf-8") + assert raw_manifest.count(needle) == 1 + manifest_path.write_text(raw_manifest.replace(needle, replacement, 1), encoding="utf-8") + + with pytest.raises(compiler.CompilerError, match="duplicate JSON object key"): + compiler.compile_source_packet(*paths) + + +def test_v3_duplicate_schema_key_fails_before_identity_or_output( + tmp_path: Path, + capsys: pytest.CaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + artifact, text, manifest_path, extractor_spec = _write_v3_inputs(tmp_path) + schema_line = f'"schema": "{compiler.MANIFEST_SCHEMA_V3}",' + raw_manifest = manifest_path.read_text(encoding="utf-8") + manifest_path.write_text( + raw_manifest.replace(schema_line, f'{schema_line}\n "schema": "{compiler.MANIFEST_SCHEMA_V3}",', 1), + encoding="utf-8", + ) + + def unexpected_identity(*_args, **_kwargs): + raise AssertionError("duplicate-key manifest reached proposal identity") + + monkeypatch.setattr(compiler, "_v3_identity_material", unexpected_identity) + output = tmp_path / "must-not-exist.json" + status = compiler.main( + [ + "--artifact", + str(artifact), + "--text", + str(text), + "--manifest", + str(manifest_path), + "--extractor-spec", + str(extractor_spec), + "--output", + str(output), + ] + ) + + assert status == 2 + assert json.loads(capsys.readouterr().out)["reason"] == "input_validation_failed" + assert not output.exists() + + +def test_v3_rejects_duplicate_semantic_keys_and_duplicate_generated_ids( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + duplicate_claim = _manifest_v3() + duplicate_claim["claims"].append(copy.deepcopy(duplicate_claim["claims"][0])) + with pytest.raises(compiler.CompilerError, match="duplicate claim_key"): + _compile_v3(tmp_path / "claim-key", duplicate_claim) + + duplicate_evidence = _manifest_v3() + duplicate_evidence["claims"][0]["evidence"].append(copy.deepcopy(duplicate_evidence["claims"][0]["evidence"][0])) + with pytest.raises(compiler.CompilerError, match="evidence_key duplicates"): + _compile_v3(tmp_path / "evidence-key", duplicate_evidence) + + monkeypatch.setattr( + compiler, + "stable_uuid_v3", + lambda _row_kind, _packet_identity_sha256, _semantic_key: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + ) + with pytest.raises(compiler.CompilerError, match="duplicate deterministic ids"): + _compile_v3(tmp_path / "generated-id") + + def test_rejects_normalized_child_that_replaces_atomic_claim_text_with_quote( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_export_kb_transition_replay_material.py b/tests/test_export_kb_transition_replay_material.py new file mode 100644 index 0000000..7ca2b71 --- /dev/null +++ b/tests/test_export_kb_transition_replay_material.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import argparse +import copy +import json +import os +import stat +import sys +from pathlib import Path + +import pytest + +from ops import run_local_genesis_ledger_rebuild as genesis_rebuild + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import export_kb_transition_replay_material as exporter # noqa: E402 +import kb_apply_replay_receipt as replay_receipt # noqa: E402 + +PROPOSAL_ID = "70000000-0000-4000-8000-000000000001" +REVIEWER_ID = "70000000-0000-4000-8000-000000000002" +SERVICE_ID = "70000000-0000-4000-8000-000000000003" +CLAIM_A = "70000000-0000-4000-8000-000000000004" +CLAIM_B = "70000000-0000-4000-8000-000000000005" + + +def _material() -> dict: + payload = { + "apply_payload": { + "from_claim": CLAIM_A, + "to_claim": CLAIM_B, + "edge_type": "supports", + "weight": 0.75, + }, + "title": "Private exact transition packet", + } + approved = { + "id": PROPOSAL_ID, + "proposal_type": "add_edge", + "status": "approved", + "proposed_by_handle": "leo", + "proposed_by_agent_id": REVIEWER_ID, + "channel": "private_export_test", + "source_ref": "private://transition/source", + "rationale": "Retain the exact approved row for deterministic reconstruction.", + "payload": payload, + "reviewed_by_handle": "m3ta", + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_at": "2026-07-18T10:01:00+00:00", + "review_note": "Approved the exact strict packet.", + "applied_by_handle": None, + "applied_by_agent_id": None, + "applied_at": None, + "created_at": "2026-07-18T10:00:00+00:00", + "updated_at": "2026-07-18T10:01:00.000001+00:00", + } + applied = { + **approved, + "status": "applied", + "applied_by_handle": "kb-apply", + "applied_by_agent_id": SERVICE_ID, + "applied_at": "2026-07-18T10:02:00+00:00", + "updated_at": "2026-07-18T10:02:00.000001+00:00", + } + approval = { + "proposal_id": PROPOSAL_ID, + "proposal_type": "add_edge", + "payload": payload, + "reviewed_by_handle": "m3ta", + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_by_db_role": "kb_review", + "reviewed_at": "2026-07-18T10:01:00+00:00", + "review_note": "Approved the exact strict packet.", + } + receipt_proposal = {field: applied[field] for field in exporter.RECEIPT_PROPOSAL_FIELDS} + receipt = replay_receipt.build_replay_receipt( + receipt_proposal, + { + "claim_edges": [ + { + "id": "70000000-0000-4000-8000-000000000006", + "from_claim": CLAIM_A, + "to_claim": CLAIM_B, + "edge_type": "supports", + "weight": 0.75, + "created_by": None, + "created_at": "2026-07-18T10:01:30+00:00", + } + ] + }, + apply_sql_sha256=replay_receipt.sha256_text("exact guarded apply SQL"), + apply_sql_source="exact_executed_sql", + exported_at_utc="2026-07-18T10:03:00+00:00", + ) + return { + "artifact": exporter.MATERIAL_ARTIFACT, + "contract_version": exporter.MATERIAL_CONTRACT_VERSION, + "sequence": 1, + "approved_proposal": approved, + "approval_snapshot": approval, + "applied_proposal": applied, + "replay_receipt": receipt, + } + + +def test_material_matches_genesis_replay_row_contract() -> None: + material = exporter.validate_material( + _material(), + expected_proposal_id=PROPOSAL_ID, + expected_sequence=1, + ) + + assert exporter.MATERIAL_ARTIFACT == genesis_rebuild.MATERIAL_ARTIFACT + assert exporter.MATERIAL_CONTRACT_VERSION == genesis_rebuild.MATERIAL_CONTRACT_VERSION + assert exporter.PROPOSAL_FIELDS == genesis_rebuild.PROPOSAL_FIELDS + assert exporter.APPROVAL_FIELDS == genesis_rebuild.APPROVAL_FIELDS + genesis_rebuild._validate_proposal_rows( + material["approved_proposal"], + material["applied_proposal"], + material["approval_snapshot"], + material["replay_receipt"]["proposal"], + ) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda value: value["approved_proposal"].__setitem__("unexpected", True), "invalid fields"), + ( + lambda value: value["applied_proposal"].__setitem__("rationale", "post-approval drift"), + "exact guarded apply transition", + ), + ( + lambda value: value["approval_snapshot"].__setitem__("review_note", "different review"), + "does not match approved proposal", + ), + ( + lambda value: value["replay_receipt"]["proposal"].__setitem__("id", CLAIM_A), + "worker replay receipt", + ), + (lambda value: value.__setitem__("sequence", 0), "positive integer"), + ], +) +def test_material_validation_fails_closed_on_drift(mutation, message: str) -> None: + material = copy.deepcopy(_material()) + mutation(material) + + with pytest.raises(exporter.ExportError, match=message): + exporter.validate_material(material, expected_proposal_id=PROPOSAL_ID) + + +def test_json_parser_rejects_duplicate_and_nonfinite_fields() -> None: + with pytest.raises(exporter.ExportError, match="duplicate"): + exporter._parse_json_object('{"artifact":"a","artifact":"b"}', label="fixture") + with pytest.raises(exporter.ExportError, match="invalid numeric"): + exporter._parse_json_object('{"sequence":NaN}', label="fixture") + + +def test_private_writer_is_atomic_idempotent_and_refuses_alias_or_rerun_drift(tmp_path: Path) -> None: + material = exporter.validate_material(_material(), expected_proposal_id=PROPOSAL_ID) + protected = tmp_path / "worker-receipt.json" + protected.write_text("private worker receipt", encoding="utf-8") + protected.chmod(0o600) + output = tmp_path / "material.json" + + written, disposition = exporter.write_private_material(output, material, protected_paths=(protected,)) + assert written == output + assert disposition == "created" + assert stat.S_IMODE(output.stat().st_mode) == 0o600 + assert output.stat().st_nlink == 1 + assert json.loads(output.read_text(encoding="utf-8")) == material + + _, rerun_disposition = exporter.write_private_material(output, material, protected_paths=(protected,)) + assert rerun_disposition == "unchanged" + + drifted = copy.deepcopy(material) + drifted["sequence"] = 2 + with pytest.raises(exporter.ExportError, match="different replay material"): + exporter.write_private_material(output, drifted, protected_paths=(protected,)) + with pytest.raises(exporter.ExportError, match="aliases a protected"): + exporter.write_private_material(protected, material, protected_paths=(protected,)) + + dangling = tmp_path / "dangling-output.json" + dangling.symlink_to(tmp_path / "missing-target.json") + with pytest.raises(exporter.ExportError, match="non-symlink"): + exporter.write_private_material(dangling, material, protected_paths=(protected,)) + + hardlink = tmp_path / "hardlink.json" + os.link(output, hardlink) + with pytest.raises(exporter.ExportError, match="hard-link"): + exporter.write_private_material(output, material, protected_paths=(protected,)) + + +def test_private_receipt_loader_requires_exact_mode_and_rejects_symlinks(tmp_path: Path) -> None: + receipt = _material()["replay_receipt"] + receipt_path = tmp_path / "receipt.json" + receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + receipt_path.chmod(0o600) + assert exporter._load_private_receipt(receipt_path) == receipt + + receipt_path.chmod(0o640) + with pytest.raises(exporter.ExportError, match="exactly 0600"): + exporter._load_private_receipt(receipt_path) + + receipt_path.chmod(0o600) + receipt_link = tmp_path / "receipt-link.json" + receipt_link.symlink_to(receipt_path) + with pytest.raises(exporter.ExportError, match="non-symlink"): + exporter._load_private_receipt(receipt_link) + + +def test_transition_fetch_uses_only_read_only_narrow_function(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, bool]] = [] + rows = {key: _material()[key] for key in ("approved_proposal", "approval_snapshot", "applied_proposal")} + + def fake_run_psql(args, sql: str, password: str, *, redact_output_on_error: bool = False) -> str: + assert args.role == "kb_apply" + assert password == "private-password" + calls.append((sql, redact_output_on_error)) + return json.dumps(rows) + + monkeypatch.setattr(exporter.apply_engine, "run_psql", fake_run_psql) + args = argparse.Namespace(container="disposable", db="teleo", host="127.0.0.1", role="kb_apply") + + assert exporter.fetch_transition_rows(args, "private-password", PROPOSAL_ID) == rows + assert len(calls) == 1 + sql, redacted = calls[0] + assert "transaction isolation level repeatable read read only" in sql + assert "kb_stage.export_applied_proposal_replay_rows" in sql + assert not any(keyword in sql.lower() for keyword in ("insert ", "update ", "delete ", "truncate ")) + assert redacted is True diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index 89aff9c..f22841f 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -1022,6 +1022,37 @@ def _load_module(path: Path): return module +@pytest.mark.parametrize("local", (True, False)) +def test_vps_bridge_psql_fails_closed_on_sql_errors( + local: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_module(BRIDGE_DIR / "kb_tool.py") + captured: dict[str, object] = {} + + def fake_run(command, **kwargs): + captured["command"] = command + captured["kwargs"] = kwargs + return SimpleNamespace(returncode=3, stdout="", stderr='ERROR: column "missing" does not exist') + + monkeypatch.setattr(module.subprocess, "run", fake_run) + args = SimpleNamespace( + local=local, + container="teleo-pg", + db="teleo", + ssh="teleo@example.invalid", + ) + + with pytest.raises(SystemExit, match=r"psql command failed \(3\)"): + module.psql_json(args, "select missing from claims;") + + command = captured["command"] + if local: + assert command[-3:] == ["-q", "-v", "ON_ERROR_STOP=1"] + else: + assert command[-1].endswith("-At -q -v ON_ERROR_STOP=1") + assert captured["kwargs"]["input"] == "select missing from claims;" + + @pytest.mark.parametrize( ("value", "minimum", "maximum"), ((True, 1, 2), ("1", 1, 2), (0, 1, 2), (3, 1, 2), (1, 2, 1)), diff --git a/tests/test_kb_apply_prereqs.py b/tests/test_kb_apply_prereqs.py index e66395c..63f4ac6 100644 --- a/tests/test_kb_apply_prereqs.py +++ b/tests/test_kb_apply_prereqs.py @@ -1,18 +1,27 @@ from __future__ import annotations +import os import shutil import subprocess import time import uuid from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path import pytest REPO_ROOT = Path(__file__).resolve().parents[1] MIGRATION = REPO_ROOT / "scripts" / "kb_apply_prereqs.sql" -POSTGRES_IMAGE = "postgres:16-alpine" +POSTGRES_IMAGE = os.environ.get("LEOCLEAN_RUNTIME_POSTGRES_IMAGE", "postgres:16-alpine") DATABASE = "teleo_clone" +LEGACY_APPROVED_ID = "aaaaaaaa-0000-4000-8000-000000000001" +LEGACY_APPLIED_ID = "aaaaaaaa-0000-4000-8000-000000000002" +FRESH_PROPOSAL_ID = "aaaaaaaa-0000-4000-8000-000000000003" +REVIEWER_ID = "11111111-1111-1111-1111-111111111111" +STALE_REVIEWER_ID = "22222222-2222-2222-2222-222222222222" +LEGACY_REVIEWER_HANDLE = "m3ta" +CANONICAL_REVIEWER_HANDLE = "m3taversal" BOOTSTRAP_SQL = r""" create role kb_gate_owner nologin inherit; @@ -84,6 +93,42 @@ create table kb_stage.kb_proposal_approvals ( review_note text not null ); +insert into kb_stage.kb_proposals ( + id, proposal_type, status, payload, reviewed_by_handle, reviewed_by_agent_id, + reviewed_at, review_note, applied_by_handle, applied_by_agent_id, applied_at, updated_at +) values + ( + 'aaaaaaaa-0000-4000-8000-000000000001', 'add_edge', 'approved', + '{"apply_payload":{"from_claim":"aaaaaaaa-1000-4000-8000-000000000001"}}'::jsonb, + 'm3ta', '11111111-1111-1111-1111-111111111111', '2026-07-18T10:00:00Z', + 'Exact still-approved legacy row.', null, null, null, '2026-07-18T10:00:01Z' + ), + ( + 'aaaaaaaa-0000-4000-8000-000000000002', 'add_edge', 'applied', + '{"apply_payload":{"from_claim":"aaaaaaaa-1000-4000-8000-000000000002"}}'::jsonb, + 'm3ta', '11111111-1111-1111-1111-111111111111', '2026-07-18T09:00:00Z', + 'Applied before exact approved rows were retained.', 'kb-apply', + '44444444-4444-4444-4444-444444444444', '2026-07-18T09:01:00Z', + '2026-07-18T09:01:00Z' + ); + +insert into kb_stage.kb_proposal_approvals ( + proposal_id, proposal_type, payload, reviewed_by_handle, reviewed_by_agent_id, + reviewed_by_db_role, reviewed_at, review_note +) values + ( + 'aaaaaaaa-0000-4000-8000-000000000001', 'add_edge', + '{"apply_payload":{"from_claim":"aaaaaaaa-1000-4000-8000-000000000001"}}'::jsonb, + 'm3ta', '11111111-1111-1111-1111-111111111111', 'kb_review', + '2026-07-18T10:00:00Z', 'Exact still-approved legacy row.' + ), + ( + 'aaaaaaaa-0000-4000-8000-000000000002', 'add_edge', + '{"apply_payload":{"from_claim":"aaaaaaaa-1000-4000-8000-000000000002"}}'::jsonb, + 'm3ta', '11111111-1111-1111-1111-111111111111', 'kb_review', + '2026-07-18T09:00:00Z', 'Applied before exact approved rows were retained.' + ); + create function kb_stage.approve_strict_proposal(uuid, text, text) returns jsonb language sql security definer as $function$ select '{}'::jsonb $function$; @@ -145,8 +190,8 @@ def _apply_migration(container: str, *, check: bool = True) -> subprocess.Comple return _psql(container, MIGRATION.read_text(), check=check) -@pytest.fixture(scope="module") -def migrated_postgres() -> Iterator[str]: +@contextmanager +def _postgres_clone() -> Iterator[str]: if shutil.which("docker") is None: pytest.skip("Docker is required for the disposable PostgreSQL migration canary") @@ -155,6 +200,7 @@ def migrated_postgres() -> Iterator[str]: [ "docker", "run", + "--pull=never", "--rm", "-d", "--name", @@ -189,7 +235,6 @@ def migrated_postgres() -> Iterator[str]: pytest.fail("disposable PostgreSQL did not become ready") _psql(container, BOOTSTRAP_SQL) - _apply_migration(container) yield container finally: subprocess.run( @@ -199,10 +244,154 @@ def migrated_postgres() -> Iterator[str]: ) +@pytest.fixture(scope="module") +def migrated_postgres() -> Iterator[str]: + with _postgres_clone() as container: + _apply_migration(container) + yield container + + def _lines(completed: subprocess.CompletedProcess[str]) -> set[str]: return {line for line in completed.stdout.splitlines() if line} +def test_legacy_only_reviewer_is_canonicalized_without_rewriting_history( + migrated_postgres: str, +) -> None: + identity_rows = _psql( + migrated_postgres, + f""" +select id::text || '|' || handle || '|' || kind + from public.agents + where handle in ('{LEGACY_REVIEWER_HANDLE}', '{CANONICAL_REVIEWER_HANDLE}') + order by handle; +""", + ).stdout.splitlines() + assert identity_rows == [f"{REVIEWER_ID}|{CANONICAL_REVIEWER_HANDLE}|human"] + + historical_rows = _psql( + migrated_postgres, + f""" +select 'proposal|' || id::text || '|' || reviewed_by_handle || '|' || reviewed_by_agent_id::text + from kb_stage.kb_proposals + where id in ('{LEGACY_APPROVED_ID}'::uuid, '{LEGACY_APPLIED_ID}'::uuid) + order by id; +select 'approval|' || proposal_id::text || '|' || reviewed_by_handle || '|' || + reviewed_by_agent_id::text + from kb_stage.kb_proposal_approvals + where proposal_id in ('{LEGACY_APPROVED_ID}'::uuid, '{LEGACY_APPLIED_ID}'::uuid) + order by proposal_id; +""", + ).stdout.splitlines() + assert historical_rows == [ + f"proposal|{LEGACY_APPROVED_ID}|{LEGACY_REVIEWER_HANDLE}|{REVIEWER_ID}", + f"proposal|{LEGACY_APPLIED_ID}|{LEGACY_REVIEWER_HANDLE}|{REVIEWER_ID}", + f"approval|{LEGACY_APPROVED_ID}|{LEGACY_REVIEWER_HANDLE}|{REVIEWER_ID}", + f"approval|{LEGACY_APPLIED_ID}|{LEGACY_REVIEWER_HANDLE}|{REVIEWER_ID}", + ] + + +def test_already_current_reviewer_migration_is_idempotent() -> None: + with _postgres_clone() as container: + _psql( + container, + f"update public.agents set handle = '{CANONICAL_REVIEWER_HANDLE}' " + f"where handle = '{LEGACY_REVIEWER_HANDLE}';", + ) + _apply_migration(container) + state_sql = f""" +select principal.reviewed_by_handle || '|' || principal.reviewed_by_agent_id::text || '|' || + principal.active::text || '|' || principal.created_at::text || '|' || + (select count(*) from public.agents where handle = '{LEGACY_REVIEWER_HANDLE}')::text + from kb_stage.kb_review_principals principal + where principal.db_role = 'kb_review'; +""" + first_state = _psql(container, state_sql).stdout.strip() + _apply_migration(container) + second_state = _psql(container, state_sql).stdout.strip() + + expected_state = f"{CANONICAL_REVIEWER_HANDLE}|{REVIEWER_ID}|true|1970-01-01 00:00:00+00|0" + assert first_state == expected_state + assert second_state == first_state + + +def test_stale_reviewer_principal_is_repaired_without_resetting_created_at() -> None: + with _postgres_clone() as container: + _psql( + container, + f""" +insert into public.agents (id, handle, kind) +values ('{STALE_REVIEWER_ID}', 'stale-reviewer', 'human'); +insert into kb_stage.kb_review_principals + (db_role, reviewed_by_handle, reviewed_by_agent_id, active, created_at) +values ( + 'kb_review', 'stale-reviewer', '{STALE_REVIEWER_ID}', false, + '2001-02-03T04:05:06Z' +); +""", + ) + _apply_migration(container) + principal_state = _psql( + container, + """ +select reviewed_by_handle || '|' || reviewed_by_agent_id::text || '|' || active::text || '|' || + created_at::text + from kb_stage.kb_review_principals + where db_role = 'kb_review'; +""", + ).stdout.strip() + + expected_state = f"{CANONICAL_REVIEWER_HANDLE}|{REVIEWER_ID}|true|2001-02-03 04:05:06+00" + assert principal_state == expected_state + + +def test_migration_refuses_distinct_legacy_and_canonical_reviewers() -> None: + with _postgres_clone() as container: + _psql( + container, + f""" +insert into public.agents (id, handle, kind) +values ('{STALE_REVIEWER_ID}', '{CANONICAL_REVIEWER_HANDLE}', 'human'); +""", + ) + refused = _apply_migration(container, check=False) + identity_rows = _psql( + container, + f""" +select handle || '|' || id::text + from public.agents + where handle in ('{LEGACY_REVIEWER_HANDLE}', '{CANONICAL_REVIEWER_HANDLE}') + order by handle; +""", + ).stdout.splitlines() + + assert refused.returncode != 0 + assert "both reviewer handles m3ta" in refused.stderr + assert "exist with different agent UUIDs" in refused.stderr + assert identity_rows == [ + f"{LEGACY_REVIEWER_HANDLE}|{REVIEWER_ID}", + f"{CANONICAL_REVIEWER_HANDLE}|{STALE_REVIEWER_ID}", + ] + + +def test_migration_refuses_to_fabricate_missing_human_reviewer() -> None: + with _postgres_clone() as container: + _psql( + container, + f""" +delete from kb_stage.kb_proposal_approvals; +delete from public.agents where handle = '{LEGACY_REVIEWER_HANDLE}'; +""", + ) + refused = _apply_migration(container, check=False) + remaining_agents = _psql(container, "select count(*) from public.agents;").stdout.strip() + + assert refused.returncode != 0 + assert "required human reviewer agent is missing" in refused.stderr + assert "handle m3ta or m3taversal" in refused.stderr + assert remaining_agents == "0" + + def test_reviewer_principal_seed_timestamp_is_deterministic(migrated_postgres: str) -> None: created_at = _psql( migrated_postgres, @@ -212,6 +401,161 @@ def test_reviewer_principal_seed_timestamp_is_deterministic(migrated_postgres: s assert created_at == "1970-01-01 00:00:00+00" +def test_upgrade_backfills_only_still_exact_approved_rows(migrated_postgres: str) -> None: + snapshot_state = _psql( + migrated_postgres, + f""" +select proposal_id::text || '|' || (approved_proposal_snapshot is not null)::text || '|' || + coalesce(approved_proposal_snapshot->>'status', 'unavailable') || '|' || + coalesce(approved_proposal_snapshot->>'reviewed_by_handle', 'unavailable') + from kb_stage.kb_proposal_approvals + where proposal_id in ('{LEGACY_APPROVED_ID}'::uuid, '{LEGACY_APPLIED_ID}'::uuid) + order by proposal_id; +""", + ).stdout.splitlines() + + assert snapshot_state == [ + f"{LEGACY_APPROVED_ID}|true|approved|{LEGACY_REVIEWER_HANDLE}", + f"{LEGACY_APPLIED_ID}|false|unavailable|unavailable", + ] + unavailable = _psql( + migrated_postgres, + f""" +set session authorization kb_apply; +select kb_stage.export_applied_proposal_replay_rows('{LEGACY_APPLIED_ID}'::uuid); +""", + check=False, + ) + assert unavailable.returncode != 0 + assert "exact applied transition unavailable" in unavailable.stderr + + +def test_new_approval_preserves_exact_post_approval_row_for_apply_export(migrated_postgres: str) -> None: + payload = '{"apply_payload":{"from_claim":"aaaaaaaa-1000-4000-8000-000000000003"}}' + _psql( + migrated_postgres, + f""" +insert into kb_stage.kb_proposals (id, proposal_type, status, payload, updated_at) +values ('{FRESH_PROPOSAL_ID}'::uuid, 'add_edge', 'pending_review', '{payload}'::jsonb, + '2026-07-18T11:00:00Z'); +set session authorization kb_review; +select kb_stage.approve_strict_proposal( + '{FRESH_PROPOSAL_ID}'::uuid, 'add_edge', '{payload}'::jsonb, '{CANONICAL_REVIEWER_HANDLE}', + 'Fresh exact approval snapshot.' +); +""", + ) + approval_identity = _psql( + migrated_postgres, + f""" +select proposal.reviewed_by_handle || '|' || proposal.reviewed_by_agent_id::text || '|' || + approval.reviewed_by_handle || '|' || approval.reviewed_by_agent_id::text || '|' || + (approval.approved_proposal_snapshot->>'reviewed_by_handle') || '|' || + (approval.approved_proposal_snapshot->>'reviewed_by_agent_id') + from kb_stage.kb_proposals proposal + join kb_stage.kb_proposal_approvals approval on approval.proposal_id = proposal.id + where proposal.id = '{FRESH_PROPOSAL_ID}'::uuid; +""", + ).stdout.strip() + assert approval_identity == ( + f"{CANONICAL_REVIEWER_HANDLE}|{REVIEWER_ID}|" + f"{CANONICAL_REVIEWER_HANDLE}|{REVIEWER_ID}|" + f"{CANONICAL_REVIEWER_HANDLE}|{REVIEWER_ID}" + ) + approval_state = _psql( + migrated_postgres, + f""" +select jsonb_build_object( + 'matches', approval.approved_proposal_snapshot = to_jsonb(proposal), + 'status', approval.approved_proposal_snapshot->>'status', + 'updated_at_matches', approval.approved_proposal_snapshot->'updated_at' = to_jsonb(proposal)->'updated_at' +)::text +from kb_stage.kb_proposal_approvals approval +join kb_stage.kb_proposals proposal on proposal.id = approval.proposal_id +where proposal.id = '{FRESH_PROPOSAL_ID}'::uuid; +""", + ).stdout.strip() + assert approval_state == '{"status": "approved", "matches": true, "updated_at_matches": true}' + + direct_approval_read = _psql( + migrated_postgres, + "set session authorization kb_apply; select * from kb_stage.kb_proposal_approvals;", + check=False, + ) + assert direct_approval_read.returncode != 0 + assert "permission denied for table kb_proposal_approvals" in direct_approval_read.stderr + + premature_export = _psql( + migrated_postgres, + f""" +set session authorization kb_apply; +select kb_stage.export_applied_proposal_replay_rows('{FRESH_PROPOSAL_ID}'::uuid); +""", + check=False, + ) + assert premature_export.returncode != 0 + assert "exact applied transition unavailable" in premature_export.stderr + + _psql( + migrated_postgres, + f""" +update kb_stage.kb_proposal_approvals + set approved_proposal_snapshot = + approved_proposal_snapshot || '{{"rationale":"tampered after approval"}}'::jsonb + where proposal_id = '{FRESH_PROPOSAL_ID}'::uuid; +""", + ) + for guard_call in ( + "kb_stage.assert_approved_proposal(" + "proposal.id, proposal.proposal_type, proposal.payload, proposal.reviewed_by_handle, " + "proposal.reviewed_by_agent_id, proposal.reviewed_at, proposal.review_note)", + "kb_stage.finish_approved_proposal(" + "proposal.id, proposal.proposal_type, proposal.payload, proposal.reviewed_by_handle, " + "proposal.reviewed_by_agent_id, proposal.reviewed_at, proposal.review_note, 'kb-apply')", + ): + refused = _psql( + migrated_postgres, + f""" +set session authorization kb_apply; +select {guard_call} +from kb_stage.kb_proposals proposal where proposal.id = '{FRESH_PROPOSAL_ID}'::uuid; +""", + check=False, + ) + assert refused.returncode != 0 + assert "immutable approved snapshot" in refused.stderr + _psql( + migrated_postgres, + f""" +update kb_stage.kb_proposal_approvals approval + set approved_proposal_snapshot = to_jsonb(proposal) + from kb_stage.kb_proposals proposal + where approval.proposal_id = proposal.id + and proposal.id = '{FRESH_PROPOSAL_ID}'::uuid; +""", + ) + + exported = _psql( + migrated_postgres, + f""" +set session authorization kb_apply; +select kb_stage.assert_approved_proposal( + proposal.id, proposal.proposal_type, proposal.payload, proposal.reviewed_by_handle, + proposal.reviewed_by_agent_id, proposal.reviewed_at, proposal.review_note +) +from kb_stage.kb_proposals proposal where proposal.id = '{FRESH_PROPOSAL_ID}'::uuid; +select kb_stage.finish_approved_proposal( + proposal.id, proposal.proposal_type, proposal.payload, proposal.reviewed_by_handle, + proposal.reviewed_by_agent_id, proposal.reviewed_at, proposal.review_note, 'kb-apply' +) +from kb_stage.kb_proposals proposal where proposal.id = '{FRESH_PROPOSAL_ID}'::uuid; +select jsonb_object_keys(kb_stage.export_applied_proposal_replay_rows('{FRESH_PROPOSAL_ID}'::uuid)) +order by 1; +""", + ).stdout.splitlines() + assert exported[-3:] == ["applied_proposal", "approval_snapshot", "approved_proposal"] + + def _catalog_snapshot(container: str) -> str: return _psql( container, @@ -239,9 +583,16 @@ select 'function|' || procedure.proname || '(' || pg_catalog.oidvectortypes(proc join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner where namespace.nspname = 'kb_stage' and procedure.proname in ( - 'approve_strict_proposal', 'assert_approved_proposal', 'finish_approved_proposal' + 'approve_strict_proposal', 'assert_approved_proposal', 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ) order by procedure.proname, procedure.proargtypes::text; +select 'column|' || attribute.attname || '|' || attribute.atttypid::regtype::text || '|' || + attribute.attnotnull + from pg_catalog.pg_attribute attribute + where attribute.attrelid = 'kb_stage.kb_proposal_approvals'::regclass + and attribute.attname = 'approved_proposal_snapshot' + and not attribute.attisdropped; select 'table_acl|' || namespace.nspname || '|' || relation.relname || '|' || case when acl.grantee = 0 then 'PUBLIC' else grantee_role.rolname end || '|' || pg_catalog.upper(acl.privilege_type) || '|' || acl.is_grantable @@ -272,7 +623,8 @@ select 'function_acl|' || procedure.proname || '(' || ) acl where namespace.nspname = 'kb_stage' and procedure.proname in ( - 'approve_strict_proposal', 'assert_approved_proposal', 'finish_approved_proposal' + 'approve_strict_proposal', 'assert_approved_proposal', 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ) and acl.grantee <> procedure.proowner order by procedure.proname, procedure.proargtypes::text, acl.grantee; @@ -343,7 +695,8 @@ select procedure.proname || '(' || pg_catalog.oidvectortypes(procedure.proargtyp join pg_catalog.pg_roles owner_role on owner_role.oid = procedure.proowner where namespace.nspname = 'kb_stage' and procedure.proname in ( - 'approve_strict_proposal', 'assert_approved_proposal', 'finish_approved_proposal' + 'approve_strict_proposal', 'assert_approved_proposal', 'finish_approved_proposal', + 'export_applied_proposal_replay_rows' ); """, ) @@ -354,6 +707,7 @@ select procedure.proname || '(' || pg_catalog.oidvectortypes(procedure.proargtyp 'kb_gate_owner|true|{"search_path=pg_catalog, pg_temp"}', "finish_approved_proposal(uuid, text, jsonb, text, uuid, timestamp with time zone, text, text)|" 'kb_gate_owner|true|{"search_path=pg_catalog, pg_temp"}', + 'export_applied_proposal_replay_rows(uuid)|kb_gate_owner|true|{"search_path=pg_catalog, pg_temp"}', } assert ( _psql( @@ -431,6 +785,7 @@ select count(*) "kb_apply|EXECUTE|false", "function_acl|finish_approved_proposal(uuid, text, jsonb, text, uuid, timestamp with time zone, text, text)|" "kb_apply|EXECUTE|false", + "function_acl|export_applied_proposal_replay_rows(uuid)|kb_apply|EXECUTE|false", } @@ -470,6 +825,38 @@ def test_migration_fails_closed_on_unexpected_table_grantee(migrated_postgres: s _psql(migrated_postgres, "revoke all on kb_stage.kb_proposal_approvals from rogue_grantee;") +def test_migration_refuses_rogue_canonical_insert_before_install_and_normalizes_after_removal() -> None: + with _postgres_clone() as container: + _psql(container, "grant usage on schema public to rogue_grantee;") + for grant_sql, revoke_sql in ( + ( + "grant insert on public.claims to rogue_grantee;", + "revoke insert on public.claims from rogue_grantee;", + ), + ( + "grant insert (id) on public.claims to rogue_grantee;", + "revoke insert (id) on public.claims from rogue_grantee;", + ), + ): + _psql(container, grant_sql) + refused = _apply_migration(container, check=False) + assert refused.returncode != 0 + assert "unexpected protected table grantee" in refused.stderr + assert _psql(container, "select count(*) from public.claims;").stdout.strip() == "0" + _psql(container, revoke_sql) + + _apply_migration(container) + rogue_insert = _psql( + container, + "set session authorization rogue_grantee; " + "insert into public.claims (id) values ('99999999-9999-4999-8999-999999999999');", + check=False, + ) + assert rogue_insert.returncode != 0 + assert "permission denied for table claims" in rogue_insert.stderr + assert _psql(container, "select count(*) from public.claims;").stdout.strip() == "0" + + def test_migration_fails_closed_on_unexpected_column_grantee(migrated_postgres: str) -> None: _psql( migrated_postgres, diff --git a/tests/test_kb_apply_replay_receipt.py b/tests/test_kb_apply_replay_receipt.py index 0bc302e..f07761c 100644 --- a/tests/test_kb_apply_replay_receipt.py +++ b/tests/test_kb_apply_replay_receipt.py @@ -70,8 +70,154 @@ def _edge_row() -> dict: } +def _v3_bundle_and_rows() -> tuple[dict, dict]: + agent_id = "11111111-1111-4111-8111-111111111111" + evidence_id = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee" + assessment_id = "ffffffff-ffff-4fff-8fff-ffffffffffff" + evidence = { + "id": evidence_id, + "claim_id": CLAIM_A, + "source_id": SOURCE_A, + "polarity": "supports", + "excerpt": "The primary record supports the bounded claim.", + "locator_json": {"section": "results", "paragraph": 3}, + "source_content_hash": "a" * 64, + "rationale": "The exact locator directly supports the proposition.", + "created_by": agent_id, + } + payload = { + "contract_version": 3, + "agent_id": agent_id, + "claims": [ + { + "id": CLAIM_A, + "owner_agent_id": None, + "type": "empirical", + "proposition": "The V3 apply path retains exact provenance.", + "body_markdown": "This bounded claim is supported by one content-addressed source.", + "scope": "Disposable V3 writer verification.", + "access_scope": "collective_internal", + "status": "active", + "revision_criteria": "Revise if the cited bytes or locator cannot be reproduced.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["v3", "replay"], + "created_by": agent_id, + } + ], + "sources": [ + { + "id": SOURCE_A, + "source_type": "paper", + "canonical_url": "https://example.test/v3-source", + "storage_uri": None, + "excerpt": "Content-addressed primary record.", + "content_hash": "a" * 64, + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-18T10:00:00+00:00", + "created_by": agent_id, + } + ], + "evidence": [evidence], + "edges": [], + "assessments": [ + { + "id": assessment_id, + "claim_id": CLAIM_A, + "support_tier": "strong", + "challenge_tier": "none", + "overall_state": "support_dominant", + "rationale": "The exact primary record supports the bounded claim.", + "evidence_set_hash": ap._v3_evidence_set_hash(CLAIM_A, [evidence]), + "supersedes_assessment_id": None, + } + ], + "reasoning_tools": [], + } + rows = { + "claims": [ + { + "id": CLAIM_A, + "owner_agent_id": None, + "type": "empirical", + "text": payload["claims"][0]["proposition"], + "proposition": payload["claims"][0]["proposition"], + "body_markdown": payload["claims"][0]["body_markdown"], + "scope": payload["claims"][0]["scope"], + "access_scope": "collective_internal", + "status": "active", + "admission_state": "admitted", + "revision_criteria": payload["claims"][0]["revision_criteria"], + "revision_kind": "original", + "supersedes_id": None, + "confidence": None, + "tags": ["v3", "replay"], + "created_by": agent_id, + "superseded_by": None, + "accepted_by_proposal_id": PROPOSAL_ID, + "created_at": "2026-07-18T10:01:00+00:00", + "updated_at": "2026-07-18T10:01:00+00:00", + } + ], + "sources": [ + { + "id": SOURCE_A, + "source_type": "paper", + "url": payload["sources"][0]["canonical_url"], + "storage_path": None, + "excerpt": payload["sources"][0]["excerpt"], + "hash": "a" * 64, + "created_by": agent_id, + "captured_at": payload["sources"][0]["captured_at"], + "canonical_url": payload["sources"][0]["canonical_url"], + "storage_uri": None, + "content_hash": "a" * 64, + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "accepted_by_proposal_id": PROPOSAL_ID, + "created_at": "2026-07-18T10:01:00+00:00", + } + ], + "claim_evidence": [ + { + "id": evidence_id, + "claim_id": CLAIM_A, + "source_id": SOURCE_A, + "role": "grounds", + "weight": None, + "created_by": agent_id, + "excerpt": evidence["excerpt"], + "locator_json": evidence["locator_json"], + "source_content_hash": "a" * 64, + "rationale": evidence["rationale"], + "polarity": "supports", + "accepted_from_proposal_id": PROPOSAL_ID, + "created_at": "2026-07-18T10:01:00+00:00", + } + ], + "claim_edges": [], + "claim_evidence_assessments": [ + { + **payload["assessments"][0], + "accepted_by_proposal_id": PROPOSAL_ID, + "created_at": "2026-07-18T10:01:00+00:00", + } + ], + "reasoning_tools": [], + } + return payload, rows + + def test_add_edge_postflight_query_selects_exact_semantic_key() -> None: sql = receipt.build_postflight_sql(_edge_proposal()) + assert sql.startswith("set timezone = 'UTC';") assert "public.claim_edges" in sql assert CLAIM_A in sql assert CLAIM_B in sql @@ -144,6 +290,42 @@ def test_receipt_hash_is_recoverable_and_independent_of_export_time() -> None: assert first["exported_at_utc"] != second["exported_at_utc"] +def test_fractional_timestamp_spellings_normalize_before_semantic_receipt_comparison() -> None: + payload, rows = _v3_bundle_and_rows() + payload["sources"][0]["captured_at"] = "2026-07-18T12:00:00.123400+02:00" + rows["sources"][0]["captured_at"] = "2026-07-18T10:00:00.1234Z" + proposal = _applied("approve_claim", payload) + proposal["reviewed_at"] = "2026-07-18T12:01:00.5+02:00" + proposal["applied_at"] = "2026-07-18T05:01:01.000001-05:00" + + data = _build_receipt( + proposal, + rows, + exported_at_utc="2026-07-18T12:02:00.25+02:00", + ) + + assert data["proposal"]["reviewed_at"] == "2026-07-18T10:01:00.500000Z" + assert data["proposal"]["applied_at"] == "2026-07-18T10:01:01.000001Z" + assert data["canonical_rows"]["sources"][0]["captured_at"] == "2026-07-18T10:00:00.123400Z" + assert data["exported_at_utc"] == "2026-07-18T10:02:00.250000Z" + assert receipt.validate_replay_receipt(data)["replay_ready"] is True + + +def test_legacy_uuid_spellings_canonicalize_without_postcommit_receipt_loss() -> None: + proposal = _edge_proposal() + proposal["payload"]["apply_payload"]["from_claim"] = CLAIM_A.upper() + proposal["payload"]["apply_payload"]["to_claim"] = "{" + CLAIM_B + "}" + row = _edge_row() + row["from_claim"] = CLAIM_A + row["to_claim"] = CLAIM_B + + data = _build_receipt(proposal, {"claim_edges": [row]}) + + assert data["canonical_rows"]["claim_edges"][0]["from_claim"] == CLAIM_A + assert data["canonical_rows"]["claim_edges"][0]["to_claim"] == CLAIM_B + assert receipt.validate_replay_receipt(data)["replay_ready"] is True + + def test_attach_evidence_receipt_matches_every_payload_controlled_field() -> None: proposal = _applied( "attach_evidence", @@ -271,6 +453,39 @@ def test_approve_claim_receipt_matches_all_supported_canonical_row_types() -> No } +def test_v3_approve_claim_receipt_binds_every_admitted_row_family() -> None: + payload, rows = _v3_bundle_and_rows() + proposal = _applied("approve_claim", payload) + + sql = receipt.build_postflight_sql(proposal) + for table in ( + "public.claims", + "public.sources", + "public.claim_evidence", + "public.claim_edges", + "public.claim_evidence_assessments", + ): + assert table in sql + + data = _build_receipt(proposal, rows) + assert data["expected_row_counts"] == { + "claims": 1, + "sources": 1, + "claim_evidence": 1, + "claim_edges": 0, + "claim_evidence_assessments": 1, + "reasoning_tools": 0, + } + assert data["checks"]["semantic_rows_match_strict_payload"] is True + + +def test_v3_approve_claim_receipt_rejects_assessment_drift() -> None: + payload, rows = _v3_bundle_and_rows() + rows["claim_evidence_assessments"][0]["evidence_set_hash"] = "b" * 64 + with pytest.raises(ValueError, match="semantic mismatch"): + _build_receipt(_applied("approve_claim", payload), rows) + + def test_revise_strategy_receipt_requires_the_fresh_active_rows() -> None: agent_id = "11111111-1111-4111-8111-111111111111" payload = { @@ -337,6 +552,14 @@ def test_receipt_refuses_legacy_applied_proposal_without_strict_payload() -> Non receipt.build_postflight_sql(proposal) +def test_receipt_refuses_explicit_null_legacy_contract_version() -> None: + proposal = _edge_proposal() + proposal["payload"]["apply_payload"]["contract_version"] = None + + with pytest.raises(ValueError, match="must omit contract_version"): + receipt.build_postflight_sql(proposal) + + def test_private_receipt_is_atomically_written_mode_0600(tmp_path: Path) -> None: data = _build_receipt(_edge_proposal(), {"claim_edges": [_edge_row()]}) output = tmp_path / "private" / "receipt.json" @@ -379,6 +602,7 @@ def test_apply_capture_writes_receipt_without_printing_canonical_rows( assert data["replay_ready"] is True assert data["apply_engine"]["source"] == "reconstructed_current_engine" assert len(seen) == 1 + assert seen[0].startswith("set timezone = 'UTC';") assert "public.claim_edges" in seen[0] args.receipt_out = str(tmp_path / "exact-executed.json") diff --git a/tests/test_kb_proposal_review_packet.py b/tests/test_kb_proposal_review_packet.py index 1bbd126..0f3cdf8 100644 --- a/tests/test_kb_proposal_review_packet.py +++ b/tests/test_kb_proposal_review_packet.py @@ -1,13 +1,16 @@ """Tests for scripts/kb_proposal_review_packet.py.""" -from pathlib import Path +import copy import sys +from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "scripts")) import kb_proposal_review_packet as rp # noqa: E402 +from tests import test_compile_kb_source_packet as source_compiler_fixture # noqa: E402 + def test_approved_freeform_packet_is_not_worker_applyable(): proposal = { @@ -73,3 +76,32 @@ def test_pending_packet_needs_human_review(): assert packet["review_state"] == "needs_human_review" assert packet["worker_applyable"] is False assert "approve, reject, or request edits" in packet["next_admin_action"] + + +def test_v3_source_packet_exposes_exact_rows_normalization_and_hashes(tmp_path: Path): + bundle = source_compiler_fixture._compile_v3(tmp_path / "source") + proposal = copy.deepcopy(bundle["strict_child_proposal"]) + + packet = rp.classify_proposal(proposal) + + apply_payload = proposal["payload"]["apply_payload"] + assert packet["review_state"] == "needs_human_review" + assert packet["payload_summary"]["contract_version"] == 3 + assert packet["payload_summary"]["strict_row_counts"] == { + "assessments": 1, + "claims": 1, + "edges": 0, + "evidence": 1, + "reasoning_tools": 0, + "sources": 1, + } + assert packet["strict_review_material"]["rows"]["claims"] == apply_payload["claims"] + assert packet["strict_review_material"]["rows"]["sources"] == apply_payload["sources"] + assert packet["strict_review_material"]["rows"]["evidence"] == apply_payload["evidence"] + assert packet["strict_review_material"]["rows"]["assessments"] == apply_payload["assessments"] + assert ( + packet["strict_review_material"]["normalization_manifest"] + == apply_payload["normalization_manifest"] + ) + assert packet["strict_review_material"]["apply_payload_sha256"] == rp.canonical_sha256(apply_payload) + assert packet["strict_review_material"]["proposal_payload_sha256"] == rp.canonical_sha256(proposal["payload"]) diff --git a/tests/test_proposal_apply_lifecycle.py b/tests/test_proposal_apply_lifecycle.py index c0390ee..2c24de1 100644 --- a/tests/test_proposal_apply_lifecycle.py +++ b/tests/test_proposal_apply_lifecycle.py @@ -416,6 +416,13 @@ def test_fresh_preflight_requires_every_owned_target_to_be_absent_and_is_determi lifecycle.build_fresh_preflight(proposal, incomplete) +def test_compensating_rollback_refuses_immutable_v3_admissions() -> None: + proposal = _proposal_before() + proposal["payload"]["apply_payload"]["contract_version"] = 3 + with pytest.raises(ValueError, match="V3 admitted rows are immutable"): + lifecycle.build_fresh_preflight(proposal, _empty_targets()) + + def test_fresh_owned_apply_consumes_timestamped_preflight_and_never_adopts_conflicts(): proposal, _approval, preflight, _receipt, _rollback_sql = _materials() diff --git a/tests/test_run_leo_integrated_learning_lifecycle.py b/tests/test_run_leo_integrated_learning_lifecycle.py index ffee6de..145fa30 100644 --- a/tests/test_run_leo_integrated_learning_lifecycle.py +++ b/tests/test_run_leo_integrated_learning_lifecycle.py @@ -1,9 +1,12 @@ from __future__ import annotations +import hashlib import json +import os import shutil import subprocess import sys +import uuid from pathlib import Path import pytest @@ -45,6 +48,30 @@ def _docker_available() -> bool: return result.returncode == 0 +def _commit_test_repository(path: Path, files: dict[str, str]) -> None: + subprocess.run(["git", "init", "-q", str(path)], check=True) + for relative_path, contents in files.items(): + target = path / relative_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(contents, encoding="utf-8") + subprocess.run(["git", "-C", str(path), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(path), + "-c", + "user.name=Lifecycle Test", + "-c", + "user.email=lifecycle-test@example.invalid", + "commit", + "-qm", + "test engine", + ], + check=True, + ) + + def test_source_packet_recompiles_to_retained_expectation(tmp_path: Path) -> None: packet = _write_source_packet(tmp_path) @@ -67,6 +94,34 @@ def test_source_packet_refuses_retained_packet_drift(tmp_path: Path) -> None: lifecycle.compile_source_packet(packet) +def test_execution_identity_requires_exact_committed_dependency_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _commit_test_repository(repo, {"scripts/engine.py": "print('committed')\n"}) + engine = repo / "scripts" / "engine.py" + monkeypatch.setattr(lifecycle, "REPO_ROOT", repo) + monkeypatch.setattr(lifecycle, "ENGINE_INPUT_PATHS", {"engine.runner": engine}) + + identity = lifecycle.execution_identity() + assert identity["engine_paths_match_head"] is True + assert identity["engine_relpaths"] == ["scripts/engine.py"] + assert identity["engine_head_sha256"]["engine.runner"] == hashlib.sha256(engine.read_bytes()).hexdigest() + + engine.write_text("print('dirty')\n", encoding="utf-8") + with pytest.raises(lifecycle.LifecycleError, match="does not match"): + lifecycle.execution_identity() + + +def test_engine_dependency_closure_includes_transitive_imports() -> None: + assert { + "engine.proposal_normalizer", + "engine.rich_proposal_plan", + "engine.clone_checkpoint", + } <= lifecycle.ENGINE_INPUT_PATHS.keys() + + def test_exact_identities_adds_observed_evidence_ids(tmp_path: Path) -> None: packet = _write_source_packet(tmp_path) compiled, _expected, _paths = lifecycle.compile_source_packet(packet) @@ -106,6 +161,598 @@ def test_lifecycle_requires_explicit_approve_action(tmp_path: Path) -> None: ) +def _application_role(*, complete_settings: bool = True) -> dict: + role = { + "name": "kb_apply", + "can_login": True, + "superuser": False, + "create_db": False, + "create_role": False, + "inherit": False, + "replication": False, + "bypass_rls": False, + "connection_limit": -1, + "default_transaction_read_only": None, + } + if complete_settings: + role["settings"] = [] + role["database_settings"] = [] + return role + + +def _minimal_loaded_manifest(*, complete_settings: bool = True) -> dict: + return { + "singleton": { + "identity": {"database": lifecycle.DATABASE, "transaction_read_only": "on"}, + "application_roles": {"items": [_application_role(complete_settings=complete_settings)]}, + "role_memberships": {"items": []}, + "object_acl": {"items": []}, + }, + "tables": {"public.fixture": {"row_count": 0, "rowset_md5": "0" * 32}}, + "performance": {}, + } + + +def _write_exporter_receipt( + path: Path, + dump: Path, + manifest_path: Path, + loaded_manifest: dict, + *, + source: dict[str, str] | None = None, +) -> dict: + source = source or { + "ssh_target": "disposable-test", + "container": "teleo-test-source", + "database": lifecycle.DATABASE, + "service": "disposable-postgres-test", + } + snapshot = { + "captured_at_utc": "2026-07-18T00:00:00+00:00", + "exported_snapshot_id": "00000003-00000001-1", + "txid_snapshot": "1:1:", + "wal_lsn": "0/1", + "system_identifier": "123456789", + } + authorization_ref = "codex-delegation:disposable-genesis-test" + manifest_sha256 = lifecycle.sha256_file(manifest_path) + dump_sha256 = lifecycle.sha256_file(dump) + manifest_sql_sha256 = lifecycle.sha256_file(lifecycle.PARITY_SQL_PATH) + source_context_sha256 = hashlib.sha256(b"disposable-source-context").hexdigest() + binding = lifecycle.snapshot_capture.build_provenance_binding( + run_id="disposable-genesis-test", + authorization_ref=authorization_ref, + source=source, + snapshot=snapshot, + dump_sha256=dump_sha256, + manifest_sql_sha256=manifest_sql_sha256, + source_manifest_sha256=manifest_sha256, + source_context_sha256=source_context_sha256, + ) + service_state = { + "ActiveState": "active", + "SubState": "running", + "MainPID": "123", + "NRestarts": "0", + "ActiveEnterTimestamp": "2026-07-18T00:00:00+00:00", + } + receipt = { + "artifact": lifecycle.TRUSTED_EXPORTER_ARTIFACT, + "schema": lifecycle.snapshot_capture.SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "receipt_version": 2, + "status": "pass", + "run_id": "disposable-genesis-test", + "capture_authorization_ref": authorization_ref, + "source": source, + "snapshot_exported": True, + "snapshot": snapshot, + "dump": {"path": str(dump), "bytes": dump.stat().st_size, "sha256": dump_sha256}, + "manifest": { + "path": str(manifest_path), + "sha256": manifest_sha256, + "manifest_sql_sha256": manifest_sql_sha256, + "table_count": len(loaded_manifest["tables"]), + "total_rows": sum(int(row["row_count"]) for row in loaded_manifest["tables"].values()), + "application_roles": loaded_manifest["singleton"]["application_roles"]["items"], + }, + "source_context": {"path": "source-context.txt", "sha256": source_context_sha256}, + "source_service": {"before": service_state, "after": service_state, "unchanged": True}, + "provenance_binding": binding, + "service_unchanged": True, + "production_db_mutated": False, + "telegram_send_attempted": False, + } + path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") + path.chmod(0o600) + return receipt + + +def _trust_test_exporter_receipt( + monkeypatch: pytest.MonkeyPatch, + path: Path, + receipt: dict, +) -> None: + registry = dict(lifecycle.TRUSTED_EXPORTER_RECEIPTS) + registry[lifecycle.sha256_file(path)] = { + "run_id": receipt["run_id"], + "capture_authorization_ref": receipt["capture_authorization_ref"], + "source": receipt["source"], + "dump_sha256": receipt["dump"]["sha256"], + "manifest_sha256": receipt["manifest"]["sha256"], + "manifest_sql_sha256": receipt["manifest"]["manifest_sql_sha256"], + "provenance_binding_sha256": receipt["provenance_binding"]["sha256"], + } + monkeypatch.setattr(lifecycle, "TRUSTED_EXPORTER_RECEIPTS", registry) + + +def _genesis_cli(tmp_path: Path) -> list[str]: + return [ + "--source-packet", + str(tmp_path / "packet"), + "--output", + str(tmp_path / "lifecycle-receipt.json"), + "--review-action", + "approve", + "--genesis-dump", + str(tmp_path / "genesis.dump"), + "--genesis-manifest", + str(tmp_path / "manifest.jsonl"), + "--genesis-exporter-receipt", + str(tmp_path / "exporter-receipt.json"), + ] + + +def test_genesis_cli_requires_complete_exporter_receipt_set(tmp_path: Path) -> None: + with pytest.raises(SystemExit): + lifecycle.parse_args(_genesis_cli(tmp_path)[:-2]) + + +def test_genesis_cli_rejects_caller_only_hash_flags(tmp_path: Path) -> None: + with pytest.raises(SystemExit): + lifecycle.parse_args([*_genesis_cli(tmp_path), "--genesis-dump-sha256", "a" * 64]) + + +def test_genesis_cli_rejects_moving_image_tag(tmp_path: Path) -> None: + with pytest.raises(SystemExit): + lifecycle.parse_args([*_genesis_cli(tmp_path), "--image", "postgres:16-alpine"]) + + +def test_exporter_receipt_dump_drift_fails_before_docker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + dump = tmp_path / "genesis.dump" + manifest = tmp_path / "manifest.jsonl" + exporter_receipt = tmp_path / "exporter-receipt.json" + dump.write_bytes(b"PGDMP-real-enough-for-signature-validation") + manifest.write_text("{}\n", encoding="utf-8") + loaded_manifest = _minimal_loaded_manifest() + receipt = _write_exporter_receipt(exporter_receipt, dump, manifest, loaded_manifest) + _trust_test_exporter_receipt(monkeypatch, exporter_receipt, receipt) + dump.write_bytes(dump.read_bytes() + b"drift") + monkeypatch.setattr(lifecycle, "load_manifest", lambda _path: loaded_manifest) + + with pytest.raises(lifecycle.LifecycleError, match="dump_hash_from_exporter_receipt"): + lifecycle.validate_genesis_inputs( + dump, + manifest, + exporter_receipt, + lifecycle.DEFAULT_GENESIS_IMAGE, + ) + + +def test_exporter_receipt_rebound_source_identity_fails_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + dump = tmp_path / "genesis.dump" + manifest = tmp_path / "manifest.jsonl" + exporter_receipt = tmp_path / "exporter-receipt.json" + dump.write_bytes(b"PGDMP-real-enough-for-signature-validation") + manifest.write_text("{}\n", encoding="utf-8") + loaded_manifest = _minimal_loaded_manifest() + receipt = _write_exporter_receipt(exporter_receipt, dump, manifest, loaded_manifest) + _trust_test_exporter_receipt(monkeypatch, exporter_receipt, receipt) + receipt["source"]["database"] = "attacker_rebound" + receipt["provenance_binding"] = lifecycle.snapshot_capture.build_provenance_binding( + run_id=receipt["run_id"], + authorization_ref=receipt["capture_authorization_ref"], + source=receipt["source"], + snapshot=receipt["snapshot"], + dump_sha256=receipt["dump"]["sha256"], + manifest_sql_sha256=receipt["manifest"]["manifest_sql_sha256"], + source_manifest_sha256=receipt["manifest"]["sha256"], + source_context_sha256=receipt["source_context"]["sha256"], + ) + exporter_receipt.write_text(json.dumps(receipt), encoding="utf-8") + exporter_receipt.chmod(0o600) + monkeypatch.setattr(lifecycle, "load_manifest", lambda _path: loaded_manifest) + + with pytest.raises(lifecycle.LifecycleError, match="trusted_exporter_receipt_hash"): + lifecycle.validate_genesis_inputs( + dump, + manifest, + exporter_receipt, + lifecycle.DEFAULT_GENESIS_IMAGE, + ) + + +def test_legacy_manifest_role_settings_projection_is_narrow_and_nonmutating() -> None: + manifest = { + "singleton": { + "application_roles": { + "items": [ + { + "name": "kb_observatory_read", + "default_transaction_read_only": "on", + }, + { + "name": "kb_apply", + "default_transaction_read_only": None, + }, + ] + } + } + } + + projected = lifecycle.manifest_for_role_replay(manifest) + + assert "settings" not in manifest["singleton"]["application_roles"]["items"][0] + assert projected["singleton"]["application_roles"]["items"][0]["settings"] == [ + {"name": "default_transaction_read_only", "value": "on"} + ] + assert projected["singleton"]["application_roles"]["items"][0]["database_settings"] == [] + assert projected["singleton"]["application_roles"]["items"][1]["settings"] == [] + assert lifecycle.role_settings_capture_complete(manifest) is False + + +def test_private_staging_is_read_only_and_original_path_drift_is_not_consumed(tmp_path: Path) -> None: + packet = _write_source_packet(tmp_path) + dump = tmp_path / "genesis.dump" + manifest = tmp_path / "manifest.jsonl" + exporter_receipt = tmp_path / "exporter-receipt.json" + dump.write_bytes(b"PGDMP-staged-input") + manifest.write_text("{}\n", encoding="utf-8") + exporter_receipt.write_text("{}\n", encoding="utf-8") + exporter_receipt.chmod(0o600) + + staged = lifecycle.stage_lifecycle_inputs(packet, dump, manifest, exporter_receipt) + try: + before = lifecycle.verify_staged_inputs(staged) + dump.write_bytes(b"PGDMP-mutated-original") + (packet / "proposal-packet.json").write_text("{}\n", encoding="utf-8") + after = lifecycle.verify_staged_inputs(staged) + + assert before == after + assert staged.genesis_dump.read_bytes() == b"PGDMP-staged-input" + assert staged.root.stat().st_mode & 0o777 == 0o500 + assert all( + (staged.root / row["relative_path"]).stat().st_mode & 0o777 == 0o400 for row in staged.files.values() + ) + finally: + cleanup = lifecycle.cleanup_staged_inputs(staged.root) + + assert cleanup["root_absent"] is True + + +def test_private_staging_detects_consumed_byte_mutation(tmp_path: Path) -> None: + packet = _write_source_packet(tmp_path) + dump = tmp_path / "genesis.dump" + manifest = tmp_path / "manifest.jsonl" + exporter_receipt = tmp_path / "exporter-receipt.json" + dump.write_bytes(b"PGDMP-staged-input") + manifest.write_text("{}\n", encoding="utf-8") + exporter_receipt.write_text("{}\n", encoding="utf-8") + exporter_receipt.chmod(0o600) + staged = lifecycle.stage_lifecycle_inputs(packet, dump, manifest, exporter_receipt) + staged_dump = staged.genesis_dump + try: + staged_dump.chmod(0o600) + staged_dump.write_bytes(b"PGDMP-mutated-staging") + staged_dump.chmod(0o400) + with pytest.raises(lifecycle.LifecycleError, match="staging bytes drifted"): + lifecycle.verify_staged_inputs(staged) + finally: + cleanup = lifecycle.cleanup_staged_inputs(staged.root) + + assert cleanup["root_absent"] is True + + +def test_genesis_initialization_cannot_call_synthetic_bootstrap(tmp_path: Path) -> None: + class RestoreOnlyPostgres: + def __init__(self) -> None: + self.restored = False + + def restore_genesis(self, _inputs): + self.restored = True + return {"parity": {"status": "pass"}} + + def psql(self, sql: str) -> str: + assert sql.startswith("alter role kb_review password") + return "" + + postgres = RestoreOnlyPostgres() + + result = lifecycle.initialize_database( + postgres, + genesis=object(), + review_password="unused", + apply_password="unused", + ) + + assert postgres.restored is True + assert result["mode"] == "exporter_receipt_bound_genesis_restore" + assert result["synthetic_schema_bootstrap_skipped"] is True + assert result["canonical_payload_preseed_performed"] is False + + +def test_fresh_initial_state_rejects_proposal_and_payload_preseed() -> None: + empty_rows = {family: [] for family in lifecycle.apply_lifecycle.ROW_TABLES} + with pytest.raises(lifecycle.LifecycleError, match="proposal or approval preseed"): + lifecycle.require_fresh_initial_state( + {"state": {"proposal": {"id": "preseed"}, "immutable_approval": None, "canonical_rows": empty_rows}} + ) + + preseed_rows = {family: [] for family in lifecycle.apply_lifecycle.ROW_TABLES} + preseed_rows["claims"] = [{"id": "preseed"}] + with pytest.raises(lifecycle.LifecycleError, match="canonical payload target preseed"): + lifecycle.require_fresh_initial_state( + {"state": {"proposal": None, "immutable_approval": None, "canonical_rows": preseed_rows}} + ) + + +def test_payload_controlled_fingerprint_ignores_generated_fields_but_detects_payload_drift() -> None: + rows = {family: [] for family in lifecycle.PAYLOAD_CONTROLLED_FIELDS} + rows["claims"] = [ + { + "id": "11111111-1111-4111-8111-111111111111", + "type": "fact", + "text": "Stable payload text", + "status": "open", + "confidence": 0.7, + "tags": ["test"], + "created_by": None, + "superseded_by": None, + "created_at": "2026-01-01T00:00:00+00:00", + } + ] + first = lifecycle.payload_controlled_fingerprint(rows) + rows["claims"][0]["created_at"] = "2027-01-01T00:00:00+00:00" + rows["claims"][0]["updated_at"] = "2027-01-01T00:00:01+00:00" + second = lifecycle.payload_controlled_fingerprint(rows) + rows["claims"][0]["text"] = "Drifted payload text" + drifted = lifecycle.payload_controlled_fingerprint(rows) + + assert first["sha256"] == second["sha256"] + assert drifted["sha256"] != first["sha256"] + + +def _fake_genesis_run(run_number: int) -> dict: + controlled = { + "schema": "livingip.leoPayloadControlledRowsFingerprint.v1", + "sha256": "c" * 64, + "table_sha256": {family: "d" * 64 for family in lifecycle.PAYLOAD_CONTROLLED_FIELDS}, + "row_counts": {family: 0 for family in lifecycle.PAYLOAD_CONTROLLED_FIELDS}, + "rows": {family: [] for family in lifecycle.PAYLOAD_CONTROLLED_FIELDS}, + } + return { + "status": "pass", + "run_number": run_number, + "source": { + "artifact_sha256": "1" * 64, + "manifest_canonical_sha256": "2" * 64, + "retained_proposal_packet_file_sha256": "3" * 64, + "retained_proposal_packet_canonical_sha256": "4" * 64, + }, + "compiled": {"bundle_sha256": "4" * 64}, + "identities": { + "proposal_id": "11111111-1111-4111-8111-111111111111", + "claim_ids": [], + "source_ids": [], + "planned_evidence_keys": [], + "evidence_ids": [f"generated-{run_number}"], + "edge_ids": [], + }, + "phases": { + "staged_pending_review": {"proposal": {"status": "pending_review"}}, + "approved_unapplied": {"proposal": {"status": "approved"}}, + "applied": { + "proposal": {"status": "applied", "applied_at": f"2026-01-0{run_number}T00:00:00+00:00"}, + "payload_controlled_fingerprint": controlled, + "worker_action": { + "exact_candidate_id": "11111111-1111-4111-8111-111111111111", + "credential_role": "kb_apply", + "retry_failure_count": 1, + "failure_state_after_success": {}, + "private_receipt_mode": "0600", + }, + "apply_receipt": { + "hashes": {"apply_payload_sha256": "5" * 64}, + "expected_row_counts": controlled["row_counts"], + "actual_row_counts": controlled["row_counts"], + }, + }, + "recompiled_readback": { + "bundle_sha256": "4" * 64, + "matches_initial_compile": True, + }, + }, + "database_initialization": { + "genesis": { + "dump": {"sha256": "6" * 64}, + "manifest": {"sha256": "7" * 64}, + "exporter_receipt": { + "sha256": "8" * 64, + "provenance_binding_sha256": "9" * 64, + }, + "image": {"reference": lifecycle.DEFAULT_GENESIS_IMAGE}, + "parity": { + "status": "pass", + "problems": [], + "claim_scope": "full_captured_manifest_including_role_and_database_settings", + "role_global_parity": { + "status": "proven", + "captured_source_settings_present": True, + "reason": None, + }, + }, + "key_counts": {"public.claims": 10}, + "canonical_schema_table_count": 10, + } + }, + "cleanup": {"all": True}, + } + + +def _mocked_genesis_case(tmp_path: Path) -> tuple[Path, Path, Path, Path]: + packet = _write_source_packet(tmp_path) + dump = tmp_path / "genesis.dump" + manifest = tmp_path / "genesis-manifest.jsonl" + exporter_receipt = tmp_path / "exporter-receipt.json" + dump.write_bytes(b"PGDMPx") + manifest.write_text("{}\n", encoding="utf-8") + _write_exporter_receipt(exporter_receipt, dump, manifest, _minimal_loaded_manifest()) + return packet, dump, manifest, exporter_receipt + + +def _fake_validate_genesis( + dump: Path, + manifest: Path, + exporter_receipt: Path, + image: str, + **paths, +) -> lifecycle.GenesisInputs: + payload = json.loads(exporter_receipt.read_text(encoding="utf-8")) + return lifecycle.GenesisInputs( + dump=dump, + dump_sha256=lifecycle.sha256_file(dump), + dump_bytes=dump.stat().st_size, + manifest_path=manifest, + manifest_sha256=lifecycle.sha256_file(manifest), + manifest_bytes=manifest.stat().st_size, + manifest=_minimal_loaded_manifest(), + image=image, + exporter_receipt_path=exporter_receipt, + exporter_receipt_sha256=lifecycle.sha256_file(exporter_receipt), + exporter_receipt=payload, + source_identity=payload["source"], + capture_authorization_ref=payload["capture_authorization_ref"], + provenance_binding_sha256=payload["provenance_binding"]["sha256"], + trust_root_receipt_sha256=lifecycle.sha256_file(exporter_receipt), + role_settings_capture_complete=True, + parity_sql_path=paths["parity_sql_path"], + guard_prerequisites_path=paths["guard_prerequisites_path"], + approve_script_path=paths["approve_script_path"], + apply_worker_script_path=paths["apply_worker_script_path"], + apply_script_path=paths["apply_script_path"], + ) + + +def test_genesis_lifecycle_repeats_semantically_and_writes_private_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + packet, dump, manifest, exporter_receipt = _mocked_genesis_case(tmp_path) + calls: list[int] = [] + + monkeypatch.setattr(lifecycle, "validate_genesis_inputs", _fake_validate_genesis) + + def fake_run(*_args, run_number: int, **_kwargs): + calls.append(run_number) + return _fake_genesis_run(run_number) + + monkeypatch.setattr(lifecycle, "run_lifecycle", fake_run) + monkeypatch.setattr( + lifecycle, + "verify_run_group_cleanup", + lambda run_group: {"checked": True, "run_group": run_group, "container_names": [], "all_absent": True}, + ) + output = tmp_path / "genesis-lifecycle-receipt.json" + + receipt = lifecycle.run_genesis_lifecycle( + packet, + output, + review_action="approve", + genesis_dump=dump, + genesis_manifest=manifest, + genesis_exporter_receipt=exporter_receipt, + image=lifecycle.DEFAULT_GENESIS_IMAGE, + ) + + assert calls == [1, 2] + assert receipt["status"] == "pass" + assert receipt["repeatability"]["semantic_match"] is True + assert len(set(receipt["repeatability"]["semantic_fingerprints_sha256"])) == 1 + assert receipt["repeatability"]["identical_post_apply_fingerprints"] is True + assert receipt["checks"]["both_runs_consumed_preflight_bound_bytes"] is True + assert receipt["cleanup"]["zero_staging_leftovers"] is True + assert receipt["cleanup"]["zero_orphan_containers"] is True + assert output.stat().st_mode & 0o777 == 0o600 + assert json.loads(output.read_text(encoding="utf-8")) == receipt + + +def test_genesis_lifecycle_fails_closed_on_semantic_repeatability_drift( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + packet, dump, manifest, exporter_receipt = _mocked_genesis_case(tmp_path) + monkeypatch.setattr(lifecycle, "validate_genesis_inputs", _fake_validate_genesis) + + def drifted_run(*_args, run_number: int, **_kwargs): + run = _fake_genesis_run(run_number) + if run_number == 2: + run["phases"]["applied"]["payload_controlled_fingerprint"]["sha256"] = "e" * 64 + return run + + monkeypatch.setattr(lifecycle, "run_lifecycle", drifted_run) + monkeypatch.setattr( + lifecycle, + "verify_run_group_cleanup", + lambda run_group: {"checked": True, "run_group": run_group, "container_names": [], "all_absent": True}, + ) + + receipt = lifecycle.run_genesis_lifecycle( + packet, + tmp_path / "repeatability-drift-receipt.json", + review_action="approve", + genesis_dump=dump, + genesis_manifest=manifest, + genesis_exporter_receipt=exporter_receipt, + image=lifecycle.DEFAULT_GENESIS_IMAGE, + ) + + assert receipt["status"] == "fail" + assert receipt["checks"]["both_runs_passed"] is True + assert receipt["checks"]["both_runs_match_semantically"] is False + assert receipt["checks"]["identical_post_apply_fingerprints"] is False + assert len(set(receipt["repeatability"]["semantic_fingerprints_sha256"])) == 2 + + +def test_genesis_lifecycle_fails_closed_on_orphan_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + packet, dump, manifest, exporter_receipt = _mocked_genesis_case(tmp_path) + monkeypatch.setattr(lifecycle, "validate_genesis_inputs", _fake_validate_genesis) + monkeypatch.setattr( + lifecycle, + "run_lifecycle", + lambda *_args, run_number, **_kwargs: _fake_genesis_run(run_number), + ) + monkeypatch.setattr( + lifecycle, + "verify_run_group_cleanup", + lambda run_group: { + "checked": True, + "run_group": run_group, + "container_names": ["orphaned-container"], + "all_absent": False, + }, + ) + + receipt = lifecycle.run_genesis_lifecycle( + packet, + tmp_path / "failed-cleanup-receipt.json", + review_action="approve", + genesis_dump=dump, + genesis_manifest=manifest, + genesis_exporter_receipt=exporter_receipt, + image=lifecycle.DEFAULT_GENESIS_IMAGE, + ) + + assert receipt["status"] == "fail" + assert receipt["checks"]["zero_orphan_containers"] is False + + @pytest.mark.skipif(not _docker_available(), reason="Docker daemon is required for the isolated lifecycle") def test_disposable_postgres_lifecycle_end_to_end(tmp_path: Path) -> None: packet = _write_source_packet(tmp_path) @@ -124,10 +771,143 @@ def test_disposable_postgres_lifecycle_end_to_end(tmp_path: Path) -> None: assert receipt["phases"]["approved_unapplied"]["proposal"]["status"] == "approved" assert receipt["phases"]["approved_unapplied"]["proposal"]["applied_at"] is None assert receipt["phases"]["applied"]["proposal"]["status"] == "applied" + worker = receipt["phases"]["applied"]["worker_action"] + assert worker["report_only"]["expected_marker_present"] is True + assert worker["expected_retry_failure"]["returncode"] == 1 + assert worker["retry_failure_count"] == 1 + assert worker["failure_state_after_success"] == {} + assert worker["enabled_retry"]["expected_marker_present"] is True + assert worker["post_apply_exact_refusal"]["returncode"] == 1 + assert worker["post_apply_exact_refusal"]["expected_refusal_marker_present"] is True + assert worker["private_receipt_mode"] == "0600" assert receipt["phases"]["applied"]["apply_receipt"]["replay_ready"] is True assert receipt["phases"]["recompiled_readback"]["matches_initial_compile"] is True assert receipt["identities"]["evidence_ids"] assert len(receipt["checks"]) >= 20 assert all(receipt["checks"].values()) assert receipt["cleanup"]["all"] is True + assert output.stat().st_mode & 0o777 == 0o600 + assert json.loads(output.read_text(encoding="utf-8")) == receipt + + +def _build_real_docker_genesis(tmp_path: Path) -> tuple[Path, Path, Path]: + run_id = uuid.uuid4().hex[:10] + source = lifecycle.DisposablePostgres( + f"leo-real-genesis-source-{os.getpid()}-{run_id}", + "disposable-admin-password", + image=lifecycle.DEFAULT_GENESIS_IMAGE, + genesis_backed=True, + run_group=f"leo-real-genesis-source-{run_id}", + ) + dump = tmp_path / "real-genesis.dump" + manifest_path = tmp_path / "real-genesis-manifest.jsonl" + exporter_receipt = tmp_path / "real-genesis-exporter-receipt.json" + try: + source.start() + source.psql(lifecycle.bootstrap_sql("review-password", "apply-password")) + source.psql( + "alter role kb_apply set statement_timeout to '17s';\n" + "alter role kb_review in database teleo set lock_timeout to '19s';" + ) + manifest_output = source.psql(lifecycle.PARITY_SQL_PATH.read_text(encoding="utf-8")) + manifest_path.write_text(manifest_output + "\n", encoding="utf-8") + manifest_path.chmod(0o600) + container_dump = "/tmp/leo-real-genesis.dump" + source._run( + [ + "exec", + source.name, + "pg_dump", + "-U", + "postgres", + "-d", + lifecycle.DATABASE, + "--format=custom", + "--compress=9", + "--no-owner", + f"--file={container_dump}", + ] + ) + source._run(["cp", f"{source.name}:{container_dump}", str(dump)]) + dump.chmod(0o600) + loaded_manifest = lifecycle.load_manifest(manifest_path) + roles = {role["name"]: role for role in loaded_manifest["singleton"]["application_roles"]["items"]} + assert roles["kb_apply"]["settings"] == [{"name": "statement_timeout", "value": "17s"}] + assert roles["kb_review"]["database_settings"] == [{"name": "lock_timeout", "value": "19s"}] + _write_exporter_receipt( + exporter_receipt, + dump, + manifest_path, + loaded_manifest, + source={ + "ssh_target": "disposable-docker-test", + "container": source.name, + "database": lifecycle.DATABASE, + "service": "disposable-postgres-test", + }, + ) + finally: + cleanup = source.cleanup() + assert all( + ( + cleanup["container_absent"], + cleanup["name_readback_clear"], + cleanup["instance_label_readback_clear"], + cleanup["run_group_readback_clear"], + cleanup["network_mode_was_none"], + cleanup["tmpfs_data_dir_was_present"], + cleanup["docker_volume_mounts_observed"] == [], + ) + ) + return dump, manifest_path, exporter_receipt + + +@pytest.mark.skipif(not _docker_available(), reason="Docker daemon is required for the real genesis lifecycle") +def test_real_docker_genesis_restore_worker_parity_and_repeatability( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + packet = _write_source_packet(tmp_path) + dump, manifest, exporter_receipt = _build_real_docker_genesis(tmp_path) + receipt_payload = json.loads(exporter_receipt.read_text(encoding="utf-8")) + _trust_test_exporter_receipt(monkeypatch, exporter_receipt, receipt_payload) + output = tmp_path / "real-genesis-lifecycle-receipt.json" + + receipt = lifecycle.run_genesis_lifecycle( + packet, + output, + review_action="approve", + review_note="Approved exact Docker genesis payload for regression coverage.", + genesis_dump=dump, + genesis_manifest=manifest, + genesis_exporter_receipt=exporter_receipt, + image=lifecycle.DEFAULT_GENESIS_IMAGE, + ) + + assert receipt["status"] == "pass", receipt.get("error") + assert receipt["execution"]["git_head"] + assert receipt["execution"]["engine_set_sha256"] + assert receipt["execution"]["engine_paths_match_head"] is True + assert receipt["execution"]["engine_sha256"] == receipt["execution"]["engine_head_sha256"] + assert receipt["pre_docker_validation"]["trusted_exporter_receipt_validated"] is True + assert receipt["pre_docker_validation"]["committed_engine_dependency_closure_verified"] is True + assert receipt["checks"]["both_runs_consumed_preflight_bound_bytes"] is True + assert receipt["checks"]["committed_engine_dependency_closure_verified"] is True + assert receipt["checks"]["portable_receipt_has_no_absolute_paths"] is True + assert lifecycle.absolute_path_value_pointers(receipt) == [] + assert receipt["repeatability"]["identical_post_apply_fingerprints"] is True + assert len(set(receipt["repeatability"]["post_apply_fingerprints_sha256"])) == 1 + assert receipt["cleanup"]["zero_orphan_containers"] is True + assert receipt["cleanup"]["zero_staging_leftovers"] is True + for run in receipt["runs"]: + assert run["status"] == "pass", run.get("error") + assert ( + run["consumed_inputs"]["preflight_manifest_sha256"] + == receipt["immutable_staging"]["preflight_manifest_sha256"] + ) + assert run["consumed_inputs"]["executing_git_head"] == receipt["execution"]["git_head"] + assert run["consumed_inputs"]["engine_set_sha256"] == receipt["execution"]["engine_set_sha256"] + assert run["database_initialization"]["genesis"]["parity"]["role_global_parity"]["status"] == "proven" + assert run["phases"]["applied"]["worker_action"]["retry_failure_count"] == 1 + assert run["phases"]["applied"]["worker_action"]["private_receipt_mode"] == "0600" + assert output.stat().st_mode & 0o777 == 0o600 assert json.loads(output.read_text(encoding="utf-8")) == receipt diff --git a/tests/test_run_local_corpus_knowledge_rebuild.py b/tests/test_run_local_corpus_knowledge_rebuild.py new file mode 100644 index 0000000..7a7d11b --- /dev/null +++ b/tests/test_run_local_corpus_knowledge_rebuild.py @@ -0,0 +1,620 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import math +import os +import stat +import subprocess +import sys +from collections.abc import Callable, Iterator +from pathlib import Path + +import pytest + +from ops import run_local_corpus_knowledge_rebuild as rebuild + +REPO_ROOT = Path(__file__).resolve().parents[1] +CLI = REPO_ROOT / "ops" / "run_local_corpus_knowledge_rebuild.py" +FIXTURE_ROOT = REPO_ROOT / "tests" / "fixtures" / "teleo_corpus_rebuild" + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _load_json(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + assert isinstance(value, dict) + return value + + +def _run_cli(source_root: Path, output: Path, *extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(CLI), + "--source-root", + str(source_root), + "--output", + str(output), + *extra, + ], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, + ) + + +def _run_verify(bundle: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(CLI), "--verify-bundle", str(bundle)], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, + ) + + +def _tree_bytes(root: Path) -> dict[str, bytes]: + return {path.relative_to(root).as_posix(): path.read_bytes() for path in sorted(root.rglob("*")) if path.is_file()} + + +def _all_keys(value: object) -> Iterator[str]: + if isinstance(value, dict): + for key, child in value.items(): + yield key + yield from _all_keys(child) + elif isinstance(value, list): + for child in value: + yield from _all_keys(child) + + +def _make_source(root: Path, name: str = "source.md", content: bytes = b"# Source\n\nExact text.\n") -> Path: + root.mkdir(parents=True, exist_ok=True) + source = root / name + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(content) + return source + + +def test_cli_builds_deterministic_source_inventory_without_claims_or_proposals(tmp_path: Path) -> None: + first_output = tmp_path / "bundle-a" + second_output = tmp_path / "bundle-b" + + first = _run_cli(FIXTURE_ROOT, first_output, "--mode", "source-inventory-only") + second = _run_cli(FIXTURE_ROOT, second_output) + + assert first.returncode == second.returncode == 0, (first.stderr, first.stdout, second.stderr, second.stdout) + first_status = json.loads(first.stdout) + second_status = json.loads(second.stdout) + assert first_status["status"] == second_status["status"] == "pass" + assert first_status["source_count"] == second_status["source_count"] == 2 + assert first_status["work_item_count"] == second_status["work_item_count"] == 3 + assert first_status["bundle_tree_sha256"] == second_status["bundle_tree_sha256"] + assert _tree_bytes(first_output) == _tree_bytes(second_output) + assert rebuild.INCOMPLETE_MARKER not in _tree_bytes(first_output) + + source_manifest = _load_json(first_output / rebuild.SOURCE_MANIFEST_PATH) + work_index = _load_json(first_output / rebuild.EXTRACTION_WORK_INDEX_PATH) + receipt = _load_json(first_output / rebuild.RECEIPT_PATH) + assert source_manifest["schema"] == rebuild.SOURCE_MANIFEST_SCHEMA + assert work_index["schema"] == rebuild.EXTRACTION_WORK_INDEX_SCHEMA + assert receipt["schema"] == rebuild.RECEIPT_SCHEMA + assert source_manifest["source_snapshot_semantics"] == { + "concurrent_directory_additions_or_removals_may_be_absent": True, + "directory_snapshot_atomic": False, + "guarantee": "each listed path is bound to the exact bytes copied from one stable file descriptor", + } + assert [row["relative_path"] for row in source_manifest["sources"]] == [ + "forgejo/project-record.md", + "operator-notes.txt", + ] + assert [row["section_count"] for row in source_manifest["sources"]] == [2, 1] + assert work_index["work_type"] == rebuild.WORK_TYPE + assert all(item["work_type"] == rebuild.WORK_TYPE for item in work_index["items"]) + + forbidden_keys = {"candidate", "candidates", "claim", "claims", "proposal", "proposals", "proposition"} + for document in (source_manifest, work_index, receipt): + assert forbidden_keys.isdisjoint(_all_keys(document)) + for key, expected in rebuild.NO_WRITE_FLAGS.items(): + assert document[key] is expected + + sources_by_path = {row["relative_path"]: row for row in source_manifest["sources"]} + for relative_path, source in sources_by_path.items(): + raw = (FIXTURE_ROOT / relative_path).read_bytes() + snapshot = first_output / source["snapshot_path"] + assert snapshot.read_bytes() == raw + assert source["bytes"] == len(raw) + assert source["artifact_sha256"] == _sha256(raw) + assert source["format_syntax_validated"] is False + for locator in source["sections"]: + start = locator["utf8_byte_start"] + end = locator["utf8_byte_end"] + quote = raw[start:end] + assert quote + assert locator["quote_sha256"] == _sha256(quote) + assert locator["line_start"] == raw[:start].decode("utf-8").count("\n") + 1 + assert locator["line_end"] == raw[: end - 1].decode("utf-8").count("\n") + 1 + + artifacts = receipt["outputs"]["artifacts"] + assert receipt["outputs"]["artifact_count"] == len(artifacts) + assert receipt["hashes"]["bundle_tree_sha256"] == rebuild.canonical_sha256(artifacts) + receipt_preimage = copy.deepcopy(receipt) + receipt_preimage["hashes"].pop("receipt_content_sha256") + assert receipt["hashes"]["receipt_content_sha256"] == rebuild.canonical_sha256(receipt_preimage) + assert rebuild.verify_bundle(first_output)["status"] == "pass" + + persisted = b"".join(_tree_bytes(first_output).values()) + assert str(FIXTURE_ROOT).encode() not in persisted + assert str(first_output).encode() not in persisted + for path in first_output.rglob("*"): + expected_mode = 0o600 if path.is_file() else 0o700 + assert stat.S_IMODE(path.stat().st_mode) == expected_mode + + +def test_receipt_is_published_last(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + source_root = tmp_path / "sources" + _make_source(source_root) + output = tmp_path / "bundle" + original = rebuild._write_private_json + writes: list[str] = [] + + def record(path: Path, value: object) -> None: + writes.append(path.name) + if path.name == rebuild.RECEIPT_PATH: + assert (output / rebuild.INCOMPLETE_MARKER).exists() + original(path, value) + + monkeypatch.setattr(rebuild, "_write_private_json", record) + rebuild.build_source_inventory_bundle(source_root, output) + + assert writes[-1] == rebuild.RECEIPT_PATH + assert rebuild.INCOMPLETE_MARKER not in writes + assert not (output / rebuild.INCOMPLETE_MARKER).exists() + + +def test_verifier_rejects_rehashed_metadata_overclaim(tmp_path: Path) -> None: + output = tmp_path / "bundle" + rebuild.build_source_inventory_bundle(FIXTURE_ROOT, output) + receipt_path = output / rebuild.RECEIPT_PATH + receipt = _load_json(receipt_path) + receipt["canonical_database_rebuilt"] = True + receipt["hashes"].pop("receipt_content_sha256") + receipt["hashes"]["receipt_content_sha256"] = rebuild.canonical_sha256(receipt) + receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + + completed = _run_verify(output) + + assert completed.returncode == 2 + assert json.loads(completed.stdout)["reason"] == "invalid_bundle_schema" + + +METADATA_PATHS = ( + rebuild.SOURCE_MANIFEST_PATH, + rebuild.EXTRACTION_WORK_INDEX_PATH, + rebuild.RECEIPT_PATH, +) + + +def _inject_duplicate_top_level_key(raw: str, _metadata_path: str) -> str: + return raw.replace("{\n", '{\n "schema": "contradictory-shadow",\n', 1) + + +def _inject_duplicate_nested_key(raw: str, metadata_path: str) -> str: + anchors = { + rebuild.SOURCE_MANIFEST_PATH: ( + '"source_snapshot_semantics": {\n', + ' "directory_snapshot_atomic": true,\n', + ), + rebuild.EXTRACTION_WORK_INDEX_PATH: ( + '"source_manifest": {\n', + ' "path": "contradictory-shadow.json",\n', + ), + rebuild.RECEIPT_PATH: ( + '"hashes": {\n', + f' "bundle_tree_sha256": "{"0" * 64}",\n', + ), + } + anchor, duplicate = anchors[metadata_path] + assert anchor in raw + return raw.replace(anchor, anchor + duplicate, 1) + + +def _inject_nonfinite(raw: str, constant: str) -> str: + return raw.replace("{\n", f'{{\n "nonfinite_shadow": {constant},\n', 1) + + +@pytest.mark.parametrize("metadata_path", METADATA_PATHS) +@pytest.mark.parametrize( + ("mutator", "expected_reason"), + ( + (_inject_duplicate_top_level_key, "duplicate_json_key"), + (_inject_duplicate_nested_key, "duplicate_json_key"), + (lambda raw, _path: _inject_nonfinite(raw, "NaN"), "invalid_json_value"), + (lambda raw, _path: _inject_nonfinite(raw, "Infinity"), "invalid_json_value"), + (lambda raw, _path: _inject_nonfinite(raw, "-Infinity"), "invalid_json_value"), + ), +) +def test_verifier_rejects_ambiguous_or_nonfinite_metadata_before_verification( + tmp_path: Path, + metadata_path: str, + mutator: Callable[[str, str], str], + expected_reason: str, +) -> None: + output = tmp_path / "bundle" + rebuild.build_source_inventory_bundle(FIXTURE_ROOT, output) + target = output / metadata_path + target.write_text(mutator(target.read_text(encoding="utf-8"), metadata_path), encoding="utf-8") + + completed = _run_verify(output) + + assert completed.returncode == 2 + assert json.loads(completed.stdout)["reason"] == expected_reason + + +@pytest.mark.parametrize("value", (math.nan, math.inf, -math.inf)) +def test_canonical_inventory_json_writers_reject_nonfinite_numbers(value: float) -> None: + for writer in (rebuild.canonical_json_bytes, rebuild.rendered_json_bytes): + with pytest.raises(rebuild.RebuildError) as error: + writer({"value": value}) + assert error.value.code == "invalid_json_value" + + +def test_verify_mode_is_read_only(tmp_path: Path) -> None: + output = tmp_path / "bundle" + completed = _run_cli(FIXTURE_ROOT, output) + assert completed.returncode == 0, completed.stdout + before = {path: (path.stat().st_mtime_ns, path.read_bytes()) for path in output.rglob("*") if path.is_file()} + + verified = _run_verify(output) + + assert verified.returncode == 0, (verified.stdout, verified.stderr) + assert json.loads(verified.stdout)["operation"] == "verify" + after = {path: (path.stat().st_mtime_ns, path.read_bytes()) for path in output.rglob("*") if path.is_file()} + assert after == before + + +def test_post_read_source_mutation_does_not_change_copied_snapshot_or_overclaim_atomicity( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_root = tmp_path / "sources" + source = _make_source(source_root, content=b"# Original\n\nCopied bytes.\n") + original = source.read_bytes() + output = tmp_path / "bundle" + + def mutate(relative_path: str, _source_root: Path, _snapshot: Path) -> None: + assert relative_path == "source.md" + source.write_bytes(b"# Mutated\n\nLater bytes.\n") + + monkeypatch.setattr(rebuild, "_after_source_snapshot", mutate) + result = rebuild.build_source_inventory_bundle(source_root, output) + + manifest = _load_json(output / rebuild.SOURCE_MANIFEST_PATH) + row = manifest["sources"][0] + assert (output / row["snapshot_path"]).read_bytes() == original + assert source.read_bytes() != original + assert row["artifact_sha256"] == _sha256(original) + assert manifest["source_snapshot_semantics"]["directory_snapshot_atomic"] is False + assert result["verification"]["status"] == "pass" + + +def test_destination_reservation_race_never_replaces_or_cleans_other_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_root = tmp_path / "sources" + _make_source(source_root) + output = tmp_path / "bundle" + original_mkdir = rebuild.os.mkdir + raced = False + + def racing_mkdir(path: os.PathLike[str] | str, mode: int = 0o777, *, dir_fd: int | None = None) -> None: + nonlocal raced + if Path(path) == output and not raced and dir_fd is None: + raced = True + original_mkdir(output, mode) + (output / "other-owner.txt").write_text("retain", encoding="utf-8") + original_mkdir(path, mode, dir_fd=dir_fd) + + monkeypatch.setattr(rebuild.os, "mkdir", racing_mkdir) + with pytest.raises(rebuild.RebuildError) as error: + rebuild.build_source_inventory_bundle(source_root, output) + + assert error.value.code == "output_exists" + assert (output / "other-owner.txt").read_text(encoding="utf-8") == "retain" + assert set(path.name for path in output.iterdir()) == {"other-owner.txt"} + + +def test_existing_destination_is_never_replaced(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + _make_source(source_root) + output = tmp_path / "bundle" + output.mkdir() + marker = output / "retain.txt" + marker.write_text("owned elsewhere", encoding="utf-8") + + with pytest.raises(rebuild.RebuildError) as error: + rebuild.build_source_inventory_bundle(source_root, output) + + assert error.value.code == "output_exists" + assert marker.read_text(encoding="utf-8") == "owned elsewhere" + + +def test_output_identity_replacement_fails_before_writing_replacement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_root = tmp_path / "sources" + _make_source(source_root) + output = tmp_path / "bundle" + moved = tmp_path / "moved-reservation" + + def replace(reserved: rebuild.ReservedOutput) -> None: + assert reserved.path == output + output.rename(moved) + output.mkdir() + (output / "other-owner.txt").write_text("retain", encoding="utf-8") + + monkeypatch.setattr(rebuild, "_after_output_reservation", replace) + with pytest.raises(rebuild.RebuildError) as error: + rebuild.build_source_inventory_bundle(source_root, output) + + assert error.value.code == "output_binding_changed" + assert set(path.name for path in output.iterdir()) == {"other-owner.txt"} + assert set(path.name for path in moved.iterdir()) == {rebuild.INCOMPLETE_MARKER} + + +def test_failure_after_reservation_removes_only_reserved_output_and_marker(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + _make_source(source_root, content=b"text\x00binary\n") + output = tmp_path / "bundle" + + with pytest.raises(rebuild.RebuildError) as error: + rebuild.build_source_inventory_bundle(source_root, output) + + assert error.value.code == "binary_control_character" + assert not output.exists() + + +def test_lf_and_crlf_locators_are_exact_and_cr_only_is_rejected(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + source_root.mkdir() + lf = b"# One\nalpha\n# Two\nbeta\n" + crlf = b"# One\r\nalpha\r\n# Two\r\nbeta\r\n" + (source_root / "lf.md").write_bytes(lf) + (source_root / "crlf.md").write_bytes(crlf) + output = tmp_path / "bundle" + + rebuild.build_source_inventory_bundle(source_root, output) + manifest = _load_json(output / rebuild.SOURCE_MANIFEST_PATH) + by_path = {row["relative_path"]: row for row in manifest["sources"]} + assert by_path["lf.md"]["newline_style"] == "lf" + assert by_path["crlf.md"]["newline_style"] == "crlf" + for name, raw in (("lf.md", lf), ("crlf.md", crlf)): + assert [(row["line_start"], row["line_end"]) for row in by_path[name]["sections"]] == [(1, 2), (3, 4)] + for locator in by_path[name]["sections"]: + quote = raw[locator["utf8_byte_start"] : locator["utf8_byte_end"]] + assert _sha256(quote) == locator["quote_sha256"] + + cr_root = tmp_path / "cr-only" + _make_source(cr_root, content=b"# One\ralpha\r# Two\rbeta\r") + with pytest.raises(rebuild.RebuildError) as error: + rebuild.build_source_inventory_bundle(cr_root, tmp_path / "cr-bundle") + assert error.value.code == "unsupported_newline" + assert not (tmp_path / "cr-bundle").exists() + + +@pytest.mark.parametrize("content", [b"nul\x00byte\n", b"escape\x1bbyte\n", b"delete\x7fbyte\n"]) +def test_binary_control_characters_are_rejected(tmp_path: Path, content: bytes) -> None: + source_root = tmp_path / "sources" + _make_source(source_root, content=content) + with pytest.raises(rebuild.RebuildError) as error: + rebuild.build_source_inventory_bundle(source_root, tmp_path / "bundle") + assert error.value.code == "binary_control_character" + assert not (tmp_path / "bundle").exists() + + +@pytest.mark.parametrize( + ("content", "limits", "expected_code"), + [ + (b"123456\n", rebuild.ResourceLimits(max_source_bytes=4), "source_byte_limit_exceeded"), + (b"123456\n", rebuild.ResourceLimits(max_total_bytes=4, max_source_bytes=4), "source_byte_limit_exceeded"), + ( + b"# One\na\n# Two\nb\n", + rebuild.ResourceLimits(max_sections_per_source=1, max_work_items=1), + "section_limit_exceeded", + ), + ], +) +def test_resource_limits_fail_closed( + tmp_path: Path, + content: bytes, + limits: rebuild.ResourceLimits, + expected_code: str, +) -> None: + source_root = tmp_path / "sources" + _make_source(source_root, content=content) + with pytest.raises(rebuild.RebuildError) as error: + rebuild.build_source_inventory_bundle(source_root, tmp_path / "bundle", limits=limits) + assert error.value.code == expected_code + assert not (tmp_path / "bundle").exists() + + +def test_total_source_count_and_work_item_limits_fail_closed(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + _make_source(source_root, "a.txt", b"a\n") + _make_source(source_root, "b.txt", b"b\n") + + with pytest.raises(rebuild.RebuildError) as count_error: + rebuild.build_source_inventory_bundle( + source_root, + tmp_path / "count-bundle", + limits=rebuild.ResourceLimits(max_source_count=1), + ) + assert count_error.value.code == "source_count_limit_exceeded" + + with pytest.raises(rebuild.RebuildError) as work_error: + rebuild.build_source_inventory_bundle( + source_root, + tmp_path / "work-bundle", + limits=rebuild.ResourceLimits(max_work_items=1, max_sections_per_source=1), + ) + assert work_error.value.code == "work_item_limit_exceeded" + + with pytest.raises(rebuild.RebuildError) as total_error: + rebuild.build_source_inventory_bundle( + source_root, + tmp_path / "total-bundle", + limits=rebuild.ResourceLimits(max_source_bytes=2, max_total_bytes=3), + ) + assert total_error.value.code == "total_byte_limit_exceeded" + + +def test_limit_overrides_remain_hard_bounded(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + _make_source(source_root) + completed = _run_cli( + source_root, + tmp_path / "bundle", + "--max-source-bytes", + str(rebuild.HARD_MAX_SOURCE_BYTES + 1), + ) + assert completed.returncode == 2 + assert json.loads(completed.stdout)["reason"] == "invalid_resource_limit" + assert not (tmp_path / "bundle").exists() + + +def test_identical_source_bytes_share_one_content_addressed_snapshot(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + content = b"same exact bytes\n" + _make_source(source_root, "a.txt", content) + _make_source(source_root, "nested/b.txt", content) + output = tmp_path / "bundle" + + rebuild.build_source_inventory_bundle(source_root, output) + manifest = _load_json(output / rebuild.SOURCE_MANIFEST_PATH) + assert manifest["source_count"] == 2 + assert manifest["unique_snapshot_count"] == 1 + assert manifest["duplicate_content_source_count"] == 1 + assert len({row["snapshot_path"] for row in manifest["sources"]}) == 1 + assert len(list((output / "snapshots" / "sha256").rglob("*.utf8"))) == 1 + assert _load_json(output / rebuild.EXTRACTION_WORK_INDEX_PATH)["work_item_count"] == 2 + + +def test_source_rows_use_global_utf8_relative_path_order(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + _make_source(source_root, "a-z.txt", b"sibling\n") + _make_source(source_root, "a/x.txt", b"nested\n") + _make_source(source_root, "aa.txt", b"later\n") + output = tmp_path / "bundle" + + rebuild.build_source_inventory_bundle(source_root, output) + + rows = _load_json(output / rebuild.SOURCE_MANIFEST_PATH)["sources"] + assert [row["relative_path"] for row in rows] == ["a-z.txt", "a/x.txt", "aa.txt"] + assert rebuild.verify_bundle(output)["status"] == "pass" + + +def test_mismatched_existing_content_addressed_snapshot_is_rejected(tmp_path: Path) -> None: + output = tmp_path / "bundle" + prefix = output / "snapshots" / "sha256" + prefix.mkdir(parents=True) + partial = output / "snapshots" / ".partial-00000001" + content = b"expected\n" + partial.write_bytes(content) + digest = _sha256(content) + destination = output / Path(rebuild._snapshot_path_for_hash(digest)) + destination.parent.mkdir() + destination.write_bytes(b"mismatch\n") + + with pytest.raises(rebuild.RebuildError) as error: + rebuild._install_snapshot(output, partial, digest, len(content)) + + assert error.value.code == "snapshot_collision" + assert destination.read_bytes() == b"mismatch\n" + assert partial.read_bytes() == content + + +@pytest.mark.parametrize("target", ["snapshot", "manifest", "index", "receipt"]) +def test_verifier_rejects_tampered_bundle(tmp_path: Path, target: str) -> None: + output = tmp_path / "bundle" + rebuild.build_source_inventory_bundle(FIXTURE_ROOT, output) + manifest = _load_json(output / rebuild.SOURCE_MANIFEST_PATH) + if target == "snapshot": + path = output / manifest["sources"][0]["snapshot_path"] + path.write_bytes(path.read_bytes() + b"tamper") + elif target == "manifest": + path = output / rebuild.SOURCE_MANIFEST_PATH + document = _load_json(path) + document["source_count"] += 1 + path.write_text(json.dumps(document), encoding="utf-8") + elif target == "index": + path = output / rebuild.EXTRACTION_WORK_INDEX_PATH + document = _load_json(path) + document["items"][0]["work_type"] = "semantic_claim" + path.write_text(json.dumps(document), encoding="utf-8") + else: + path = output / rebuild.RECEIPT_PATH + document = _load_json(path) + document["database_write_performed"] = True + path.write_text(json.dumps(document), encoding="utf-8") + + completed = _run_verify(output) + + assert completed.returncode == 2 + assert json.loads(completed.stdout)["status"] == "rejected" + + +def test_inventory_rejects_symlinks_unsupported_files_invalid_utf8_and_empty_text(tmp_path: Path) -> None: + source_root = tmp_path / "symlinks" + target = _make_source(source_root) + link = source_root / "alias.md" + try: + link.symlink_to(target.name) + except OSError as exc: # pragma: no cover - platform capability + pytest.skip(f"symlinks unavailable: {exc}") + with pytest.raises(rebuild.RebuildError) as symlink_error: + rebuild.build_source_inventory_bundle(source_root, tmp_path / "symlink-bundle") + assert symlink_error.value.code == "symlink_rejected" + + cases = [ + ("unsupported", "unsafe.pdf", b"%PDF-1.7\n", "unsupported_source_file"), + ("invalid", "invalid.md", b"invalid: \xff\n", "invalid_utf8_source"), + ("empty", "empty.txt", b" \n\t\n", "empty_source_file"), + ] + for directory, filename, content, expected in cases: + root = tmp_path / directory + _make_source(root, filename, content) + with pytest.raises(rebuild.RebuildError) as error: + rebuild.build_source_inventory_bundle(root, tmp_path / f"{directory}-bundle") + assert error.value.code == expected + + +def test_rejects_output_overlap_and_unsupported_mode(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + _make_source(source_root) + with pytest.raises(rebuild.RebuildError) as overlap: + rebuild.build_source_inventory_bundle(source_root, source_root / "bundle") + assert overlap.value.code == "output_overlap" + + completed = _run_cli(source_root, tmp_path / "bundle", "--mode", "candidate-only") + assert completed.returncode == 2 + assert json.loads(completed.stdout)["reason"] == "unsupported_mode" + assert not (tmp_path / "bundle").exists() + + +def test_structured_extensions_are_inventoried_as_utf8_not_claimed_as_format_validated(tmp_path: Path) -> None: + source_root = tmp_path / "sources" + _make_source(source_root, "opaque.json", b"not actually json\n") + output = tmp_path / "bundle" + + rebuild.build_source_inventory_bundle(source_root, output) + + row = _load_json(output / rebuild.SOURCE_MANIFEST_PATH)["sources"][0] + assert row["sectioning_strategy"] == "whole_utf8_source" + assert row["format_syntax_validated"] is False + assert row["section_count"] == 1 diff --git a/tests/test_run_local_genesis_ledger_rebuild.py b/tests/test_run_local_genesis_ledger_rebuild.py index 63ca9c1..8e7df65 100644 --- a/tests/test_run_local_genesis_ledger_rebuild.py +++ b/tests/test_run_local_genesis_ledger_rebuild.py @@ -37,6 +37,22 @@ RECEIPT_STRATEGY_ID = "14141414-1414-4414-8414-141414141414" RECEIPT_NODE_A_ID = "15151515-1515-4515-8515-151515151515" RECEIPT_NODE_B_ID = "16161616-1616-4616-8616-161616161616" PRIVATE_MARKER = "PRIVATE-REPLAY-MATERIAL-MUST-NOT-LEAK" +EXPORTER_MATERIAL_FIELDS = frozenset( + { + "artifact", + "contract_version", + "sequence", + "approved_proposal", + "approval_snapshot", + "applied_proposal", + "replay_receipt", + } +) +V3_PROPOSAL_ID = "17171717-1717-4717-8717-171717171717" +V3_CLAIM_ID = "18181818-1818-4818-8818-181818181818" +V3_SOURCE_ID = "20202020-2020-4020-8020-202020202020" +V3_EVIDENCE_ID = "21212121-2121-4121-8121-212121212121" +V3_ASSESSMENT_ID = "23232323-2323-4323-8323-232323232323" def sha256_file(path: Path) -> str: @@ -45,6 +61,11 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def sha256_json(value: object) -> str: + encoded = json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def write_json(path: Path, value: object) -> Path: path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") return path @@ -270,6 +291,272 @@ def replay_material() -> dict: } +def v3_replay_material() -> dict: + evidence = { + "id": V3_EVIDENCE_ID, + "claim_id": V3_CLAIM_ID, + "source_id": V3_SOURCE_ID, + "polarity": "supports", + "excerpt": "The exact source bytes support this bounded reconstruction claim.", + "locator_json": {"section": "v3-rebuild", "paragraph": 1}, + "source_content_hash": "a" * 64, + "rationale": "The content-addressed source and exact locator support the proposition.", + "created_by": REVIEWER_ID, + } + apply_payload = { + "contract_version": 3, + "agent_id": REVIEWER_ID, + "claims": [ + { + "id": V3_CLAIM_ID, + "owner_agent_id": None, + "type": "empirical", + "proposition": "V3 knowledge can be reconstructed from genesis plus an exact proposal ledger.", + "body_markdown": "The reconstruction replays review, admission, provenance, and assessment rows.", + "scope": "Networkless disposable PostgreSQL reconstruction.", + "access_scope": "collective_internal", + "status": "active", + "revision_criteria": "Revise if exact final parity or trigger restoration cannot be reproduced.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["v3", "reconstruction"], + "created_by": REVIEWER_ID, + } + ], + "sources": [ + { + "id": V3_SOURCE_ID, + "source_type": "paper", + "canonical_url": "https://example.test/v3-reconstruction", + "storage_uri": None, + "excerpt": "Exact source bytes for the disposable V3 reconstruction.", + "content_hash": "a" * 64, + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-18T10:00:00+00:00", + "created_by": REVIEWER_ID, + } + ], + "evidence": [evidence], + "edges": [], + "assessments": [ + { + "id": V3_ASSESSMENT_ID, + "claim_id": V3_CLAIM_ID, + "support_tier": "strong", + "challenge_tier": "none", + "overall_state": "support_dominant", + "rationale": "The exact source and locator support the bounded reconstruction proposition.", + "evidence_set_hash": rebuild.apply_engine._v3_evidence_set_hash(V3_CLAIM_ID, [evidence]), + "supersedes_assessment_id": None, + } + ], + "reasoning_tools": [], + } + payload = {"title": "V3 reconstruction fixture", "apply_payload": apply_payload} + approved = { + "id": V3_PROPOSAL_ID, + "proposal_type": "approve_claim", + "status": "approved", + "proposed_by_handle": "fixture-agent", + "proposed_by_agent_id": REVIEWER_ID, + "channel": "v3_reconstruction_fixture", + "source_ref": "private://fixture/v3-source", + "rationale": PRIVATE_MARKER, + "payload": payload, + "reviewed_by_handle": "m3ta", + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_at": "2026-07-18T10:01:00+00:00", + "review_note": "Reviewed the exact V3 reconstruction payload.", + "applied_by_handle": None, + "applied_by_agent_id": None, + "applied_at": None, + "created_at": "2026-07-18T10:00:30+00:00", + "updated_at": "2026-07-18T10:01:00+00:00", + } + applied = { + **approved, + "status": "applied", + "applied_by_handle": "kb-apply", + "applied_by_agent_id": SERVICE_ID, + "applied_at": "2026-07-18T10:02:00+00:00", + "updated_at": "2026-07-18T10:02:00.000001+00:00", + } + approval = { + "proposal_id": V3_PROPOSAL_ID, + "proposal_type": "approve_claim", + "payload": payload, + "reviewed_by_handle": approved["reviewed_by_handle"], + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_by_db_role": "kb_review", + "reviewed_at": approved["reviewed_at"], + "review_note": approved["review_note"], + } + created_at = "2026-07-18T10:01:30+00:00" + rows = { + "claims": [ + { + "id": V3_CLAIM_ID, + "owner_agent_id": None, + "type": "empirical", + "text": apply_payload["claims"][0]["proposition"], + "proposition": apply_payload["claims"][0]["proposition"], + "body_markdown": apply_payload["claims"][0]["body_markdown"], + "scope": apply_payload["claims"][0]["scope"], + "access_scope": "collective_internal", + "status": "active", + "admission_state": "admitted", + "revision_criteria": apply_payload["claims"][0]["revision_criteria"], + "revision_kind": "original", + "supersedes_id": None, + "confidence": None, + "tags": ["v3", "reconstruction"], + "created_by": REVIEWER_ID, + "superseded_by": None, + "accepted_by_proposal_id": V3_PROPOSAL_ID, + "created_at": created_at, + "updated_at": created_at, + } + ], + "sources": [ + { + "id": V3_SOURCE_ID, + "source_type": "paper", + "url": apply_payload["sources"][0]["canonical_url"], + "storage_path": None, + "excerpt": apply_payload["sources"][0]["excerpt"], + "hash": "a" * 64, + "created_by": REVIEWER_ID, + "captured_at": apply_payload["sources"][0]["captured_at"], + "canonical_url": apply_payload["sources"][0]["canonical_url"], + "storage_uri": None, + "content_hash": "a" * 64, + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "accepted_by_proposal_id": V3_PROPOSAL_ID, + "created_at": created_at, + } + ], + "claim_evidence": [ + { + "id": V3_EVIDENCE_ID, + "claim_id": V3_CLAIM_ID, + "source_id": V3_SOURCE_ID, + "role": "grounds", + "weight": None, + "created_by": REVIEWER_ID, + "excerpt": evidence["excerpt"], + "locator_json": evidence["locator_json"], + "source_content_hash": "a" * 64, + "rationale": evidence["rationale"], + "polarity": "supports", + "accepted_from_proposal_id": V3_PROPOSAL_ID, + "created_at": created_at, + } + ], + "claim_edges": [], + "claim_evidence_assessments": [ + { + **apply_payload["assessments"][0], + "accepted_by_proposal_id": V3_PROPOSAL_ID, + "created_at": created_at, + } + ], + "reasoning_tools": [], + } + proposal = applied_envelope(applied) + apply_sql = rebuild.apply_engine.build_apply_sql(proposal, applied["applied_by_handle"]) + receipt = rebuild.replay_receipt.build_replay_receipt( + proposal, + rows, + apply_sql_sha256=rebuild.replay_receipt.sha256_text(apply_sql), + apply_sql_source="exact_executed_sql", + exported_at_utc="2026-07-18T10:03:00+00:00", + ) + return { + "artifact": rebuild.MATERIAL_ARTIFACT, + "contract_version": rebuild.MATERIAL_CONTRACT_VERSION, + "sequence": 1, + "approved_proposal": approved, + "approval_snapshot": approval, + "applied_proposal": applied, + "replay_receipt": receipt, + } + + +def exporter_shaped_v3_replay_material() -> dict: + material = copy.deepcopy(v3_replay_material()) + payload_rows = ( + material["approved_proposal"], + material["approval_snapshot"], + material["applied_proposal"], + material["replay_receipt"]["proposal"], + ) + for row in payload_rows: + row["payload"]["apply_payload"]["sources"][0]["captured_at"] = "2026-07-18T12:00:00+02:00" + + approved = material["approved_proposal"] + approved.update( + created_at="2026-07-18T10:00:30.000000Z", + reviewed_at="2026-07-18T10:01:00.000000Z", + updated_at="2026-07-18T10:01:00.000000Z", + ) + applied = material["applied_proposal"] + applied.update( + created_at=approved["created_at"], + reviewed_at=approved["reviewed_at"], + applied_at="2026-07-18T10:02:00.000000Z", + updated_at="2026-07-18T10:02:00.000001Z", + ) + material["approval_snapshot"]["reviewed_at"] = approved["reviewed_at"] + + canonical_rows = material["replay_receipt"]["canonical_rows"] + for table_rows in canonical_rows.values(): + for row in table_rows: + if "captured_at" in row: + row["captured_at"] = "2026-07-18T10:00:00.000000Z" + if "created_at" in row: + row["created_at"] = "2026-07-18T10:01:30.000000Z" + if "updated_at" in row: + row["updated_at"] = "2026-07-18T10:01:30.000000Z" + + proposal = applied_envelope(applied) + semantic_proposal = copy.deepcopy(proposal) + semantic_proposal["payload"]["apply_payload"]["sources"][0]["captured_at"] = canonical_rows["sources"][0][ + "captured_at" + ] + receipt = rebuild.replay_receipt.build_replay_receipt( + semantic_proposal, + canonical_rows, + apply_sql_sha256=rebuild.replay_receipt.sha256_text("snapshot exporter fixture apply SQL"), + apply_sql_source="exact_executed_sql", + exported_at_utc="2026-07-18T10:03:00.000000Z", + ) + receipt["proposal"]["payload"] = proposal["payload"] + replay_material = { + "receipt_contract_version": receipt["receipt_contract_version"], + "apply_engine": receipt["apply_engine"], + "proposal": receipt["proposal"], + "canonical_rows": receipt["canonical_rows"], + } + receipt["hashes"] = { + "apply_payload_sha256": sha256_json(proposal["payload"]["apply_payload"]), + "proposal_payload_sha256": sha256_json(proposal["payload"]), + "canonical_rows_sha256": sha256_json(receipt["canonical_rows"]), + "row_sha256": {table: [sha256_json(row) for row in rows] for table, rows in receipt["canonical_rows"].items()}, + "table_sha256": {table: sha256_json(rows) for table, rows in receipt["canonical_rows"].items()}, + "replay_material_sha256": sha256_json(replay_material), + } + material["replay_receipt"] = receipt + return material + + def revise_strategy_replay_material(*, apply_sql_source: str = "exact_executed_sql", version: int = 2) -> dict: approved, applied, approval = revise_strategy_proposal_rows() proposal = applied_envelope(applied) @@ -386,9 +673,45 @@ def write_bundle(tmp_path: Path, *, material: dict | None = None) -> argparse.Na max_target_source_ratio=100.0, docker_bin="docker", output=tmp_path / "receipt.json", + admission_manifest=None, + admission_manifest_sha256=None, ) +def attach_admission_manifest( + args: argparse.Namespace, + *, + admission_scope: str = "collective_core", + basis: str = "forgejo_primary_source", +) -> Path: + ledger = json.loads(args.ledger.read_text(encoding="utf-8")) + ledger_entry = ledger["entries"][0] + material = json.loads((args.ledger.parent / ledger_entry["material"]).read_text(encoding="utf-8")) + manifest = { + "artifact": rebuild.ADMISSION_ARTIFACT, + "contract_version": rebuild.ADMISSION_CONTRACT_VERSION, + "ledger_sha256": args.ledger_sha256, + "reviewed_by_handle": "m3taversal", + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_at": "2026-07-18T14:00:00+00:00", + "review_note": "Admit this exact source-backed V3 proposal into the clean genesis fixture.", + "entries": [ + { + "sequence": 1, + "proposal_id": material["applied_proposal"]["id"], + "material_sha256": ledger_entry["sha256"], + "replay_material_sha256": ledger_entry["replay_material_sha256"], + "admission_scope": admission_scope, + "basis": basis, + } + ], + } + path = write_json(args.ledger.parent / "admission-manifest.json", manifest) + args.admission_manifest = path + args.admission_manifest_sha256 = sha256_file(path) + return path + + def rewrite_material_and_ledger(args: argparse.Namespace, material: dict) -> None: material_path = args.ledger.parent / "entry-0001.json" write_json(material_path, material) @@ -408,12 +731,77 @@ def test_load_bundle_validates_every_pin_and_current_apply_engine(tmp_path: Path assert bundle.final_manifest_sha256 == sha256_file(tmp_path / "final-manifest.jsonl") assert bundle.engine_hashes == rebuild._engine_hashes() assert len(bundle.entries) == 1 + assert bundle.admission == { + "status": "forensic_only", + "canonical_retrieval_eligible": False, + "reason": "no explicit admission manifest was supplied", + "entry_count": 1, + } entry = bundle.entries[0] assert entry.applied_proposal["proposal_type"] == "add_edge" assert entry.current_apply_sql_sha256 == hashlib.sha256(entry.current_apply_sql.encode("utf-8")).hexdigest() assert "kb_stage.assert_approved_proposal" in entry.current_apply_sql +def test_hash_pinned_v3_review_metadata_selects_material_without_granting_authority(tmp_path: Path) -> None: + args = write_bundle(tmp_path, material=exporter_shaped_v3_replay_material()) + manifest_path = attach_admission_manifest(args) + + bundle = rebuild.load_bundle(args) + + assert bundle.admission_manifest_path == manifest_path + assert bundle.admission_manifest_sha256 == sha256_file(manifest_path) + assert bundle.admission["status"] == "review_metadata_supplied" + assert bundle.admission["canonical_retrieval_eligible"] is False + assert bundle.admission["cryptographically_authenticated"] is False + assert bundle.admission["review_metadata_provenance"] == rebuild.REVIEW_METADATA_PROVENANCE + assert "not authenticated" in bundle.admission["reason"] + assert bundle.admission["reviewed_by_handle"] == "m3taversal" + assert ( + bundle.admission["review_note_sha256"] + == hashlib.sha256(b"Admit this exact source-backed V3 proposal into the clean genesis fixture.").hexdigest() + ) + assert bundle.admission["entries"][0]["basis"] == "forgejo_primary_source" + + +def test_admission_rejects_forged_reviewer_identity(tmp_path: Path) -> None: + args = write_bundle(tmp_path, material=exporter_shaped_v3_replay_material()) + manifest_path = attach_admission_manifest(args) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["reviewed_by_handle"] = "forged-reviewer" + write_json(manifest_path, manifest) + args.admission_manifest_sha256 = sha256_file(manifest_path) + + with pytest.raises(rebuild.ReconstructionError) as exc_info: + rebuild.load_bundle(args) + + assert exc_info.value.code == "invalid_admission_review" + + +def test_admission_rejects_rehashed_duplicate_json_fields(tmp_path: Path) -> None: + args = write_bundle(tmp_path, material=exporter_shaped_v3_replay_material()) + manifest_path = attach_admission_manifest(args) + raw = manifest_path.read_text(encoding="utf-8") + duplicate = '{"artifact":"teleo_ledger_admission_manifest",' + raw.lstrip()[1:] + manifest_path.write_text(duplicate, encoding="utf-8") + args.admission_manifest_sha256 = sha256_file(manifest_path) + + with pytest.raises(rebuild.ReconstructionError) as exc_info: + rebuild.load_bundle(args) + + assert exc_info.value.code == "invalid_json_artifact" + + +def test_admission_rejects_legacy_rows_instead_of_blessing_forensic_history(tmp_path: Path) -> None: + args = write_bundle(tmp_path) + attach_admission_manifest(args) + + with pytest.raises(rebuild.ReconstructionError) as exc_info: + rebuild.load_bundle(args) + + assert exc_info.value.code == "legacy_admission_forbidden" + + def test_dump_hash_mismatch_fails_before_container_start(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: args = write_bundle(tmp_path) args.genesis_dump.write_bytes(b"PGDMPtampered") @@ -531,7 +919,9 @@ def test_revise_strategy_accepts_timezone_aware_closed_timestamp_boundaries( entry = rebuild.load_bundle(write_bundle(tmp_path, material=material)).entries[0] - assert entry.receipt["canonical_rows"]["strategies"][0]["created_at"] == transaction_timestamp + assert entry.receipt["canonical_rows"]["strategies"][0]["created_at"] == ( + rebuild.replay_receipt.canonical_utc_timestamp(transaction_timestamp) + ) def test_revise_strategy_rejects_reviewed_at_without_timezone(tmp_path: Path) -> None: @@ -540,13 +930,8 @@ def test_revise_strategy_rejects_reviewed_at_without_timezone(tmp_path: Path) -> for proposal_row in (material["approved_proposal"], material["applied_proposal"]): proposal_row["reviewed_at"] = reviewed_at material["approval_snapshot"]["reviewed_at"] = reviewed_at - rehash_revise_strategy_receipt(material) - - with pytest.raises(rebuild.ReconstructionError) as exc_info: - rebuild.load_bundle(write_bundle(tmp_path, material=material)) - - assert exc_info.value.code == "invalid_mutating_receipt" - assert "reviewed_at must include a timezone" in exc_info.value.safe_message + with pytest.raises(ValueError, match="reviewed_at must be a timezone-aware ISO-8601 timestamp"): + rehash_revise_strategy_receipt(material) @pytest.mark.parametrize( @@ -598,7 +983,7 @@ def test_revise_strategy_transition_builders_preserve_historical_prestate(tmp_pa assert PRIOR_RETIRED_NODE_ID not in sql assert PRIOR_STRATEGY_ID in sql assert RECEIPT_STRATEGY_ID in sql - assert "updated_at = E'2026-07-14T01:00:30+00:00'::timestamptz" in sql + assert "updated_at = E'2026-07-14T01:00:30.000000Z'::timestamptz" in sql def test_revise_strategy_transition_builders_allow_empty_prestate(tmp_path: Path) -> None: @@ -722,6 +1107,89 @@ def test_seed_sql_preserves_exact_rows_and_uses_only_fixed_tables(tmp_path: Path assert "public.strategies" not in sql +def test_seed_sql_retains_the_exact_approved_row_outside_the_v1_approval_contract( + tmp_path: Path, +) -> None: + entry = rebuild.load_bundle(write_bundle(tmp_path)).entries[0] + approval_seed = { + **entry.approval_snapshot, + "approved_proposal_snapshot": entry.approved_proposal, + } + + assert rebuild._jsonb_literal(approval_seed) in rebuild.build_seed_sql(entry) + assert "jsonb_set" in rebuild.build_seed_sql(entry) + assert "to_jsonb(proposal)" in rebuild.build_seed_sql(entry) + readback_sql = rebuild.build_proposal_readback_sql(entry.approved_proposal["id"]) + assert "to_jsonb(a) - 'approved_proposal_snapshot'" in readback_sql + assert "'approved_proposal_snapshot', (select a.approved_proposal_snapshot" in readback_sql + + +def test_v3_seed_and_timestamp_normalization_preserve_contract_guards(tmp_path: Path) -> None: + entry = rebuild.load_bundle(write_bundle(tmp_path, material=v3_replay_material())).entries[0] + + seed_sql = rebuild.build_seed_sql(entry) + normalization_sql = rebuild.build_applied_timestamp_normalization_sql(entry) + canonical_normalization_sql = rebuild.build_v3_canonical_timestamp_normalization_sql(entry) + + assert entry.contract_version == 3 + assert "approved_proposal_snapshot" in seed_sql + assert "public.claims" not in seed_sql + assert "public.sources" not in seed_sql + assert "captured_at = pinned.captured_at" in canonical_normalization_sql + assert "created_at = pinned.created_at" in canonical_normalization_sql + assert f"accepted_by_proposal_id = E'{V3_PROPOSAL_ID}'::uuid" in canonical_normalization_sql + assert "disable trigger teleo_v3_source_immutable" in canonical_normalization_sql + assert "enable trigger teleo_v3_source_immutable" in canonical_normalization_sql + assert "disable trigger all" not in canonical_normalization_sql + assert "trigger.tgenabled = 'O'" in canonical_normalization_sql + assert "disable trigger teleo_v3_guard_linked_proposal_decision" in normalization_sql + assert "disable trigger all" not in normalization_sql + assert normalization_sql.index("disable trigger") < normalization_sql.index("update kb_stage.kb_proposals") + assert normalization_sql.index("update kb_stage.kb_proposals") < normalization_sql.index("enable trigger") + assert "trigger.tgenabled = 'O'" in normalization_sql + + +def test_exporter_shaped_material_validates_storage_provenance_and_typed_source_timestamp( + tmp_path: Path, +) -> None: + material = exporter_shaped_v3_replay_material() + entry = rebuild.load_bundle(write_bundle(tmp_path, material=material)).entries[0] + + assert frozenset(material) == EXPORTER_MATERIAL_FIELDS + assert len(entry.approval_snapshot) == 8 + assert entry.approved_proposal["payload"]["apply_payload"]["sources"][0]["captured_at"].endswith("+02:00") + assert entry.receipt["canonical_rows"]["sources"][0]["captured_at"] == "2026-07-18T10:00:00.000000Z" + + approval_seed = { + **entry.approval_snapshot, + "approved_proposal_snapshot": entry.approved_proposal, + } + assert rebuild._jsonb_literal(approval_seed) in rebuild.build_seed_sql(entry) + assert "public.sources" not in rebuild.build_seed_sql(entry) + readback_sql = rebuild.build_proposal_readback_sql(entry.approved_proposal["id"]) + assert "to_jsonb(a) - 'approved_proposal_snapshot'" in readback_sql + assert "'approved_proposal_snapshot', (select a.approved_proposal_snapshot" in readback_sql + + actual_rows = copy.deepcopy(entry.receipt["canonical_rows"]) + actual_rows["sources"][0]["captured_at"] = "2026-07-18T12:00:00+02:00" + rebuild._assert_exact_canonical_rows( + actual_rows, + entry.receipt["canonical_rows"], + code="fixture_mismatch", + action="exporter-shaped fixture", + ) + + +def test_contract_order_rejects_v2_after_v3() -> None: + v3 = type("Entry", (), {"contract_version": 3})() + v2 = type("Entry", (), {"contract_version": 2})() + + rebuild._validate_contract_order([v2, v3]) + with pytest.raises(rebuild.ReconstructionError, match="cannot appear") as exc_info: + rebuild._validate_contract_order([v3, v2]) + assert exc_info.value.code == "contract_order_mismatch" + + def test_private_psql_failure_output_is_replaced_with_a_fingerprint( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1069,6 +1537,11 @@ def source_postgres() -> Iterator[tuple[str, str]]: ) if migration.returncode != 0: pytest.fail(f"could not apply guarded prerequisites: {migration.stderr}") + docker_psql( + container, + database, + "alter table kb_stage.kb_proposal_approvals add column if not exists approved_proposal_snapshot jsonb;", + ) yield container, database finally: subprocess.run( @@ -1176,6 +1649,7 @@ def seed_proposal_and_approval( approved: dict, approval: dict, ) -> dict: + approval_seed = rebuild._approval_seed_row(approved, approval) sql = f"""begin; set local standard_conforming_strings = on; insert into kb_stage.kb_proposals @@ -1184,7 +1658,7 @@ select seeded.* from jsonb_populate_record( ) seeded; insert into kb_stage.kb_proposal_approvals select seeded.* from jsonb_populate_record( - null::kb_stage.kb_proposal_approvals, {rebuild._jsonb_literal(approval)} + null::kb_stage.kb_proposal_approvals, {rebuild._jsonb_literal(approval_seed)} ) seeded; commit; """ @@ -1208,6 +1682,7 @@ def run_genesis_ledger_command( artifact_stem: str, container_prefix: str, run_number: int, + admit: bool = False, ) -> tuple[subprocess.CompletedProcess[str], dict, Path]: ledger = { "artifact": rebuild.LEDGER_ARTIFACT, @@ -1233,6 +1708,32 @@ def run_genesis_ledger_command( }, } ledger_path = write_json(tmp_path / f"{artifact_stem}-ledger-run-{run_number}.json", ledger) + admission_path: Path | None = None + if admit: + material = json.loads(material_path.read_text(encoding="utf-8")) + ledger_entry = ledger["entries"][0] + admission_path = write_json( + tmp_path / f"{artifact_stem}-admission-run-{run_number}.json", + { + "artifact": rebuild.ADMISSION_ARTIFACT, + "contract_version": rebuild.ADMISSION_CONTRACT_VERSION, + "ledger_sha256": sha256_file(ledger_path), + "reviewed_by_handle": "m3taversal", + "reviewed_by_agent_id": REVIEWER_ID, + "reviewed_at": "2026-07-18T14:00:00+00:00", + "review_note": "Admit this exact source-backed lifecycle fixture.", + "entries": [ + { + "sequence": 1, + "proposal_id": material["applied_proposal"]["id"], + "material_sha256": ledger_entry["sha256"], + "replay_material_sha256": ledger_entry["replay_material_sha256"], + "admission_scope": "collective_core", + "basis": "forgejo_primary_source", + } + ], + }, + ) output = tmp_path / f"{artifact_stem}-reconstruction-receipt-run-{run_number}.json" command = [ sys.executable, @@ -1258,6 +1759,15 @@ def run_genesis_ledger_command( "--output", str(output), ] + if admission_path is not None: + command.extend( + [ + "--admission-manifest", + str(admission_path), + "--admission-manifest-sha256", + sha256_file(admission_path), + ] + ) completed = subprocess.run( command, cwd=rebuild.REPO_ROOT, @@ -1468,3 +1978,110 @@ def test_live_revise_strategy_rebuild_is_exact_repeatable_and_leaves_no_containe first["ledger"]["entries"][0]["replay_material_sha256"] == second["ledger"]["entries"][0]["replay_material_sha256"] ) + + +def test_live_v3_exporter_material_genesis_plus_ledger_rebuild_is_exact_and_restores_guard( + tmp_path: Path, + source_postgres: tuple[str, str], +) -> None: + source_container, source_database = source_postgres + database = f"teleo_v3_{uuid.uuid4().hex[:12]}" + docker_psql( + source_container, + "postgres", + f'create database "{database}" template "{source_database}";', + ) + try: + genesis_dump, genesis_manifest = capture_source_snapshot( + source_container, + database, + tmp_path, + stem="v3-genesis", + ) + docker_psql(source_container, database, "alter role kb_apply nologin;") + docker_psql(source_container, database, rebuild.V3_CONTRACT_PATH.read_text(encoding="utf-8")) + docker_psql(source_container, database, rebuild.GUARD_PREREQUISITES_PATH.read_text(encoding="utf-8")) + + material = exporter_shaped_v3_replay_material() + material_path = write_json(tmp_path / "v3-entry-0001.json", material) + entry = rebuild._validate_entry_material( + material, + expected_sequence=1, + expected_replay_hash=material["replay_receipt"]["hashes"]["replay_material_sha256"], + material_path=material_path, + material_sha256=sha256_file(material_path), + ) + docker_psql(source_container, database, rebuild.build_seed_sql(entry)) + docker_psql(source_container, database, entry.current_apply_sql, role="kb_apply") + docker_psql( + source_container, + database, + rebuild.build_v3_canonical_timestamp_normalization_sql(entry), + ) + docker_psql(source_container, database, rebuild.build_applied_timestamp_normalization_sql(entry)) + source_readback = json.loads( + docker_psql( + source_container, + database, + rebuild.build_proposal_readback_sql(V3_PROPOSAL_ID), + ).stdout.strip() + ) + rebuild._assert_exact_proposal_readback( + source_readback, + { + "proposal": material["applied_proposal"], + "approval_snapshot": material["approval_snapshot"], + "approved_proposal_snapshot": material["approved_proposal"], + }, + code="fixture_mismatch", + action="exporter-shaped source proposal", + ) + assert len(source_readback["approval_snapshot"]) == 8 + assert source_readback["approved_proposal_snapshot"]["payload"] == material["approved_proposal"]["payload"] + + final_manifest = tmp_path / "v3-final-manifest.jsonl" + capture_manifest(source_container, database, final_manifest) + completed, receipt, output = run_genesis_ledger_command( + tmp_path=tmp_path, + database=database, + genesis_dump=genesis_dump, + genesis_manifest=genesis_manifest, + material_path=material_path, + final_manifest=final_manifest, + artifact_stem="v3", + container_prefix="teleo-genesis-ledger-v3", + run_number=1, + admit=True, + ) + + assert receipt["status"] == "pass" + assert receipt["genesis_parity"]["status"] == "pass" + assert receipt["v3_contract"]["status"] == "pass" + assert receipt["admission"]["status"] == "review_metadata_supplied" + assert receipt["admission"]["canonical_retrieval_eligible"] is False + assert receipt["admission"]["cryptographically_authenticated"] is False + assert receipt["v3_contract"]["cutover"] == { + "active_apply_sessions": 0, + "apply_role_login_disabled": True, + } + assert receipt["v3_contract"]["catalog"] == { + "assessment_table_present": True, + "apply_role_login_restored": True, + "cutover_guard_enabled": True, + "decision_guard_enabled": True, + } + assert receipt["ledger"]["entries"][0]["contract_version"] == 3 + assert receipt["ledger"]["entries"][0]["guarded_apply_executed"] is True + assert receipt["ledger"]["entries"][0]["canonical_rows_exact"] is True + assert receipt["final_parity"]["status"] == "pass" + assert receipt["cleanup"]["container_absent"] is True + assert PRIVATE_MARKER not in completed.stdout + assert PRIVATE_MARKER not in output.read_text(encoding="utf-8") + assert stat.S_IMODE(output.stat().st_mode) == 0o600 + finally: + docker_psql( + source_container, + "postgres", + f'drop database if exists "{database}" with (force);', + check=False, + ) diff --git a/tests/test_shard_corpus_extraction_work.py b/tests/test_shard_corpus_extraction_work.py new file mode 100644 index 0000000..e02273f --- /dev/null +++ b/tests/test_shard_corpus_extraction_work.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +import copy +import json +import os +import stat +import subprocess +import sys +from collections.abc import Callable, Iterator +from pathlib import Path + +import pytest + +from ops import run_local_corpus_knowledge_rebuild as rebuild +from ops import shard_corpus_extraction_work as sharder + +REPO_ROOT = Path(__file__).resolve().parents[1] +CLI = REPO_ROOT / "ops" / "shard_corpus_extraction_work.py" +FIXTURE_ROOT = REPO_ROOT / "tests" / "fixtures" / "teleo_corpus_rebuild" +PRIVATE_TEXT = "PRIVATE SOURCE BODY MUST NOT ENTER SHARDS" + + +def _load_json(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + assert isinstance(value, dict) + return value + + +def _write_json(path: Path, value: object) -> None: + path.write_bytes(sharder.rendered_json_bytes(value)) + path.chmod(0o600) + + +def _tree_bytes(root: Path) -> dict[str, bytes]: + return {path.relative_to(root).as_posix(): path.read_bytes() for path in sorted(root.rglob("*")) if path.is_file()} + + +def _all_keys(value: object) -> Iterator[str]: + if isinstance(value, dict): + for key, child in value.items(): + yield key + yield from _all_keys(child) + elif isinstance(value, list): + for child in value: + yield from _all_keys(child) + + +def _build_inventory(tmp_path: Path, source_root: Path = FIXTURE_ROOT, *, name: str = "inventory") -> Path: + output = tmp_path / name + rebuild.build_source_inventory_bundle(source_root, output) + return output + + +def _make_section_source(root: Path, count: int, *, private_text: str = PRIVATE_TEXT) -> Path: + root.mkdir(parents=True, exist_ok=True) + body = "".join(f"# Section {index:04d}\n\n{private_text} {index:04d}.\n\n" for index in range(count)) + source = root / "source.md" + source.write_text(body, encoding="utf-8") + return source + + +def _run_cli(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(CLI), *args], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, + ) + + +def _shard_documents(bundle: Path) -> list[tuple[Path, dict]]: + manifest = _load_json(bundle / sharder.MANIFEST_PATH) + return [(bundle / record["path"], _load_json(bundle / record["path"])) for record in manifest["shards"]] + + +def _rehash_bundle(bundle: Path) -> None: + manifest_path = bundle / sharder.MANIFEST_PATH + receipt_path = bundle / sharder.RECEIPT_PATH + manifest = _load_json(manifest_path) + for record in manifest["shards"]: + shard_path = bundle / record["path"] + document = _load_json(shard_path) + items = document["items"] + work_ids = [item["work_item_id"] for item in items] + document["work_item_count"] = len(items) + document["work_item_ids_sha256"] = sharder.canonical_sha256(work_ids) + _write_json(shard_path, document) + raw = shard_path.read_bytes() + record.update( + bytes=len(raw), + sha256=sharder.sha256_bytes(raw), + work_item_count=len(items), + work_item_ids_sha256=sharder.canonical_sha256(work_ids), + ) + _write_json(manifest_path, manifest) + + receipt = _load_json(receipt_path) + artifact_paths = [sharder.MANIFEST_PATH, *(record["path"] for record in manifest["shards"])] + artifacts = sharder._artifact_records(bundle, artifact_paths) + receipt["outputs"] = { + "artifact_count": len(artifacts), + "artifacts": artifacts, + "manifest_path": sharder.MANIFEST_PATH, + } + receipt["hashes"] = { + "bundle_tree_sha256": sharder.canonical_sha256(artifacts), + "manifest_file_sha256": sharder.sha256_bytes(manifest_path.read_bytes()), + } + receipt["hashes"]["receipt_content_sha256"] = sharder.canonical_sha256(receipt) + _write_json(receipt_path, receipt) + + +def test_build_is_deterministic_private_complete_and_source_text_free(tmp_path: Path) -> None: + source_root = tmp_path / "source" + _make_section_source(source_root, 12) + inventory_bundle = _build_inventory(tmp_path, source_root) + first = tmp_path / "shards-a" + second = tmp_path / "shards-b" + + first_result = sharder.build_shard_bundle(inventory_bundle, first, 4)["verification"] + second_result = sharder.build_shard_bundle(inventory_bundle, second, 4)["verification"] + + assert first_result["status"] == second_result["status"] == "pass" + assert first_result["work_item_count"] == second_result["work_item_count"] == 12 + assert first_result["shard_count"] == second_result["shard_count"] == 4 + assert first_result["bundle_tree_sha256"] == second_result["bundle_tree_sha256"] + assert _tree_bytes(first) == _tree_bytes(second) + persisted = b"".join(_tree_bytes(first).values()) + assert PRIVATE_TEXT.encode() not in persisted + assert str(source_root).encode() not in persisted + assert str(inventory_bundle).encode() not in persisted + assert str(first).encode() not in persisted + + forbidden_keys = {"claim", "claims", "proposal", "proposals", "admitted", "database_rows"} + for document in [_load_json(first / sharder.MANIFEST_PATH), _load_json(first / sharder.RECEIPT_PATH)]: + assert forbidden_keys.isdisjoint(_all_keys(document)) + for key, expected in sharder.NO_WRITE_FLAGS.items(): + assert document[key] is expected + + for path in first.rglob("*"): + expected_mode = 0o600 if path.is_file() else 0o700 + assert stat.S_IMODE(path.stat().st_mode) == expected_mode + assert stat.S_IMODE(first.stat().st_mode) == 0o700 + assert not (first / sharder.INCOMPLETE_MARKER).exists() + + +def test_hash_assignment_covers_every_exact_inventory_item_once(tmp_path: Path) -> None: + source_root = tmp_path / "source" + _make_section_source(source_root, 40) + inventory_bundle = _build_inventory(tmp_path, source_root) + output = tmp_path / "shards" + sharder.build_shard_bundle(inventory_bundle, output, 7) + inventory_items = _load_json(inventory_bundle / rebuild.EXTRACTION_WORK_INDEX_PATH)["items"] + + assigned: list[dict] = [] + for index, (_path, document) in enumerate(_shard_documents(output)): + ids = [item["work_item_id"] for item in document["items"]] + assert ids == sorted(ids, key=lambda value: value.encode("utf-8")) + assert all(sharder.assignment_for(work_id, 7) == index for work_id in ids) + assigned.extend(document["items"]) + + assert len(assigned) == len(inventory_items) + assert {item["work_item_id"] for item in assigned} == {item["work_item_id"] for item in inventory_items} + assert {item["work_item_id"]: item for item in assigned} == {item["work_item_id"]: item for item in inventory_items} + + +@pytest.mark.parametrize("shard_count", [0, 1025]) +def test_shard_count_must_be_within_hard_bounds(tmp_path: Path, shard_count: int) -> None: + inventory_bundle = _build_inventory(tmp_path) + with pytest.raises(sharder.ShardError) as error: + sharder.build_shard_bundle(inventory_bundle, tmp_path / "shards", shard_count) + assert error.value.code == "invalid_shard_count" + assert not (tmp_path / "shards").exists() + + +def test_shard_count_must_not_exceed_work_item_count(tmp_path: Path) -> None: + inventory_bundle = _build_inventory(tmp_path) + work_count = _load_json(inventory_bundle / rebuild.EXTRACTION_WORK_INDEX_PATH)["work_item_count"] + with pytest.raises(sharder.ShardError) as error: + sharder.build_shard_bundle(inventory_bundle, tmp_path / "shards", work_count + 1) + assert error.value.code == "invalid_shard_count" + + +def test_existing_output_is_never_overwritten(tmp_path: Path) -> None: + inventory_bundle = _build_inventory(tmp_path) + output = tmp_path / "shards" + output.mkdir(mode=0o700) + sentinel = output / "sentinel" + sentinel.write_text("preserve", encoding="utf-8") + + with pytest.raises(sharder.ShardError) as error: + sharder.build_shard_bundle(inventory_bundle, output, 2) + + assert error.value.code == "output_exists" + assert sentinel.read_text(encoding="utf-8") == "preserve" + + +def test_marker_write_failure_closes_and_removes_reserved_output( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inventory_bundle = _build_inventory(tmp_path) + output = tmp_path / "shards" + original_write = sharder._write_bytes_at + + def fail_after_write(directory_fd: int, name: str, payload: bytes) -> None: + original_write(directory_fd, name, payload) + raise OSError("synthetic marker publication failure") + + monkeypatch.setattr(sharder, "_write_bytes_at", fail_after_write) + with pytest.raises(OSError, match="synthetic marker publication failure"): + sharder.build_shard_bundle(inventory_bundle, output, 2) + + assert not output.exists() + + +def test_receipt_is_published_while_incomplete_marker_exists( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inventory_bundle = _build_inventory(tmp_path) + output = tmp_path / "shards" + + def inspect(reserved: sharder.ReservedOutput) -> None: + marker = os.stat(sharder.INCOMPLETE_MARKER, dir_fd=reserved.root_fd, follow_symlinks=False) + assert stat.S_ISREG(marker.st_mode) + with pytest.raises(FileNotFoundError): + os.stat(sharder.RECEIPT_PATH, dir_fd=reserved.root_fd, follow_symlinks=False) + + monkeypatch.setattr(sharder, "_before_receipt_publication", inspect) + sharder.build_shard_bundle(inventory_bundle, output, 2) + assert not (output / sharder.INCOMPLETE_MARKER).exists() + assert (output / sharder.RECEIPT_PATH).is_file() + + +def test_destination_swap_does_not_overwrite_or_cleanup_replacement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inventory_bundle = _build_inventory(tmp_path) + output = tmp_path / "shards" + moved = tmp_path / "reserved-moved" + + def swap(_reserved: sharder.ReservedOutput) -> None: + output.rename(moved) + output.mkdir(mode=0o700) + sentinel = output / "replacement-sentinel" + sentinel.write_text("preserve replacement", encoding="utf-8") + sentinel.chmod(0o600) + + monkeypatch.setattr(sharder, "_after_output_reservation", swap) + with pytest.raises(sharder.ShardError) as error: + sharder.build_shard_bundle(inventory_bundle, output, 2) + + assert error.value.code == "output_binding_changed" + assert (output / "replacement-sentinel").read_text(encoding="utf-8") == "preserve replacement" + assert moved.is_dir() + assert list(moved.iterdir()) == [] + + +def test_inventory_change_before_publication_rejects_and_cleans_owned_output( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + inventory_bundle = _build_inventory(tmp_path) + output = tmp_path / "shards" + + def mutate(root: Path) -> None: + receipt = root / rebuild.RECEIPT_PATH + receipt.write_bytes(receipt.read_bytes() + b" ") + + monkeypatch.setattr(sharder, "_before_final_publication", mutate) + with pytest.raises(sharder.ShardError) as error: + sharder.build_shard_bundle(inventory_bundle, output, 2) + + assert error.value.code == "inventory_changed" + assert not output.exists() + + +def _mutate_unknown(bundle: Path) -> None: + for path, document in _shard_documents(bundle): + if document["items"]: + document["items"][0]["work_item_id"] = "f" * 64 + document["items"].sort(key=lambda item: item["work_item_id"].encode("utf-8")) + _write_json(path, document) + return + raise AssertionError("fixture unexpectedly produced no shard items") + + +def _mutate_missing(bundle: Path) -> None: + for path, document in _shard_documents(bundle): + if document["items"]: + document["items"].pop() + _write_json(path, document) + return + raise AssertionError("fixture unexpectedly produced no shard items") + + +def _mutate_duplicate(bundle: Path) -> None: + for path, document in _shard_documents(bundle): + if document["items"]: + document["items"].append(copy.deepcopy(document["items"][0])) + document["items"].sort(key=lambda item: item["work_item_id"].encode("utf-8")) + _write_json(path, document) + return + raise AssertionError("fixture unexpectedly produced no shard items") + + +def _mutate_overclaim(bundle: Path) -> None: + receipt_path = bundle / sharder.RECEIPT_PATH + receipt = _load_json(receipt_path) + receipt["database_write_performed"] = True + _write_json(receipt_path, receipt) + + +@pytest.mark.parametrize( + ("mutator", "expected_codes"), + [ + (_mutate_unknown, {"unknown_work_item"}), + (_mutate_missing, {"missing_work_item"}), + (_mutate_duplicate, {"duplicate_work_item", "invalid_shard_order"}), + (_mutate_overclaim, {"false_sharding_claim"}), + ], +) +def test_verifier_rejects_fully_rehashed_semantic_tampering( + tmp_path: Path, + mutator: Callable[[Path], None], + expected_codes: set[str], +) -> None: + source_root = tmp_path / "source" + _make_section_source(source_root, 20) + inventory_bundle = _build_inventory(tmp_path, source_root) + output = tmp_path / "shards" + sharder.build_shard_bundle(inventory_bundle, output, 4) + + mutator(output) + _rehash_bundle(output) + + with pytest.raises(sharder.ShardError) as error: + sharder.verify_shard_bundle(inventory_bundle, output) + assert error.value.code in expected_codes + + +def test_verifier_rejects_wrong_inventory_even_with_same_cardinality(tmp_path: Path) -> None: + first_source = tmp_path / "source-a" + second_source = tmp_path / "source-b" + _make_section_source(first_source, 8, private_text="first corpus") + _make_section_source(second_source, 8, private_text="second corpus") + first_inventory = _build_inventory(tmp_path, first_source, name="inventory-a") + second_inventory = _build_inventory(tmp_path, second_source, name="inventory-b") + output = tmp_path / "shards" + sharder.build_shard_bundle(first_inventory, output, 3) + + with pytest.raises(sharder.ShardError) as error: + sharder.verify_shard_bundle(second_inventory, output) + assert error.value.code == "inventory_mismatch" + + +def test_verifier_rejects_extra_file_and_incomplete_marker(tmp_path: Path) -> None: + inventory_bundle = _build_inventory(tmp_path) + extra_output = tmp_path / "extra-shards" + sharder.build_shard_bundle(inventory_bundle, extra_output, 2) + extra = extra_output / "extra.json" + extra.write_text("{}\n", encoding="utf-8") + extra.chmod(0o600) + with pytest.raises(sharder.ShardError) as extra_error: + sharder.verify_shard_bundle(inventory_bundle, extra_output) + assert extra_error.value.code == "unexpected_bundle_entry" + + incomplete_output = tmp_path / "incomplete-shards" + sharder.build_shard_bundle(inventory_bundle, incomplete_output, 2) + marker = incomplete_output / sharder.INCOMPLETE_MARKER + marker.write_text("incomplete\n", encoding="utf-8") + marker.chmod(0o600) + with pytest.raises(sharder.ShardError) as incomplete_error: + sharder.verify_shard_bundle(inventory_bundle, incomplete_output) + assert incomplete_error.value.code == "incomplete_shard_bundle" + + +def test_verifier_rejects_symlinked_shard(tmp_path: Path) -> None: + inventory_bundle = _build_inventory(tmp_path) + output = tmp_path / "shards" + sharder.build_shard_bundle(inventory_bundle, output, 2) + manifest = _load_json(output / sharder.MANIFEST_PATH) + target = output / manifest["shards"][0]["path"] + retained = target.with_suffix(".retained") + target.rename(retained) + try: + target.symlink_to(retained.name) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + with pytest.raises(sharder.ShardError) as error: + sharder.verify_shard_bundle(inventory_bundle, output) + assert error.value.code == "unsafe_bundle_entry" + + +def test_cli_build_and_verify_return_path_free_receipts(tmp_path: Path) -> None: + inventory_bundle = _build_inventory(tmp_path) + output = tmp_path / "shards" + build = _run_cli( + "--inventory-bundle", + str(inventory_bundle), + "--output-dir", + str(output), + "--shard-count", + "2", + ) + verify = _run_cli( + "--inventory-bundle", + str(inventory_bundle), + "--verify-bundle", + str(output), + ) + + assert build.returncode == verify.returncode == 0, (build.stderr, build.stdout, verify.stderr, verify.stdout) + build_status = json.loads(build.stdout) + verify_status = json.loads(verify.stdout) + assert build_status["status"] == verify_status["status"] == "pass" + assert build_status["bundle_tree_sha256"] == verify_status["bundle_tree_sha256"] + assert build_status["work_item_count"] == verify_status["work_item_count"] == 3 + assert str(inventory_bundle) not in build.stdout + verify.stdout + assert str(output) not in build.stdout + verify.stdout + for key, expected in sharder.NO_WRITE_FLAGS.items(): + assert build_status[key] is verify_status[key] is expected + + +def test_cli_rejects_invalid_mode_arguments_without_output(tmp_path: Path) -> None: + inventory_bundle = _build_inventory(tmp_path) + result = _run_cli( + "--inventory-bundle", + str(inventory_bundle), + "--verify-bundle", + str(tmp_path / "missing"), + "--shard-count", + "2", + ) + status = json.loads(result.stdout) + assert result.returncode == 2 + assert status["status"] == "rejected" + assert status["reason"] == "invalid_arguments" diff --git a/tests/test_stage_normalized_proposal.py b/tests/test_stage_normalized_proposal.py index a4309a8..524d870 100644 --- a/tests/test_stage_normalized_proposal.py +++ b/tests/test_stage_normalized_proposal.py @@ -13,6 +13,8 @@ sys.path.insert(0, str(REPO_ROOT / "scripts")) import stage_normalized_proposal as stage # noqa: E402 +from tests import test_compile_kb_source_packet as source_compiler_fixture # noqa: E402 + PARENT_ID = "77777777-7777-4777-8777-777777777701" AGENT_ID = "11111111-1111-4111-8111-111111111111" DOCUMENT = ( @@ -211,6 +213,131 @@ def test_stage_normalized_child_binds_to_full_disposable_container_id( assert result["database_receipt"]["created"] is True +def test_stage_normalized_child_does_not_require_an_ambient_production_container( + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = { + "id": "clone-container-id", + "name": "composition-clone", + "labels": {stage.bound.DISPOSABLE_LABEL_KEY: stage.bound.DISPOSABLE_LABEL_VALUE}, + "network_mode": "none", + "ports": {}, + "mount_sources": [], + } + + def identity(name: str) -> dict[str, Any]: + if name == "composition-clone": + return target + raise stage.bound.CheckpointError("docker inspect teleo-pg: no such object: teleo-pg") + + monkeypatch.setattr(stage.bound, "container_identity", identity) + child = stage.prepare_normalized_child(_parent()) + monkeypatch.setattr( + stage.bound, + "_psql_json", + lambda container, database, sql: { + "created": True, + "proposal_id": child["id"], + "proposal_type": child["proposal_type"], + "status": "pending_review", + "source_ref": child["source_ref"], + "payload_sha256": child["payload_sha256"], + }, + ) + + result = stage.stage_normalized_child("composition-clone", "teleo", _parent()) + + assert result["bound_container_id"] == target["id"] + + +def test_stage_normalized_child_refuses_unexpected_production_inspection_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = { + "id": "clone-container-id", + "name": "composition-clone", + "labels": {stage.bound.DISPOSABLE_LABEL_KEY: stage.bound.DISPOSABLE_LABEL_VALUE}, + "network_mode": "none", + "ports": {}, + "mount_sources": [], + } + + def identity(name: str) -> dict[str, Any]: + if name == "composition-clone": + return target + raise stage.bound.CheckpointError("docker inspect teleo-pg: permission denied") + + monkeypatch.setattr(stage.bound, "container_identity", identity) + + with pytest.raises(stage.bound.CheckpointError, match="permission denied"): + stage.stage_normalized_child("composition-clone", "teleo", _parent()) + + +def test_compiled_v3_child_revalidates_exact_bundle_and_rejects_tampering(tmp_path: Path) -> None: + bundle = source_compiler_fixture._compile_v3(tmp_path / "source") + + child = stage.prepare_compiled_v3_child(bundle) + + assert child == bundle["strict_child_proposal"] + assert child is not bundle["strict_child_proposal"] + assert child["payload"]["apply_payload"]["contract_version"] == 3 + + tampered = copy.deepcopy(bundle) + tampered["strict_child_proposal"]["payload"]["apply_payload"]["claims"][0]["proposition"] += " tampered" + with pytest.raises(stage.bound.CheckpointError, match="bundle content hash does not match"): + stage.prepare_compiled_v3_child(tampered) + + +def test_stage_compiled_v3_bundle_binds_to_disposable_container( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + bundle = source_compiler_fixture._compile_v3(tmp_path / "source") + child = bundle["strict_child_proposal"] + target = { + "id": "clone-container-id", + "name": "composition-clone", + "labels": {stage.bound.DISPOSABLE_LABEL_KEY: stage.bound.DISPOSABLE_LABEL_VALUE}, + "network_mode": "none", + "ports": {}, + "mount_sources": ["/clone-data"], + } + production = { + "id": "production-container-id", + "name": stage.bound.PRODUCTION_CONTAINER, + "labels": {}, + "network_mode": "bridge", + "ports": {}, + "mount_sources": ["/production-data"], + } + monkeypatch.setattr( + stage.bound, + "container_identity", + lambda name: target if name == "composition-clone" else production, + ) + observed: dict[str, Any] = {} + + def fake_psql(container: str, database: str, sql: str) -> dict[str, Any]: + observed.update({"container": container, "database": database, "sql": sql}) + return { + "created": True, + "proposal_id": child["id"], + "proposal_type": child["proposal_type"], + "status": "pending_review", + "source_ref": child["source_ref"], + "payload_sha256": child["payload_sha256"], + } + + monkeypatch.setattr(stage.bound, "_psql_json", fake_psql) + + result = stage.stage_compiled_v3_bundle("composition-clone", "teleo", bundle) + + assert observed["container"] == "clone-container-id" + assert observed["database"] == "teleo" + assert "insert into kb_stage.kb_proposals" in observed["sql"].lower() + assert result["child"]["id"] == child["id"] + + def test_main_rejects_the_production_container_without_reading_the_packet(tmp_path: Path) -> None: missing = tmp_path / "missing.json" diff --git a/tests/test_teleo_v3_epistemic_contract.py b/tests/test_teleo_v3_epistemic_contract.py new file mode 100644 index 0000000..07ab772 --- /dev/null +++ b/tests/test_teleo_v3_epistemic_contract.py @@ -0,0 +1,799 @@ +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +import pytest + +from ops import verify_teleo_v3_epistemic_contract as verifier + +REPO_ROOT = Path(__file__).resolve().parents[1] +MIGRATION_SQL = REPO_ROOT / "ops" / "teleo_v3_epistemic_contract.sql" +ROLLBACK_SQL = REPO_ROOT / "ops" / "rollback_teleo_v3_epistemic_contract.sql" +SECRET_STARTUP_MARKER = "secret-startup-marker" + + +def test_install_migration_is_additive_and_legacy_compatible() -> None: + sql = MIGRATION_SQL.read_text(encoding="utf-8").lower() + + assert "add column if not exists" in sql + assert "create table if not exists public.claim_evidence_assessments" in sql + assert "admission_state text not null default 'legacy'" in sql + assert "admission_state = 'legacy'" in sql + assert "admission_state = 'admitted'" in sql + assert "accepted_by_proposal_id" in sql + assert "accepted_from_proposal_id" in sql + assert "teleo_v3_require_linked_insert" in sql + assert ( + "create or replace function kb_stage.teleo_v3_require_accepted_proposal()\nreturns trigger\nlanguage plpgsql\nsecurity definer" + in sql + ) + assert ( + "create or replace function kb_stage.teleo_v3_enforce_apply_contract_cutover()\n" + "returns trigger\nlanguage plpgsql\nsecurity definer" in sql + ) + assert "alter function kb_stage.teleo_v3_require_accepted_proposal() owner to kb_gate_owner" in sql + assert "revoke all on function kb_stage.teleo_v3_require_accepted_proposal()" in sql + assert "alter function kb_stage.teleo_v3_enforce_apply_contract_cutover() owner to kb_gate_owner" in sql + assert "revoke all on function kb_stage.teleo_v3_enforce_apply_contract_cutover()" in sql + assert "from public, kb_apply, kb_review" in sql + assert "security definer trigger invariants failed" in sql + assert "kb_gate_owner must be nologin noinherit and unprivileged" in sql + for attribute in ( + "not role.rolcanlogin", + "not role.rolinherit", + "not role.rolsuper", + "not role.rolcreatedb", + "not role.rolcreaterole", + "not role.rolreplication", + "not role.rolbypassrls", + ): + assert attribute in sql + assert "teleo_v3_reject_delete" in sql + assert "teleo_v3_proposal_apply_receipt_check" in sql + assert "create or replace trigger teleo_v3_claim_immutable" in sql + assert "create or replace trigger teleo_v3_guard_linked_proposal_decision" in sql + assert "'status', 'admission_state'" in sql + assert "'updated_at', 'accepted_by_proposal_id'" in sql + assert "legacy is reserved for rows present at migration" in sql + assert "cutover order: deploy the cutover-aware writer" in sql + assert "alter role kb_apply nologin" in sql + assert "active kb_apply sessions must exit" in sql + assert "retains % pending/approved v2 proposal(s) for explicit migration or rejection" in sql + assert "teleo_v3_00_enforce_apply_contract_cutover" in sql + assert "teleo_v3_enforce_apply_contract_cutover" in sql + assert sql.count("create or replace trigger teleo_v3_") == 24 + assert re.search(r"(?m)^\s*create trigger teleo_v3_", sql) is None + assert "actual.tgenabled = 'o'" in sql + assert "actual.tgtype = expected.trigger_type" in sql + assert "actual.update_columns = expected.update_columns" in sql + assert "actual.when_expression is not distinct from expected.when_expression" in sql + assert "actual.argument_hex = expected.argument_hex" in sql + assert "actual.tgconstraint = 0::pg_catalog.oid" in sql + assert "actual.tgoldtable is null" in sql + assert "actual.tgnewtable is null" in sql + assert "actual.tgfoid" in sql + assert "actual.function_security_definer = expected.function_security_definer" in sql + assert "actual.function_source_sha256 = expected.function_source_sha256" in sql + assert "acl.grantee <> procedure.proowner" in sql + assert "protected role membership drift must be removed" in sql + assert "teleo_v3_contract_postflight_query_begin" in sql + assert "alter function %i.%i() owner to kb_gate_owner" in sql + assert "jsonb_typeof(new.payload->'apply_payload'->'contract_version') = 'number'" in sql + assert "migrate or reject it" in sql + assert "for share" in sql + assert "source_content_hash" in sql + assert "locator_json" in sql + assert "relation_type in ('supports', 'challenges', 'depends_on', 'contradicts')" in sql + assert "preexisting v3 schema objects lack installation ownership" in sql + assert "create table if not exists kb_stage.teleo_v3_installation_ownership" in sql + assert "installation_token = 'collision-free-install-v1'" in sql + assert "alter table kb_stage.teleo_v3_installation_ownership owner to kb_gate_owner" in sql + + destructive_patterns = ( + r"\bdrop\s+", + r"\btruncate\b", + r"\bdelete\s+from\b", + r"\bupdate\s+public\.", + r"\balter\s+type\b", + ) + for pattern in destructive_patterns: + assert re.search(pattern, sql) is None, pattern + + +def test_rollback_refuses_silent_v3_data_loss_before_dropping_contract() -> None: + sql = ROLLBACK_SQL.read_text(encoding="utf-8").lower() + + guard_position = sql.index("teleo_v3 rollback refused") + first_drop_position = sql.index("drop trigger") + assert guard_position < first_drop_position + assert "claim_evidence_assessments contains v3 data" in sql + assert "claims contains admitted or proposal-linked v3 data" in sql + assert "drop column if exists proposition" in sql + assert "drop function if exists kb_stage.teleo_v3_require_accepted_proposal" in sql + assert "drop function if exists kb_stage.teleo_v3_require_linked_insert" in sql + assert "drop function if exists kb_stage.teleo_v3_reject_delete" in sql + assert "drop function if exists kb_stage.teleo_v3_acquire_protected_relation_locks" in sql + assert "drop function if exists kb_stage.teleo_v3_enforce_apply_contract_cutover" in sql + assert "drop constraint if exists teleo_v3_proposal_apply_receipt_check" in sql + assert "installation ownership ledger is not exact" in sql + assert "v3-named objects lack exact installation ownership" in sql + assert "drop table if exists kb_stage.teleo_v3_installation_ownership" in sql + + +@pytest.mark.parametrize( + ("logs", "probe_returncode", "probe_stdout", "expected"), + ( + (verifier.POSTGRES_READY_LOG_LINE, 0, "1\n", False), + (f"{verifier.POSTGRES_READY_LOG_LINE}\n{verifier.POSTGRES_READY_LOG_LINE}", 2, "", False), + (f"{verifier.POSTGRES_READY_LOG_LINE}\n{verifier.POSTGRES_READY_LOG_LINE}", 0, "", False), + (f"{verifier.POSTGRES_READY_LOG_LINE}\n{verifier.POSTGRES_READY_LOG_LINE}", 0, "1\n", True), + ), +) +def test_postgres_final_startup_requires_second_transition_and_target_database( + logs: str, + probe_returncode: int, + probe_stdout: str, + expected: bool, +) -> None: + probe = subprocess.CompletedProcess(["psql"], probe_returncode, stdout=probe_stdout, stderr="") + + assert verifier.postgres_final_startup_ready(logs, probe) is expected + + +def _startup_test_postgres() -> verifier.DisposablePostgres: + postgres = object.__new__(verifier.DisposablePostgres) + postgres.docker = "docker" + postgres.name = "teleo-v3-contract-test" + postgres.password = SECRET_STARTUP_MARKER + postgres.started = True + postgres.cleanup_required = True + postgres.environment = {} + return postgres + + +@pytest.mark.parametrize("as_bytes", (False, True)) +def test_startup_detail_redacts_secret_before_tail_limit(as_bytes: bool) -> None: + postgres = _startup_test_postgres() + postgres.password = "K8VN3PQ7RM2TX6ZC9WB4HJ5DF1GS0LAE8YU3IO2N" + context = "context=database-unavailable|" + trailer = "|stage=target_database_probe|kind=timeout" + suffix_length = 500 - len(context) - len("") + suffix = ("x" * (suffix_length - len(trailer))) + trailer + diagnostic = context + postgres.password + suffix + value: str | bytes = diagnostic.encode() if as_bytes else diagnostic + + assert len(diagnostic) > 500 + assert len(diagnostic) - 500 > len(context) + + detail = postgres._redacted_startup_detail(value) + + assert detail.startswith(f"{context}") + assert detail.endswith(trailer) + assert postgres.password not in detail + assert all(postgres.password[offset : offset + 4] not in detail for offset in range(len(postgres.password) - 3)) + + +def test_docker_run_timeout_is_secret_safe_and_reconciles_owned_side_effect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + postgres = _startup_test_postgres() + postgres.password = "K8VN3PQ7RM2TX6ZC9WB4HJ5DF1GS0LAE8YU3IO2N" + postgres.started = False + postgres.cleanup_required = False + owned_container_id = "a" * 64 + owned_container_exists = [True] + calls: list[tuple[list[str], dict[str, str] | None]] = [] + context = "context=docker-daemon-accepted-request|" + trailer = "|stage=docker_run|kind=timeout" + suffix_length = 500 - len(context) - len("") + timeout_stderr = context + postgres.password + ("x" * (suffix_length - len(trailer))) + trailer + + def fake_run( + arguments: list[str], + *, + input_text: str | None = None, + check: bool = True, + timeout: float = 120, + environment: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + del input_text, check + calls.append((list(arguments), environment)) + if arguments[0] == "run": + assert postgres.cleanup_required is True + assert postgres.started is False + assert postgres.password not in " ".join(arguments) + assert environment is not None + assert environment["POSTGRES_PASSWORD"] == postgres.password + secret_bearing_command = ["docker", *arguments, f"adjacent={postgres.password}"] + raise subprocess.TimeoutExpired( + secret_bearing_command, + timeout, + stderr=timeout_stderr, + ) + if arguments[:2] == ["container", "ls"] and arguments[-1] == "{{.ID}}": + stdout = f"{owned_container_id}\n" if owned_container_exists[0] else "" + return subprocess.CompletedProcess(arguments, 0, stdout=stdout, stderr="") + if arguments[:2] == ["rm", "--force"]: + assert arguments == ["rm", "--force", owned_container_id] + owned_container_exists[0] = False + return subprocess.CompletedProcess(arguments, 0, stdout=owned_container_id, stderr="") + if arguments[0] == "inspect": + return subprocess.CompletedProcess( + arguments, + 0 if owned_container_exists[0] else 1, + stdout="[]" if owned_container_exists[0] else "", + stderr="" if owned_container_exists[0] else "not found", + ) + if arguments[:2] == ["container", "ls"] and arguments[-1] == "{{.Names}}": + stdout = f"{postgres.name}\n" if owned_container_exists[0] else "" + return subprocess.CompletedProcess(arguments, 0, stdout=stdout, stderr="") + raise AssertionError(f"unexpected docker arguments: {arguments}") + + monkeypatch.setattr(postgres, "run", fake_run) + monkeypatch.setattr(verifier.time, "sleep", lambda _seconds: None) + + with pytest.raises(verifier.VerificationError) as exc_info: + postgres.start() + cleanup = postgres.cleanup() + + error = str(exc_info.value) + assert "launch state is ambiguous; cleanup_required=true" in error + assert "stage=docker_run kind=timeout timeout_seconds=120" in error + assert error.endswith(trailer) + assert exc_info.value.__suppress_context__ is True + assert postgres.password not in error + assert all(postgres.password[offset : offset + 4] not in error for offset in range(len(postgres.password) - 3)) + assert postgres.started is False + assert cleanup["cleanup_reconciliation_attempted"] is True + assert cleanup["remove_attempted"] is True + assert cleanup["remove_returncodes"] == [0] + assert cleanup["owned_container_ids_observed"] == [owned_container_id] + assert cleanup["container_absent"] is True + assert cleanup["name_readback_clear"] is True + assert cleanup["instance_label_readback_clear"] is True + assert cleanup["stable_absence_readback"] is True + ownership_call = next(arguments for arguments, _environment in calls if arguments[-1] == "{{.ID}}") + assert f"name=^/{postgres.name}$" in ownership_call + assert f"label={verifier.CONTAINER_LABEL}" in ownership_call + assert f"label=livingip.canary.instance={postgres.name}" in ownership_call + + +def test_docker_run_normal_launch_marks_cleanup_before_execution( + monkeypatch: pytest.MonkeyPatch, +) -> None: + postgres = _startup_test_postgres() + postgres.started = False + postgres.cleanup_required = False + container_id = "b" * 64 + waited_for_startup: list[bool] = [] + + def fake_run( + arguments: list[str], + *, + input_text: str | None = None, + check: bool = True, + timeout: float = 120, + environment: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + del input_text, check, timeout + if arguments[0] == "run": + assert postgres.cleanup_required is True + assert postgres.started is False + assert postgres.password not in " ".join(arguments) + assert environment is not None + assert environment["POSTGRES_PASSWORD"] == postgres.password + return subprocess.CompletedProcess(arguments, 0, stdout=f"{container_id}\n", stderr="") + assert arguments == ["inspect", postgres.name] + inspected = [ + { + "Config": { + "Image": verifier.POSTGRES_IMAGE, + "Labels": { + "livingip.canary": "teleo-v3-epistemic-contract", + "livingip.canary.instance": postgres.name, + }, + }, + "HostConfig": { + "NetworkMode": "none", + "Tmpfs": {"/var/lib/postgresql/data": "rw,nosuid,nodev,size=384m"}, + }, + "Image": "sha256:postgres-image", + "Mounts": [], + } + ] + return subprocess.CompletedProcess(arguments, 0, stdout=json.dumps(inspected), stderr="") + + def fake_wait_for_final_startup() -> None: + assert postgres.cleanup_required is True + assert postgres.started is True + waited_for_startup.append(True) + + monkeypatch.setattr(postgres, "run", fake_run) + monkeypatch.setattr(postgres, "_wait_for_final_startup", fake_wait_for_final_startup) + + environment = postgres.start() + + assert waited_for_startup == [True] + assert postgres.started is True + assert postgres.cleanup_required is True + assert environment["container_name"] == postgres.name + assert environment["network_mode"] == "none" + assert environment["tmpfs_data_dir"] is True + assert environment["docker_volume_mounts"] == [] + + +def test_cleanup_never_removes_foreign_exact_name_without_instance_labels( + monkeypatch: pytest.MonkeyPatch, +) -> None: + postgres = _startup_test_postgres() + postgres.started = False + postgres.cleanup_required = True + calls: list[list[str]] = [] + + def fake_run( + arguments: list[str], + *, + input_text: str | None = None, + check: bool = True, + timeout: float = 120, + environment: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + del input_text, check, timeout, environment + calls.append(list(arguments)) + if arguments[:2] == ["container", "ls"] and arguments[-1] == "{{.ID}}": + return subprocess.CompletedProcess(arguments, 0, stdout="", stderr="") + if arguments[0] == "inspect": + return subprocess.CompletedProcess(arguments, 0, stdout="[]", stderr="") + if arguments[:2] == ["container", "ls"] and arguments[-1] == "{{.Names}}": + has_instance_filter = f"label=livingip.canary.instance={postgres.name}" in arguments + stdout = "" if has_instance_filter else f"{postgres.name}\n" + return subprocess.CompletedProcess(arguments, 0, stdout=stdout, stderr="") + raise AssertionError(f"unexpected docker arguments: {arguments}") + + monkeypatch.setattr(postgres, "run", fake_run) + monkeypatch.setattr(verifier.time, "sleep", lambda _seconds: None) + + cleanup = postgres.cleanup() + + assert not any(arguments[:2] == ["rm", "--force"] for arguments in calls) + assert cleanup["remove_attempted"] is False + assert cleanup["foreign_name_collision_detected"] is True + assert cleanup["container_absent"] is False + assert cleanup["name_readback_clear"] is False + assert cleanup["instance_label_readback_clear"] is True + assert cleanup["stable_absence_readback"] is False + + +@pytest.mark.parametrize("hung_stage", ("docker_logs", "target_database_probe")) +def test_postgres_startup_commands_are_capped_to_remaining_deadline( + monkeypatch: pytest.MonkeyPatch, + hung_stage: str, +) -> None: + postgres = _startup_test_postgres() + now = [100.0] + observed_timeouts: list[float] = [] + ready_logs = f"{verifier.POSTGRES_READY_LOG_LINE}\n{verifier.POSTGRES_READY_LOG_LINE}\n" + + def fake_run( + arguments: list[str], + *, + input_text: str | None = None, + check: bool = True, + timeout: float = 120, + ) -> subprocess.CompletedProcess[str]: + del input_text, check + assert arguments == ["logs", postgres.name] + if hung_stage == "docker_logs": + observed_timeouts.append(timeout) + now[0] += timeout + raise subprocess.TimeoutExpired(arguments, timeout, stderr=SECRET_STARTUP_MARKER) + return subprocess.CompletedProcess(arguments, 0, stdout=ready_logs, stderr="") + + def fake_psql( + sql: str, + *, + check: bool = True, + timeout: float = 120, + ) -> subprocess.CompletedProcess[str]: + del check + assert sql == "select 1;" + observed_timeouts.append(timeout) + now[0] += timeout + raise subprocess.TimeoutExpired(["psql"], timeout, stderr=SECRET_STARTUP_MARKER) + + monkeypatch.setattr(verifier.time, "monotonic", lambda: now[0]) + monkeypatch.setattr(postgres, "run", fake_run) + monkeypatch.setattr(postgres, "psql", fake_psql) + + with pytest.raises(verifier.VerificationError) as exc_info: + postgres._wait_for_final_startup(startup_timeout=3.0, probe_timeout=120.0, poll_interval=0) + + assert observed_timeouts == [3.0] + assert f"stage={hung_stage} kind=timeout timeout_seconds=3" in str(exc_info.value) + assert SECRET_STARTUP_MARKER not in str(exc_info.value) + + +@pytest.mark.parametrize( + ("failed_stage", "expected_context"), + ( + ("docker_logs", "stage=docker_logs kind=nonzero returncode=17 detail=daemon "), + ( + "target_database_probe", + "stage=target_database_probe kind=nonzero returncode=2 detail=database missing ", + ), + ), +) +def test_postgres_startup_failure_reports_sanitized_terminal_context( + monkeypatch: pytest.MonkeyPatch, + failed_stage: str, + expected_context: str, +) -> None: + postgres = _startup_test_postgres() + now = [200.0] + ready_logs = f"{verifier.POSTGRES_READY_LOG_LINE}\n{verifier.POSTGRES_READY_LOG_LINE}\n" + + def fake_run( + arguments: list[str], + *, + input_text: str | None = None, + check: bool = True, + timeout: float = 120, + ) -> subprocess.CompletedProcess[str]: + del input_text, check, timeout + assert arguments == ["logs", postgres.name] + if failed_stage == "docker_logs": + return subprocess.CompletedProcess( + arguments, + 17, + stdout="", + stderr=f"daemon {SECRET_STARTUP_MARKER}", + ) + return subprocess.CompletedProcess(arguments, 0, stdout=ready_logs, stderr="") + + def fake_psql( + sql: str, + *, + check: bool = True, + timeout: float = 120, + ) -> subprocess.CompletedProcess[str]: + del check, timeout + assert sql == "select 1;" + return subprocess.CompletedProcess( + ["psql"], + 2, + stdout="", + stderr=f"database missing {SECRET_STARTUP_MARKER}", + ) + + def fake_sleep(seconds: float) -> None: + now[0] += seconds + + monkeypatch.setattr(verifier.time, "monotonic", lambda: now[0]) + monkeypatch.setattr(verifier.time, "sleep", fake_sleep) + monkeypatch.setattr(postgres, "run", fake_run) + monkeypatch.setattr(postgres, "psql", fake_psql) + + with pytest.raises(verifier.VerificationError) as exc_info: + postgres._wait_for_final_startup(startup_timeout=0.25, probe_timeout=10.0, poll_interval=0.25) + + error = str(exc_info.value) + assert expected_context in error + assert "within 0.25s after 1 attempt(s)" in error + assert SECRET_STARTUP_MARKER not in error + + +@pytest.mark.skipif(not verifier.docker_available(), reason="Docker daemon is required") +def test_preexisting_v3_named_objects_are_never_adopted_or_rolled_back() -> None: + postgres = verifier.DisposablePostgres() + cleanup: dict[str, object] = {} + try: + postgres.start() + postgres.psql(verifier.BOOTSTRAP_SQL) + postgres.psql( + """ +alter table public.claims + add column owner_agent_id uuid, + add column proposition text, + add column body_markdown text, + add column scope text, + add column access_scope text, + add column admission_state text, + add column revision_criteria text, + add column revision_kind text, + add column supersedes_id uuid, + add column accepted_by_proposal_id uuid; +update public.claims + set owner_agent_id = '11111111-1111-4111-8111-111111111111'::uuid, + proposition = 'claims.proposition', + body_markdown = 'claims.body_markdown', + scope = 'claims.scope', + access_scope = 'claims.access_scope', + admission_state = 'claims.admission_state', + revision_criteria = 'claims.revision_criteria', + revision_kind = 'claims.revision_kind', + supersedes_id = '20000000-0000-4000-8000-000000000002'::uuid, + accepted_by_proposal_id = '30000000-0000-4000-8000-000000000003'::uuid + where id = '20000000-0000-4000-8000-000000000001'::uuid; + +alter table public.sources + add column canonical_url text, + add column storage_uri text, + add column content_hash text, + add column access_scope text, + add column ingestion_origin text, + add column provenance_status text, + add column source_class text, + add column auto_promotion_policy text, + add column accepted_by_proposal_id uuid; +update public.sources + set canonical_url = 'sources.canonical_url', + storage_uri = 'sources.storage_uri', + content_hash = 'sources.content_hash', + access_scope = 'sources.access_scope', + ingestion_origin = 'sources.ingestion_origin', + provenance_status = 'sources.provenance_status', + source_class = 'sources.source_class', + auto_promotion_policy = 'sources.auto_promotion_policy', + accepted_by_proposal_id = '30000000-0000-4000-8000-000000000003'::uuid + where id = '20000000-0000-4000-8000-000000000003'::uuid; + +alter table public.claim_evidence + add column excerpt text, + add column locator_json jsonb, + add column source_content_hash text, + add column rationale text, + add column polarity text, + add column accepted_from_proposal_id uuid; +update public.claim_evidence + set excerpt = 'claim_evidence.excerpt', + locator_json = '{"sentinel":"claim_evidence.locator_json"}'::jsonb, + source_content_hash = 'claim_evidence.source_content_hash', + rationale = 'claim_evidence.rationale', + polarity = 'claim_evidence.polarity', + accepted_from_proposal_id = '30000000-0000-4000-8000-000000000003'::uuid + where id = '20000000-0000-4000-8000-000000000004'::uuid; + +alter table public.claim_edges + add column relation_type text, + add column rationale text, + add column scope_or_conditions text, + add column accepted_by_proposal_id uuid; +update public.claim_edges + set relation_type = 'claim_edges.relation_type', + rationale = 'claim_edges.rationale', + scope_or_conditions = 'claim_edges.scope_or_conditions', + accepted_by_proposal_id = '30000000-0000-4000-8000-000000000003'::uuid + where id = '20000000-0000-4000-8000-000000000005'::uuid; + +create table public.claim_evidence_assessments ( + id uuid primary key, + legacy_marker text +); +""" + ) + state_sql = """ +select jsonb_build_object( + 'claims', ( + select to_jsonb(row_data) from ( + select owner_agent_id, proposition, body_markdown, scope, access_scope, admission_state, + revision_criteria, revision_kind, supersedes_id, accepted_by_proposal_id + from public.claims + where id = '20000000-0000-4000-8000-000000000001'::uuid + ) row_data + ), + 'sources', ( + select to_jsonb(row_data) from ( + select canonical_url, storage_uri, content_hash, access_scope, ingestion_origin, + provenance_status, source_class, auto_promotion_policy, accepted_by_proposal_id + from public.sources + where id = '20000000-0000-4000-8000-000000000003'::uuid + ) row_data + ), + 'claim_evidence', ( + select to_jsonb(row_data) from ( + select excerpt, locator_json, source_content_hash, rationale, polarity, accepted_from_proposal_id + from public.claim_evidence + where id = '20000000-0000-4000-8000-000000000004'::uuid + ) row_data + ), + 'claim_edges', ( + select to_jsonb(row_data) from ( + select relation_type, rationale, scope_or_conditions, accepted_by_proposal_id + from public.claim_edges + where id = '20000000-0000-4000-8000-000000000005'::uuid + ) row_data + ), + 'assessment_columns', ( + select jsonb_agg(column_name order by ordinal_position) + from information_schema.columns + where table_schema = 'public' and table_name = 'claim_evidence_assessments' + ), + 'assessment_rows', (select count(*) from public.claim_evidence_assessments), + 'ownership_ledger', to_regclass('kb_stage.teleo_v3_installation_ownership')::text +)::text; +""" + before = postgres.psql_json(state_sql) + + install = postgres.apply_file(MIGRATION_SQL, check=False) + rollback = postgres.apply_file(ROLLBACK_SQL, check=False) + after = postgres.psql_json(state_sql) + + assert install.returncode != 0 + assert "preexisting V3 schema objects lack installation ownership" in install.stderr + assert rollback.returncode != 0 + assert "V3-named objects lack exact installation ownership" in rollback.stderr + assert before == after + assert set(before["claims"]) == { + "owner_agent_id", + "proposition", + "body_markdown", + "scope", + "access_scope", + "admission_state", + "revision_criteria", + "revision_kind", + "supersedes_id", + "accepted_by_proposal_id", + } + assert set(before["sources"]) == { + "canonical_url", + "storage_uri", + "content_hash", + "access_scope", + "ingestion_origin", + "provenance_status", + "source_class", + "auto_promotion_policy", + "accepted_by_proposal_id", + } + assert set(before["claim_evidence"]) == { + "excerpt", + "locator_json", + "source_content_hash", + "rationale", + "polarity", + "accepted_from_proposal_id", + } + assert set(before["claim_edges"]) == { + "relation_type", + "rationale", + "scope_or_conditions", + "accepted_by_proposal_id", + } + assert all(value is not None for value in before["claims"].values()) + assert all(value is not None for value in before["sources"].values()) + assert all(value is not None for value in before["claim_evidence"].values()) + assert all(value is not None for value in before["claim_edges"].values()) + assert before["assessment_columns"] == ["id", "legacy_marker"] + assert before["assessment_rows"] == 0 + assert before["ownership_ledger"] is None + finally: + cleanup = postgres.cleanup() + + assert cleanup["container_absent"] is True + assert cleanup["name_readback_clear"] is True + assert cleanup["instance_label_readback_clear"] is True + + +@pytest.mark.skipif(not verifier.docker_available(), reason="Docker daemon is required") +def test_partial_v3_contract_is_never_adopted_or_rolled_back() -> None: + postgres = verifier.DisposablePostgres() + cleanup: dict[str, object] = {} + try: + postgres.start() + postgres.psql(verifier.BOOTSTRAP_SQL) + postgres.psql( + """ +create function kb_stage.teleo_v3_partial_sentinel() +returns text +language sql +immutable +as $$ select 'legacy-partial-sentinel'::text $$; +""" + ) + state_sql = """ +select jsonb_build_object( + 'sentinel_result', kb_stage.teleo_v3_partial_sentinel(), + 'sentinel_definition', pg_get_functiondef('kb_stage.teleo_v3_partial_sentinel()'::regprocedure), + 'ownership_ledger', to_regclass('kb_stage.teleo_v3_installation_ownership')::text +)::text; +""" + before = postgres.psql_json(state_sql) + + install = postgres.apply_file(MIGRATION_SQL, check=False) + rollback = postgres.apply_file(ROLLBACK_SQL, check=False) + after = postgres.psql_json(state_sql) + + assert install.returncode != 0 + assert "partial or altered V3 contract lacks exact installation ownership" in install.stderr + assert rollback.returncode != 0 + assert "V3-named objects lack exact installation ownership" in rollback.stderr + assert before == after + assert before["sentinel_result"] == "legacy-partial-sentinel" + assert "teleo_v3_partial_sentinel" in before["sentinel_definition"] + assert before["ownership_ledger"] is None + finally: + cleanup = postgres.cleanup() + + assert cleanup["container_absent"] is True + assert cleanup["name_readback_clear"] is True + assert cleanup["instance_label_readback_clear"] is True + + +def test_verifier_has_no_external_database_target_surface(tmp_path: Path) -> None: + args = verifier.parse_args(["--output", str(tmp_path / "receipt.json")]) + assert args.output == tmp_path / "receipt.json" + + source = Path(verifier.__file__).read_text(encoding="utf-8") + assert "DATABASE_URL" not in source + assert "PGHOST" not in source + assert '"--network",\n "none"' in source + assert '"--tmpfs"' in source + assert 'production_access_attempted": False' in source + + +def test_failure_receipt_requires_exact_sqlstate() -> None: + completed = subprocess.CompletedProcess( + args=["psql"], + returncode=3, + stdout="", + stderr="ERROR: 23514: expected constraint failure\n", + ) + receipt = verifier.failure_receipt(completed, "23514") + + assert receipt["refused"] is True + assert receipt["observed_sqlstate"] == "23514" + assert receipt["expected_sqlstate_observed"] is True + + +def test_verifier_exercises_concurrent_proposal_downgrade_lock() -> None: + source = Path(verifier.__file__).read_text(encoding="utf-8") + + assert "proposal_decision_lock_race_check" in source + assert "teleo_v3_proposal_lock_holder" in source + assert "set statement_timeout = '750ms'" in source + assert 'failure_receipt(downgrade, "57014")' in source + + +def test_verifier_asserts_security_definer_owner_search_path_and_acl() -> None: + source = Path(verifier.__file__).read_text(encoding="utf-8") + + assert '"owner_name": "kb_gate_owner"' in source + assert '"security_definer": True' in source + assert '"config": ["search_path=pg_catalog, pg_temp"]' in source + assert '"owner_only_execute": True' in source + + +def test_verifier_cleanup_bypass_is_bounded_to_disposable_superuser_transaction() -> None: + source = Path(verifier.__file__).read_text(encoding="utf-8") + + assert "set local session_replication_role = replica" in source + assert '"isolated-disposable-superuser-transaction"' in source + assert "Production writers must never receive" in source + + +@pytest.mark.skipif(not verifier.docker_available(), reason="Docker daemon is required") +def test_disposable_postgres_contract_end_to_end(tmp_path: Path) -> None: + output = tmp_path / "teleo-v3-epistemic-contract-receipt.json" + + receipt = verifier.run_verification(output) + + assert receipt["status"] == "pass", receipt.get("error") + assert receipt["required_tier"] == "T2_runtime" + assert receipt["current_tier"] == "T2_runtime_disposable_postgresql" + assert receipt["production_access_attempted"] is False + assert receipt["production_migration_shipped"] is False + assert receipt["environment"]["network_mode"] == "none" + assert receipt["environment"]["tmpfs_data_dir"] is True + assert receipt["environment"]["docker_volume_mounts"] == [] + assert len(receipt["checks"]) >= 20 + assert all(receipt["checks"].values()), receipt["checks"] + assert all(item["refused"] for item in receipt["adversarial"].values()) + assert all(item["expected_sqlstate_observed"] for item in receipt["adversarial"].values()) + assert receipt["rollback_with_data"]["refused"] is True + assert receipt["cleanup"]["container_absent"] is True + assert receipt["cleanup"]["instance_label_readback_clear"] is True + assert json.loads(output.read_text(encoding="utf-8")) == receipt diff --git a/tests/test_v3_apply_catalog_serialization_postgres.py b/tests/test_v3_apply_catalog_serialization_postgres.py new file mode 100644 index 0000000..cc02cc5 --- /dev/null +++ b/tests/test_v3_apply_catalog_serialization_postgres.py @@ -0,0 +1,467 @@ +from __future__ import annotations + +import subprocess +import time + +import pytest + +from ops import verify_teleo_v3_epistemic_contract as contract +from tests import test_v3_apply_worker_postgres as worker + +CATALOG_DDL_ADVISORY_LOCK_KEY = 6072343533146485505 +TEST_APPLY_BARRIER_LOCK_KEY = 6072343533146485599 + + +def _start_psql( + postgres: contract.DisposablePostgres, + sql: str, + *, + role: str = "postgres", + password: str | None = None, + keep_stdin_open: bool = False, +) -> subprocess.Popen[str]: + assert postgres.docker is not None + command = [postgres.docker, "exec"] + if password is not None: + command.extend(["-e", f"PGPASSWORD={password}"]) + command.extend( + [ + "-i", + postgres.name, + "psql", + "--no-psqlrc", + "--set=ON_ERROR_STOP=1", + "--set=VERBOSITY=verbose", + "--username", + role, + f"--dbname={contract.DATABASE}", + ] + ) + if password is not None: + command.extend(["--host=127.0.0.1"]) + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert process.stdin is not None + process.stdin.write(sql) + if keep_stdin_open: + process.stdin.flush() + else: + process.stdin.close() + process.stdin = None + return process + + +def _finish(process: subprocess.Popen[str], *, timeout: float = 20) -> subprocess.CompletedProcess[str]: + stdout, stderr = process.communicate(timeout=timeout) + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) + + +def _stop(process: subprocess.Popen[str] | None) -> None: + if process is None or process.poll() is not None: + return + process.terminate() + try: + process.communicate(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=3) + + +def _wait_for_activity( + postgres: contract.DisposablePostgres, + application_name: str, + *, + wait_event_type: str, + wait_event: str, + timeout: float = 8, +) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + waiting = postgres.psql( + f""" +select exists ( + select 1 + from pg_catalog.pg_stat_activity + where application_name = '{application_name}' + and wait_event_type = '{wait_event_type}' + and wait_event = '{wait_event}' +); +""" + ).stdout.strip() + if waiting == "t": + return + time.sleep(0.05) + raise AssertionError(f"{application_name} did not wait on {wait_event_type}/{wait_event}") + + +def _approved_apply_sql( + postgres: contract.DisposablePostgres, + ids: dict[str, str], +) -> str: + payload = worker._payload(ids) + packet = worker._insert_pending_proposal(postgres, payload, ids) + worker._approve_proposal(postgres, ids["proposal"], packet) + proposal = postgres.psql_json( + f""" +select pg_catalog.jsonb_build_object( + 'id', id::text, + 'proposal_type', proposal_type, + 'status', status, + 'payload', payload, + 'reviewed_by_handle', reviewed_by_handle, + 'reviewed_by_agent_id', reviewed_by_agent_id::text, + 'reviewed_at', pg_catalog.to_char( + reviewed_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"' + ), + 'review_note', review_note, + 'applied_by_handle', applied_by_handle, + 'applied_by_agent_id', applied_by_agent_id::text, + 'applied_at', applied_at +)::text +from kb_stage.kb_proposals +where id = '{ids["proposal"]}'::uuid; +""" + ) + return worker.apply.build_apply_sql(proposal, worker.apply.SERVICE_AGENT_HANDLE) + + +def _instrument_apply_after_catalog_guard(sql: str, application_name: str) -> str: + guard_end = "$v3_contract_guard$;" + assert sql.count(guard_end) == 1 + return sql.replace( + "begin;\n", + f"begin;\nset local application_name = '{application_name}';\n", + 1, + ).replace( + guard_end, + guard_end + f"\nselect pg_catalog.pg_advisory_xact_lock({TEST_APPLY_BARRIER_LOCK_KEY});", + 1, + ) + + +def _instrument_apply_before_transaction(sql: str, application_name: str) -> str: + return f"set application_name = '{application_name}';\n{sql}" + + +def _start_barrier_holder(postgres: contract.DisposablePostgres) -> subprocess.Popen[str]: + holder = _start_psql( + postgres, + f""" +set application_name = 'teleo_v3_catalog_barrier_holder'; +select pg_catalog.pg_advisory_lock({TEST_APPLY_BARRIER_LOCK_KEY}); +""", + keep_stdin_open=True, + ) + _wait_for_activity( + postgres, + "teleo_v3_catalog_barrier_holder", + wait_event_type="Client", + wait_event="ClientRead", + ) + return holder + + +def _release_barrier( + postgres: contract.DisposablePostgres, + holder: subprocess.Popen[str], +) -> None: + released = postgres.psql( + """ +select pg_catalog.pg_terminate_backend(pid) + from pg_catalog.pg_stat_activity + where application_name = 'teleo_v3_catalog_barrier_holder'; +""" + ) + assert released.stdout.strip() == "t" + if holder.stdin is not None: + holder.stdin.close() + holder.stdin = None + _finish(holder, timeout=5) + + +def _install_and_build_apply( + postgres: contract.DisposablePostgres, + ids: dict[str, str], +) -> str: + environment = postgres.start() + assert environment["network_mode"] == "none" + postgres.psql(contract.BOOTSTRAP_SQL) + postgres.psql(worker._supplemental_bootstrap()) + postgres.apply_file(worker.PREREQUISITES) + worker._install_v3_contract(postgres) + return _approved_apply_sql(postgres, ids) + + +@pytest.mark.skipif(not contract.docker_available(), reason="Docker daemon is required") +def test_trigger_ddl_waits_for_apply_transaction_relation_locks() -> None: + postgres = contract.DisposablePostgres() + holder: subprocess.Popen[str] | None = None + apply_process: subprocess.Popen[str] | None = None + ddl_process: subprocess.Popen[str] | None = None + try: + apply_sql = _install_and_build_apply(postgres, worker.RACE_IDS) + holder = _start_barrier_holder(postgres) + apply_process = _start_psql( + postgres, + _instrument_apply_after_catalog_guard(apply_sql, "teleo_v3_guarded_apply"), + role="kb_apply", + password=worker.APPLY_PASSWORD, + ) + _wait_for_activity( + postgres, + "teleo_v3_guarded_apply", + wait_event_type="Lock", + wait_event="advisory", + ) + + ddl_process = _start_psql( + postgres, + """ +set application_name = 'teleo_v3_uncoordinated_trigger_ddl'; +create or replace trigger teleo_v3_00_enforce_apply_contract_cutover +before insert or update of proposal_type, payload, status on kb_stage.kb_proposals +for each row when (false) +execute function kb_stage.teleo_v3_enforce_apply_contract_cutover(); +""", + ) + _wait_for_activity( + postgres, + "teleo_v3_uncoordinated_trigger_ddl", + wait_event_type="Lock", + wait_event="relation", + ) + assert ddl_process.poll() is None + assert worker._contract_state(postgres) == "V3" + + _release_barrier(postgres, holder) + holder = None + apply_result = _finish(apply_process) + apply_process = None + assert apply_result.returncode == 0, apply_result.stderr + ddl_result = _finish(ddl_process) + ddl_process = None + assert ddl_result.returncode == 0, ddl_result.stderr + assert postgres.psql_json( + f""" +select pg_catalog.jsonb_build_object( + 'proposal_status', ( + select status from kb_stage.kb_proposals + where id = '{worker.RACE_IDS["proposal"]}'::uuid + ), + 'contract_state', ({worker.apply.build_v3_contract_state_query()}) +)::text; +""" + ) == {"proposal_status": "applied", "contract_state": "INVALID"} + finally: + if holder is not None and holder.poll() is None: + _release_barrier(postgres, holder) + _stop(ddl_process) + _stop(apply_process) + _stop(holder) + cleanup = postgres.cleanup() + assert cleanup["container_absent"] is True + assert cleanup["instance_label_readback_clear"] is True + + +@pytest.mark.skipif(not contract.docker_available(), reason="Docker daemon is required") +def test_controlled_function_migration_waits_for_apply_shared_advisory_lock() -> None: + postgres = contract.DisposablePostgres() + holder: subprocess.Popen[str] | None = None + apply_process: subprocess.Popen[str] | None = None + migration_process: subprocess.Popen[str] | None = None + try: + apply_sql = _install_and_build_apply(postgres, worker.RACE_IDS) + holder = _start_barrier_holder(postgres) + apply_process = _start_psql( + postgres, + _instrument_apply_after_catalog_guard(apply_sql, "teleo_v3_shared_lock_apply"), + role="kb_apply", + password=worker.APPLY_PASSWORD, + ) + _wait_for_activity( + postgres, + "teleo_v3_shared_lock_apply", + wait_event_type="Lock", + wait_event="advisory", + ) + + migration_process = _start_psql( + postgres, + f""" +begin; +set local application_name = 'teleo_v3_controlled_function_migration'; +select pg_catalog.pg_advisory_xact_lock({CATALOG_DDL_ADVISORY_LOCK_KEY}); +create or replace function kb_stage.teleo_v3_validate_claim_edge() +returns trigger +language plpgsql +set search_path = pg_catalog, pg_temp +as $function$ +begin + return new; +end +$function$; +commit; +""", + ) + _wait_for_activity( + postgres, + "teleo_v3_controlled_function_migration", + wait_event_type="Lock", + wait_event="advisory", + ) + assert migration_process.poll() is None + assert worker._contract_state(postgres) == "V3" + + _release_barrier(postgres, holder) + holder = None + apply_result = _finish(apply_process) + apply_process = None + assert apply_result.returncode == 0, apply_result.stderr + migration_result = _finish(migration_process) + migration_process = None + assert migration_result.returncode == 0, migration_result.stderr + assert worker._contract_state(postgres) == "INVALID" + finally: + if holder is not None and holder.poll() is None: + _release_barrier(postgres, holder) + _stop(migration_process) + _stop(apply_process) + _stop(holder) + cleanup = postgres.cleanup() + assert cleanup["container_absent"] is True + assert cleanup["instance_label_readback_clear"] is True + + +@pytest.mark.skipif(not contract.docker_available(), reason="Docker daemon is required") +def test_migration_first_apply_resnapshots_and_refuses_post_migration_catalog_drift() -> None: + postgres = contract.DisposablePostgres() + holder: subprocess.Popen[str] | None = None + apply_process: subprocess.Popen[str] | None = None + migration_process: subprocess.Popen[str] | None = None + try: + apply_sql = _install_and_build_apply(postgres, worker.RACE_IDS) + holder = _start_barrier_holder(postgres) + migration_process = _start_psql( + postgres, + f""" +begin; +set local application_name = 'teleo_v3_migration_first_holder'; +select pg_catalog.pg_advisory_xact_lock({CATALOG_DDL_ADVISORY_LOCK_KEY}); +select pg_catalog.pg_advisory_xact_lock({TEST_APPLY_BARRIER_LOCK_KEY}); +create or replace function kb_stage.teleo_v3_validate_claim_edge() +returns trigger +language plpgsql +set search_path = pg_catalog, pg_temp +as $function$ +begin + return new; +end +$function$; +commit; +""", + ) + _wait_for_activity( + postgres, + "teleo_v3_migration_first_holder", + wait_event_type="Lock", + wait_event="advisory", + ) + + apply_process = _start_psql( + postgres, + _instrument_apply_before_transaction(apply_sql, "teleo_v3_migration_first_apply"), + role="kb_apply", + password=worker.APPLY_PASSWORD, + ) + _wait_for_activity( + postgres, + "teleo_v3_migration_first_apply", + wait_event_type="Lock", + wait_event="advisory", + ) + + _release_barrier(postgres, holder) + holder = None + migration_result = _finish(migration_process) + migration_process = None + assert migration_result.returncode == 0, migration_result.stderr + apply_result = _finish(apply_process) + apply_process = None + assert apply_result.returncode != 0 + assert "V3 catalog contract is INVALID" in apply_result.stderr + assert postgres.psql_json( + f""" +select pg_catalog.jsonb_build_object( + 'proposal_status', ( + select status from kb_stage.kb_proposals + where id = '{worker.RACE_IDS["proposal"]}'::uuid + ), + 'applied_at', ( + select applied_at from kb_stage.kb_proposals + where id = '{worker.RACE_IDS["proposal"]}'::uuid + ), + 'claims', ( + select count(*) from public.claims + where accepted_by_proposal_id = '{worker.RACE_IDS["proposal"]}'::uuid + ), + 'sources', ( + select count(*) from public.sources + where accepted_by_proposal_id = '{worker.RACE_IDS["proposal"]}'::uuid + ), + 'evidence', ( + select count(*) from public.claim_evidence + where accepted_from_proposal_id = '{worker.RACE_IDS["proposal"]}'::uuid + ), + 'edges', ( + select count(*) from public.claim_edges + where accepted_by_proposal_id = '{worker.RACE_IDS["proposal"]}'::uuid + ), + 'assessments', ( + select count(*) from public.claim_evidence_assessments + where accepted_by_proposal_id = '{worker.RACE_IDS["proposal"]}'::uuid + ), + 'contract_state', ({worker.apply.build_v3_contract_state_query()}) +)::text; +""" + ) == { + "proposal_status": "approved", + "applied_at": None, + "claims": 0, + "sources": 0, + "evidence": 0, + "edges": 0, + "assessments": 0, + "contract_state": "INVALID", + } + lock_cleanup = postgres.psql_json( + f""" +with acquired as ( + select pg_catalog.pg_try_advisory_lock({CATALOG_DDL_ADVISORY_LOCK_KEY}) as value +), released as ( + select pg_catalog.pg_advisory_unlock({CATALOG_DDL_ADVISORY_LOCK_KEY}) as value + from acquired where value +) +select pg_catalog.jsonb_build_object( + 'exclusive_lock_acquired', (select value from acquired), + 'exclusive_lock_released', coalesce((select value from released), false) +)::text; +""" + ) + assert lock_cleanup == { + "exclusive_lock_acquired": True, + "exclusive_lock_released": True, + } + finally: + if holder is not None and holder.poll() is None: + _release_barrier(postgres, holder) + _stop(migration_process) + _stop(apply_process) + _stop(holder) + cleanup = postgres.cleanup() + assert cleanup["container_absent"] is True + assert cleanup["instance_label_readback_clear"] is True diff --git a/tests/test_v3_apply_worker_postgres.py b/tests/test_v3_apply_worker_postgres.py new file mode 100644 index 0000000..c420b23 --- /dev/null +++ b/tests/test_v3_apply_worker_postgres.py @@ -0,0 +1,1545 @@ +from __future__ import annotations + +import hashlib +import json +import re +import stat +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from ops import run_local_genesis_ledger_rebuild as genesis_rebuild +from ops import verify_teleo_v3_epistemic_contract as contract + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import apply_proposal as apply # noqa: E402 +import export_kb_transition_replay_material as material_exporter # noqa: E402 +import kb_apply_replay_receipt as replay # noqa: E402 + +PREREQUISITES = REPO_ROOT / "scripts" / "kb_apply_prereqs.sql" +MIGRATION = REPO_ROOT / "ops" / "teleo_v3_epistemic_contract.sql" +WORKER = REPO_ROOT / "scripts" / "apply_worker.py" +APPLY_SCRIPT = REPO_ROOT / "scripts" / "apply_proposal.py" +MATERIAL_EXPORTER = REPO_ROOT / "scripts" / "export_kb_transition_replay_material.py" + +PROPOSAL_ID = "90000000-0000-4000-8000-000000000001" +CLAIM_ID = "90000000-0000-4000-8000-000000000002" +SOURCE_ID = "90000000-0000-4000-8000-000000000003" +EVIDENCE_ID = "90000000-0000-4000-8000-000000000004" +ASSESSMENT_ID = "90000000-0000-4000-8000-000000000005" +REVIEWER_AGENT_ID = "90000000-0000-4000-8000-000000000006" +CANONICAL_REVIEWER_HANDLE = "m3taversal" +LEGACY_REVIEWER_HANDLE = "m3ta" +SUPPORT_CLAIM_ID = "90000000-0000-4000-8000-000000000007" +EDGE_ID = "90000000-0000-4000-8000-000000000008" +V2_PROPOSAL_ID = "90000000-0000-4000-8000-000000000009" +V2_REVISE_PROPOSAL_ID = "90000000-0000-4000-8000-000000000014" +MALFORMED_V3_PROPOSAL_ID = "90000000-0000-4000-8000-000000000010" +HYBRID_V3_PROPOSAL_ID = "90000000-0000-4000-8000-000000000011" +SUPPORT_ASSESSMENT_ID = "90000000-0000-4000-8000-000000000012" +SUPPORT_EVIDENCE_ID = "90000000-0000-4000-8000-000000000013" + +PRESEED_IDS = { + "proposal": "91000000-0000-4000-8000-000000000001", + "claim": "91000000-0000-4000-8000-000000000002", + "source": "91000000-0000-4000-8000-000000000003", + "evidence": "91000000-0000-4000-8000-000000000004", + "assessment": "91000000-0000-4000-8000-000000000005", + "support_claim": "91000000-0000-4000-8000-000000000007", + "edge": "91000000-0000-4000-8000-000000000008", + "support_assessment": "91000000-0000-4000-8000-000000000012", + "support_evidence": "91000000-0000-4000-8000-000000000013", +} +MISMATCH_IDS = { + "proposal": "92000000-0000-4000-8000-000000000001", + "claim": "92000000-0000-4000-8000-000000000002", + "source": "92000000-0000-4000-8000-000000000003", + "evidence": "92000000-0000-4000-8000-000000000004", + "assessment": "92000000-0000-4000-8000-000000000005", + "support_claim": "92000000-0000-4000-8000-000000000007", + "edge": "92000000-0000-4000-8000-000000000008", + "support_assessment": "92000000-0000-4000-8000-000000000012", + "support_evidence": "92000000-0000-4000-8000-000000000013", +} +RACE_IDS = { + "proposal": "93000000-0000-4000-8000-000000000001", + "claim": "93000000-0000-4000-8000-000000000002", + "source": "93000000-0000-4000-8000-000000000003", + "evidence": "93000000-0000-4000-8000-000000000004", + "assessment": "93000000-0000-4000-8000-000000000005", + "support_claim": "93000000-0000-4000-8000-000000000007", + "edge": "93000000-0000-4000-8000-000000000008", + "support_assessment": "93000000-0000-4000-8000-000000000012", + "support_evidence": "93000000-0000-4000-8000-000000000013", +} +SUPERSEDED_MISMATCH_IDS = { + "proposal": "94000000-0000-4000-8000-000000000001", + "claim": "94000000-0000-4000-8000-000000000002", + "source": "94000000-0000-4000-8000-000000000003", + "evidence": "94000000-0000-4000-8000-000000000004", + "assessment": "94000000-0000-4000-8000-000000000005", + "support_claim": "94000000-0000-4000-8000-000000000007", + "edge": "94000000-0000-4000-8000-000000000008", + "support_assessment": "94000000-0000-4000-8000-000000000012", + "support_evidence": "94000000-0000-4000-8000-000000000013", +} +DEFAULT_IDS = { + "proposal": PROPOSAL_ID, + "claim": CLAIM_ID, + "source": SOURCE_ID, + "evidence": EVIDENCE_ID, + "assessment": ASSESSMENT_ID, + "support_claim": SUPPORT_CLAIM_ID, + "edge": EDGE_ID, + "support_assessment": SUPPORT_ASSESSMENT_ID, + "support_evidence": SUPPORT_EVIDENCE_ID, +} +APPLY_PASSWORD = "disposable-v3-apply-password" +REVIEW_PASSWORD = "disposable-v3-review-password" + + +def _role_psql( + postgres: contract.DisposablePostgres, + role: str, + password: str, + sql: str, + *, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [ + "docker", + "exec", + "-e", + f"PGPASSWORD={password}", + "-e", + "PGOPTIONS=-c timezone=UTC", + "-i", + postgres.name, + "psql", + "--no-psqlrc", + "--set=ON_ERROR_STOP=1", + "--set=VERBOSITY=verbose", + "--tuples-only", + "--no-align", + "--host=127.0.0.1", + f"--username={role}", + f"--dbname={contract.DATABASE}", + ], + input=sql, + text=True, + capture_output=True, + check=False, + ) + if check and result.returncode != 0: + raise AssertionError(f"psql as {role} failed:\n{result.stderr}") + return result + + +def _payload(ids: dict[str, str] = DEFAULT_IDS) -> dict: + content_hash = hashlib.sha256(ids["proposal"].encode("ascii")).hexdigest() + evidence = { + "id": ids["evidence"], + "claim_id": ids["claim"], + "source_id": ids["source"], + "polarity": "supports", + "excerpt": "The exact source bytes support the bounded proposition.", + "locator_json": {"section": "worker-canary", "paragraph": 1}, + "source_content_hash": content_hash, + "rationale": "The exact locator directly supports the claim body.", + "created_by": contract.AGENT_ID, + } + support_evidence = { + "id": ids["support_evidence"], + "claim_id": ids["support_claim"], + "source_id": ids["source"], + "polarity": "illustrates", + "excerpt": "The exact source bytes illustrate typed receipt recovery.", + "locator_json": {"section": "worker-canary", "paragraph": 2}, + "source_content_hash": content_hash, + "rationale": "The second exact locator illustrates the conceptual claim.", + "created_by": contract.AGENT_ID, + } + evidence_rows = [evidence, support_evidence] + return { + "contract_version": 3, + "agent_id": contract.AGENT_ID, + "claims": [ + { + "id": ids["claim"], + "owner_agent_id": None, + "type": "empirical", + "proposition": "The V3 worker preserves an exact reviewed provenance chain.", + "body_markdown": "One content-addressed source and exact locator support this bounded claim.", + "scope": "Network-isolated disposable PostgreSQL worker canary.", + "access_scope": "collective_internal", + "status": "active", + "revision_criteria": "Revise if the source bytes, locator, or review receipt cannot be reproduced.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["v3", "worker-canary"], + "created_by": contract.AGENT_ID, + }, + { + "id": ids["support_claim"], + "owner_agent_id": None, + "type": "conceptual", + "proposition": "Receipt recovery depends on canonical typed values.", + "body_markdown": "Canonical UUID and timestamp values bind replay to PostgreSQL rows.", + "scope": "Network-isolated disposable PostgreSQL worker canary.", + "access_scope": "collective_internal", + "status": "active", + "revision_criteria": "Revise if typed canonicalization is no longer deterministic.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["v3", "worker-canary"], + "created_by": contract.AGENT_ID, + }, + ], + "sources": [ + { + "id": ids["source"], + "source_type": "paper", + "canonical_url": "https://example.test/v3-worker-canary", + "storage_uri": None, + "excerpt": "Exact disposable source bytes.", + "content_hash": content_hash, + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-18T14:00:00.123400+02:00", + "created_by": contract.AGENT_ID, + } + ], + "evidence": evidence_rows, + "edges": [ + { + "id": ids["edge"], + "from_claim": ids["support_claim"], + "to_claim": ids["claim"], + "relation_type": "supports", + "rationale": "Typed canonicalization supports deterministic receipt recovery.", + "scope_or_conditions": "The reviewed V3 apply and receipt contracts.", + "created_by": contract.AGENT_ID, + } + ], + "assessments": [ + { + "id": ids["assessment"], + "claim_id": ids["claim"], + "support_tier": "strong", + "challenge_tier": "none", + "overall_state": "support_dominant", + "rationale": "The exact source and locator directly support the bounded claim.", + "evidence_set_hash": apply._v3_evidence_set_hash(ids["claim"], evidence_rows), + "supersedes_assessment_id": None, + }, + { + "id": ids["support_assessment"], + "claim_id": ids["support_claim"], + "support_tier": None, + "challenge_tier": None, + "overall_state": "insufficient", + "rationale": "One exact locator illustrates the conceptual support claim.", + "evidence_set_hash": apply._v3_evidence_set_hash(ids["support_claim"], evidence_rows), + "supersedes_assessment_id": None, + }, + ], + "reasoning_tools": [], + } + + +def _supplemental_bootstrap() -> str: + return f""" +alter role kb_review login password '{REVIEW_PASSWORD}'; +alter role kb_apply login password '{APPLY_PASSWORD}'; + +create table public.strategies ( + id uuid primary key, + agent_id uuid not null, + active boolean not null default true +); +create table public.strategy_nodes (id uuid primary key); +create table public.reasoning_tools (id uuid primary key); + +insert into public.agents (id, handle, kind) +values ('{REVIEWER_AGENT_ID}', '{CANONICAL_REVIEWER_HANDLE}', 'human'); +""" + + +def _packet(payload: dict, title: str) -> dict: + return {"title": title, "apply_payload": payload} + + +def _insert_pending_proposal( + postgres: contract.DisposablePostgres, + payload: dict, + ids: dict[str, str] = DEFAULT_IDS, +) -> dict: + title = f"V3 worker disposable canary {ids['proposal']}" + packet = _packet(payload, title) + postgres.psql( + f""" +insert into kb_stage.kb_proposals ( + id, proposal_type, status, proposed_by_handle, proposed_by_agent_id, + channel, rationale, payload, created_at, updated_at +) values ( + '{ids["proposal"]}'::uuid, 'approve_claim', 'pending_review', 'leo', + '{contract.AGENT_ID}'::uuid, 'disposable_worker_canary', + 'Prove the exact restricted V3 worker lifecycle.', + {apply.sql_literal(json.dumps(packet, separators=(",", ":"), sort_keys=True))}::jsonb, + '2026-07-18T12:01:00+00:00', '2026-07-18T12:01:00+00:00' +); +""" + ) + return packet + + +def _approve_proposal( + postgres: contract.DisposablePostgres, + proposal_id: str, + packet: dict, +) -> None: + approved = _role_psql( + postgres, + "kb_review", + REVIEW_PASSWORD, + f""" +select kb_stage.approve_strict_proposal( + '{proposal_id}'::uuid, + 'approve_claim', + {apply.sql_literal(json.dumps(packet, separators=(",", ":"), sort_keys=True))}::jsonb, + '{CANONICAL_REVIEWER_HANDLE}', + 'Approved exact V3 disposable worker packet.' +)::text; +""", + ) + assert '"status": "approved"' in approved.stdout + + +def _insert_historical_v2_queue(postgres: contract.DisposablePostgres) -> None: + payload = { + "apply_payload": { + "from_claim": contract.LEGACY_CLAIM_ID, + "to_claim": contract.LEGACY_RELATED_CLAIM_ID, + "edge_type": "supports", + "weight": None, + } + } + revise_payload = { + "apply_payload": { + "agent_id": contract.AGENT_ID, + "strategy": { + "diagnosis": "A disabled V3 guard must never reopen V2 strategy execution.", + "guiding_policy": "Refuse catalog drift before selecting queued work.", + "proximate_objectives": ["zero V2 writes under drift"], + }, + "strategy_nodes": [ + { + "node_type": "policy", + "title": "Fail closed", + "body": "Keep the retained V2 packet review-visible without executing it.", + "rank": 1, + } + ], + } + } + postgres.psql( + f""" +insert into kb_stage.kb_proposals ( + id, proposal_type, status, proposed_by_handle, proposed_by_agent_id, + channel, rationale, payload, reviewed_by_handle, reviewed_by_agent_id, + reviewed_at, review_note, created_at, updated_at +) values ( + '{V2_PROPOSAL_ID}'::uuid, 'add_edge', 'approved', 'leo', + '{contract.AGENT_ID}'::uuid, 'retained_genesis_queue', + 'Historical approved V2 packet remains visible across V3 cutover.', + {apply.sql_literal(json.dumps(payload, separators=(",", ":"), sort_keys=True))}::jsonb, + '{LEGACY_REVIEWER_HANDLE}', '{REVIEWER_AGENT_ID}'::uuid, '2026-07-18T11:55:00Z', + 'Retain for explicit migration or rejection.', + '2026-07-18T11:54:00Z', '2026-07-18T11:55:00Z' +); + +insert into kb_stage.kb_proposals ( + id, proposal_type, status, proposed_by_handle, proposed_by_agent_id, + channel, rationale, payload, reviewed_by_handle, reviewed_by_agent_id, + reviewed_at, review_note, created_at, updated_at +) values ( + '{V2_REVISE_PROPOSAL_ID}'::uuid, 'revise_strategy', 'approved', 'leo', + '{contract.AGENT_ID}'::uuid, 'retained_genesis_queue', + 'Historical approved V2 strategy packet remains visible across V3 cutover.', + {apply.sql_literal(json.dumps(revise_payload, separators=(",", ":"), sort_keys=True))}::jsonb, + '{LEGACY_REVIEWER_HANDLE}', '{REVIEWER_AGENT_ID}'::uuid, '2026-07-18T11:55:30Z', + 'Retain for explicit migration or rejection.', + '2026-07-18T11:54:30Z', '2026-07-18T11:55:30Z' +); +""" + ) + + +def _preseed_v3_rows( + postgres: contract.DisposablePostgres, + payload: dict, + ids: dict[str, str], + *, + mismatch_source_excerpt: bool = False, +) -> None: + bundle = apply._normalize_v3_approve_claim(payload) + source = bundle["sources"][0] + source_excerpt = "Conflicting preseeded source text." if mismatch_source_excerpt else source["excerpt"] + claim_values = ",\n".join( + f"""( + {apply.sql_literal(row["id"])}::uuid, {apply.sql_literal(row["owner_agent_id"])}::uuid, + {apply.sql_literal(row["type"])}, {apply.sql_literal(row["proposition"])}, + {apply.sql_literal(row["proposition"])}, {apply.sql_literal(row["body_markdown"])}, + {apply.sql_literal(row["scope"])}, {apply.sql_literal(row["access_scope"])}, + {apply.sql_literal(row["status"])}, 'admitted', {apply.sql_literal(row["revision_criteria"])}, + 'original', null, null, {apply._text_array(row["tags"])}, + {apply.sql_literal(row["created_by"])}::uuid, null, {apply.sql_literal(ids["proposal"])}::uuid, + '2026-07-18T12:03:00.654321Z', '2026-07-18T12:03:00.654321Z' +)""" + for row in bundle["claims"] + ) + evidence_values = ",\n".join( + f"""( + {apply.sql_literal(row["id"])}::uuid, {apply.sql_literal(row["claim_id"])}::uuid, + {apply.sql_literal(row["source_id"])}::uuid, {apply.sql_literal(row["role"])}::evidence_role, null, + {apply.sql_literal(row["created_by"])}::uuid, {apply.sql_literal(row["excerpt"])}, + {apply.sql_literal(json.dumps(row["locator_json"], separators=(",", ":"), sort_keys=True))}::jsonb, + {apply.sql_literal(row["source_content_hash"])}, {apply.sql_literal(row["rationale"])}, + {apply.sql_literal(row["polarity"])}, {apply.sql_literal(ids["proposal"])}::uuid, + '2026-07-18T12:04:00.654321Z' +)""" + for row in bundle["evidence"] + ) + edge = bundle["edges"][0] + assessment_values = ",\n".join( + f"""( + {apply.sql_literal(row["id"])}::uuid, {apply.sql_literal(row["claim_id"])}::uuid, + {apply.sql_literal(row["support_tier"])}, {apply.sql_literal(row["challenge_tier"])}, + {apply.sql_literal(row["overall_state"])}, {apply.sql_literal(row["rationale"])}, + {apply.sql_literal(row["evidence_set_hash"])}, {apply.sql_literal(ids["proposal"])}::uuid, + {apply.sql_literal(row["supersedes_assessment_id"])}::uuid, '2026-07-18T12:06:00.654321Z' +)""" + for row in bundle["assessments"] + ) + postgres.psql( + f""" +insert into public.sources ( + id, source_type, url, storage_path, excerpt, hash, created_by, captured_at, + canonical_url, storage_uri, content_hash, access_scope, ingestion_origin, + provenance_status, source_class, auto_promotion_policy, accepted_by_proposal_id, created_at +) values ( + {apply.sql_literal(source["id"])}::uuid, {apply.sql_literal(source["source_type"])}, + {apply.sql_literal(source["canonical_url"])}, {apply.sql_literal(source["storage_uri"])}, + {apply.sql_literal(source_excerpt)}, {apply.sql_literal(source["content_hash"])}, + {apply.sql_literal(source["created_by"])}::uuid, {apply.sql_literal(source["captured_at"])}::timestamptz, + {apply.sql_literal(source["canonical_url"])}, {apply.sql_literal(source["storage_uri"])}, + {apply.sql_literal(source["content_hash"])}, {apply.sql_literal(source["access_scope"])}, + {apply.sql_literal(source["ingestion_origin"])}, {apply.sql_literal(source["provenance_status"])}, + {apply.sql_literal(source["source_class"])}, {apply.sql_literal(source["auto_promotion_policy"])}, + {apply.sql_literal(ids["proposal"])}::uuid, '2026-07-18T12:02:00.654321Z' +); + +insert into public.claims ( + id, owner_agent_id, type, text, proposition, body_markdown, scope, access_scope, + status, admission_state, revision_criteria, revision_kind, supersedes_id, + confidence, tags, created_by, superseded_by, accepted_by_proposal_id, created_at, updated_at +) values +{claim_values}; + +insert into public.claim_evidence ( + id, claim_id, source_id, role, weight, created_by, excerpt, locator_json, + source_content_hash, rationale, polarity, accepted_from_proposal_id, created_at +) values +{evidence_values}; + +insert into public.claim_edges ( + id, from_claim, to_claim, edge_type, weight, created_by, relation_type, + rationale, scope_or_conditions, accepted_by_proposal_id, created_at +) values ( + {apply.sql_literal(edge["id"])}::uuid, {apply.sql_literal(edge["from_claim"])}::uuid, + {apply.sql_literal(edge["to_claim"])}::uuid, 'supports', null, + {apply.sql_literal(edge["created_by"])}::uuid, {apply.sql_literal(edge["relation_type"])}, + {apply.sql_literal(edge["rationale"])}, {apply.sql_literal(edge["scope_or_conditions"])}, + {apply.sql_literal(ids["proposal"])}::uuid, '2026-07-18T12:05:00.654321Z' +); + +insert into public.claim_evidence_assessments ( + id, claim_id, support_tier, challenge_tier, overall_state, rationale, + evidence_set_hash, accepted_by_proposal_id, supersedes_assessment_id, created_at +) values +{assessment_values}; +""" + ) + + +def _install_v3_contract(postgres: contract.DisposablePostgres) -> None: + postgres.psql("alter role kb_apply nologin;") + postgres.apply_file(MIGRATION) + postgres.apply_file(PREREQUISITES) + + +def _install_spoofed_cutover_catalog(postgres: contract.DisposablePostgres) -> None: + postgres.psql( + """ +create function kb_stage.teleo_v3_enforce_apply_contract_cutover() +returns trigger language plpgsql as $function$ +begin + return new; +end +$function$; + +create function kb_stage.teleo_v3_spoof_cutover_target() +returns trigger language plpgsql as $function$ +begin + return new; +end +$function$; + +create trigger teleo_v3_00_enforce_apply_contract_cutover +before insert or update of proposal_type, payload, status on kb_stage.kb_proposals +for each row execute function kb_stage.teleo_v3_spoof_cutover_target(); +""" + ) + + +def _reduce_spoofed_cutover_to_partial_function(postgres: contract.DisposablePostgres) -> None: + postgres.psql( + """ +drop trigger teleo_v3_00_enforce_apply_contract_cutover on kb_stage.kb_proposals; +drop function kb_stage.teleo_v3_spoof_cutover_target(); +""" + ) + + +def _drop_partial_cutover_function(postgres: contract.DisposablePostgres) -> None: + postgres.psql( + """ +drop function kb_stage.teleo_v3_enforce_apply_contract_cutover(); +""" + ) + + +def _start_active_apply_session( + postgres: contract.DisposablePostgres, +) -> subprocess.Popen[str]: + assert postgres.docker is not None + holder = subprocess.Popen( + [ + postgres.docker, + "exec", + "-e", + f"PGPASSWORD={APPLY_PASSWORD}", + "-e", + "PGOPTIONS=-c timezone=UTC", + "-i", + postgres.name, + "psql", + "--no-psqlrc", + "--set=ON_ERROR_STOP=1", + "--host=127.0.0.1", + "--username=kb_apply", + f"--dbname={contract.DATABASE}", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert holder.stdin is not None + holder.stdin.write("set application_name = 'teleo_v2_active_holder'; select pg_sleep(30);\n") + holder.stdin.close() + holder.stdin = None + deadline = time.monotonic() + 8 + while time.monotonic() < deadline: + active = postgres.psql_json( + """ +select jsonb_build_object( + 'active', exists ( + select 1 from pg_stat_activity + where application_name = 'teleo_v2_active_holder' + and usename = 'kb_apply' + and wait_event = 'PgSleep' + ) +)::text; +""" + )["active"] + if active: + return holder + time.sleep(0.1) + holder.terminate() + holder.wait(timeout=3) + raise AssertionError("kb_apply session did not reach the active cutover gate") + + +def _terminate_active_apply_session( + postgres: contract.DisposablePostgres, + holder: subprocess.Popen[str], +) -> None: + postgres.psql( + "select pg_terminate_backend(pid) from pg_stat_activity where application_name = 'teleo_v2_active_holder';" + ) + try: + holder.communicate(timeout=5) + except subprocess.TimeoutExpired: + holder.terminate() + holder.wait(timeout=3) + + +def _worker_command( + postgres: contract.DisposablePostgres, + secrets_file: Path, + receipt_dir: Path, + state_file: Path, + contract_version: int, + *, + apply_script: Path = APPLY_SCRIPT, +) -> list[str]: + return [ + sys.executable, + str(WORKER), + "--enable", + "--contract-version", + str(contract_version), + "--max-per-tick", + "1", + "--secrets-file", + str(secrets_file), + "--failure-state-file", + str(state_file), + "--receipt-dir", + str(receipt_dir), + "--apply-script", + str(apply_script), + "--container", + postgres.name, + "--db", + contract.DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_apply", + ] + + +def _contract_state(postgres: contract.DisposablePostgres) -> str: + return postgres.psql(apply.build_v3_contract_state_query() + ";").stdout.strip() + + +def _assert_invalid_contract_workers( + postgres: contract.DisposablePostgres, + secrets_file: Path, + receipt_dir: Path, + state_file: Path, +) -> None: + assert _contract_state(postgres) == "INVALID" + for contract_version in (2, 3): + worker = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, contract_version), + text=True, + capture_output=True, + check=False, + ) + assert worker.returncode == 1 + assert "contract integrity is INVALID" in worker.stderr + assert "no candidate selection or writes attempted" in worker.stderr + + +def _write_catalog_race_apply_wrapper(tmp_path: Path) -> Path: + wrapper = tmp_path / "shape-drift-cutover-before-real-apply.py" + wrapper.write_text( + f"""import os +import subprocess +import sys + +arguments = sys.argv[1:] +container = arguments[arguments.index("--container") + 1] +database = arguments[arguments.index("--db") + 1] +subprocess.run( + [ + "docker", "exec", "-i", container, "psql", "--no-psqlrc", + "--set=ON_ERROR_STOP=1", "--username=postgres", f"--dbname={{database}}", + ], + input=( + "create or replace trigger teleo_v3_00_enforce_apply_contract_cutover " + "before insert or update of proposal_type, payload, status on kb_stage.kb_proposals " + "for each row when (false) execute function " + "kb_stage.teleo_v3_enforce_apply_contract_cutover();" + ), + text=True, + check=True, +) +os.execv(sys.executable, [sys.executable, {str(APPLY_SCRIPT)!r}, *arguments]) +""", + encoding="utf-8", + ) + return wrapper + + +@pytest.mark.skipif(not contract.docker_available(), reason="Docker daemon is required") +def test_v3_restricted_worker_applies_once_and_retains_exact_receipt(tmp_path: Path) -> None: + postgres = contract.DisposablePostgres() + cleanup: dict = {} + try: + environment = postgres.start() + assert environment["network_mode"] == "none" + postgres.psql(contract.BOOTSTRAP_SQL) + postgres.psql(_supplemental_bootstrap()) + postgres.apply_file(PREREQUISITES) + secrets_file = tmp_path / "kb-apply.env" + secrets_file.write_text(f"KB_APPLY_DB_PASSWORD={APPLY_PASSWORD}\n", encoding="utf-8") + secrets_file.chmod(0o600) + receipt_dir = tmp_path / "receipts" + state_file = tmp_path / "worker-state.json" + _insert_historical_v2_queue(postgres) + + postgres.psql("alter role kb_gate_owner login inherit superuser createdb createrole replication bypassrls;") + role_drift_refusal = postgres.apply_file(MIGRATION, check=False) + assert role_drift_refusal.returncode != 0 + assert "55000" in role_drift_refusal.stderr + assert "kb_gate_owner must be NOLOGIN NOINHERIT and unprivileged" in role_drift_refusal.stderr + assert postgres.psql("select to_regclass('public.claim_evidence_assessments') is null;").stdout.strip() == "t" + postgres.psql( + "alter role kb_gate_owner nologin noinherit nosuperuser nocreatedb nocreaterole noreplication nobypassrls;" + ) + + _install_spoofed_cutover_catalog(postgres) + for contract_version in (2, 3): + spoofed_worker = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, contract_version), + text=True, + capture_output=True, + check=False, + ) + assert spoofed_worker.returncode == 1 + assert "contract integrity is INVALID" in spoofed_worker.stderr + assert "no candidate selection or writes attempted" in spoofed_worker.stderr + refused_cutover = postgres.apply_file(MIGRATION, check=False) + assert refused_cutover.returncode != 0 + assert "55000" in refused_cutover.stderr + assert "stop the V2 worker" in refused_cutover.stderr + preinstall = postgres.psql_json( + """ +select jsonb_build_object( + 'assessment_table', to_regclass('public.claim_evidence_assessments')::text, + 'cutover_function', to_regprocedure('kb_stage.teleo_v3_enforce_apply_contract_cutover()')::text, + 'trigger_target', ( + select procedure.proname + from pg_trigger trigger_row + join pg_proc procedure on procedure.oid = trigger_row.tgfoid + where trigger_row.tgrelid = 'kb_stage.kb_proposals'::regclass + and trigger_row.tgname = 'teleo_v3_00_enforce_apply_contract_cutover' + ), + 'v2_status', (select status from kb_stage.kb_proposals where id = + '90000000-0000-4000-8000-000000000009'::uuid) +)::text; +""" + ) + assert preinstall == { + "assessment_table": None, + "cutover_function": "kb_stage.teleo_v3_enforce_apply_contract_cutover()", + "trigger_target": "teleo_v3_spoof_cutover_target", + "v2_status": "approved", + } + _reduce_spoofed_cutover_to_partial_function(postgres) + for contract_version in (2, 3): + partial_worker = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, contract_version), + text=True, + capture_output=True, + check=False, + ) + assert partial_worker.returncode == 1 + assert "contract integrity is INVALID" in partial_worker.stderr + _drop_partial_cutover_function(postgres) + + active_holder = _start_active_apply_session(postgres) + try: + postgres.psql("alter role kb_apply nologin;") + active_refusal = postgres.apply_file(MIGRATION, check=False) + assert active_refusal.returncode != 0 + assert "55000" in active_refusal.stderr + assert "active kb_apply sessions must exit" in active_refusal.stderr + assert ( + postgres.psql("select to_regclass('public.claim_evidence_assessments') is null;").stdout.strip() == "t" + ) + finally: + _terminate_active_apply_session(postgres, active_holder) + + _install_v3_contract(postgres) + + postgres.psql("alter table kb_stage.kb_proposals disable trigger teleo_v3_00_enforce_apply_contract_cutover;") + for contract_version in (2, 3): + disabled_worker = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, contract_version), + text=True, + capture_output=True, + check=False, + ) + assert disabled_worker.returncode == 1 + assert "contract integrity is INVALID" in disabled_worker.stderr + assert ( + postgres.psql( + f"select count(*) from public.strategies where agent_id = '{contract.AGENT_ID}'::uuid;" + ).stdout.strip() + == "0" + ) + assert ( + postgres.psql( + f"select count(*) from public.claim_edges where from_claim = '{contract.LEGACY_CLAIM_ID}'::uuid " + f"and to_claim = '{contract.LEGACY_RELATED_CLAIM_ID}'::uuid and edge_type = 'supports';" + ).stdout.strip() + == "0" + ) + disabled_refusal = postgres.apply_file(MIGRATION, check=False) + assert disabled_refusal.returncode != 0 + assert "55000" in disabled_refusal.stderr + assert "stop the V2 worker" in disabled_refusal.stderr + postgres.psql("alter role kb_apply nologin;") + postgres.apply_file(MIGRATION) + postgres.apply_file(PREREQUISITES) + restored_cutover = postgres.psql_json( + """ +select jsonb_build_object( + 'enabled', trigger_row.tgenabled, + 'exact_target', trigger_row.tgfoid = + 'kb_stage.teleo_v3_enforce_apply_contract_cutover()'::regprocedure +)::text +from pg_trigger trigger_row +where trigger_row.tgrelid = 'kb_stage.kb_proposals'::regclass + and trigger_row.tgname = 'teleo_v3_00_enforce_apply_contract_cutover'; +""" + ) + assert restored_cutover == {"enabled": "O", "exact_target": True} + + postgres.psql( + """ +create or replace trigger teleo_v3_00_enforce_apply_contract_cutover +before insert or update of proposal_type, payload, status on kb_stage.kb_proposals +for each row when (false) +execute function kb_stage.teleo_v3_enforce_apply_contract_cutover(); +""" + ) + _assert_invalid_contract_workers(postgres, secrets_file, receipt_dir, state_file) + conditional_cutover_refusal = postgres.apply_file(MIGRATION, check=False) + assert conditional_cutover_refusal.returncode != 0 + assert "stop the V2 worker" in conditional_cutover_refusal.stderr + _install_v3_contract(postgres) + assert _contract_state(postgres) == "V3" + + postgres.psql( + """ +create or replace trigger teleo_v3_00_enforce_apply_contract_cutover +before insert or update of status on kb_stage.kb_proposals +for each row execute function kb_stage.teleo_v3_enforce_apply_contract_cutover(); +""" + ) + _assert_invalid_contract_workers(postgres, secrets_file, receipt_dir, state_file) + _install_v3_contract(postgres) + assert _contract_state(postgres) == "V3" + + postgres.psql( + """ +create or replace trigger teleo_v3_guard_linked_proposal_decision +before update of status on kb_stage.kb_proposals +for each row when (false) +execute function kb_stage.teleo_v3_guard_linked_proposal_decision(); +""" + ) + _assert_invalid_contract_workers(postgres, secrets_file, receipt_dir, state_file) + _install_v3_contract(postgres) + assert _contract_state(postgres) == "V3" + + postgres.psql( + """ +drop trigger teleo_v3_00_claim_require_linked_insert on public.claims; +create trigger teleo_v3_00_claim_require_linked_insert +before update on public.sources +for each row execute function kb_stage.teleo_v3_validate_claim_edge(); +""" + ) + _assert_invalid_contract_workers(postgres, secrets_file, receipt_dir, state_file) + postgres.psql("alter role kb_apply nologin;") + wrong_table_migration = postgres.apply_file(MIGRATION, check=False) + assert wrong_table_migration.returncode != 0 + assert "installed trigger catalog is INVALID" in wrong_table_migration.stderr + assert _contract_state(postgres) == "INVALID" + postgres.psql("drop trigger teleo_v3_00_claim_require_linked_insert on public.sources;") + postgres.apply_file(MIGRATION) + postgres.apply_file(PREREQUISITES) + assert _contract_state(postgres) == "V3" + assert ( + postgres.psql( + f"select count(*) from public.strategies where agent_id = '{contract.AGENT_ID}'::uuid;" + ).stdout.strip() + == "0" + ) + assert ( + postgres.psql( + f"select count(*) from public.claim_edges where from_claim = '{contract.LEGACY_CLAIM_ID}'::uuid " + f"and to_claim = '{contract.LEGACY_RELATED_CLAIM_ID}'::uuid and edge_type = 'supports';" + ).stdout.strip() + == "0" + ) + + race_payload = _payload(RACE_IDS) + race_packet = _insert_pending_proposal(postgres, race_payload, RACE_IDS) + _approve_proposal(postgres, RACE_IDS["proposal"], race_packet) + + postgres.psql("alter role kb_gate_owner login;") + unsafe_owner = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert unsafe_owner.returncode == 1 + assert "contract integrity is INVALID" in unsafe_owner.stderr + postgres.psql("alter role kb_gate_owner nologin;") + + postgres.psql("grant kb_gate_owner to kb_apply;") + membership_drift = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert membership_drift.returncode == 1 + assert "contract integrity is INVALID" in membership_drift.stderr + postgres.psql("revoke kb_gate_owner from kb_apply;") + + postgres.psql("grant execute on function kb_stage.teleo_v3_enforce_apply_contract_cutover() to kb_apply;") + acl_drift = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert acl_drift.returncode == 1 + assert "contract integrity is INVALID" in acl_drift.stderr + postgres.psql("revoke execute on function kb_stage.teleo_v3_enforce_apply_contract_cutover() from kb_apply;") + + postgres.psql("alter function kb_stage.teleo_v3_enforce_apply_contract_cutover() owner to postgres;") + function_owner_drift = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert function_owner_drift.returncode == 1 + assert "contract integrity is INVALID" in function_owner_drift.stderr + postgres.psql( + "alter function kb_stage.teleo_v3_enforce_apply_contract_cutover() owner to kb_gate_owner; " + "revoke all on function kb_stage.teleo_v3_enforce_apply_contract_cutover() " + "from public, kb_apply, kb_review;" + ) + + postgres.psql("alter function kb_stage.teleo_v3_validate_claim_edge() owner to postgres;") + _assert_invalid_contract_workers(postgres, secrets_file, receipt_dir, state_file) + postgres.psql("alter function kb_stage.teleo_v3_validate_claim_edge() owner to kb_gate_owner;") + + postgres.psql( + """ +create or replace function kb_stage.teleo_v3_guard_immutable_fields() +returns trigger +language plpgsql +security invoker +set search_path = pg_catalog, pg_temp +as $function$ +begin + return new; +end +$function$; +""" + ) + _assert_invalid_contract_workers(postgres, secrets_file, receipt_dir, state_file) + _install_v3_contract(postgres) + assert _contract_state(postgres) == "V3" + + race_wrapper = _write_catalog_race_apply_wrapper(tmp_path) + raced_apply = subprocess.run( + _worker_command( + postgres, + secrets_file, + receipt_dir, + state_file, + 3, + apply_script=race_wrapper, + ), + text=True, + capture_output=True, + check=False, + ) + assert raced_apply.returncode == 1 + assert "V3 catalog contract is INVALID" in raced_apply.stderr + assert not (receipt_dir / f"{RACE_IDS['proposal']}.json").exists() + assert postgres.psql_json( + f""" +select jsonb_build_object( + 'status', (select status from kb_stage.kb_proposals where id = '{RACE_IDS["proposal"]}'::uuid), + 'applied_at', (select applied_at from kb_stage.kb_proposals where id = '{RACE_IDS["proposal"]}'::uuid), + 'claims', ( + select count(*) from public.claims where accepted_by_proposal_id = '{RACE_IDS["proposal"]}'::uuid + ), + 'sources', ( + select count(*) from public.sources where accepted_by_proposal_id = '{RACE_IDS["proposal"]}'::uuid + ), + 'evidence', ( + select count(*) from public.claim_evidence where accepted_from_proposal_id = '{RACE_IDS["proposal"]}'::uuid + ), + 'edges', ( + select count(*) from public.claim_edges where accepted_by_proposal_id = '{RACE_IDS["proposal"]}'::uuid + ), + 'assessments', ( + select count(*) from public.claim_evidence_assessments + where accepted_by_proposal_id = '{RACE_IDS["proposal"]}'::uuid + ) +)::text; +""" + ) == { + "status": "approved", + "applied_at": None, + "claims": 0, + "sources": 0, + "evidence": 0, + "edges": 0, + "assessments": 0, + } + _install_v3_contract(postgres) + postgres.psql( + f"update kb_stage.kb_proposals set status = 'rejected' where id = '{RACE_IDS['proposal']}'::uuid;" + ) + + payload = _payload() + packet = _insert_pending_proposal(postgres, payload) + _approve_proposal(postgres, PROPOSAL_ID, packet) + material_path = tmp_path / "private-material" / "entry-0001.json" + assert not material_path.exists() + approved_snapshot = postgres.psql_json( + f""" +select jsonb_build_object( + 'status', proposal.status, + 'snapshot_present', approval.approved_proposal_snapshot is not null, + 'snapshot_matches', approval.approved_proposal_snapshot = to_jsonb(proposal), + 'snapshot_updated_at', approval.approved_proposal_snapshot->'updated_at', + 'proposal_updated_at', to_jsonb(proposal)->'updated_at' +)::text +from kb_stage.kb_proposals proposal +join kb_stage.kb_proposal_approvals approval on approval.proposal_id = proposal.id +where proposal.id = '{PROPOSAL_ID}'::uuid; +""" + ) + assert approved_snapshot == { + "status": "approved", + "snapshot_present": True, + "snapshot_matches": True, + "snapshot_updated_at": approved_snapshot["proposal_updated_at"], + "proposal_updated_at": approved_snapshot["proposal_updated_at"], + } + + v2 = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 2), + text=True, + capture_output=True, + check=False, + ) + assert v2.returncode == 0, v2.stderr + assert "SKIP contract v2 execution" in v2.stderr + assert f"retained V2 proposal {V2_PROPOSAL_ID} (add_edge)" in v2.stderr + assert "remain review-visible" in v2.stderr + assert ( + postgres.psql(f"select status from kb_stage.kb_proposals where id = '{PROPOSAL_ID}'::uuid;").stdout.strip() + == "approved" + ) + retained_v2 = postgres.psql_json( + f""" +select jsonb_build_object( + 'status', (select status from kb_stage.kb_proposals where id = '{V2_PROPOSAL_ID}'::uuid), + 'revise_status', (select status from kb_stage.kb_proposals where id = '{V2_REVISE_PROPOSAL_ID}'::uuid), + 'strategy_count', (select count(*) from public.strategies + where agent_id = '{contract.AGENT_ID}'::uuid), + 'legacy_edge_count', (select count(*) from public.claim_edges + where from_claim = '{contract.LEGACY_CLAIM_ID}'::uuid + and to_claim = '{contract.LEGACY_RELATED_CLAIM_ID}'::uuid + and edge_type = 'supports') +)::text; +""" + ) + assert retained_v2 == { + "status": "approved", + "revise_status": "approved", + "strategy_count": 0, + "legacy_edge_count": 0, + } + + v3 = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert v3.returncode == 0, v3.stderr + assert f"applied {PROPOSAL_ID} (approve_claim)" in v3.stdout + + receipt_path = receipt_dir / f"{PROPOSAL_ID}.json" + assert stat.S_IMODE(receipt_path.stat().st_mode) == 0o600 + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + replay.validate_replay_receipt(receipt, expected_proposal_id=PROPOSAL_ID) + assert receipt["expected_row_counts"] == { + "claims": 2, + "sources": 1, + "claim_evidence": 2, + "claim_edges": 1, + "claim_evidence_assessments": 2, + "reasoning_tools": 0, + } + assert receipt["canonical_rows"]["sources"][0]["captured_at"] == "2026-07-18T12:00:00.123400Z" + assert re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z", receipt["proposal"]["reviewed_at"]) + assert re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z", receipt["proposal"]["applied_at"]) + + exported = subprocess.run( + [ + sys.executable, + str(MATERIAL_EXPORTER), + PROPOSAL_ID, + "--replay-receipt", + str(receipt_path), + "--sequence", + "1", + "--output", + str(material_path), + "--secrets-file", + str(secrets_file), + "--container", + postgres.name, + "--db", + contract.DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_apply", + ], + text=True, + capture_output=True, + check=False, + ) + assert exported.returncode == 0, exported.stderr + assert exported.stdout.strip() == f"replay material created: proposal={PROPOSAL_ID} sequence=1" + assert str(material_path) not in exported.stdout + assert "Private exact" not in exported.stdout + assert stat.S_IMODE(material_path.stat().st_mode) == 0o600 + material = json.loads(material_path.read_text(encoding="utf-8")) + material = material_exporter.validate_material( + material, + expected_proposal_id=PROPOSAL_ID, + expected_sequence=1, + ) + assert set(material) == material_exporter.MATERIAL_FIELDS + genesis_rebuild._validate_proposal_rows( + material["approved_proposal"], + material["applied_proposal"], + material["approval_snapshot"], + material["replay_receipt"]["proposal"], + ) + validated_entry = genesis_rebuild._validate_entry_material( + material, + expected_sequence=1, + expected_replay_hash=material["replay_receipt"]["hashes"]["replay_material_sha256"], + material_path=material_path, + material_sha256=hashlib.sha256(material_path.read_bytes()).hexdigest(), + ) + assert validated_entry.sequence == 1 + assert validated_entry.applied_proposal["id"] == PROPOSAL_ID + assert material["approved_proposal"]["updated_at"] == replay.canonical_utc_timestamp( + approved_snapshot["proposal_updated_at"], + path="approved_snapshot.proposal_updated_at", + ) + + exported_again = subprocess.run( + [ + sys.executable, + str(MATERIAL_EXPORTER), + PROPOSAL_ID, + "--replay-receipt", + str(receipt_path), + "--sequence", + "1", + "--output", + str(material_path), + "--secrets-file", + str(secrets_file), + "--container", + postgres.name, + "--db", + contract.DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_apply", + ], + text=True, + capture_output=True, + check=False, + ) + assert exported_again.returncode == 0, exported_again.stderr + assert exported_again.stdout.strip() == f"replay material unchanged: proposal={PROPOSAL_ID} sequence=1" + + readback = postgres.psql_json( + f""" +select jsonb_build_object( + 'proposal_status', (select status from kb_stage.kb_proposals where id = '{PROPOSAL_ID}'::uuid), + 'applied_at_present', (select applied_at is not null from kb_stage.kb_proposals where id = '{PROPOSAL_ID}'::uuid), + 'claims', (select count(*) from public.claims where accepted_by_proposal_id = '{PROPOSAL_ID}'::uuid), + 'sources', (select count(*) from public.sources where accepted_by_proposal_id = '{PROPOSAL_ID}'::uuid), + 'evidence', (select count(*) from public.claim_evidence where accepted_from_proposal_id = '{PROPOSAL_ID}'::uuid), + 'edges', (select count(*) from public.claim_edges where accepted_by_proposal_id = '{PROPOSAL_ID}'::uuid), + 'assessments', (select count(*) from public.claim_evidence_assessments where accepted_by_proposal_id = '{PROPOSAL_ID}'::uuid) +)::text; +""" + ) + assert readback == { + "proposal_status": "applied", + "applied_at_present": True, + "claims": 2, + "sources": 1, + "evidence": 2, + "edges": 1, + "assessments": 2, + } + + original_canonical_hash = receipt["hashes"]["canonical_rows_sha256"] + original_payload_hash = receipt["hashes"]["proposal_payload_sha256"] + receipt_path.unlink() + recovered = subprocess.run( + [ + sys.executable, + str(APPLY_SCRIPT), + PROPOSAL_ID, + "--expected-contract-version", + "3", + "--receipt-only", + "--secrets-file", + str(secrets_file), + "--container", + postgres.name, + "--db", + contract.DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_apply", + "--receipt-out", + str(receipt_path), + ], + text=True, + capture_output=True, + check=False, + ) + assert recovered.returncode == 0, recovered.stderr + recovered_receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + replay.validate_replay_receipt(recovered_receipt, expected_proposal_id=PROPOSAL_ID) + assert recovered_receipt["apply_engine"]["source"] == "reconstructed_current_engine" + assert recovered_receipt["hashes"]["canonical_rows_sha256"] == original_canonical_hash + assert recovered_receipt["hashes"]["proposal_payload_sha256"] == original_payload_hash + assert recovered_receipt["canonical_rows"]["sources"][0]["captured_at"] == ("2026-07-18T12:00:00.123400Z") + assert stat.S_IMODE(receipt_path.stat().st_mode) == 0o600 + + second = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert second.returncode == 0, second.stderr + assert "no approved+applyable contract v3 proposals" in second.stdout + + postgres.psql(f"update kb_stage.kb_proposals set status = 'rejected' where id = '{V2_PROPOSAL_ID}'::uuid;") + assert ( + postgres.psql( + f"select status from kb_stage.kb_proposals where id = '{V2_PROPOSAL_ID}'::uuid;" + ).stdout.strip() + == "rejected" + ) + + malformed_insert = f""" +insert into kb_stage.kb_proposals ( + id, proposal_type, status, rationale, payload, reviewed_by_handle, + reviewed_by_agent_id, reviewed_at, review_note +) values ( + '{MALFORMED_V3_PROPOSAL_ID}'::uuid, 'approve_claim', 'approved', + 'Malformed typed V3 contract.', '{{"apply_payload": {{"contract_version": "3"}}}}'::jsonb, + '{CANONICAL_REVIEWER_HANDLE}', '{REVIEWER_AGENT_ID}'::uuid, '2026-07-18T12:10:00Z', 'Must be refused.' +); +""" + hybrid_insert = f""" +insert into kb_stage.kb_proposals ( + id, proposal_type, status, rationale, payload, reviewed_by_handle, + reviewed_by_agent_id, reviewed_at, review_note +) values ( + '{HYBRID_V3_PROPOSAL_ID}'::uuid, 'add_edge', 'approved', + 'Hybrid proposal type and V3 version.', '{{"apply_payload": {{"contract_version": 3}}}}'::jsonb, + '{CANONICAL_REVIEWER_HANDLE}', '{REVIEWER_AGENT_ID}'::uuid, '2026-07-18T12:11:00Z', 'Must be refused.' +); +""" + for invalid_sql in (malformed_insert, hybrid_insert): + refused = postgres.psql(invalid_sql, check=False) + assert refused.returncode != 0 + assert "55000" in refused.stderr + assert "migrate or reject" in refused.stderr + + postgres.psql( + "begin; set local session_replication_role = replica;\n" + malformed_insert + hybrid_insert + "commit;" + ) + malformed_tick = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert malformed_tick.returncode == 0, malformed_tick.stderr + assert "no approved+applyable contract v3 proposals" in malformed_tick.stdout + assert postgres.psql_json( + f""" +select jsonb_build_object( + 'malformed', (select status from kb_stage.kb_proposals where id = '{MALFORMED_V3_PROPOSAL_ID}'::uuid), + 'hybrid', (select status from kb_stage.kb_proposals where id = '{HYBRID_V3_PROPOSAL_ID}'::uuid) +)::text; +""" + ) == {"malformed": "approved", "hybrid": "approved"} + + gate_security = postgres.psql_json( + """ +select jsonb_build_object( + 'owner', owner_role.rolname, + 'security_definer', procedure.prosecdef, + 'search_path', procedure.proconfig, + 'non_owner_execute', exists ( + select 1 from aclexplode(coalesce(procedure.proacl, acldefault('f', procedure.proowner))) acl + where acl.privilege_type = 'EXECUTE' and acl.grantee <> procedure.proowner + ) +)::text +from pg_proc procedure +join pg_namespace namespace on namespace.oid = procedure.pronamespace +join pg_roles owner_role on owner_role.oid = procedure.proowner +where namespace.nspname = 'kb_stage' + and procedure.proname = 'teleo_v3_require_accepted_proposal'; +""" + ) + assert gate_security == { + "owner": "kb_gate_owner", + "security_definer": True, + "search_path": ["search_path=pg_catalog, pg_temp"], + "non_owner_execute": False, + } + cutover_security = postgres.psql_json( + """ +select jsonb_build_object( + 'owner', owner_role.rolname, + 'security_definer', procedure.prosecdef, + 'search_path', procedure.proconfig, + 'non_owner_execute', exists ( + select 1 from aclexplode(coalesce(procedure.proacl, acldefault('f', procedure.proowner))) acl + where acl.privilege_type = 'EXECUTE' and acl.grantee <> procedure.proowner + ) +)::text +from pg_proc procedure +join pg_namespace namespace on namespace.oid = procedure.pronamespace +join pg_roles owner_role on owner_role.oid = procedure.proowner +where namespace.nspname = 'kb_stage' + and procedure.proname = 'teleo_v3_enforce_apply_contract_cutover'; +""" + ) + assert cutover_security == gate_security + + rejected_delete = postgres.psql( + f"delete from public.claims where id = '{CLAIM_ID}'::uuid;", + check=False, + ) + assert rejected_delete.returncode != 0 + assert "55000" in rejected_delete.stderr + finally: + cleanup = postgres.cleanup() + + assert cleanup["container_absent"] is True + assert cleanup["name_readback_clear"] is True + assert cleanup["instance_label_readback_clear"] is True + + +@pytest.mark.skipif(not contract.docker_available(), reason="Docker daemon is required") +def test_v3_worker_replays_exact_preseeded_rows_and_rejects_conflicts(tmp_path: Path) -> None: + postgres = contract.DisposablePostgres() + cleanup: dict = {} + try: + environment = postgres.start() + assert environment["network_mode"] == "none" + postgres.psql(contract.BOOTSTRAP_SQL) + postgres.psql(_supplemental_bootstrap()) + postgres.apply_file(PREREQUISITES) + _install_v3_contract(postgres) + + secrets_file = tmp_path / "kb-apply.env" + secrets_file.write_text(f"KB_APPLY_DB_PASSWORD={APPLY_PASSWORD}\n", encoding="utf-8") + secrets_file.chmod(0o600) + receipt_dir = tmp_path / "receipts" + state_file = tmp_path / "worker-state.json" + + preseed_payload = _payload(PRESEED_IDS) + preseed_packet = _insert_pending_proposal(postgres, preseed_payload, PRESEED_IDS) + _approve_proposal(postgres, PRESEED_IDS["proposal"], preseed_packet) + _preseed_v3_rows(postgres, preseed_payload, PRESEED_IDS) + + exact_replay = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert exact_replay.returncode == 0, exact_replay.stderr + assert f"applied {PRESEED_IDS['proposal']} (approve_claim)" in exact_replay.stdout + + exact_receipt_path = receipt_dir / f"{PRESEED_IDS['proposal']}.json" + exact_receipt = json.loads(exact_receipt_path.read_text(encoding="utf-8")) + replay.validate_replay_receipt(exact_receipt, expected_proposal_id=PRESEED_IDS["proposal"]) + assert exact_receipt["expected_row_counts"] == { + "claims": 2, + "sources": 1, + "claim_evidence": 2, + "claim_edges": 1, + "claim_evidence_assessments": 2, + "reasoning_tools": 0, + } + assert exact_receipt["canonical_rows"]["sources"][0]["created_at"] == ("2026-07-18T12:02:00.654321Z") + assert exact_receipt["canonical_rows"]["claims"][0]["created_at"] == ("2026-07-18T12:03:00.654321Z") + assert stat.S_IMODE(exact_receipt_path.stat().st_mode) == 0o600 + assert ( + postgres.psql( + f"select status from kb_stage.kb_proposals where id = '{PRESEED_IDS['proposal']}'::uuid;" + ).stdout.strip() + == "applied" + ) + + no_replay = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert no_replay.returncode == 0, no_replay.stderr + assert "no approved+applyable contract v3 proposals" in no_replay.stdout + + mismatch_payload = _payload(MISMATCH_IDS) + mismatch_packet = _insert_pending_proposal(postgres, mismatch_payload, MISMATCH_IDS) + _approve_proposal(postgres, MISMATCH_IDS["proposal"], mismatch_packet) + _preseed_v3_rows( + postgres, + mismatch_payload, + MISMATCH_IDS, + mismatch_source_excerpt=True, + ) + + conflicting_replay = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert conflicting_replay.returncode == 1 + assert "source row does not match strict payload" in conflicting_replay.stderr + assert not (receipt_dir / f"{MISMATCH_IDS['proposal']}.json").exists() + mismatch_readback = postgres.psql_json( + f""" +select jsonb_build_object( + 'status', (select status from kb_stage.kb_proposals where id = '{MISMATCH_IDS["proposal"]}'::uuid), + 'applied_at', (select applied_at from kb_stage.kb_proposals where id = '{MISMATCH_IDS["proposal"]}'::uuid), + 'source_excerpt', (select excerpt from public.sources where id = '{MISMATCH_IDS["source"]}'::uuid), + 'claims', (select count(*) from public.claims where accepted_by_proposal_id = '{MISMATCH_IDS["proposal"]}'::uuid), + 'sources', (select count(*) from public.sources where accepted_by_proposal_id = '{MISMATCH_IDS["proposal"]}'::uuid), + 'evidence', (select count(*) from public.claim_evidence where accepted_from_proposal_id = '{MISMATCH_IDS["proposal"]}'::uuid), + 'edges', (select count(*) from public.claim_edges where accepted_by_proposal_id = '{MISMATCH_IDS["proposal"]}'::uuid), + 'assessments', (select count(*) from public.claim_evidence_assessments where accepted_by_proposal_id = '{MISMATCH_IDS["proposal"]}'::uuid) +)::text; +""" + ) + assert mismatch_readback == { + "status": "approved", + "applied_at": None, + "source_excerpt": "Conflicting preseeded source text.", + "claims": 2, + "sources": 1, + "evidence": 2, + "edges": 1, + "assessments": 2, + } + state_file.write_text( + json.dumps({MISMATCH_IDS["proposal"]: 3}, sort_keys=True) + "\n", + encoding="utf-8", + ) + + superseded_payload = _payload(SUPERSEDED_MISMATCH_IDS) + superseded_packet = _insert_pending_proposal( + postgres, + superseded_payload, + SUPERSEDED_MISMATCH_IDS, + ) + _approve_proposal(postgres, SUPERSEDED_MISMATCH_IDS["proposal"], superseded_packet) + _preseed_v3_rows(postgres, superseded_payload, SUPERSEDED_MISMATCH_IDS) + postgres.psql( + f""" +begin; +set local session_replication_role = replica; +update public.claims + set superseded_by = '{SUPERSEDED_MISMATCH_IDS["support_claim"]}'::uuid + where id = '{SUPERSEDED_MISMATCH_IDS["claim"]}'::uuid; +commit; +""" + ) + + superseded_conflict = subprocess.run( + _worker_command(postgres, secrets_file, receipt_dir, state_file, 3), + text=True, + capture_output=True, + check=False, + ) + assert superseded_conflict.returncode == 1 + assert "claim row does not match strict payload" in superseded_conflict.stderr + assert not (receipt_dir / f"{SUPERSEDED_MISMATCH_IDS['proposal']}.json").exists() + assert postgres.psql_json( + f""" +select jsonb_build_object( + 'status', (select status from kb_stage.kb_proposals + where id = '{SUPERSEDED_MISMATCH_IDS["proposal"]}'::uuid), + 'applied_at', (select applied_at from kb_stage.kb_proposals + where id = '{SUPERSEDED_MISMATCH_IDS["proposal"]}'::uuid), + 'superseded_by', (select superseded_by from public.claims + where id = '{SUPERSEDED_MISMATCH_IDS["claim"]}'::uuid), + 'claims', (select count(*) from public.claims + where accepted_by_proposal_id = '{SUPERSEDED_MISMATCH_IDS["proposal"]}'::uuid), + 'sources', (select count(*) from public.sources + where accepted_by_proposal_id = '{SUPERSEDED_MISMATCH_IDS["proposal"]}'::uuid) +)::text; +""" + ) == { + "status": "approved", + "applied_at": None, + "superseded_by": SUPERSEDED_MISMATCH_IDS["support_claim"], + "claims": 2, + "sources": 1, + } + assert json.loads(state_file.read_text(encoding="utf-8")) == { + MISMATCH_IDS["proposal"]: 3, + SUPERSEDED_MISMATCH_IDS["proposal"]: 1, + } + finally: + cleanup = postgres.cleanup() + + assert cleanup["container_absent"] is True + assert cleanup["name_readback_clear"] is True + assert cleanup["instance_label_readback_clear"] is True diff --git a/tests/test_v3_source_rebuild_lifecycle_postgres.py b/tests/test_v3_source_rebuild_lifecycle_postgres.py new file mode 100644 index 0000000..90e7718 --- /dev/null +++ b/tests/test_v3_source_rebuild_lifecycle_postgres.py @@ -0,0 +1,701 @@ +from __future__ import annotations + +import hashlib +import json +import secrets +import stat +import subprocess +import sys +import time +import uuid +from pathlib import Path + +import pytest + +from ops import run_local_corpus_knowledge_rebuild as corpus_rebuild +from ops import run_local_genesis_ledger_rebuild as genesis_rebuild +from ops import verify_teleo_v3_epistemic_contract as v3_contract +from scripts import compile_kb_source_packet as compiler +from tests import test_run_local_genesis_ledger_rebuild as genesis_test +from tests import test_v3_apply_worker_postgres as worker_test + +REPO_ROOT = Path(__file__).resolve().parents[1] +CORPUS_REBUILD = REPO_ROOT / "ops" / "run_local_corpus_knowledge_rebuild.py" +SOURCE_COMPILER = REPO_ROOT / "scripts" / "compile_kb_source_packet.py" +PROPOSAL_STAGER = REPO_ROOT / "scripts" / "stage_normalized_proposal.py" +REVIEW_PACKET = REPO_ROOT / "scripts" / "kb_proposal_review_packet.py" +APPROVER = REPO_ROOT / "scripts" / "approve_proposal.py" +APPLY_WORKER = REPO_ROOT / "scripts" / "apply_worker.py" +MATERIAL_EXPORTER = REPO_ROOT / "scripts" / "export_kb_transition_replay_material.py" +LEDGER_BUILDER = REPO_ROOT / "ops" / "build_v3_genesis_ledger_bundle.py" +GENESIS_REBUILDER = REPO_ROOT / "ops" / "run_local_genesis_ledger_rebuild.py" + +SOURCE_CONTAINER_LABEL = "livingip.test.v3-source-rebuild-lifecycle" +STAGING_LABEL = "com.livingip.leo.checkpoint=disposable" +DATABASE = "teleo_v3_source_lifecycle" +SYNTHETIC_REVIEW_NOTE = "SYNTHETIC TEST ONLY: operator-supplied fixture approval; this is not human review proof." + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _run(command: list[str], *, timeout: int = 240) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + command, + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=False, + timeout=timeout, + ) + assert completed.returncode == 0, ( + f"command failed ({completed.returncode}): {command[0]} {Path(command[1]).name if len(command) > 1 else ''}\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + return completed + + +def _docker(*arguments: str, check: bool = True, timeout: int = 120) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + ["docker", *arguments], + text=True, + capture_output=True, + check=False, + timeout=timeout, + ) + if check: + assert completed.returncode == 0, completed.stderr or completed.stdout + return completed + + +def _write_private(path: Path, content: str | bytes) -> Path: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + path.parent.chmod(0o700) + if isinstance(content, bytes): + path.write_bytes(content) + else: + path.write_text(content, encoding="utf-8") + path.chmod(0o600) + return path + + +def _start_postgres(name: str, database: str, run_label: str, *, staging_target: bool) -> str: + command = [ + "run", + "--detach", + "--rm", + "--network", + "none", + "--name", + name, + "--label", + f"{SOURCE_CONTAINER_LABEL}={run_label}", + "--tmpfs", + "/var/lib/postgresql/data:rw,nosuid,nodev,noexec,size=384m", + "--env", + f"POSTGRES_DB={database}", + "--env", + "POSTGRES_HOST_AUTH_METHOD=trust", + ] + if staging_target: + command.extend(("--label", STAGING_LABEL)) + command.append(v3_contract.POSTGRES_IMAGE) + container_id = _docker(*command).stdout.strip() + assert container_id + + for _ in range(180): + ready = _docker( + "exec", + name, + "pg_isready", + "-U", + "postgres", + "-h", + "127.0.0.1", + "-d", + database, + check=False, + timeout=10, + ) + if ready.returncode == 0: + break + time.sleep(0.25) + else: + pytest.fail(f"disposable PostgreSQL did not become ready: {name}") + + inspected = json.loads(_docker("inspect", name).stdout)[0] + assert inspected["HostConfig"]["NetworkMode"] == "none" + assert inspected["NetworkSettings"]["Ports"] == {} + assert inspected["Mounts"] == [] + assert "/var/lib/postgresql/data" in inspected["HostConfig"]["Tmpfs"] + assert inspected["Config"]["Labels"][SOURCE_CONTAINER_LABEL] == run_label + if staging_target: + assert inspected["Config"]["Labels"]["com.livingip.leo.checkpoint"] == "disposable" + return container_id + + +def _remove_container(name: str) -> None: + _docker("rm", "--force", name, check=False) + + +def _assert_container_absent(name: str) -> None: + assert _docker("inspect", name, check=False).returncode != 0 + names = _docker( + "container", + "ls", + "--all", + "--filter", + f"name=^/{name}$", + "--format", + "{{.Names}}", + ) + assert names.stdout.strip() == "" + + +def _capture_dump(container: str, database: str, destination: Path) -> None: + remote = f"/tmp/{destination.name}" + _docker( + "exec", + container, + "pg_dump", + "-U", + "postgres", + "-d", + database, + "--format=custom", + f"--file={remote}", + ) + _docker("cp", f"{container}:{remote}", str(destination)) + destination.chmod(0o600) + + +def _capture_manifest(container: str, database: str, destination: Path) -> None: + genesis_test.capture_manifest(container, database, destination) + destination.chmod(0o600) + + +def _json_stdout(completed: subprocess.CompletedProcess[str]) -> dict: + value = json.loads(completed.stdout) + assert isinstance(value, dict) + return value + + +def _assert_private_files(root: Path) -> None: + files = [path for path in root.rglob("*") if path.is_file()] + assert files + for path in files: + assert stat.S_IMODE(path.stat().st_mode) == 0o600, path + + +@pytest.mark.skipif(not v3_contract.docker_available(), reason="Docker daemon is required") +def test_v3_source_rebuild_lifecycle_is_exact_networkless_and_cleanup_proven(tmp_path: Path) -> None: + if _docker("image", "inspect", v3_contract.POSTGRES_IMAGE, check=False).returncode != 0: + pytest.skip("the pinned PostgreSQL image must already exist; this test never pulls over the network") + run_id = uuid.uuid4().hex[:12] + source_container = f"teleo-v3-source-{run_id}" + rebuild_prefix = f"teleo-v3-rebuild-{run_id[:8]}" + private_root = tmp_path / "private" + private_root.mkdir(mode=0o700) + password = secrets.token_hex(24) + source_started = False + rebuild_container: str | None = None + + try: + corpus_root = private_root / "corpus" + source_text = ( + "# Canonical review boundary\n\n" + "A staged knowledge proposal remains non-canonical until explicit review and guarded apply both succeed.\n" + ) + source_path = _write_private(corpus_root / "review-boundary.md", source_text) + inventory = private_root / "source-inventory" + + built_inventory = _run( + [ + sys.executable, + str(CORPUS_REBUILD), + "--source-root", + str(corpus_root), + "--output", + str(inventory), + "--mode", + "source-inventory-only", + ] + ) + inventory_status = _json_stdout(built_inventory) + assert inventory_status["status"] == "pass" + assert inventory_status["source_count"] == 1 + verified_inventory = _run( + [ + sys.executable, + str(CORPUS_REBUILD), + "--verify-bundle", + str(inventory), + ] + ) + assert _json_stdout(verified_inventory)["status"] == "pass" + + source_manifest = json.loads((inventory / corpus_rebuild.SOURCE_MANIFEST_PATH).read_text(encoding="utf-8")) + assert len(source_manifest["sources"]) == 1 + source_row = source_manifest["sources"][0] + snapshot = inventory / source_row["snapshot_path"] + assert snapshot.read_bytes() == source_path.read_bytes() + assert source_row["artifact_sha256"] == _sha256(snapshot) + assert snapshot.name == f"{source_row['artifact_sha256']}.utf8" + + quote = ( + "A staged knowledge proposal remains non-canonical until explicit review and guarded apply both succeed." + ) + text_bytes = snapshot.read_bytes() + quote_bytes = quote.encode("utf-8") + quote_start = text_bytes.index(quote_bytes) + artifact_sha256 = _sha256(snapshot) + extractor_spec_bytes = ( + b"livingip synthetic lifecycle extractor specification\n" + b"name=synthetic-v3-source-lifecycle\n" + b"version=1.0.0\n" + b"quotes=exact-utf8-byte-locators\n" + b"approval=test-operator-supplied-not-human-proof\n" + ) + extractor_spec = _write_private(private_root / "extractor.spec", extractor_spec_bytes) + extraction_manifest = { + "schema": compiler.MANIFEST_SCHEMA_V3, + "artifact_sha256": artifact_sha256, + "extracted_text_sha256": artifact_sha256, + "extractor": { + "name": "synthetic-v3-source-lifecycle", + "version": "1.0.0", + "spec_sha256": hashlib.sha256(extractor_spec_bytes).hexdigest(), + }, + "agent_id": v3_contract.AGENT_ID, + "source": { + "identity": f"document:sha256:{artifact_sha256}", + "source_key": "canonical_review_boundary", + "source_type": "article", + "title": "Canonical review boundary", + "locator": f"artifact://sha256/{artifact_sha256}", + "access_scope": "collective_internal", + "ingestion_origin": "operator_upload", + "provenance_status": "verified", + "source_class": "primary_record", + "auto_promotion_policy": "human_required", + "captured_at": "2026-07-19T00:00:00+00:00", + }, + "claims": [ + { + "claim_key": "review_and_apply_precede_canonical_state", + "type": "empirical", + "proposition": ( + "A staged knowledge proposal is not canonical until explicit review and guarded apply succeed." + ), + "body_markdown": "The exact retained source states both gates and their order.", + "scope": "The exact content-addressed source snapshot used by this disposable lifecycle.", + "access_scope": "collective_internal", + "owner_agent_id": None, + "status": "active", + "revision_criteria": "Revise if the exact source bytes or guarded apply contract changes.", + "revision_kind": "original", + "supersedes_id": None, + "tags": ["genesis", "provenance", "review"], + "evidence": [ + { + "evidence_key": "exact_review_boundary_sentence", + "quote": quote, + "polarity": "supports", + "locator": { + "unit": "utf8_bytes", + "start": quote_start, + "end": quote_start + len(quote_bytes), + }, + "rationale": "The exact byte-bound sentence directly supports the bounded proposition.", + } + ], + "assessment": { + "support_tier": "strong", + "challenge_tier": "none", + "overall_state": "support_dominant", + "rationale": "One exact verified primary-record sentence directly supports the claim.", + "supersedes_assessment_id": None, + }, + } + ], + } + manifest_path = _write_private( + private_root / "synthetic-extraction-manifest.json", + json.dumps(extraction_manifest, indent=2, sort_keys=True) + "\n", + ) + compiled_bundle_path = private_root / "compiled-v3-bundle.json" + compiled = _run( + [ + sys.executable, + str(SOURCE_COMPILER), + "--artifact", + str(snapshot), + "--text", + str(snapshot), + "--manifest", + str(manifest_path), + "--extractor-spec", + str(extractor_spec), + "--output", + str(compiled_bundle_path), + ] + ) + assert _json_stdout(compiled)["status"] == "pending_review" + compiled_bundle = json.loads(compiled_bundle_path.read_text(encoding="utf-8")) + child = compiled_bundle["strict_child_proposal"] + proposal_id = child["id"] + apply_payload = child["payload"]["apply_payload"] + assert compiled_bundle["hashes"]["artifact_sha256"] == artifact_sha256 + assert apply_payload["sources"][0]["content_hash"] == artifact_sha256 + + source_container_id = _start_postgres(source_container, DATABASE, run_id, staging_target=True) + source_started = True + + genesis_test.docker_psql(source_container, DATABASE, v3_contract.BOOTSTRAP_SQL) + genesis_test.docker_psql(source_container, DATABASE, worker_test._supplemental_bootstrap()) + genesis_test.docker_psql( + source_container, + DATABASE, + (f"alter role kb_review login password '{password}';alter role kb_apply login password '{password}';"), + ) + genesis_test.docker_psql( + source_container, + DATABASE, + worker_test.PREREQUISITES.read_text(encoding="utf-8"), + ) + + genesis_dump = private_root / "baseline-genesis.dump" + genesis_manifest = private_root / "baseline-genesis-manifest.jsonl" + _capture_dump(source_container, DATABASE, genesis_dump) + _capture_manifest(source_container, DATABASE, genesis_manifest) + + genesis_test.docker_psql(source_container, DATABASE, "alter role kb_apply nologin;") + genesis_test.docker_psql( + source_container, + DATABASE, + worker_test.MIGRATION.read_text(encoding="utf-8"), + ) + genesis_test.docker_psql( + source_container, + DATABASE, + worker_test.PREREQUISITES.read_text(encoding="utf-8"), + ) + + secrets_file = _write_private( + private_root / "database-secrets.env", + f"KB_APPLY_DB_PASSWORD={password}\nKB_REVIEW_DB_PASSWORD={password}\n", + ) + staged = _run( + [ + sys.executable, + str(PROPOSAL_STAGER), + "--container", + source_container, + "--db", + DATABASE, + "--compiled-v3-bundle", + str(compiled_bundle_path), + ] + ) + stage_status = _json_stdout(staged) + assert stage_status == { + "bound_container_id": source_container_id, + "contract_version": 3, + "created": True, + "payload_sha256": child["payload_sha256"], + "proposal_id": proposal_id, + "status": "pending_review", + } + + reviewed = _run( + [ + sys.executable, + str(REVIEW_PACKET), + "--proposal-id", + proposal_id, + "--format", + "json", + "--secrets-file", + str(secrets_file), + "--container", + source_container, + "--db", + DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_review", + ] + ) + packets = _json_stdout(reviewed)["packets"] + assert len(packets) == 1 + packet = packets[0] + assert packet["proposal_id"] == proposal_id + assert packet["review_state"] == "needs_human_review" + assert packet["strict_review_material"]["rows"] == { + key: apply_payload[key] + for key in ("claims", "sources", "evidence", "edges", "assessments", "reasoning_tools") + } + assert packet["strict_review_material"]["normalization_manifest"] == apply_payload["normalization_manifest"] + + approved = _run( + [ + sys.executable, + str(APPROVER), + proposal_id, + "--reviewed-by", + "m3taversal", + "--review-note", + SYNTHETIC_REVIEW_NOTE, + "--secrets-file", + str(secrets_file), + "--container", + source_container, + "--db", + DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_review", + ] + ) + approval_status = _json_stdout(approved) + assert approval_status["id"] == proposal_id + assert approval_status["status"] == "approved" + assert approval_status["reviewed_by_handle"] == "m3taversal" + + receipts = private_root / "apply-receipts" + failure_state = private_root / "apply-worker-state.json" + applied = _run( + [ + sys.executable, + str(APPLY_WORKER), + "--enable", + "--contract-version", + "3", + "--proposal-id", + proposal_id, + "--max-per-tick", + "1", + "--secrets-file", + str(secrets_file), + "--failure-state-file", + str(failure_state), + "--receipt-dir", + str(receipts), + "--container", + source_container, + "--db", + DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_apply", + ] + ) + assert f"applied {proposal_id} (approve_claim)" in applied.stdout + assert json.loads(failure_state.read_text(encoding="utf-8")) == {} + failure_state.unlink() + replay_receipt = receipts / f"{proposal_id}.json" + receipt = json.loads(replay_receipt.read_text(encoding="utf-8")) + assert receipt["proposal"]["reviewed_by_handle"] == "m3taversal" + assert receipt["canonical_rows"]["sources"][0]["content_hash"] == artifact_sha256 + assert receipt["canonical_rows"]["claims"][0]["proposition"] == extraction_manifest["claims"][0]["proposition"] + + material_path = private_root / "entry-0001.json" + exported = _run( + [ + sys.executable, + str(MATERIAL_EXPORTER), + proposal_id, + "--replay-receipt", + str(replay_receipt), + "--sequence", + "1", + "--output", + str(material_path), + "--secrets-file", + str(secrets_file), + "--container", + source_container, + "--db", + DATABASE, + "--host", + "127.0.0.1", + "--role", + "kb_apply", + ] + ) + assert exported.stdout.strip() == f"replay material created: proposal={proposal_id} sequence=1" + material = json.loads(material_path.read_text(encoding="utf-8")) + assert material["contract_version"] == genesis_rebuild.MATERIAL_CONTRACT_VERSION + assert material["applied_proposal"]["payload"]["apply_payload"]["contract_version"] == 3 + assert material["approved_proposal"]["review_note"] == SYNTHETIC_REVIEW_NOTE + assert material["approval_snapshot"]["reviewed_by_db_role"] == "kb_review" + + final_manifest = private_root / "final-parity-manifest.jsonl" + _capture_manifest(source_container, DATABASE, final_manifest) + + _remove_container(source_container) + source_started = False + _assert_container_absent(source_container) + labeled_sources = _docker( + "container", + "ls", + "--all", + "--filter", + f"label={SOURCE_CONTAINER_LABEL}={run_id}", + "--format", + "{{.Names}}", + ) + assert labeled_sources.stdout.strip() == "" + + review_note_file = _write_private(private_root / "admission-review-note.txt", SYNTHETIC_REVIEW_NOTE + "\n") + ledger_path = private_root / "v3-genesis-ledger.json" + admission_path = private_root / "v3-admission-manifest.json" + built_ledger = _run( + [ + sys.executable, + str(LEDGER_BUILDER), + "--genesis-dump", + str(genesis_dump), + "--genesis-manifest", + str(genesis_manifest), + "--final-manifest", + str(final_manifest), + "--material", + str(material_path), + "--output-ledger", + str(ledger_path), + "--emit-admission-manifest", + "--admission-output", + str(admission_path), + "--reviewed-by", + "m3taversal", + "--reviewed-by-agent-id", + worker_test.REVIEWER_AGENT_ID, + "--reviewed-at", + "2026-07-19T00:00:00+00:00", + "--review-note-file", + str(review_note_file), + "--admission-scope", + "knowledge_commons", + "--admission-basis", + "reviewed_primary_source", + ] + ) + ledger_status = _json_stdout(built_ledger) + assert ledger_status["status"] == "pass" + assert ledger_status["action"] == "build" + ledger_sha256 = _sha256(ledger_path) + admission_sha256 = _sha256(admission_path) + admission = json.loads(admission_path.read_text(encoding="utf-8")) + assert admission["reviewed_by_handle"] == "m3taversal" + assert admission["review_note"] == SYNTHETIC_REVIEW_NOTE + "\n" + + verified_ledger = _run( + [ + sys.executable, + str(LEDGER_BUILDER), + "--genesis-dump", + str(genesis_dump), + "--genesis-manifest", + str(genesis_manifest), + "--verify-ledger", + str(ledger_path), + "--ledger-sha256", + ledger_sha256, + "--admission-manifest", + str(admission_path), + "--admission-manifest-sha256", + admission_sha256, + ] + ) + verify_status = _json_stdout(verified_ledger) + assert verify_status["status"] == "pass" + assert verify_status["action"] == "verify" + + reconstruction_receipt = private_root / "reconstruction-receipt.json" + rebuilt = _run( + [ + sys.executable, + str(GENESIS_REBUILDER), + "--genesis-dump", + str(genesis_dump), + "--genesis-manifest", + str(genesis_manifest), + "--ledger", + str(ledger_path), + "--ledger-sha256", + ledger_sha256, + "--admission-manifest", + str(admission_path), + "--admission-manifest-sha256", + admission_sha256, + "--database", + DATABASE, + "--container-prefix", + rebuild_prefix, + "--tmpfs-mb", + "384", + "--max-target-query-ms", + "5000", + "--max-target-source-ratio", + "100", + "--output", + str(reconstruction_receipt), + ], + timeout=360, + ) + assert _json_stdout(rebuilt)["status"] == "pass" + reconstruction = json.loads(reconstruction_receipt.read_text(encoding="utf-8")) + rebuild_container = reconstruction["container"]["name"] + assert reconstruction["status"] == "pass" + assert reconstruction["genesis_parity"]["status"] == "pass" + assert reconstruction["genesis_parity"]["problems"] == [] + assert reconstruction["v3_contract"]["status"] == "pass" + assert reconstruction["ledger"]["entries"][0]["proposal_exact"] is True + assert reconstruction["ledger"]["entries"][0]["canonical_rows_exact"] is True + assert reconstruction["admission"]["status"] == "review_metadata_supplied" + assert reconstruction["admission"]["canonical_retrieval_eligible"] is False + assert reconstruction["admission"]["cryptographically_authenticated"] is False + assert reconstruction["final_parity"]["status"] == "pass" + assert reconstruction["final_parity"]["problems"] == [] + assert reconstruction["cleanup"]["container_absent"] is True + assert reconstruction["safety"] == { + "network_access_available_to_container": False, + "persistent_postgres_storage_used": False, + "private_replay_material_emitted": False, + "production_database_touched": False, + "vps_gcp_telegram_or_live_db_used": False, + } + _assert_container_absent(rebuild_container) + rebuilt_labels = _docker( + "container", + "ls", + "--all", + "--filter", + f"label={genesis_rebuild.canonical_rebuild.CANARY_LABEL}={rebuild_container}", + "--format", + "{{.Names}}", + ) + assert rebuilt_labels.stdout.strip() == "" + _assert_private_files(private_root) + finally: + if source_started: + _remove_container(source_container) + + _assert_container_absent(source_container) + if rebuild_container is not None: + _assert_container_absent(rebuild_container) + orphaned = _docker( + "container", + "ls", + "--all", + "--filter", + f"label={SOURCE_CONTAINER_LABEL}={run_id}", + "--format", + "{{.Names}}", + ) + assert orphaned.stdout.strip() == ""