From 37c0ed091d01034dfd8bef74a38b865a68eedccd Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 04:11:29 +0200 Subject: [PATCH 1/5] feat(ops): bind snapshots to private GCP parity lifecycle --- .agents/skills/teleo-gcp-parity-ops/SKILL.md | 107 ++- .github/workflows/ci.yml | 7 + docs/gcp-kb-cloudsql-restore-runbook.md | 231 +++++- .../gcp-canonical-parity-live-20260715.json | 134 ++++ ...cp-canonical-parity-teardown-20260715.json | 36 + ops/attest_gcp_reasoning_compute.py | 172 ++++ ...capture_vps_canonical_postgres_snapshot.py | 190 ++++- ops/private_receipt_io.py | 34 + ...restore_gcp_generated_postgres_snapshot.py | 604 +++++++++++++- ops/verify_gcp_canonical_lifecycle.py | 754 ++++++++++++++++++ ops/verify_postgres_parity_manifest.py | 29 +- ops/verify_vps_canonical_snapshot_delta.py | 284 +++++++ tests/test_attest_gcp_reasoning_compute.py | 128 +++ ...capture_vps_canonical_postgres_snapshot.py | 137 ++++ ...restore_gcp_generated_postgres_snapshot.py | 421 +++++++++- tests/test_verify_gcp_canonical_lifecycle.py | 703 ++++++++++++++++ tests/test_verify_postgres_parity_manifest.py | 4 + ...est_verify_vps_canonical_snapshot_delta.py | 259 ++++++ 18 files changed, 4123 insertions(+), 111 deletions(-) create mode 100644 docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json create mode 100644 docs/reports/leo-working-state-20260709/gcp-canonical-parity-teardown-20260715.json create mode 100644 ops/attest_gcp_reasoning_compute.py create mode 100644 ops/private_receipt_io.py create mode 100644 ops/verify_gcp_canonical_lifecycle.py create mode 100644 ops/verify_vps_canonical_snapshot_delta.py create mode 100644 tests/test_attest_gcp_reasoning_compute.py create mode 100644 tests/test_verify_gcp_canonical_lifecycle.py create mode 100644 tests/test_verify_vps_canonical_snapshot_delta.py diff --git a/.agents/skills/teleo-gcp-parity-ops/SKILL.md b/.agents/skills/teleo-gcp-parity-ops/SKILL.md index 6d9191e..2a0b41f 100644 --- a/.agents/skills/teleo-gcp-parity-ops/SKILL.md +++ b/.agents/skills/teleo-gcp-parity-ops/SKILL.md @@ -11,27 +11,35 @@ Restore a copy of the VPS canonical Leo database to GCP, prove exact schema, row, role, extension, performance, and private-connectivity parity, then run the real GCP Leo read/reasoning path without Telegram sends or DB mutation. +The required tier is T3: a live private GCP staging restore, readback, +reasoning, compute attestation, and cleanup receipt. Local restore proof or an +access-gate report does not satisfy this tier. + ## Operator Paths -The direct alias is passwordless and was live-verified on 2026-07-14: +The direct alias was passwordless and live-verified on 2026-07-14: ```bash ssh teleo-gcp-staging ``` -It uses `/Users/user/.ssh/google_compute_engine` for the configured operator, +It uses `~/.ssh/google_compute_engine` for the configured operator, disables password and keyboard-interactive authentication, and does not store a -Google password. `sudo -n` also works. The route remains -firewall-source-dependent, so verify `ssh -o BatchMode=yes teleo-gcp-staging true` before a -long run instead of assuming retained access is current. +Google password. That is historical proof, not current reachability. The route +remains firewall-source-dependent. On +2026-07-15, both direct SSH and the private TCP route timed out; verify +`ssh -o BatchMode=yes teleo-gcp-staging true` before a long run and fail closed +if it does not return successfully. The intended secondary path is `.github/workflows/gcp-iap-operator.yml`, using short-lived GitHub OIDC, IAP, OS Login, and fixed reviewed operations. It is -merged but not bootstrapped: workflow run `29208215340` failed at auth with -`invalid_target` because provider `teleo-iap-operator` is absent, disabled, or -deleted. Do not call this path working until a live `status` run passes. +merged but not bootstrapped: the current 2026-07-15 check still lacks the +reviewed operator WIF/IAP authority. Provider `teleo-iap-operator` is absent, +disabled, or deleted (`invalid_target`), and the available artifact identity is intentionally not +a Cloud SQL/Compute operator. Do not call this path working until a live `status` run passes +with the intended operator identity. -Verified target: +Historically verified and intended target; reverify before any new lifecycle: - project `teleo-501523`; - VM `teleo-prod-1` in `europe-west6-a`; @@ -45,8 +53,9 @@ variable, then unset it. ## Two Different Databases -- Canonical collective knowledge is Cloud SQL database `teleo_canonical`, with - `public.*` and `kb_stage.*` tables. +- Canonical collective knowledge remains VPS PostgreSQL database `teleo`, with + `public.*` and `kb_stage.*` tables. Persistent Cloud SQL database + `teleo_canonical` is a staging copy and has not been promoted. - Hermes conversation continuity is `state.db` plus session JSONL files. The `leoclean-cloudsql-memory-sync.service` snapshot path copies this runtime memory; it does not populate or prove canonical claims, sources, evidence, @@ -54,7 +63,7 @@ variable, then unset it. Do not describe a passing Hermes memory sync as canonical KB parity. -## Current Verified State - 2026-07-14 +## Retained Historical Proof - 2026-07-14 - The newest captured VPS database has `39` tables and `52,167` rows, including claims `1837`, sources `4145`, claim evidence `4670`, claim edges `4916`, and @@ -110,17 +119,54 @@ Track these independently: Use: - `ops/capture_vps_canonical_postgres_snapshot.py` for a single source dump and - manifest; + manifest, with a required retained `--authorization-ref` and v2 provenance + receipt binding the exported snapshot identity plus dump/manifest hashes; it + requires identical healthy structured source-service states before/after + (`active/running`, positive PID, nonnegative restart count); - `ops/restore_gcp_generated_postgres_snapshot.py restore --execute` for a receipt-bound private-TLS restore into a bounded `teleo_clone_*` database; + pass the exact capture `--source-receipt`, and require its live Cloud SQL + control-plane check to prove public IP disabled before clone creation; it + writes `target-manifest.jsonl`, `restore-receipt.json`, and the generated + `gcp-private-connectivity.json` under + `/var/lib/teleo-gcp-restore-runs//`; - `ops/restore_gcp_generated_postgres_snapshot.py cleanup --execute` for exact clone cleanup with live-service and rollback readback; - `ops/postgres_parity_manifest.sql` for row/catalog/role/performance readback; -- `ops/verify_postgres_parity_manifest.py --scope gcp_staging` for exact parity; +- `ops/verify_postgres_parity_manifest.py --scope gcp_staging` for exact parity, + always retaining the canonical output name `gcp-parity.json`; - `scripts/run_gcp_generated_db_direct_claim_suite.py` for the adapter-free, read-only, no-send six-response replay against a generated `teleo_clone_*`. - `scripts/run_gcp_generated_db_blind_claim_canary.py` for the bounded ID-free claim challenge, source receipt, reasoning, no-write, and cleanup proof. +- `ops/attest_gcp_reasoning_compute.py` on the replay VM after blind reasoning + and before cleanup, binding the reasoning receipt SHA-256 to live GCE metadata + in `reasoning-compute-attestation.json`; +- a second authorized source capture after cleanup, followed by + `ops/verify_vps_canonical_snapshot_delta.py`, to retain the explicit + post-snapshot VPS delta in `source-delta-receipt.json`; +- `ops/verify_gcp_canonical_lifecycle.py` after cleanup to bind the exact source, + current source, source delta, restore, `gcp_staging` parity, ID-free reasoning, + reasoning-compute attestation, and cleanup receipts into one fail-closed T3 + lifecycle verdict. Pass `--max-postflight-age-seconds 900` and + `--max-lifecycle-age-seconds 3600`; the latter bounds both cleanup-to-verdict + age and restore-to-postflight span. + +With `TARGET_DB`, `RESTORE_RUN_ID`, and `PRIVATE_RUN_DIR` set as shown in the +runbook, run the replay-host attestation after the blind receipt passes and +before the receipt-bound cleanup: + +```bash +sudo python3 ops/attest_gcp_reasoning_compute.py \ + --reasoning-receipt "$PRIVATE_RUN_DIR/blind-reasoning-receipt.json" \ + --restore-run-id "$RESTORE_RUN_ID" \ + --target-db "$TARGET_DB" \ + --project teleo-501523 \ + --expected-compute-instance teleo-prod-1 \ + --expected-compute-zone europe-west6-a \ + --expected-private-network projects/teleo-501523/global/networks/teleo-staging-net \ + --output "$PRIVATE_RUN_DIR/reasoning-compute-attestation.json" +``` The replay must use the Hermes virtualenv, not system Python: @@ -129,8 +175,16 @@ The replay must use the Hermes virtualenv, not system Python: scripts/run_gcp_generated_db_direct_claim_suite.py ... ``` -The replay is not complete until the generated clone, temporary profile, -children, upload/run directories, and any temporary client are absent. +The lifecycle is not complete until the generated clone, temporary profile, +children, transient upload/bundle directories, and any temporary client are +absent. Preserve the private receipt run directory and every useful lifecycle +input, supporting receipt, and verifier output; do not treat that evidence as +transient cleanup. The canonical restore streams through `pg_restore` and +creates no GCS import object. + +Static inspection or a dry run is insufficient for these helpers: verify a new +capture against the live VPS, run the focused tests, and require a passing live +lifecycle receipt before describing the workflow as current parity. ## Safety And Claim Ceiling @@ -147,12 +201,17 @@ delivery, GCP canonical mutation, ongoing replication, or production cutover. ## Current Access And Next Action -Direct SSH and passwordless sudo are currently working. Local `gcloud` still -requires account reauthentication for control-plane metadata, so the current -private-IP/TLS proof comes from a live database connection while the public-IP- -disabled control-plane receipt remains dated 2026-07-12. +As of 2026-07-15, direct SSH and private TCP timed out, local `gcloud` for +`billy@livingip.xyz` requires reauthentication, and the intended operator +WIF/IAP authority is unavailable. The artifact-builder WIF identity must not be +expanded or mistaken for database-operator access. Current retained T2/source +and local restore proof does not satisfy the required T3 lifecycle. -The next production decision is not another restore drill. It is whether to -promote a newly verified snapshot into persistent GCP `teleo_canonical` and -repoint the production read adapter. Until that decision is explicit, keep GCP -classified as staging and do not expose Cloud SQL publicly. +The next action after an authorized operator route is restored is: newest +authorized VPS capture -> disposable private `teleo_clone_*` restore -> +generated private-connectivity receipt -> `gcp-parity.json` -> no-send +m3taversal blind reasoning -> live GCE reasoning attestation -> receipt-bound +clone cleanup -> postflight VPS capture/delta -> eight-receipt lifecycle verdict +within the 900/3600-second freshness bounds. Do not promote +`teleo_canonical`, expose Cloud SQL publicly, send Telegram, or mutate canonical +knowledge. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91dada8..ccb4925 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,17 +42,21 @@ jobs: lib/llm.py \ lib/post_extract.py \ ops/apply_gcp_iam_split.py \ + ops/attest_gcp_reasoning_compute.py \ ops/capture_vps_canonical_postgres_snapshot.py \ ops/check_gcp_infra_readiness.py \ ops/run_gcp_infra_execute_canary.py \ ops/apply_gcp_runtime_baseline.py \ ops/check_gcp_service_communications.py \ ops/plan_gcp_iam_split.py \ + ops/private_receipt_io.py \ ops/redact_sqlite_postgres_restore_canary.py \ ops/restore_gcp_generated_postgres_snapshot.py \ ops/sqlite_to_postgres_dump.py \ ops/verify_gcp_cloudsql_restore_readback.py \ + ops/verify_gcp_canonical_lifecycle.py \ ops/verify_postgres_parity_manifest.py \ + ops/verify_vps_canonical_snapshot_delta.py \ telegram/approvals.py \ hermes-agent/leoclean-bin/kb_tool.py \ hermes-agent/leoclean-bin/cloudsql_memory_tool.py \ @@ -72,6 +76,7 @@ jobs: scripts/working_leo_m3taversal_oos_benchmark.py \ scripts/working_leo_open_ended_benchmark.py \ tests/test_agent_routing.py \ + tests/test_attest_gcp_reasoning_compute.py \ tests/test_assemble_telegram_visible_direct_claim_capture_receipt.py \ tests/test_build_working_leo_m3taversal_outcome_sandbox.py \ tests/test_decision_engine_replay.py \ @@ -96,6 +101,8 @@ jobs: tests/test_verify_leo_db_first_oos_canary.py \ tests/test_compile_kb_source_packet.py \ tests/test_verify_postgres_parity_manifest.py \ + tests/test_verify_gcp_canonical_lifecycle.py \ + tests/test_verify_vps_canonical_snapshot_delta.py \ tests/test_working_leo_m3taversal_oos_benchmark.py \ tests/test_working_leo_open_ended_benchmark.py \ tests/test_phase1b_end_to_end.py \ diff --git a/docs/gcp-kb-cloudsql-restore-runbook.md b/docs/gcp-kb-cloudsql-restore-runbook.md index 2015807..f22aa21 100644 --- a/docs/gcp-kb-cloudsql-restore-runbook.md +++ b/docs/gcp-kb-cloudsql-restore-runbook.md @@ -53,12 +53,21 @@ python3 ops/capture_vps_canonical_postgres_snapshot.py \ --ssh-target root@77.42.65.182 \ --ssh-key ~/.ssh/livingip_hetzner_20260604_ed25519 \ --run-id canonical- \ + --authorization-ref codex-delegation:THREAD_ID \ --output-dir ``` -The capture fails closed if the source service changes while it runs. It +Replace `THREAD_ID` with the exact retained authorization reference for the +capture. The v2 receipt binds that reference, exported snapshot ID, TXID +snapshot, WAL LSN, source system identifier, dump hash, manifest hash, reviewed +manifest-SQL hash, and source-context hash. A dump and manifest without this +passing receipt are not eligible for GCP restore. + +The capture fails closed unless the structured source-service states before +and after the snapshot are identical and healthy: `ActiveState=active`, +`SubState=running`, a positive `MainPID`, and a nonnegative `NRestarts`. It retains a private custom dump, dump SHA-256, object TOC, catalog/data manifest, -and before/after service state. It never restarts Leo or writes to the source +and both service-state objects. It never restarts Leo or writes to the source database. Prove that this exact snapshot can rebuild a blank Postgres target before using @@ -79,16 +88,70 @@ then removes the container and proves it is absent. A passing local receipt is the exact-recovery preflight; it is not semantic recompilation from raw source documents. -Run `ops/postgres_parity_manifest.sql` against the isolated restored target, -then compare source and target: +Choose one bounded suffix and use it consistently on the GCP replay VM. The +restore helper retains its mode-0700 evidence directory at the fixed private +root: ```bash -python3 ops/verify_postgres_parity_manifest.py \ +RUN_SUFFIX=20260715t010000z +TARGET_DB="teleo_clone_${RUN_SUFFIX}" +RESTORE_RUN_ID="gcp-restore-${RUN_SUFFIX}" +PRIVATE_RUN_DIR="/var/lib/teleo-gcp-restore-runs/${RESTORE_RUN_ID}" +``` + +Use a generated target database such as `teleo_clone_`. Never import a +drill into `teleo`, `teleo_kb`, or `teleo_canonical`. Database isolation does +not isolate cluster-global roles or extensions, so verify those separately and +do not run the Docker-only gate bootstrap against the shared Cloud SQL instance. + +The canonical restore command must include the v2 capture receipt. It also runs +a live Cloud SQL control-plane preflight before resolving the database secret or +creating the clone, and fails closed unless the exact instance is runnable, +PostgreSQL 16, private-networked, bound to the expected RFC1918 host, and has +public IP disabled. A GCE metadata preflight independently binds the operator +runtime to `teleo-prod-1` in `europe-west6-a` on `teleo-staging-net`; a +hard-coded compute label is not accepted: + +```bash +sudo python3 ops/restore_gcp_generated_postgres_snapshot.py restore \ + --execute \ + --target-db "$TARGET_DB" \ + --run-id "$RESTORE_RUN_ID" \ + --dump /teleo-canonical.dump \ + --expected-dump-sha256 \ + --source-manifest /source-manifest.jsonl \ + --source-receipt /receipt.json \ + --expected-source-receipt-sha256 \ + --expected-capture-authorization-ref codex-delegation:THREAD_ID \ + --project teleo-501523 \ + --cloudsql-instance teleo-pgvector-standby \ + --expected-private-network projects/teleo-501523/global/networks/teleo-staging-net \ + --expected-compute-instance teleo-prod-1 \ + --expected-compute-zone europe-west6-a \ + --expected-source-ssh-target root@77.42.65.182 \ + --expected-source-container teleo-pg \ + --expected-source-database teleo \ + --expected-source-service leoclean-gateway.service +``` + +The restore writes both `target-manifest.jsonl` and +`gcp-private-connectivity.json` as mode-0600 files in its private run +directory. The latter is composed from live GCE metadata, Cloud SQL +control-plane checks, and the private TLS database identity; do not hand-compose +the connectivity receipt. The restore also records identical healthy +before/after state for `leoclean-gcp-prod-parallel.service`, including its PID +and restart count. + +Compare the captured target manifest and generated connectivity receipt to the +authorized source: + +```bash +sudo python3 ops/verify_postgres_parity_manifest.py \ --source /source-manifest.jsonl \ - --target /target-manifest.jsonl \ + --target "$PRIVATE_RUN_DIR/target-manifest.jsonl" \ --scope gcp_staging \ - --connectivity-proof /gcp-private-connectivity.json \ - --output /gcp-parity.json + --connectivity-proof "$PRIVATE_RUN_DIR/gcp-private-connectivity.json" \ + --output "$PRIVATE_RUN_DIR/gcp-parity.json" ``` The verifier checks all table row counts and collation-independent row hashes, @@ -98,14 +161,100 @@ password-free application-role attributes, and bounded query timings. In GCP scope it also requires a receipt proving a staging compute source, a private server address, TLS, and public-IP-disabled instance metadata. -Use a generated target database such as `teleo_clone_`. Never import a -drill into `teleo`, `teleo_kb`, or `teleo_canonical`. Database isolation does -not isolate cluster-global roles or extensions, so verify those separately and -do not run the Docker-only gate bootstrap against the shared Cloud SQL instance. +After the parity verifier passes, run the no-send m3taversal blind reasoning +replay from staging compute against that generated database, attest the replay +host from live GCE metadata, and only then delete the generated database. This +canonical path streams the custom dump through `pg_restore`; it does not create +a GCS import object. -After the parity verifier passes, run the no-send operator composition replay from -staging compute against that generated database. Only then delete the generated -database and uploaded import object and retain cleanup readback. +Run the bounded ID-free reasoning canary with the generated parity receipt. The +script re-executes itself in the Hermes virtualenv when that runtime is present: + +```bash +sudo python3 scripts/run_gcp_generated_db_blind_claim_canary.py \ + --execute \ + --target-db "$TARGET_DB" \ + --cloudsql-tool hermes-agent/leoclean-bin/cloudsql_memory_tool.py \ + --manifest-sql ops/postgres_parity_manifest.sql \ + --parity-receipt "$PRIVATE_RUN_DIR/gcp-parity.json" \ + --output "$PRIVATE_RUN_DIR/blind-reasoning-receipt.json" \ + --run-id "gcp-blind-claim-${RUN_SUFFIX}" +``` + +Before cleanup, bind that reasoning receipt to live metadata from the same GCE +replay host: + +```bash +sudo python3 ops/attest_gcp_reasoning_compute.py \ + --reasoning-receipt "$PRIVATE_RUN_DIR/blind-reasoning-receipt.json" \ + --restore-run-id "$RESTORE_RUN_ID" \ + --target-db "$TARGET_DB" \ + --project teleo-501523 \ + --expected-compute-instance teleo-prod-1 \ + --expected-compute-zone europe-west6-a \ + --expected-private-network projects/teleo-501523/global/networks/teleo-staging-net \ + --output "$PRIVATE_RUN_DIR/reasoning-compute-attestation.json" +``` + +After the compute attestation passes, run the receipt-bound cleanup command +retained by the restore. Then capture the VPS source again under a distinct +postflight authorization reference. Run this capture from the same trusted +source-access lane used for the baseline; it need not run on the GCP replay VM. +The postflight capture must complete after cleanup, so the final verdict can +state whether the VPS changed after the restored snapshot boundary. Compare the +two source captures: + +```bash +python3 ops/capture_vps_canonical_postgres_snapshot.py \ + --execute \ + --ssh-target root@77.42.65.182 \ + --ssh-key ~/.ssh/livingip_hetzner_20260604_ed25519 \ + --run-id canonical-${RUN_SUFFIX}-postflight \ + --authorization-ref codex-delegation:THREAD_ID:postflight \ + --output-dir +``` + +```bash +sudo python3 ops/verify_vps_canonical_snapshot_delta.py \ + --baseline-receipt /receipt.json \ + --baseline-manifest /source-manifest.jsonl \ + --baseline-authorization-ref codex-delegation:THREAD_ID \ + --current-receipt /receipt.json \ + --current-manifest /source-manifest.jsonl \ + --current-authorization-ref codex-delegation:THREAD_ID:postflight \ + --output "$PRIVATE_RUN_DIR/source-delta-receipt.json" +``` + +Finally, verify that all eight receipts belong to one bound lifecycle. This +verifier rejects local-scope parity, stale or mismatched source hashes, a +postflight source capture that predates reasoning or cleanup, hand-authored +reasoning booleans without real retrieval/tool receipts, an unattested replay +host, compute or Cloud SQL topology drift, and any cleanup receipt that leaves +the clone present. The postflight capture may be at most 900 seconds old at +verification; the GCP lifecycle age and restore-to-postflight span must each be +at most 3600 seconds: + +```bash +sudo python3 ops/verify_gcp_canonical_lifecycle.py \ + --source-receipt /receipt.json \ + --current-source-receipt /receipt.json \ + --source-delta-receipt "$PRIVATE_RUN_DIR/source-delta-receipt.json" \ + --restore-receipt "$PRIVATE_RUN_DIR/restore-receipt.json" \ + --parity-receipt "$PRIVATE_RUN_DIR/gcp-parity.json" \ + --reasoning-receipt "$PRIVATE_RUN_DIR/blind-reasoning-receipt.json" \ + --reasoning-compute-receipt "$PRIVATE_RUN_DIR/reasoning-compute-attestation.json" \ + --cleanup-receipt "$PRIVATE_RUN_DIR/cleanup-receipt.json" \ + --max-postflight-age-seconds 900 \ + --max-lifecycle-age-seconds 3600 \ + --output "$PRIVATE_RUN_DIR/lifecycle-verification.json" +``` + +Retain the private receipt run directory and every useful lifecycle input, +supporting receipt, and verifier output. Remove only transient upload/bundle +directories, temporary Hermes profiles, helper clients, and child processes +after their absence is verified. There is no canonical-path GCS object to +delete. Do not remove the evidence directory until its useful receipts are +integrated or explicitly rejected. ## Legacy SQLite Source Backup Canary @@ -236,11 +385,15 @@ A successful restore or replication canary must retain: - dump timestamp or replication slot timestamp; - source schema/database name. - transfer proof: - - dump object path in a versioned bucket, or logical replication subscription details; + - for the canonical path, the v2 capture receipt, dump SHA-256, and exact + private staging path used by streaming `pg_restore`; + - for a legacy import or replication path, a versioned dump-object generation + or logical replication subscription details; - row/table counts before import where available. - target proof: - - `teleo-pgvector-standby` readback; - - `teleo_kb` database readback; + - `teleo-pgvector-standby` control-plane readback; + - generated `teleo_clone_*` identity for canonical parity, or `teleo_kb` only + for the explicitly legacy SQLite drill; - extension readback for `vector` if the restored schema needs pgvector; - representative query readback for core KB tables. - failure boundary: @@ -295,28 +448,26 @@ Retain only redacted connection metadata. Do not commit or paste credentials. ## Current Blocker -As of 2026-07-11, the canonical Postgres exported-snapshot capture and isolated -local restore parity pass. Live GCP restore and staging replay do not. +As of 2026-07-15, a newest authorized canonical Postgres capture and isolated +local restore can pass, but the required T3 live private GCP lifecycle cannot be +started from the current operator route: -- GitHub WIF works for `sa-artifact-builder`, but that identity is intentionally - limited to Artifact Registry and cannot inspect or mutate Cloud SQL/Compute. -- The configured `sa-teleo-readiness` and `sa-teleo-restore-drill` identities - return IAM 404 and do not exist. -- The local privileged `billy@livingip.xyz` gcloud session requires password - reauthentication. No password was entered or inspected. -- Direct VM SSH is closed to the current egress `/32`; IAP requires the same - privileged GCP authentication. +- direct VM SSH and the private TCP route timed out; +- local `gcloud` for `billy@livingip.xyz` requires reauthentication; no + password was entered or inspected; +- the intended operator WIF/IAP authority is unavailable because provider + `teleo-iap-operator` is absent, disabled, or deleted; +- the working artifact-builder WIF identity is intentionally limited and must + not be expanded or mistaken for Cloud SQL/Compute operator access. -That is why the readiness checker still reports: +The authorized recovery is a local +`gcloud auth login billy@livingip.xyz --force` without sharing credentials, or +administrator convergence of the reviewed operator WIF/IAP bootstrap. Until a +route passes live readback, keep the claim at T2/source-local proof; do not +promote staging or expose Cloud SQL publicly. -- `kb_source_restore_access = blocked` -- `kb_restore_or_replication = blocked` - -The immediate operator CTA is to complete -`gcloud auth login billy@livingip.xyz --force` locally without sharing the -password, or apply the reviewed IAM split with an authorized GCP administrator. -The next non-user action is: - -canonical `teleo` snapshot -> generated Cloud SQL database -> full parity and -private-connectivity verifier -> no-send Cory composition replay from staging -compute -> delete the generated database/object -> retain cleanup proof. +Once access is restored, run: newest authorized `teleo` snapshot -> disposable +private `teleo_clone_*` restore -> generated connectivity proof -> +`gcp-parity.json` -> no-send m3taversal blind reasoning -> live GCE reasoning +attestation -> receipt-bound clone cleanup -> postflight VPS capture and delta +-> eight-receipt lifecycle verdict within the 900/3600-second freshness bounds. diff --git a/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json b/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json new file mode 100644 index 0000000..258e20a --- /dev/null +++ b/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json @@ -0,0 +1,134 @@ +{ + "artifact": "gcp_canonical_parity_live_attempt", + "schema": "livingip.gcpCanonicalParityLiveAttempt.v1", + "generated_at_utc": "2026-07-15T01:50:20Z", + "status": "gated_external", + "required_tier": "T3_live_private_gcp_staging", + "current_tier": "T2_current_source_and_isolated_restore_plus_live_access_gate", + "current_canary": "Restore the newest authorized canonical snapshot into one private generated Cloud SQL database, prove exact parity and ID-free Leo reasoning from staging compute, then drop the generated database and verify absence.", + "current_source": { + "run_id": "canonical-20260715t014930z", + "captured_at_utc": "2026-07-15T01:49:39.104116+00:00", + "capture_authorization_ref": "codex-delegation:019f47af-f2cc-7c10-b928-a9283afa2621", + "receipt_sha256": "7d96b052591d36df3b67dc6291b0f74a7bbeb9602c9cc424779e7c21194d59a5", + "dump_sha256": "6c56a841c1159a17f9404c2d63e3e93ae8ff55e842bf311d6d1bff21e9b95c9a", + "manifest_sha256": "39f06c6b9fc806a89ecd19294be29c58c499c65234611c7a483ddefa3c78d7d3", + "provenance_binding_sha256": "fb587093a76b61af670b7d968d32b78d6a87572faa78ca1ad93ef6cba1c9b776", + "source_system_identifier": "7649789040005668902", + "table_count": 39, + "total_rows": 52167, + "service": { + "active_state": "active", + "sub_state": "running", + "main_pid": 347406, + "restart_count": 0, + "unchanged": true + }, + "production_database_mutated": false, + "telegram_send_attempted": false + }, + "source_stability": { + "baseline_run_id": "canonical-20260715t014814z", + "current_run_id": "canonical-20260715t014930z", + "receipt_sha256": "d761a2b8b0313ee03c9d01e50fcca709af779578281cf10de5aed4709e914d72", + "status": "pass", + "delta_detected": false, + "table_count": 39, + "total_rows": 52167, + "total_row_delta": 0, + "changed_tables": [], + "changed_structures": [], + "table_fingerprint_sha256": "f50d86fc3ce699f602bdc0029485bd901fd98bd8114eecd5dcc4346a0f52112a", + "structure_fingerprint_sha256": "61bf482330a652ebdb419de3fb78196f2c7d25d1a482b100f385e8102c8e8d52" + }, + "isolated_restore": { + "receipt_sha256": "181901afc91bebee7c343500c601ed1e5b7a5e42a5b81583d445109bfc680d86", + "status": "pass", + "source_table_count": 39, + "target_table_count": 39, + "source_total_rows": 52167, + "target_total_rows": 52167, + "table_mismatches": [], + "structural_hashes_equal": true, + "constraint_hash": "8b26d26fda56f09325e0603aae48051da08dfc6c35322fa88880acdabb8ad859", + "application_role_mismatches": {}, + "target_extra_application_roles": [], + "required_extension_mismatches": {}, + "performance_problems": [], + "network_mode": "none", + "persistent_postgres_storage_used": false, + "container_absent_after_test": true + }, + "permission_profile": { + "approval_policy": "never", + "sandbox_mode": "danger-full-access", + "network": "enabled" + }, + "config_readback": { + "gcloud_sdk_version": "551.0.0", + "configured_account": "billy@livingip.xyz", + "configured_project": "teleo-501523", + "configured_account_status": "failed_noninteractive_reauthentication", + "application_default_credentials_status": "failed_noninteractive_reauthentication", + "expected_cloudsql_instance_not_currently_verified": "teleo-pgvector-standby", + "expected_private_network_not_currently_verified": "teleo-staging-net", + "expected_private_address_not_currently_verified": "10.61.0.3", + "expected_staging_compute_not_currently_verified": "teleo-prod-1/europe-west6-a", + "public_ip_disabled_currently_verified": false, + "generated_database_inventory_currently_verified": false + }, + "attempted_no_approval_routes": [ + "configured privileged gcloud identity", + "three other cached gcloud identities", + "application-default credentials", + "direct SSH to the retained staging address", + "local gcloud IAP SSH dry run", + "local and VPS TCP probes to the expected private Cloud SQL address", + "fixed GitHub IAP status workflow receipt", + "five GitHub readiness workflow receipts across candidate service accounts", + "current successful Artifact Registry-only WIF workflow" + ], + "exact_gate": "The privileged local Google OAuth session requires human reauthentication, the fixed IAP workflow names an unavailable teleo-iap-operator WIF provider, and no existing noninteractive service-account route can obtain a Cloud SQL/Compute-capable token. Current Cloud SQL topology and generated-database inventory cannot be read, so the private restore cannot start.", + "why_autonomous_repair_stops": "Google reauthentication may require a password, MFA, passkey, or consent that Codex must not retrieve or enter. Recreating or rebinding the WIF/IAP identities requires an already-authorized GCP administrator, and no safe noninteractive identity currently has that authority.", + "next_non_user_action": "After operator-controlled Google reauthentication or administrator convergence of the reviewed least-privilege IAP operator, read the current Cloud SQL and GCE topology, stream the receipt-bound dump to a teleo_clone_* database from teleo-prod-1, run exact parity, blind reasoning, live GCE metadata attestation, cleanup, postflight source capture, and the eight-receipt lifecycle verifier.", + "gcp_effects_from_this_attempt": { + "cloudsql_database_created": false, + "gcp_resource_created": false, + "gcp_resource_modified": false, + "public_ip_exposure_changed": false, + "staging_promoted": false, + "telegram_sent": false + }, + "proof_receipts": { + "retained_private_runtime_artifacts": true, + "source_receipt": { + "filename": "receipt.json", + "sha256": "7d96b052591d36df3b67dc6291b0f74a7bbeb9602c9cc424779e7c21194d59a5" + }, + "source_manifest": { + "filename": "source-manifest.jsonl", + "sha256": "39f06c6b9fc806a89ecd19294be29c58c499c65234611c7a483ddefa3c78d7d3" + }, + "source_delta": { + "filename": "source-stability-delta.json", + "sha256": "d761a2b8b0313ee03c9d01e50fcca709af779578281cf10de5aed4709e914d72" + }, + "isolated_restore": { + "filename": "local-rebuild-receipt.json", + "sha256": "181901afc91bebee7c343500c601ed1e5b7a5e42a5b81583d445109bfc680d86" + }, + "route_recheck": { + "filename": "route-recheck.json", + "sha256": "0fda5e6d58d72dcb084dc277bc59312bf62c002c728e249a9f16e79ae096037b" + } + }, + "strongest_claim_allowed": "The newest authorized VPS snapshot is stable and exactly reconstructs in an isolated PostgreSQL 16 canary; current private GCP restore, reasoning, and cleanup are not proven because the live operator route is externally gated.", + "not_proven": [ + "current Cloud SQL topology or public-IP-disabled state", + "current generated-database inventory", + "private staging-to-Cloud-SQL TLS connectivity", + "current Cloud SQL restore parity", + "blind Leo reasoning against the generated Cloud SQL database", + "generated Cloud SQL database teardown" + ] +} diff --git a/docs/reports/leo-working-state-20260709/gcp-canonical-parity-teardown-20260715.json b/docs/reports/leo-working-state-20260709/gcp-canonical-parity-teardown-20260715.json new file mode 100644 index 0000000..490afad --- /dev/null +++ b/docs/reports/leo-working-state-20260709/gcp-canonical-parity-teardown-20260715.json @@ -0,0 +1,36 @@ +{ + "artifact": "gcp_canonical_parity_attempt_teardown", + "schema": "livingip.gcpCanonicalParityAttemptTeardown.v1", + "generated_at_utc": "2026-07-15T01:50:20Z", + "status": "no_cloud_resource_created", + "required_tier": "T3_live_private_gcp_staging", + "required_tier_met": false, + "cloudsql_clone_created": false, + "cloudsql_clone_dropped": false, + "cloudsql_clone_absence_readback": "not_applicable_no_create", + "gcs_object_created": false, + "compute_resource_created": false, + "cleanup_required": false, + "cleanup_performed": false, + "orphan_risk_from_this_attempt": false, + "public_ip_exposure_changed": false, + "staging_promoted": false, + "local_restore_container": { + "name": "teleo-canonical-rebuild-20260715015004-4ec8d1daac32", + "force_remove_exit_code": 0, + "container_absent": true, + "persistent_storage_used": false, + "network_access_available": false + }, + "current_gcp_orphan_inventory_verified": false, + "exact_gate": "No authenticated Cloud SQL/Compute-capable route was available, so execution stopped before any GCP resource or generated database was created.", + "proof_paths": { + "live_attempt": "docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json", + "route_recheck_filename": "route-recheck.json", + "route_recheck_sha256": "0fda5e6d58d72dcb084dc277bc59312bf62c002c728e249a9f16e79ae096037b", + "local_cleanup_receipt_filename": "local-rebuild-receipt.json", + "local_cleanup_receipt_sha256": "181901afc91bebee7c343500c601ed1e5b7a5e42a5b81583d445109bfc680d86", + "runtime_receipts_retained_privately": true + }, + "strongest_claim_allowed": "This attempt introduced no GCP cleanup debt and its isolated local restore container was removed; it is not a teardown receipt for a live disposable Cloud SQL database." +} diff --git a/ops/attest_gcp_reasoning_compute.py b/ops/attest_gcp_reasoning_compute.py new file mode 100644 index 0000000..e9d5124 --- /dev/null +++ b/ops/attest_gcp_reasoning_compute.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Bind a no-send reasoning receipt to live GCE metadata on the replay host.""" + +from __future__ import annotations + +import argparse +import json +import stat +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +try: + from .private_receipt_io import write_private_json + from .restore_gcp_generated_postgres_snapshot import ( + DEFAULT_COMPUTE_INSTANCE, + DEFAULT_COMPUTE_ZONE, + DEFAULT_PRIVATE_NETWORK, + DEFAULT_PROJECT, + RUN_ID_RE, + TARGET_DATABASE_RE, + gce_compute_identity, + sha256_file, + ) +except ImportError: # pragma: no cover - direct script execution on the GCP VM + from private_receipt_io import write_private_json + from restore_gcp_generated_postgres_snapshot import ( + DEFAULT_COMPUTE_INSTANCE, + DEFAULT_COMPUTE_ZONE, + DEFAULT_PRIVATE_NETWORK, + DEFAULT_PROJECT, + RUN_ID_RE, + TARGET_DATABASE_RE, + gce_compute_identity, + sha256_file, + ) + +REASONING_SCHEMA = "livingip.gcpGeneratedDbBlindClaimCanary.v1" +ATTESTATION_SCHEMA = "livingip.gcpReasoningComputeAttestation.v1" + + +def utc_now() -> str: + return datetime.now(UTC).isoformat() + + +def parse_timestamp(value: Any, label: str) -> datetime: + text = str(value or "") + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError as exc: + raise ValueError(f"{label} is not an ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise ValueError(f"{label} must include a timezone") + return parsed.astimezone(UTC) + + +def load_reasoning_receipt(path: Path) -> dict[str, Any]: + try: + mode = path.lstat().st_mode + except OSError as exc: + raise ValueError(f"reasoning receipt is unavailable: {exc}") from exc + if not stat.S_ISREG(mode) or path.is_symlink(): + raise ValueError("reasoning receipt must be a regular non-symlink file") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError("reasoning receipt is invalid JSON") from exc + if not isinstance(payload, dict): + raise ValueError("reasoning receipt must be a JSON object") + return payload + + +def attest_reasoning_compute( + *, + reasoning_receipt_path: Path, + restore_run_id: str, + target_database: str, + project: str = DEFAULT_PROJECT, + expected_compute_instance: str = DEFAULT_COMPUTE_INSTANCE, + expected_compute_zone: str = DEFAULT_COMPUTE_ZONE, + expected_private_network: str = DEFAULT_PRIVATE_NETWORK, + metadata_timeout: float = 10.0, + generated_at_utc: str | None = None, +) -> dict[str, Any]: + if not RUN_ID_RE.fullmatch(restore_run_id): + raise ValueError("restore run id is invalid") + if not TARGET_DATABASE_RE.fullmatch(target_database): + raise ValueError("target database must be a disposable teleo_clone_* database") + reasoning_receipt_path = reasoning_receipt_path.resolve() + reasoning = load_reasoning_receipt(reasoning_receipt_path) + started = parse_timestamp(reasoning.get("generated_at_utc"), "reasoning start") + completed = parse_timestamp(reasoning.get("completed_at_utc"), "reasoning completion") + checks = { + "schema": reasoning.get("schema") == REASONING_SCHEMA, + "status": reasoning.get("status") == "pass" and not reasoning.get("errors"), + "target_database": reasoning.get("target_database") == target_database, + "source_compute": reasoning.get("source_compute") == expected_compute_instance, + "chronology": started <= completed, + "no_telegram_send": reasoning.get("posted_to_telegram") is False, + "no_database_write": reasoning.get("database_write_attempted") is False, + } + failed = sorted(name for name, passed in checks.items() if not passed) + if failed: + raise ValueError("reasoning receipt failed attestation preflight: " + ", ".join(failed)) + + compute = gce_compute_identity( + project, + expected_compute_instance, + expected_compute_zone, + expected_private_network, + timeout=metadata_timeout, + ) + return { + "artifact": "gcp_reasoning_compute_attestation", + "schema": ATTESTATION_SCHEMA, + "status": "pass", + "generated_at_utc": generated_at_utc or utc_now(), + "restore_run_id": restore_run_id, + "target_database": target_database, + "reasoning_receipt": { + "sha256": sha256_file(reasoning_receipt_path), + "schema": reasoning["schema"], + "generated_at_utc": reasoning["generated_at_utc"], + "completed_at_utc": reasoning["completed_at_utc"], + "claimed_source_compute": reasoning["source_compute"], + }, + "compute": compute, + "checks": {**checks, "gce_metadata_identity": all(compute.get("checks", {}).values())}, + "method": "gce_metadata_server_v1", + "mutation": { + "cloudsql_changed": False, + "compute_changed": False, + "database_changed": False, + "telegram_message_sent": False, + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reasoning-receipt", required=True, type=Path) + parser.add_argument("--restore-run-id", required=True) + parser.add_argument("--target-db", required=True) + parser.add_argument("--project", default=DEFAULT_PROJECT) + parser.add_argument("--expected-compute-instance", default=DEFAULT_COMPUTE_INSTANCE) + parser.add_argument("--expected-compute-zone", default=DEFAULT_COMPUTE_ZONE) + parser.add_argument("--expected-private-network", default=DEFAULT_PRIVATE_NETWORK) + parser.add_argument("--metadata-timeout", default=10.0, type=float) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + try: + receipt = attest_reasoning_compute( + reasoning_receipt_path=args.reasoning_receipt, + restore_run_id=args.restore_run_id, + target_database=args.target_db, + project=args.project, + expected_compute_instance=args.expected_compute_instance, + expected_compute_zone=args.expected_compute_zone, + expected_private_network=args.expected_private_network, + metadata_timeout=args.metadata_timeout, + ) + write_private_json(args.output, receipt) + except (OSError, ValueError, RuntimeError) as exc: + parser.error(str(exc)) + print(json.dumps(receipt, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/capture_vps_canonical_postgres_snapshot.py b/ops/capture_vps_canonical_postgres_snapshot.py index 055d933..9e02dea 100755 --- a/ops/capture_vps_canonical_postgres_snapshot.py +++ b/ops/capture_vps_canonical_postgres_snapshot.py @@ -25,15 +25,20 @@ except ImportError: # pragma: no cover - direct script execution SAFE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") SAFE_RUN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{7,63}\Z") -SAFE_SSH_TARGET_RE = re.compile( - r"(?:[A-Za-z0-9][A-Za-z0-9._-]{0,63}@)?[A-Za-z0-9][A-Za-z0-9.-]{0,252}\Z" -) +SAFE_AUTHORIZATION_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/@+-]{7,255}\Z") +SAFE_SSH_TARGET_RE = re.compile(r"(?:[A-Za-z0-9][A-Za-z0-9._-]{0,63}@)?[A-Za-z0-9][A-Za-z0-9.-]{0,252}\Z") +SNAPSHOT_ID_RE = re.compile(r"[0-9A-F]+-[0-9A-F]+-[0-9]+\Z") +TXID_SNAPSHOT_RE = re.compile(r"[0-9]+:[0-9]+:(?:[0-9]+(?:,[0-9]+)*)?\Z") +WAL_LSN_RE = re.compile(r"[0-9A-F]+/[0-9A-F]+\Z") +SYSTEM_IDENTIFIER_RE = re.compile(r"[0-9]+\Z") +SOURCE_SNAPSHOT_RECEIPT_SCHEMA = "livingip.sourceSnapshotReceipt.v2" EXPECTED_FILES = { "teleo-canonical.dump", "teleo-canonical.dump.sha256", "teleo-canonical.toc", "source-manifest.jsonl", "source-context.txt", + "source-snapshot.json", "service-before.txt", "service-after.txt", } @@ -47,6 +52,81 @@ def sha256(path: Path) -> str: return digest.hexdigest() +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def canonical_sha256(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return sha256_bytes(payload.encode()) + + +def load_snapshot_metadata(path: Path) -> dict[str, str]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError("source snapshot metadata is unreadable") from exc + if not isinstance(payload, dict): + raise RuntimeError("source snapshot metadata must be an object") + fields = { + "exported_snapshot_id": SNAPSHOT_ID_RE, + "txid_snapshot": TXID_SNAPSHOT_RE, + "wal_lsn": WAL_LSN_RE, + "system_identifier": SYSTEM_IDENTIFIER_RE, + } + normalized: dict[str, str] = {} + for field, pattern in fields.items(): + value = str(payload.get(field) or "") + if not pattern.fullmatch(value): + raise RuntimeError(f"source snapshot metadata has invalid {field}") + normalized[field] = value + captured_at_utc = str(payload.get("captured_at_utc") or "") + if not captured_at_utc or any(character in captured_at_utc for character in "\r\n\x00"): + raise RuntimeError("source snapshot metadata has invalid captured_at_utc") + normalized["captured_at_utc"] = captured_at_utc + return normalized + + +def load_service_state(path: Path) -> dict[str, str]: + try: + state = dict(line.split("=", 1) for line in path.read_text(encoding="utf-8").splitlines() if "=" in line) + main_pid = int(state.get("MainPID") or 0) + restarts = int(state.get("NRestarts") or 0) + except (OSError, TypeError, ValueError) as exc: + raise RuntimeError("source service state is unreadable") from exc + if state.get("ActiveState") != "active" or state.get("SubState") != "running" or main_pid <= 0 or restarts < 0: + raise RuntimeError("source service is not active/running with a positive MainPID") + return state + + +def build_provenance_binding( + *, + run_id: str, + authorization_ref: str, + source: dict[str, str], + snapshot: dict[str, str], + dump_sha256: str, + manifest_sql_sha256: str, + source_manifest_sha256: str, + source_context_sha256: str, +) -> dict[str, Any]: + payload = { + "receipt_schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "run_id": run_id, + "capture_authorization_ref": authorization_ref, + "source": source, + "exported_snapshot_id": snapshot["exported_snapshot_id"], + "txid_snapshot": snapshot["txid_snapshot"], + "wal_lsn": snapshot["wal_lsn"], + "source_system_identifier": snapshot["system_identifier"], + "dump_sha256": dump_sha256, + "manifest_sql_sha256": manifest_sql_sha256, + "source_manifest_sha256": source_manifest_sha256, + "source_context_sha256": source_context_sha256, + } + return {"algorithm": "sha256", "payload": payload, "sha256": canonical_sha256(payload)} + + def ssh_base(identity_file: Path) -> list[str]: return [ "ssh", @@ -73,12 +153,12 @@ def remote_capture_script(*, run_id: str, container: str, database: str, service "container_dump": shlex.quote(container_dump), } return f"""set -euo pipefail -remote_root={quoted['remote_root']} -container={quoted['container']} -database={quoted['database']} -service={quoted['service']} -container_manifest={quoted['container_manifest']} -container_dump={quoted['container_dump']} +remote_root={quoted["remote_root"]} +container={quoted["container"]} +database={quoted["database"]} +service={quoted["service"]} +container_manifest={quoted["container_manifest"]} +container_dump={quoted["container_dump"]} cleanup() {{ docker exec "$container" rm -f "$container_manifest" "$container_dump" >/dev/null 2>&1 || true @@ -86,11 +166,23 @@ cleanup() {{ }} trap cleanup EXIT +assert_service_healthy() {{ + local state_file="$1" + grep -qx 'ActiveState=active' "$state_file" + grep -qx 'SubState=running' "$state_file" + local main_pid restarts + main_pid="$(sed -n 's/^MainPID=//p' "$state_file")" + restarts="$(sed -n 's/^NRestarts=//p' "$state_file")" + [[ "$main_pid" =~ ^[1-9][0-9]*$ ]] + [[ "$restarts" =~ ^[0-9]+$ ]] +}} + test -d "$remote_root" test -f "$remote_root/postgres-parity-manifest.sql" docker inspect "$container" >/dev/null docker exec "$container" pg_isready -U postgres -d "$database" >/dev/null systemctl show "$service" -p ActiveState -p SubState -p MainPID -p NRestarts -p ActiveEnterTimestamp --no-pager > "$remote_root/service-before.txt" +assert_service_healthy "$remote_root/service-before.txt" docker inspect "$container" --format 'container_id={{{{.Id}}}}\nrunning={{{{.State.Running}}}}\nnetwork_mode={{{{.HostConfig.NetworkMode}}}}' > "$remote_root/source-context.txt" printf 'database=%s\ncaptured_at_utc=%s\n' "$database" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$remote_root/source-context.txt" docker cp "$remote_root/postgres-parity-manifest.sql" "$container:$container_manifest" >/dev/null @@ -107,6 +199,19 @@ if [[ ! "$snapshot_id" =~ ^[0-9A-F]+-[0-9A-F]+-[0-9]+$ ]]; then echo "invalid exported snapshot id" >&2 exit 1 fi +printf '%s\n' "select jsonb_build_object( + 'exported_snapshot_id', '$snapshot_id', + 'txid_snapshot', txid_current_snapshot()::text, + 'wal_lsn', pg_current_wal_lsn()::text, + 'system_identifier', (select system_identifier::text from pg_control_system()), + 'captured_at_utc', clock_timestamp() +)::text;" >&3 +IFS= read -r snapshot_metadata <&4 +if [[ -z "$snapshot_metadata" ]]; then + echo "invalid source snapshot metadata" >&2 + exit 1 +fi +printf '%s\n' "$snapshot_metadata" > "$remote_root/source-snapshot.json" docker exec "$container" pg_dump \ -U postgres -d "$database" \ @@ -130,6 +235,7 @@ docker cp "$container:$container_dump" "$remote_root/teleo-canonical.dump" >/dev sha256sum "$remote_root/teleo-canonical.dump" > "$remote_root/teleo-canonical.dump.sha256" printf 'snapshot_exported=true\n' >> "$remote_root/source-context.txt" systemctl show "$service" -p ActiveState -p SubState -p MainPID -p NRestarts -p ActiveEnterTimestamp --no-pager > "$remote_root/service-after.txt" +assert_service_healthy "$remote_root/service-after.txt" cmp -s "$remote_root/service-before.txt" "$remote_root/service-after.txt" tar -C "$remote_root" -cf - \ @@ -138,12 +244,13 @@ tar -C "$remote_root" -cf - \ teleo-canonical.toc \ source-manifest.jsonl \ source-context.txt \ + source-snapshot.json \ service-before.txt \ service-after.txt """ -def upload_manifest(ssh: list[str], target: str, remote_root: str, manifest: Path) -> None: +def upload_manifest(ssh: list[str], target: str, remote_root: str, manifest: bytes) -> None: command = ( f"umask 077; rm -rf {shlex.quote(remote_root)}; " f"mkdir -m 700 {shlex.quote(remote_root)}; " @@ -151,7 +258,7 @@ def upload_manifest(ssh: list[str], target: str, remote_root: str, manifest: Pat ) completed = subprocess.run( [*ssh, target, "--", command], - input=manifest.read_bytes(), + input=manifest, capture_output=True, check=False, ) @@ -206,8 +313,10 @@ def capture(args: argparse.Namespace) -> dict[str, Any]: output_dir.mkdir(mode=0o700) remote_root = f"/tmp/teleo-canonical-snapshot-{args.run_id}" ssh = ssh_base(args.ssh_key.resolve()) + manifest_sql = args.manifest.resolve().read_bytes() + manifest_sql_sha256 = sha256_bytes(manifest_sql) try: - upload_manifest(ssh, args.ssh_target, remote_root, args.manifest.resolve()) + upload_manifest(ssh, args.ssh_target, remote_root, manifest_sql) script = remote_capture_script( run_id=args.run_id, container=args.container, @@ -233,19 +342,38 @@ def capture(args: argparse.Namespace) -> dict[str, Any]: if expected_sha != actual_sha: raise RuntimeError("captured dump SHA-256 mismatch") manifest = load_manifest(output_dir / "source-manifest.jsonl") - before = (output_dir / "service-before.txt").read_text() - after = (output_dir / "service-after.txt").read_text() + snapshot = load_snapshot_metadata(output_dir / "source-snapshot.json") + before = load_service_state(output_dir / "service-before.txt") + after = load_service_state(output_dir / "service-after.txt") + source = { + "ssh_target": args.ssh_target, + "container": args.container, + "database": args.database, + "service": args.service, + } + source_manifest_sha256 = sha256(output_dir / "source-manifest.jsonl") + source_context_sha256 = sha256(output_dir / "source-context.txt") + provenance_binding = build_provenance_binding( + run_id=args.run_id, + authorization_ref=args.authorization_ref, + source=source, + snapshot=snapshot, + dump_sha256=actual_sha, + manifest_sql_sha256=manifest_sql_sha256, + source_manifest_sha256=source_manifest_sha256, + source_context_sha256=source_context_sha256, + ) return { "artifact": "vps_canonical_postgres_exported_snapshot", + "schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "receipt_version": 2, "generated_at_utc": datetime.now(UTC).isoformat(), "status": "pass", - "source": { - "ssh_target": args.ssh_target, - "container": args.container, - "database": args.database, - "service": args.service, - }, + "run_id": args.run_id, + "capture_authorization_ref": args.authorization_ref, + "source": source, "snapshot_exported": True, + "snapshot": snapshot, "dump": { "path": str(dump_path), "bytes": dump_path.stat().st_size, @@ -254,10 +382,18 @@ def capture(args: argparse.Namespace) -> dict[str, Any]: }, "manifest": { "path": str(output_dir / "source-manifest.jsonl"), + "sha256": source_manifest_sha256, + "manifest_sql_sha256": manifest_sql_sha256, "table_count": len(manifest["tables"]), "total_rows": sum(int(row["row_count"]) for row in manifest["tables"].values()), "application_roles": manifest["singleton"]["application_roles"]["items"], }, + "source_context": { + "path": str(output_dir / "source-context.txt"), + "sha256": source_context_sha256, + }, + "source_service": {"before": before, "after": after, "unchanged": before == after}, + "provenance_binding": provenance_binding, "service_unchanged": before == after, "production_db_mutated": False, "telegram_send_attempted": False, @@ -269,7 +405,7 @@ def cleanup_failed_output(output_dir: Path, *, preexisting: bool) -> None: shutil.rmtree(output_dir) -def parse_args() -> argparse.Namespace: +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--execute", action="store_true") parser.add_argument("--ssh-target", required=True) @@ -279,9 +415,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--service", default="leoclean-gateway.service") parser.add_argument("--manifest", default=Path("ops/postgres_parity_manifest.sql"), type=Path) parser.add_argument("--run-id", required=True) + parser.add_argument("--authorization-ref", required=True) parser.add_argument("--output-dir", required=True, type=Path) parser.add_argument("--timeout", default=180, type=int) - args = parser.parse_args() + args = parser.parse_args(argv) for value, flag in ((args.container, "--container"), (args.database, "--database"), (args.service, "--service")): if not SAFE_NAME_RE.fullmatch(value): parser.error(f"{flag} contains unsafe characters") @@ -289,6 +426,8 @@ def parse_args() -> argparse.Namespace: parser.error("--ssh-target must be a host or user@host without options") if not SAFE_RUN_RE.fullmatch(args.run_id): parser.error("--run-id must be 8-64 safe characters") + if not SAFE_AUTHORIZATION_REF_RE.fullmatch(args.authorization_ref): + parser.error("--authorization-ref must be 8-256 safe metadata characters") if not args.ssh_key.is_file(): parser.error("--ssh-key must be an existing file") if not args.manifest.is_file(): @@ -298,8 +437,12 @@ def parse_args() -> argparse.Namespace: json.dumps( { "artifact": "vps_canonical_postgres_exported_snapshot_plan", + "schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, "status": "dry_run", + "run_id": args.run_id, + "capture_authorization_ref": args.authorization_ref, "source": f"{args.ssh_target}:{args.container}/{args.database}", + "manifest_sql_sha256": sha256(args.manifest.resolve()), "output_dir": str(args.output_dir), "mutates_production_db": False, }, @@ -321,8 +464,11 @@ def main() -> int: cleanup_failed_output(output_dir, preexisting=output_preexisted) payload = { "artifact": "vps_canonical_postgres_exported_snapshot", + "schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, "generated_at_utc": datetime.now(UTC).isoformat(), "status": "fail", + "run_id": args.run_id, + "capture_authorization_ref": args.authorization_ref, "error": str(exc), } print(json.dumps(payload, indent=2, sort_keys=True)) diff --git a/ops/private_receipt_io.py b/ops/private_receipt_io.py new file mode 100644 index 0000000..13dc3d6 --- /dev/null +++ b/ops/private_receipt_io.py @@ -0,0 +1,34 @@ +"""Atomic private JSON receipt output shared by GCP parity helpers.""" + +from __future__ import annotations + +import json +import os +import uuid +from pathlib import Path +from typing import Any + + +def write_private_json(path: Path, payload: dict[str, Any]) -> None: + """Write one JSON receipt as mode 0600 without chmodding existing parents.""" + parent_preexisting = path.parent.exists() + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if not path.parent.is_dir() or path.parent.is_symlink(): + raise ValueError("output parent must be a real directory") + if not parent_preexisting: + os.chmod(path.parent, 0o700) + temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp" + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + os.chmod(path, 0o600) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass diff --git a/ops/restore_gcp_generated_postgres_snapshot.py b/ops/restore_gcp_generated_postgres_snapshot.py index f1eb801..7e09f4c 100644 --- a/ops/restore_gcp_generated_postgres_snapshot.py +++ b/ops/restore_gcp_generated_postgres_snapshot.py @@ -16,10 +16,13 @@ import ipaddress import json import os import re +import shlex import signal import stat import subprocess import time +import urllib.error +import urllib.request import uuid from datetime import UTC, datetime from pathlib import Path @@ -33,17 +36,34 @@ except ImportError: # pragma: no cover - direct script execution on the GCP VM DEFAULT_HOST = "10.61.0.3" DEFAULT_PROJECT = "teleo-501523" +DEFAULT_INSTANCE = "teleo-pgvector-standby" +DEFAULT_PRIVATE_NETWORK = "projects/teleo-501523/global/networks/teleo-staging-net" +DEFAULT_COMPUTE_INSTANCE = "teleo-prod-1" +DEFAULT_COMPUTE_ZONE = "europe-west6-a" +DEFAULT_SOURCE_SSH_TARGET = "root@77.42.65.182" +DEFAULT_SOURCE_CONTAINER = "teleo-pg" +DEFAULT_SOURCE_DATABASE = "teleo" +DEFAULT_SOURCE_SERVICE = "leoclean-gateway.service" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service" DEFAULT_RUN_ROOT = Path("/var/lib/teleo-gcp-restore-runs") DEFAULT_MANIFEST_SQL = Path(__file__).with_name("postgres_parity_manifest.sql") REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" +SOURCE_SNAPSHOT_RECEIPT_SCHEMA = "livingip.sourceSnapshotReceipt.v2" TARGET_DATABASE_RE = re.compile(r"teleo_clone_[a-z0-9][a-z0-9_]{0,50}\Z") RUN_ID_RE = re.compile(r"gcp-restore-[a-z0-9][a-z0-9-]{7,47}\Z") SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") LOCALE_RE = re.compile(r"[A-Za-z0-9_.@-]{1,64}\Z") +CLOUDSQL_INSTANCE_RE = re.compile(r"[a-z][a-z0-9-]{0,97}[a-z0-9]\Z") +AUTHORIZATION_REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/@+-]{7,255}\Z") +PRIVATE_NETWORK_RE = re.compile(r"projects/[a-z][a-z0-9-]{4,28}[a-z0-9]/global/networks/[a-z][a-z0-9-]{0,61}[a-z0-9]\Z") +GCE_INSTANCE_RE = re.compile(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?\Z") +GCE_ZONE_RE = re.compile(r"[a-z]+-[a-z]+[0-9]-[a-z]\Z") +SAFE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") +SAFE_SSH_TARGET_RE = re.compile(r"(?:[A-Za-z0-9][A-Za-z0-9._-]{0,63}@)?[A-Za-z0-9][A-Za-z0-9.-]{0,252}\Z") PG_RESTORE_VERSION_RE = re.compile(r"pg_restore \(PostgreSQL\) (?P[0-9]+)(?:\.[0-9]+)*") ROLLBACK_DATABASE = "teleo_canonical_pre_20260712t1905z" +GCE_METADATA_ROOT = "http://metadata.google.internal/computeMetadata/v1" class RestoreError(RuntimeError): @@ -66,6 +86,11 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def canonical_sha256(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(payload.encode()).hexdigest() + + def validate_regular_file(path: Path, label: str) -> Path: try: mode = path.lstat().st_mode @@ -89,6 +114,104 @@ def validate_custom_dump(path: Path, expected_sha256: str) -> dict[str, Any]: return {"path": str(path), "bytes": path.stat().st_size, "sha256": actual_sha256} +def validate_source_snapshot_receipt( + path: Path, + *, + expected_receipt_sha256: str, + expected_authorization_ref: str, + expected_source: dict[str, str], + dump: dict[str, Any], + source_manifest_path: Path, + manifest_sql_path: Path, + source_manifest: dict[str, Any], +) -> dict[str, Any]: + path = validate_regular_file(path, "source snapshot receipt") + receipt_sha256 = sha256_file(path) + if receipt_sha256 != expected_receipt_sha256: + raise RestoreError("source snapshot receipt SHA-256 does not match --expected-source-receipt-sha256") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise RestoreError("source snapshot receipt is invalid JSON") from exc + if not isinstance(payload, dict): + raise RestoreError("source snapshot receipt must be a JSON object") + + snapshot = payload.get("snapshot") or {} + manifest = payload.get("manifest") or {} + source = payload.get("source") or {} + source_context = payload.get("source_context") or {} + source_service = payload.get("source_service") or {} + binding = payload.get("provenance_binding") or {} + binding_payload = binding.get("payload") or {} + source_manifest_sha256 = sha256_file(source_manifest_path) + manifest_sql_sha256 = sha256_file(manifest_sql_path) + expected_table_count = len(source_manifest["tables"]) + expected_total_rows = sum(int(row["row_count"]) for row in source_manifest["tables"].values()) + expected_binding_payload = { + "receipt_schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "run_id": payload.get("run_id"), + "capture_authorization_ref": payload.get("capture_authorization_ref"), + "source": source, + "exported_snapshot_id": snapshot.get("exported_snapshot_id"), + "txid_snapshot": snapshot.get("txid_snapshot"), + "wal_lsn": snapshot.get("wal_lsn"), + "source_system_identifier": snapshot.get("system_identifier"), + "dump_sha256": dump["sha256"], + "manifest_sql_sha256": manifest_sql_sha256, + "source_manifest_sha256": source_manifest_sha256, + "source_context_sha256": source_context.get("sha256"), + } + checks = { + "artifact": payload.get("artifact") == "vps_canonical_postgres_exported_snapshot", + "schema": payload.get("schema") == SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "receipt_version": payload.get("receipt_version") == 2, + "status": payload.get("status") == "pass", + "snapshot_exported": payload.get("snapshot_exported") is True, + "service_unchanged": payload.get("service_unchanged") is True, + "source_service_healthy": source_service.get("unchanged") is True + and service_state_is_healthy(source_service.get("before")) + and source_service.get("before") == source_service.get("after"), + "source_not_mutated": payload.get("production_db_mutated") is False, + "telegram_not_sent": payload.get("telegram_send_attempted") is False, + "run_id": bool(payload.get("run_id")), + "capture_authorization_ref": payload.get("capture_authorization_ref") == expected_authorization_ref, + "source_identity": source == expected_source, + "source_database": source.get("database") == source_manifest["singleton"]["identity"].get("database"), + "snapshot_identity": all( + bool(snapshot.get(field)) + for field in ("exported_snapshot_id", "txid_snapshot", "wal_lsn", "system_identifier", "captured_at_utc") + ), + "dump_sha256": (payload.get("dump") or {}).get("sha256") == dump["sha256"], + "dump_bytes": (payload.get("dump") or {}).get("bytes") == dump["bytes"], + "source_manifest_sha256": manifest.get("sha256") == source_manifest_sha256, + "manifest_sql_sha256": manifest.get("manifest_sql_sha256") == manifest_sql_sha256, + "source_context_sha256": bool(SHA256_RE.fullmatch(str(source_context.get("sha256") or ""))), + "table_count": manifest.get("table_count") == expected_table_count, + "total_rows": manifest.get("total_rows") == expected_total_rows, + "binding_algorithm": binding.get("algorithm") == "sha256", + "binding_payload": binding_payload == expected_binding_payload, + "binding_sha256": binding.get("sha256") == canonical_sha256(expected_binding_payload), + } + failed = sorted(name for name, passed in checks.items() if not passed) + if failed: + raise RestoreError("source snapshot receipt failed validation: " + ", ".join(failed)) + return { + "path": str(path), + "sha256": receipt_sha256, + "schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "run_id": payload["run_id"], + "capture_authorization_ref": payload["capture_authorization_ref"], + "snapshot": snapshot, + "provenance_binding_sha256": binding["sha256"], + "manifest_sha256": source_manifest_sha256, + "manifest_sql_sha256": manifest_sql_sha256, + "dump_sha256": dump["sha256"], + "table_count": expected_table_count, + "total_rows": expected_total_rows, + "checks": checks, + } + + def _run( command: list[str], *, @@ -158,6 +281,163 @@ def service_state(service: str, *, timeout: float) -> dict[str, Any]: return result +def validate_service_healthy(state: dict[str, Any], label: str) -> None: + if not service_state_is_healthy(state): + raise RestoreError(f"{label} service is not active/running with a positive MainPID") + + +def service_state_is_healthy(state: Any) -> bool: + if not isinstance(state, dict): + return False + try: + main_pid = int(state.get("MainPID") or 0) + restarts = int(state.get("NRestarts") or 0) + except (TypeError, ValueError): + return False + return ( + state.get("ActiveState") == "active" and state.get("SubState") == "running" and main_pid > 0 and restarts >= 0 + ) + + +def cloudsql_control_plane_state( + project: str, + instance: str, + expected_host: str, + expected_private_network: str, + *, + timeout: float, +) -> dict[str, Any]: + completed = _run( + [ + "gcloud", + "sql", + "instances", + "describe", + instance, + f"--project={project}", + "--format=json", + "--quiet", + ], + timeout=timeout, + ) + raw = _require_success(completed, "Cloud SQL control-plane readback") + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise RestoreError("Cloud SQL control-plane readback returned invalid JSON") from exc + settings = payload.get("settings") or {} + ip_configuration = settings.get("ipConfiguration") or {} + addresses = payload.get("ipAddresses") or [] + private_addresses = sorted( + str(row.get("ipAddress")) for row in addresses if row.get("type") == "PRIVATE" and row.get("ipAddress") + ) + private_network = str(ip_configuration.get("privateNetwork") or "") + non_private_addresses = sorted( + str(row.get("ipAddress")) for row in addresses if row.get("type") != "PRIVATE" and row.get("ipAddress") + ) + checks = { + "instance": payload.get("name") == instance, + "state": payload.get("state") == "RUNNABLE", + "postgres_16": str(payload.get("databaseVersion") or "").startswith("POSTGRES_16"), + "public_ip_disabled": ip_configuration.get("ipv4Enabled") is False, + "private_network": private_network == expected_private_network, + "expected_private_host": expected_host in private_addresses, + "no_non_private_address": not non_private_addresses, + } + failed = sorted(name for name, passed in checks.items() if not passed) + if failed: + raise RestoreError("Cloud SQL control-plane preflight failed: " + ", ".join(failed)) + return { + "project": project, + "instance": instance, + "database_version": payload.get("databaseVersion"), + "region": payload.get("region"), + "state": payload.get("state"), + "tier": settings.get("tier"), + "private_network": private_network, + "expected_private_network": expected_private_network, + "private_addresses": private_addresses, + "expected_private_host": expected_host, + "public_ip_disabled": True, + "checks": checks, + } + + +def gce_metadata_value(path: str, *, timeout: float) -> str: + request = urllib.request.Request( + f"{GCE_METADATA_ROOT}/{path}", + headers={"Metadata-Flavor": "Google"}, + method="GET", + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(request, timeout=timeout) as response: + if response.headers.get("Metadata-Flavor") != "Google": + raise RestoreError("GCE metadata response is missing Metadata-Flavor: Google") + value = response.read(4096).decode("utf-8", errors="strict").strip() + except (OSError, UnicodeError, urllib.error.URLError) as exc: + raise RestoreError(f"GCE metadata readback failed for {path}") from exc + if not value or any(character in value for character in "\r\n\x00"): + raise RestoreError(f"GCE metadata readback is invalid for {path}") + return value + + +def gce_compute_identity( + project: str, + expected_instance: str, + expected_zone: str, + expected_private_network: str, + *, + timeout: float, +) -> dict[str, Any]: + metadata = { + "project": gce_metadata_value("project/project-id", timeout=timeout), + "project_number": gce_metadata_value("project/numeric-project-id", timeout=timeout), + "instance": gce_metadata_value("instance/name", timeout=timeout), + "instance_id": gce_metadata_value("instance/id", timeout=timeout), + "zone_resource": gce_metadata_value("instance/zone", timeout=timeout), + "network_resource": gce_metadata_value("instance/network-interfaces/0/network", timeout=timeout), + "private_ip": gce_metadata_value("instance/network-interfaces/0/ip", timeout=timeout), + "service_account": gce_metadata_value("instance/service-accounts/default/email", timeout=timeout), + } + zone = metadata["zone_resource"].rsplit("/", 1)[-1] + network_name = expected_private_network.rsplit("/", 1)[-1] + expected_network_resource = f"projects/{metadata['project_number']}/networks/{network_name}" + try: + private_ip = ipaddress.ip_address(metadata["private_ip"]) + except ValueError as exc: + raise RestoreError("GCE metadata private IP is invalid") from exc + checks = { + "project": metadata["project"] == project, + "project_number": metadata["project_number"].isdigit(), + "expected_network_project": expected_private_network.startswith(f"projects/{project}/global/networks/"), + "instance": metadata["instance"] == expected_instance, + "instance_id": metadata["instance_id"].isdigit(), + "zone": zone == expected_zone, + "network": metadata["network_resource"] == expected_network_resource, + "private_ip": any( + private_ip in network + for network in ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ) + ), + "service_account": metadata["service_account"].endswith(".iam.gserviceaccount.com"), + } + failed = sorted(name for name, passed in checks.items() if not passed) + if failed: + raise RestoreError("GCE compute identity preflight failed: " + ", ".join(failed)) + return { + **metadata, + "zone": zone, + "expected_zone": expected_zone, + "expected_private_network": expected_private_network, + "expected_network_resource": expected_network_resource, + "checks": checks, + } + + def resolve_password(project: str, secret: str, *, timeout: float) -> str: completed = _run( [ @@ -310,7 +590,11 @@ def capture_target_manifest(args: argparse.Namespace, password: str) -> str: return _require_success(completed, "target parity manifest capture", secrets=(password,)) -def validate_private_identity(target_manifest: dict[str, Any], target_db: str) -> dict[str, Any]: +def validate_private_identity( + target_manifest: dict[str, Any], + target_db: str, + source_compute: str, +) -> dict[str, Any]: identity = target_manifest["singleton"]["identity"] if identity.get("database") != target_db: raise RestoreError("target manifest database does not match the generated clone") @@ -340,7 +624,7 @@ def validate_private_identity(target_manifest: dict[str, Any], target_db: str) - "ssl_version": identity.get("ssl_version"), "private_connectivity": True, "transaction_read_only": "on", - "source_compute": "teleo-prod-1", + "source_compute": source_compute, } @@ -440,6 +724,22 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: "generated_at_utc": utc_now(), "run_id": args.run_id, "target_database": args.target_db, + "execution_contract": { + "project": args.project, + "cloudsql_instance": args.cloudsql_instance, + "host": args.host, + "private_network": args.expected_private_network, + "compute_instance": args.expected_compute_instance, + "compute_zone": args.expected_compute_zone, + "source": { + "ssh_target": args.expected_source_ssh_target, + "container": args.expected_source_container, + "database": args.expected_source_database, + "service": args.expected_source_service, + }, + "password_secret": args.password_secret, + "service": args.service, + }, "status": "running", "phase": "validation", "source": {}, @@ -475,12 +775,39 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: "sha256": sha256_file(args.source_manifest), } source_manifest = load_manifest(args.source_manifest) + receipt["source"]["capture_receipt"] = validate_source_snapshot_receipt( + args.source_receipt, + expected_receipt_sha256=args.expected_source_receipt_sha256, + expected_authorization_ref=args.expected_capture_authorization_ref, + expected_source=receipt["execution_contract"]["source"], + dump=receipt["source"]["dump"], + source_manifest_path=args.source_manifest, + manifest_sql_path=args.manifest_sql, + source_manifest=source_manifest, + ) receipt["toolchain"] = validate_pg_restore_version( args.pg_restore_bin, source_manifest, timeout=args.command_timeout, ) + receipt["target"]["compute"] = gce_compute_identity( + args.project, + args.expected_compute_instance, + args.expected_compute_zone, + args.expected_private_network, + timeout=args.command_timeout, + ) receipt["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout) + validate_service_healthy(receipt["live_service"]["before"], "pre-restore") + + receipt["phase"] = "cloudsql_control_plane_preflight" + receipt["target"]["cloudsql"] = cloudsql_control_plane_state( + args.project, + args.cloudsql_instance, + args.host, + args.expected_private_network, + timeout=args.command_timeout, + ) receipt["phase"] = "credential_resolution" password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout) @@ -506,9 +833,46 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: "manifest_path": str(target_manifest_path), "manifest_bytes": target_manifest_path.stat().st_size, "manifest_sha256": sha256_file(target_manifest_path), - "connectivity": validate_private_identity(target_manifest, args.target_db), + "connectivity": validate_private_identity( + target_manifest, + args.target_db, + receipt["target"]["compute"]["instance"], + ), } ) + connectivity = receipt["target"]["connectivity"] + cloudsql = receipt["target"]["cloudsql"] + compute = receipt["target"]["compute"] + connectivity_proof = { + "artifact": "gcp_private_postgres_connectivity", + "schema": "livingip.gcpPrivatePostgresConnectivity.v2", + "generated_at_utc": utc_now(), + "status": "pass", + "restore_run_id": args.run_id, + "target_database": args.target_db, + "project": cloudsql["project"], + "cloudsql_instance": cloudsql["instance"], + "private_network": cloudsql["private_network"], + "source_compute": compute["instance"], + "source_compute_instance_id": compute["instance_id"], + "source_compute_zone": compute["zone"], + "source_compute_private_ip": compute["private_ip"], + "server_address": connectivity["server_address"], + "server_port": connectivity["server_port"], + "ssl": connectivity["ssl"], + "ssl_version": connectivity["ssl_version"], + "private_connectivity": connectivity["private_connectivity"], + "public_ip_disabled": cloudsql["public_ip_disabled"], + "cloudsql_control_plane_checks": cloudsql["checks"], + "compute_identity_checks": compute["checks"], + } + connectivity_proof_path = run_dir / "gcp-private-connectivity.json" + write_private(connectivity_proof_path, json.dumps(connectivity_proof, indent=2, sort_keys=True) + "\n") + receipt["target"]["connectivity_proof"] = { + "path": str(connectivity_proof_path), + "sha256": sha256_file(connectivity_proof_path), + "schema": connectivity_proof["schema"], + } receipt["phase"] = "parity_compare" problems, details = compare_manifests( @@ -529,6 +893,7 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: receipt["phase"] = "service_and_rollback_readback" receipt["rollback_database"] = rollback_database_state(args, password) receipt["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) + validate_service_healthy(receipt["live_service"]["after"], "post-restore") receipt["live_service"]["unchanged"] = receipt["live_service"]["before"] == receipt["live_service"]["after"] if not receipt["live_service"]["unchanged"]: raise RestoreError("live GCP service state changed during disposable restore") @@ -539,9 +904,42 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: "required": True, "attempted": False, "clone_database_remaining": 1, - "exact_command": ( - f"sudo python3 ops/restore_gcp_generated_postgres_snapshot.py cleanup --execute " - f"--target-db {args.target_db} --run-id {args.run_id}" + "exact_command": shlex.join( + [ + "sudo", + "python3", + "ops/restore_gcp_generated_postgres_snapshot.py", + "cleanup", + "--execute", + "--target-db", + args.target_db, + "--run-id", + args.run_id, + "--host", + args.host, + "--project", + args.project, + "--cloudsql-instance", + args.cloudsql_instance, + "--expected-private-network", + args.expected_private_network, + "--expected-compute-instance", + args.expected_compute_instance, + "--expected-compute-zone", + args.expected_compute_zone, + "--expected-source-ssh-target", + args.expected_source_ssh_target, + "--expected-source-container", + args.expected_source_container, + "--expected-source-database", + args.expected_source_database, + "--expected-source-service", + args.expected_source_service, + "--password-secret", + args.password_secret, + "--service", + args.service, + ] ), } except ( @@ -583,6 +981,7 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: except Exception as service_exc: receipt["live_service"]["after_error"] = str(service_exc)[-2000:] receipt["duration_seconds"] = round(time.monotonic() - started, 6) + receipt["completed_at_utc"] = utc_now() write_private(run_dir / "restore-receipt.json", json.dumps(receipt, indent=2, sort_keys=True) + "\n") password = "" return receipt @@ -592,6 +991,7 @@ def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: run_dir = secure_run_dir(args.run_root, args.run_id, create=False) restore_receipt_path = run_dir / "restore-receipt.json" restore_receipt_path = validate_regular_file(restore_receipt_path, "restore receipt") + restore_receipt_sha256 = sha256_file(restore_receipt_path) restore_receipt = json.loads(restore_receipt_path.read_text(encoding="utf-8")) if restore_receipt.get("status") != "pass": raise RestoreError("cleanup requires a passing restore receipt") @@ -600,23 +1000,66 @@ def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: if restore_receipt.get("run_id") != args.run_id: raise RestoreError("cleanup run id does not match the retained restore receipt") - before = service_state(args.service, timeout=args.command_timeout) - password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout) - try: - dropped = drop_clone(args, password) - rollback = rollback_database_state(args, password) - finally: - password = "" - after = service_state(args.service, timeout=args.command_timeout) - payload = { + expected_contract = { + "project": args.project, + "cloudsql_instance": args.cloudsql_instance, + "host": args.host, + "private_network": args.expected_private_network, + "compute_instance": args.expected_compute_instance, + "compute_zone": args.expected_compute_zone, + "source": { + "ssh_target": args.expected_source_ssh_target, + "container": args.expected_source_container, + "database": args.expected_source_database, + "service": args.expected_source_service, + }, + "password_secret": args.password_secret, + "service": args.service, + } + if restore_receipt.get("execution_contract") != expected_contract: + raise RestoreError("cleanup execution contract does not match the retained restore receipt") + restore_cloudsql = (restore_receipt.get("target") or {}).get("cloudsql") or {} + restore_topology_checks = { + "project": restore_cloudsql.get("project") == args.project, + "instance": restore_cloudsql.get("instance") == args.cloudsql_instance, + "host": restore_cloudsql.get("expected_private_host") == args.host, + "private_network": restore_cloudsql.get("expected_private_network") == args.expected_private_network, + "public_ip_disabled": restore_cloudsql.get("public_ip_disabled") is True, + } + failed_topology_checks = sorted(name for name, passed in restore_topology_checks.items() if not passed) + if failed_topology_checks: + raise RestoreError( + "cleanup target topology does not match the retained restore receipt: " + ", ".join(failed_topology_checks) + ) + restore_compute = (restore_receipt.get("target") or {}).get("compute") or {} + restore_compute_checks = { + "project": restore_compute.get("project") == args.project, + "instance": restore_compute.get("instance") == args.expected_compute_instance, + "zone": restore_compute.get("zone") == args.expected_compute_zone, + "private_network": restore_compute.get("expected_private_network") == args.expected_private_network, + "checks": bool(restore_compute.get("checks")) + and all(value is True for value in restore_compute["checks"].values()), + } + failed_compute_checks = sorted(name for name, passed in restore_compute_checks.items() if not passed) + if failed_compute_checks: + raise RestoreError( + "cleanup compute identity does not match the retained restore receipt: " + ", ".join(failed_compute_checks) + ) + + started = time.monotonic() + payload: dict[str, Any] = { "artifact": "gcp_generated_postgres_snapshot_cleanup", "generated_at_utc": utc_now(), "run_id": args.run_id, "target_database": args.target_db, - "status": "pass" if before == after and dropped["clone_database_remaining"] == 0 else "fail", - "database": dropped, - "rollback_database": rollback, - "live_service": {"before": before, "after": after, "unchanged": before == after}, + "restore_receipt": {"path": str(restore_receipt_path), "sha256": restore_receipt_sha256}, + "execution_contract": expected_contract, + "target": {}, + "status": "running", + "phase": "cloudsql_control_plane_preflight", + "database": {"clone_database_remaining": None}, + "rollback_database": {}, + "live_service": {}, "safety": { "live_database_named": False, "live_profile_modified": False, @@ -624,8 +1067,95 @@ def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: "telegram_message_sent": False, "secret_persisted": False, }, + "error": None, } - write_private(run_dir / "cleanup-receipt.json", json.dumps(payload, indent=2, sort_keys=True) + "\n") + password = "" + try: + payload["target"]["cloudsql"] = cloudsql_control_plane_state( + args.project, + args.cloudsql_instance, + args.host, + args.expected_private_network, + timeout=args.command_timeout, + ) + payload["target"]["compute"] = gce_compute_identity( + args.project, + args.expected_compute_instance, + args.expected_compute_zone, + args.expected_private_network, + timeout=args.command_timeout, + ) + cleanup_compute_mismatches = sorted( + field + for field in ( + "project", + "project_number", + "instance", + "instance_id", + "zone", + "network_resource", + "private_ip", + "service_account", + ) + if payload["target"]["compute"].get(field) != restore_compute.get(field) + ) + if cleanup_compute_mismatches: + raise RestoreError("cleanup GCE identity changed since restore: " + ", ".join(cleanup_compute_mismatches)) + payload["phase"] = "service_readback_before" + payload["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout) + validate_service_healthy(payload["live_service"]["before"], "pre-cleanup") + payload["phase"] = "credential_resolution" + password = resolve_password(args.project, args.password_secret, timeout=args.command_timeout) + payload["phase"] = "database_drop" + payload["database"] = drop_clone(args, password) + payload["phase"] = "rollback_readback" + payload["rollback_database"] = rollback_database_state(args, password) + payload["phase"] = "service_readback_after" + payload["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) + validate_service_healthy(payload["live_service"]["after"], "post-cleanup") + payload["live_service"]["unchanged"] = payload["live_service"]["before"] == payload["live_service"]["after"] + payload["status"] = ( + "pass" + if payload["live_service"]["unchanged"] and payload["database"]["clone_database_remaining"] == 0 + else "fail" + ) + payload["phase"] = "complete" + except ( + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + subprocess.TimeoutExpired, + RestoreError, + KeyboardInterrupt, + ) as exc: + payload["status"] = "fail" + payload["error"] = { + "phase": payload.get("phase"), + "type": type(exc).__name__, + "message": str(exc)[-2000:], + } + if password and payload["database"].get("clone_database_remaining") is None: + try: + payload["database"]["clone_database_remaining"] = ( + 1 if database_exists(args, password, args.target_db) else 0 + ) + except Exception as readback_exc: + payload["database"]["readback_error"] = str(readback_exc)[-2000:] + if "before" in payload["live_service"] and "after" not in payload["live_service"]: + try: + payload["live_service"]["after"] = service_state(args.service, timeout=args.command_timeout) + payload["live_service"]["unchanged"] = ( + payload["live_service"]["before"] == payload["live_service"]["after"] + ) + except Exception as service_exc: + payload["live_service"]["after_error"] = str(service_exc)[-2000:] + finally: + password = "" + payload["duration_seconds"] = round(time.monotonic() - started, 6) + payload["completed_at_utc"] = utc_now() + write_private(run_dir / "cleanup-receipt.json", json.dumps(payload, indent=2, sort_keys=True) + "\n") return payload @@ -635,6 +1165,14 @@ def add_common_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--run-id", required=True) parser.add_argument("--host", default=DEFAULT_HOST) parser.add_argument("--project", default=DEFAULT_PROJECT) + parser.add_argument("--cloudsql-instance", default=DEFAULT_INSTANCE) + parser.add_argument("--expected-private-network", default=DEFAULT_PRIVATE_NETWORK) + parser.add_argument("--expected-compute-instance", default=DEFAULT_COMPUTE_INSTANCE) + parser.add_argument("--expected-compute-zone", default=DEFAULT_COMPUTE_ZONE) + parser.add_argument("--expected-source-ssh-target", default=DEFAULT_SOURCE_SSH_TARGET) + parser.add_argument("--expected-source-container", default=DEFAULT_SOURCE_CONTAINER) + parser.add_argument("--expected-source-database", default=DEFAULT_SOURCE_DATABASE) + parser.add_argument("--expected-source-service", default=DEFAULT_SOURCE_SERVICE) parser.add_argument("--password-secret", default=DEFAULT_SECRET) parser.add_argument("--service", default=DEFAULT_SERVICE) parser.add_argument("--command-timeout", type=float, default=120.0) @@ -650,6 +1188,9 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: restore.add_argument("--dump", required=True, type=Path) restore.add_argument("--expected-dump-sha256", required=True) restore.add_argument("--source-manifest", required=True, type=Path) + restore.add_argument("--source-receipt", required=True, type=Path) + restore.add_argument("--expected-source-receipt-sha256", required=True) + restore.add_argument("--expected-capture-authorization-ref", required=True) restore.add_argument("--manifest-sql", default=DEFAULT_MANIFEST_SQL, type=Path) restore.add_argument("--pg-restore-bin", default="pg_restore") restore.add_argument("--restore-timeout", type=float, default=900.0) @@ -667,6 +1208,25 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--target-db must be a bounded lowercase teleo_clone_* identifier") if not RUN_ID_RE.fullmatch(args.run_id): parser.error("--run-id must match gcp-restore-<8-48 lowercase letters, digits, or hyphens>") + if not CLOUDSQL_INSTANCE_RE.fullmatch(args.cloudsql_instance): + parser.error("--cloudsql-instance must be a safe lowercase Cloud SQL instance name") + if not PRIVATE_NETWORK_RE.fullmatch(args.expected_private_network): + parser.error("--expected-private-network must be a full projects/.../global/networks/... resource") + if not args.expected_private_network.startswith(f"projects/{args.project}/global/networks/"): + parser.error("--expected-private-network project must match --project") + if not GCE_INSTANCE_RE.fullmatch(args.expected_compute_instance): + parser.error("--expected-compute-instance must be a safe lowercase GCE instance name") + if not GCE_ZONE_RE.fullmatch(args.expected_compute_zone): + parser.error("--expected-compute-zone must be a safe GCE zone") + if not SAFE_SSH_TARGET_RE.fullmatch(args.expected_source_ssh_target): + parser.error("--expected-source-ssh-target must be a host or user@host without options") + for value, flag in ( + (args.expected_source_container, "--expected-source-container"), + (args.expected_source_database, "--expected-source-database"), + (args.expected_source_service, "--expected-source-service"), + ): + if not SAFE_NAME_RE.fullmatch(value): + parser.error(f"{flag} contains unsafe characters") try: host = ipaddress.ip_address(args.host) except ValueError: @@ -683,6 +1243,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: if args.operation == "restore": if not SHA256_RE.fullmatch(args.expected_dump_sha256): parser.error("--expected-dump-sha256 must be 64 lowercase hexadecimal characters") + if not SHA256_RE.fullmatch(args.expected_source_receipt_sha256): + parser.error("--expected-source-receipt-sha256 must be 64 lowercase hexadecimal characters") + if not AUTHORIZATION_REF_RE.fullmatch(args.expected_capture_authorization_ref): + parser.error("--expected-capture-authorization-ref must be 8-256 safe metadata characters") for value, flag in ( (args.restore_timeout, "--restore-timeout"), (args.manifest_timeout, "--manifest-timeout"), diff --git a/ops/verify_gcp_canonical_lifecycle.py b/ops/verify_gcp_canonical_lifecycle.py new file mode 100644 index 0000000..aeb7760 --- /dev/null +++ b/ops/verify_gcp_canonical_lifecycle.py @@ -0,0 +1,754 @@ +#!/usr/bin/env python3 +"""Verify one bound source-to-GCP restore, reasoning, and cleanup lifecycle.""" + +from __future__ import annotations + +import argparse +import hashlib +import ipaddress +import json +import re +from datetime import UTC, datetime +from itertools import pairwise +from pathlib import Path +from typing import Any + +try: + from .verify_vps_canonical_snapshot_delta import DELTA_SCHEMA, parse_timestamp, write_private_json +except ImportError: # pragma: no cover - direct script execution + from verify_vps_canonical_snapshot_delta import DELTA_SCHEMA, parse_timestamp, write_private_json + +SOURCE_SCHEMA = "livingip.sourceSnapshotReceipt.v2" +BLIND_REASONING_SCHEMA = "livingip.gcpGeneratedDbBlindClaimCanary.v1" +REASONING_COMPUTE_SCHEMA = "livingip.gcpReasoningComputeAttestation.v1" +RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1" +UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b", re.I) +SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") +FORBIDDEN_HANDLE_RE = re.compile(r"\b(?:cory|m3ta)\b", re.I) +EXPECTED_BLIND_ROW_IDS = { + "2a7ae257-d01d-46f4-b813-63f81bb9c7c7", + "261c3532-fa32-47d8-a5b5-6cc45035c267", + "15740795-ecc6-40fa-9a01-3d6bc7c54f79", +} + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_receipt(path: Path, label: str) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"{label} is unreadable or invalid JSON") from exc + if not isinstance(payload, dict): + raise ValueError(f"{label} must be a JSON object") + return payload + + +def is_rfc1918(value: Any) -> bool: + try: + address = ipaddress.ip_address(str(value)) + except ValueError: + return False + return any( + address in network + for network in ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ) + ) + + +def nonempty_all_true(value: Any) -> bool: + return isinstance(value, dict) and bool(value) and all(item is True for item in value.values()) + + +def required_true_checks(value: Any, required: set[str]) -> bool: + return nonempty_all_true(value) and required <= set(value) + + +def service_healthy(value: Any) -> bool: + if not isinstance(value, dict): + return False + try: + main_pid = int(value.get("MainPID") or 0) + restarts = int(value.get("NRestarts") or 0) + except (TypeError, ValueError): + return False + return ( + value.get("ActiveState") == "active" and value.get("SubState") == "running" and main_pid > 0 and restarts >= 0 + ) + + +def parity_evidence(details: dict[str, Any]) -> dict[str, bool]: + structural = details.get("structural_hashes") or {} + required_structures = { + "schemas", + "columns", + "constraints", + "indexes", + "sequences", + "views", + "functions", + "triggers", + "types", + "policies", + } + structural_valid = required_structures <= set(structural) and all( + bool(SHA256_RE.fullmatch(str((structural.get(kind) or {}).get("source") or ""))) + and (structural.get(kind) or {}).get("source") == (structural.get(kind) or {}).get("target") + for kind in required_structures + ) + performance = details.get("performance") or [] + performance_valid = bool(performance) and all( + bool(row.get("query")) + and isinstance(row.get("source_ms"), (int, float)) + and isinstance(row.get("target_ms"), (int, float)) + and isinstance(row.get("source_ratio"), (int, float)) + for row in performance + ) + source_table_count = details.get("source_table_count") + target_table_count = details.get("target_table_count") + source_total_rows = details.get("source_total_rows") + target_total_rows = details.get("target_total_rows") + return { + "table_counts": isinstance(source_table_count, int) + and source_table_count > 0 + and source_table_count == target_table_count, + "row_counts": isinstance(source_total_rows, int) + and source_total_rows > 0 + and source_total_rows == target_total_rows, + "table_hashes": details.get("table_mismatches") == [], + "structural_hashes": structural_valid, + "roles": details.get("application_role_mismatches") == {} + and details.get("target_extra_application_roles") == [], + "extensions": details.get("required_extension_mismatches") == {}, + "performance": performance_valid and details.get("performance_problems") == [], + } + + +def blind_reasoning_evidence(reasoning: dict[str, Any], target_db: Any) -> dict[str, bool]: + result = reasoning.get("result") or {} + gateway = result.get("gateway") or {} + child = gateway.get("child_process") or {} + tool_surface = gateway.get("tool_surface") or {} + tool_trace = result.get("database_tool_trace") or {} + calls = tool_trace.get("calls") or [] + wrapper = reasoning.get("wrapper_tool_proof") or {} + invocations = wrapper.get("invocations") or [] + prompt = str(reasoning.get("prompt") or "") + reply = str(result.get("reply") or "") + outcomes = reasoning.get("outcomes") or {} + required_outcomes = ( + "asks_for_user_iteration_before_live", + "challenges_weak_support", + "decomposes_the_bundled_legal_claim", + "does_not_claim_a_database_change", + "proposes_candidate_claims", + "uses_only_m3taversal_handle", + ) + required_subcommands = {"show", "evidence"} + discovery_subcommands = {"search", "context"} + traced_subcommands = { + str(invocation.get("subcommand")) for call in calls for invocation in (call.get("database_invocations") or []) + } + wrapper_subcommands = {str(invocation.get("subcommand")) for invocation in invocations} + retrieved_row_ids = {str(row_id) for call in calls for row_id in ((call.get("result") or {}).get("row_ids") or [])} + call_row_ids_valid = bool(calls) and all( + isinstance((call.get("result") or {}).get("row_ids"), list) + and isinstance((call.get("result") or {}).get("row_id_count"), int) + and (call.get("result") or {}).get("row_id_count") >= 0 + and (call.get("result") or {}).get("row_id_count") >= len((call.get("result") or {}).get("row_ids") or []) + for call in calls + ) + call_receipts_valid = bool(calls) and all( + (call.get("result") or {}).get("nonempty") is True + and (call.get("result") or {}).get("error_detected") is False + and ((call.get("result") or {}).get("retrieval_receipt") or {}).get("schema") == RETRIEVAL_RECEIPT_SCHEMA + and bool( + SHA256_RE.fullmatch( + str(((call.get("result") or {}).get("retrieval_receipt") or {}).get("semantic_context_sha256") or "") + ) + ) + and bool( + SHA256_RE.fullmatch( + str(((call.get("result") or {}).get("retrieval_receipt") or {}).get("artifact_state_sha256") or "") + ) + ) + and str( + ((call.get("result") or {}).get("retrieval_receipt") or {}).get("read_consistency_status") or "" + ).startswith("stable") + for call in calls + ) + wrapper_invocations_valid = bool(invocations) and all( + invocation.get("database") == target_db + and invocation.get("returncode") == 0 + and ((invocation.get("database_identity") or {}).get("transaction_read_only") == "on") + and ((invocation.get("database_identity") or {}).get("default_transaction_read_only") == "on") + and ((invocation.get("database_identity") or {}).get("ssl") is True) + and is_rfc1918((invocation.get("database_identity") or {}).get("server_address")) + for invocation in invocations + ) + return { + "mode": reasoning.get("mode") == "gcp_generated_db_gatewayrunner_blind_claim_no_send", + "timestamps": bool(reasoning.get("generated_at_utc")) and bool(reasoning.get("completed_at_utc")), + "source_compute": bool(reasoning.get("source_compute")), + "blind_prompt": bool(prompt) and UUID_RE.search(prompt) is None, + "reply_nonempty": bool(reply.strip()), + "reply_uses_only_m3taversal_handle": FORBIDDEN_HANDLE_RE.search(reply) is None, + "outcomes": all(outcomes.get(name) is True for name in required_outcomes), + "gateway": gateway.get("authorized") is True + and gateway.get("handler_invoked") is True + and gateway.get("model_free_fallback_used") is False + and gateway.get("posted_to_telegram") is False, + "gateway_cleanup": child.get("exitcode") == 0 + and child.get("alive_after_readback") is False + and child.get("process_group_alive_after_readback") is False + and child.get("timed_out") is False, + "send_tool_absent": tool_surface.get("send_message_tool_enabled") is False, + "tool_trace": tool_trace.get("schema") == "livingip.leoKbToolTrace.v1" + and tool_trace.get("database_tool_call_proven") is True + and tool_trace.get("database_retrieval_receipt_proven") is True + and tool_trace.get("database_tool_calls_read_only") is True + and tool_trace.get("database_tool_call_count") == len(calls) + and tool_trace.get("database_tool_completed_count") == len(calls), + "tool_receipts": call_receipts_valid, + "expected_claim_and_sources_retrieved": call_row_ids_valid and retrieved_row_ids >= EXPECTED_BLIND_ROW_IDS, + "required_tool_sequence": required_subcommands <= traced_subcommands + and bool(discovery_subcommands & traced_subcommands), + "wrapper": wrapper.get("all_bound_to_supplied_target") is True + and wrapper.get("all_completed_successfully") is True + and wrapper.get("complete_start_end_pairing") is True + and wrapper.get("database_read_only_required") is True + and wrapper.get("default_read_only_required") is True, + "wrapper_invocations": wrapper_invocations_valid + and required_subcommands <= wrapper_subcommands + and bool(discovery_subcommands & wrapper_subcommands), + "canonical_status_unchanged": bool(reasoning.get("canonical_status_before")) + and reasoning.get("canonical_status_before") == reasoning.get("canonical_status_after"), + "database_identity_unchanged": bool(reasoning.get("database_identity_before")) + and reasoning.get("database_identity_before") == reasoning.get("database_identity_after"), + "service_unchanged": service_healthy(reasoning.get("service_before")) + and reasoning.get("service_before") == reasoning.get("service_after"), + "temporary_profile_removed": reasoning.get("temp_profile_absent") is True + and reasoning.get("temp_profile_removed") is True, + } + + +def verify_lifecycle( + *, + source_path: Path, + current_source_path: Path, + source_delta_path: Path, + restore_path: Path, + parity_path: Path, + reasoning_path: Path, + reasoning_compute_path: Path, + cleanup_path: Path, + max_postflight_age_seconds: float = 900.0, + max_lifecycle_age_seconds: float = 3600.0, + now: datetime | None = None, +) -> dict[str, Any]: + if max_postflight_age_seconds <= 0: + raise ValueError("max postflight age must be positive") + if max_lifecycle_age_seconds <= 0: + raise ValueError("max lifecycle age must be positive") + verification_time = (now or datetime.now(UTC)).astimezone(UTC) + source = load_receipt(source_path, "source receipt") + current_source = load_receipt(current_source_path, "current source receipt") + source_delta = load_receipt(source_delta_path, "source delta receipt") + restore = load_receipt(restore_path, "restore receipt") + parity = load_receipt(parity_path, "parity receipt") + reasoning = load_receipt(reasoning_path, "reasoning receipt") + reasoning_compute_attestation = load_receipt(reasoning_compute_path, "reasoning compute attestation") + cleanup = load_receipt(cleanup_path, "cleanup receipt") + + target_db = restore.get("target_database") + run_id = restore.get("run_id") + restore_source = restore.get("source") or {} + capture_binding = restore_source.get("capture_receipt") or {} + restore_target = restore.get("target") or {} + restore_connectivity = restore_target.get("connectivity") or {} + restore_connectivity_proof = restore_target.get("connectivity_proof") or {} + restore_cloudsql = restore_target.get("cloudsql") or {} + restore_compute = restore_target.get("compute") or {} + restore_parity = restore.get("parity") or {} + restore_service = restore.get("live_service") or {} + parity_details = parity.get("details") or {} + parity_evidence_checks = parity_evidence(parity_details) + restore_inline_parity_checks = parity_evidence(restore_parity.get("details") or {}) + parity_connectivity = parity.get("private_connectivity") or {} + parity_connectivity_proof = parity_connectivity.get("proof") or {} + reasoning_checks = reasoning.get("checks") or {} + reasoning_parity = reasoning.get("parity_receipt") or {} + before_fingerprint = reasoning.get("database_fingerprint_before") or {} + after_fingerprint = reasoning.get("database_fingerprint_after") or {} + cleanup_database = cleanup.get("database") or {} + cleanup_service = cleanup.get("live_service") or {} + cleanup_cloudsql = (cleanup.get("target") or {}).get("cloudsql") or {} + cleanup_compute = (cleanup.get("target") or {}).get("compute") or {} + restore_contract = restore.get("execution_contract") or {} + cleanup_contract = cleanup.get("execution_contract") or {} + producer_reasoning_checks = blind_reasoning_evidence(reasoning, target_db) + reasoning_compute = reasoning_compute_attestation.get("compute") or {} + reasoning_compute_checks = reasoning_compute_attestation.get("checks") or {} + reasoning_compute_mutation = reasoning_compute_attestation.get("mutation") or {} + delta_baseline = source_delta.get("baseline") or {} + delta_current = source_delta.get("current") or {} + delta_details = source_delta.get("delta") or {} + source_snapshot = source.get("snapshot") or {} + current_source_snapshot = current_source.get("snapshot") or {} + timeline_values = { + "source_snapshot": source_snapshot.get("captured_at_utc"), + "restore_started": restore.get("generated_at_utc"), + "restore_completed": restore.get("completed_at_utc"), + "parity_completed": parity.get("completed_at_utc"), + "reasoning_started": reasoning.get("generated_at_utc"), + "reasoning_completed": reasoning.get("completed_at_utc"), + "reasoning_compute_attested": reasoning_compute_attestation.get("generated_at_utc"), + "cleanup_started": cleanup.get("generated_at_utc"), + "cleanup_completed": cleanup.get("completed_at_utc"), + "current_source_snapshot": current_source_snapshot.get("captured_at_utc"), + } + timeline = [parse_timestamp(value, name) for name, value in timeline_values.items()] + lifecycle_chronology = all(earlier <= later for earlier, later in pairwise(timeline)) + delta_detected = delta_details.get("delta_detected") + delta_change_present = bool( + delta_details.get("added_tables") + or delta_details.get("removed_tables") + or delta_details.get("changed_tables") + or delta_details.get("changed_structures") + or delta_details.get("total_row_delta") + ) + current_source_time = parse_timestamp( + current_source_snapshot.get("captured_at_utc"), "current source snapshot time" + ) + restore_started_time = parse_timestamp(restore.get("generated_at_utc"), "restore start time") + cleanup_completed_time = parse_timestamp(cleanup.get("completed_at_utc"), "cleanup completion time") + postflight_age_seconds = (verification_time - current_source_time).total_seconds() + lifecycle_age_seconds = (verification_time - cleanup_completed_time).total_seconds() + lifecycle_span_seconds = (current_source_time - restore_started_time).total_seconds() + postflight_after_lifecycle = current_source_time >= max( + parse_timestamp(reasoning.get("completed_at_utc"), "reasoning completion time"), + parse_timestamp(cleanup.get("completed_at_utc"), "cleanup completion time"), + ) + + required_reasoning_checks = ( + "exact_blind_prompt_without_ids", + "claim_and_sources_retrieved", + "discovery_show_evidence_completed", + "retrieval_receipts_proven", + "private_tls_read_only_target", + "generated_database_unchanged", + "canonical_counts_unchanged", + "database_calls_read_only", + "no_database_write", + "no_telegram_send", + "live_service_unchanged", + "live_profile_unchanged", + "temporary_profile_removed", + ) + checks = { + "source_schema_v2": source.get("schema") == SOURCE_SCHEMA and source.get("receipt_version") == 2, + "source_pass": source.get("status") == "pass" + and source.get("snapshot_exported") is True + and (source.get("source_service") or {}).get("unchanged") is True + and service_healthy((source.get("source_service") or {}).get("before")) + and (source.get("source_service") or {}).get("before") == (source.get("source_service") or {}).get("after"), + "source_authorized": bool(source.get("capture_authorization_ref")), + "source_authorization_bound": capture_binding.get("capture_authorization_ref") + == source.get("capture_authorization_ref"), + "source_service_unchanged": source.get("service_unchanged") is True, + "source_not_mutated": source.get("production_db_mutated") is False, + "current_source_schema_v2": current_source.get("schema") == SOURCE_SCHEMA + and current_source.get("receipt_version") == 2, + "current_source_pass": current_source.get("status") == "pass" + and current_source.get("snapshot_exported") is True + and current_source.get("service_unchanged") is True + and (current_source.get("source_service") or {}).get("unchanged") is True + and service_healthy((current_source.get("source_service") or {}).get("before")) + and (current_source.get("source_service") or {}).get("before") + == (current_source.get("source_service") or {}).get("after") + and current_source.get("production_db_mutated") is False, + "current_source_identity": current_source.get("source") == source.get("source") + and current_source_snapshot.get("system_identifier") == source_snapshot.get("system_identifier"), + "source_delta_pass": source_delta.get("artifact") == "vps_canonical_snapshot_postflight_delta" + and source_delta.get("schema") == DELTA_SCHEMA + and source_delta.get("status") == "pass", + "source_delta_receipts_bound": delta_baseline.get("capture_receipt_sha256") == sha256_file(source_path) + and delta_current.get("capture_receipt_sha256") == sha256_file(current_source_path), + "source_delta_manifests_bound": delta_baseline.get("manifest_sha256") + == (source.get("manifest") or {}).get("sha256") + and delta_current.get("manifest_sha256") == (current_source.get("manifest") or {}).get("sha256"), + "source_delta_authorizations_bound": delta_baseline.get("capture_authorization_ref") + == source.get("capture_authorization_ref") + and delta_current.get("capture_authorization_ref") == current_source.get("capture_authorization_ref"), + "source_delta_explicit": isinstance(delta_details.get("delta_detected"), bool) + and isinstance(delta_details.get("added_tables"), list) + and isinstance(delta_details.get("removed_tables"), list) + and isinstance(delta_details.get("changed_tables"), list) + and isinstance(delta_details.get("changed_structures"), list) + and isinstance(delta_details.get("total_row_delta"), int), + "source_delta_baseline_shape": delta_baseline.get("run_id") == source.get("run_id") + and delta_baseline.get("captured_at_utc") == source_snapshot.get("captured_at_utc") + and delta_baseline.get("table_count") == (source.get("manifest") or {}).get("table_count") + and delta_baseline.get("total_rows") == (source.get("manifest") or {}).get("total_rows") + and bool(SHA256_RE.fullmatch(str(delta_baseline.get("table_fingerprint_sha256") or ""))) + and bool(SHA256_RE.fullmatch(str(delta_baseline.get("structure_fingerprint_sha256") or ""))), + "source_delta_current_shape": delta_current.get("run_id") == current_source.get("run_id") + and delta_current.get("captured_at_utc") == current_source_snapshot.get("captured_at_utc") + and delta_current.get("table_count") == (current_source.get("manifest") or {}).get("table_count") + and delta_current.get("total_rows") == (current_source.get("manifest") or {}).get("total_rows") + and bool(SHA256_RE.fullmatch(str(delta_current.get("table_fingerprint_sha256") or ""))) + and bool(SHA256_RE.fullmatch(str(delta_current.get("structure_fingerprint_sha256") or ""))), + "source_delta_consistent": delta_detected is delta_change_present + and delta_details.get("total_row_delta") + == (delta_current.get("total_rows") or 0) - (delta_baseline.get("total_rows") or 0) + and ( + delta_detected is True + or ( + delta_baseline.get("table_count") == delta_current.get("table_count") + and delta_baseline.get("total_rows") == delta_current.get("total_rows") + and delta_baseline.get("table_fingerprint_sha256") == delta_current.get("table_fingerprint_sha256") + and delta_baseline.get("structure_fingerprint_sha256") + == delta_current.get("structure_fingerprint_sha256") + ) + ), + "source_postflight_after_lifecycle": postflight_after_lifecycle, + "source_postflight_fresh": 0 <= postflight_age_seconds <= max_postflight_age_seconds, + "gcp_lifecycle_fresh": 0 <= lifecycle_age_seconds <= max_lifecycle_age_seconds, + "lifecycle_span_bounded": 0 <= lifecycle_span_seconds <= max_lifecycle_age_seconds, + "lifecycle_chronology": lifecycle_chronology, + "restore_pass": restore.get("artifact") == "gcp_generated_postgres_snapshot_restore" + and restore.get("status") == "pass", + "restore_target_is_disposable": isinstance(target_db, str) and target_db.startswith("teleo_clone_"), + "restore_run_id": isinstance(run_id, str) and run_id.startswith("gcp-restore-"), + "source_receipt_hash_bound": capture_binding.get("sha256") == sha256_file(source_path), + "source_provenance_bound": capture_binding.get("provenance_binding_sha256") + == (source.get("provenance_binding") or {}).get("sha256"), + "source_capture_checks": required_true_checks( + capture_binding.get("checks"), + { + "artifact", + "schema", + "receipt_version", + "status", + "snapshot_exported", + "service_unchanged", + "source_service_healthy", + "source_not_mutated", + "telegram_not_sent", + "run_id", + "capture_authorization_ref", + "source_identity", + "source_database", + "snapshot_identity", + "dump_sha256", + "dump_bytes", + "source_manifest_sha256", + "manifest_sql_sha256", + "source_context_sha256", + "table_count", + "total_rows", + "binding_algorithm", + "binding_payload", + "binding_sha256", + }, + ), + "source_dump_bound": capture_binding.get("dump_sha256") == (source.get("dump") or {}).get("sha256"), + "source_execution_contract_bound": restore_contract.get("source") == source.get("source"), + "source_manifest_bound": capture_binding.get("manifest_sha256") == (source.get("manifest") or {}).get("sha256"), + "restore_manifest_matches_parity_source": capture_binding.get("manifest_sha256") + == parity.get("source_manifest_sha256"), + "restore_target_manifest_matches_parity": restore_target.get("manifest_sha256") + == parity.get("target_manifest_sha256"), + "restore_private_tls": restore_connectivity.get("private_connectivity") is True + and restore_connectivity.get("ssl") is True + and is_rfc1918(restore_connectivity.get("server_address")), + "restore_public_ip_disabled": restore_cloudsql.get("public_ip_disabled") is True, + "restore_cloudsql_checks": required_true_checks( + restore_cloudsql.get("checks"), + { + "instance", + "state", + "postgres_16", + "public_ip_disabled", + "private_network", + "expected_private_host", + "no_non_private_address", + }, + ), + "restore_compute_checks": required_true_checks( + restore_compute.get("checks"), + { + "project", + "project_number", + "expected_network_project", + "instance", + "instance_id", + "zone", + "network", + "private_ip", + "service_account", + }, + ), + "restore_compute_contract": restore_compute.get("project") == restore_contract.get("project") + and restore_compute.get("instance") == restore_contract.get("compute_instance") + and restore_compute.get("zone") == restore_contract.get("compute_zone") + and restore_compute.get("expected_private_network") == restore_contract.get("private_network") + and bool(restore_compute.get("network_resource")) + and is_rfc1918(restore_compute.get("private_ip")) + and str(restore_compute.get("service_account") or "").endswith(".iam.gserviceaccount.com"), + "restore_connectivity_compute_bound": restore_connectivity.get("source_compute") + == restore_compute.get("instance"), + "restore_inline_parity": restore_parity.get("status") == "pass" and restore_parity.get("problems") == [], + "restore_inline_parity_evidence": all(restore_inline_parity_checks.values()), + "restore_service_healthy_unchanged": restore_service.get("unchanged") is True + and service_healthy(restore_service.get("before")) + and restore_service.get("before") == restore_service.get("after"), + "parity_pass": parity.get("artifact") == "canonical_postgres_parity_verification" + and parity.get("status") == "pass" + and parity.get("scope") == "gcp_staging" + and parity.get("problems") == [], + "parity_target_bound": parity_details.get("target_database") == target_db, + "parity_exact_evidence": all(parity_evidence_checks.values()), + "parity_connectivity_hash_bound": restore_connectivity_proof.get("sha256") + == parity.get("connectivity_proof_sha256"), + "parity_private_proof": parity_connectivity.get("required") is True + and parity_connectivity_proof.get("artifact") == "gcp_private_postgres_connectivity" + and parity_connectivity_proof.get("status") == "pass" + and parity_connectivity_proof.get("schema") == "livingip.gcpPrivatePostgresConnectivity.v2" + and parity_connectivity_proof.get("restore_run_id") == run_id + and parity_connectivity_proof.get("private_connectivity") is True + and parity_connectivity_proof.get("public_ip_disabled") is True + and parity_connectivity_proof.get("target_database") == target_db + and parity_connectivity_proof.get("project") == restore_cloudsql.get("project") + and parity_connectivity_proof.get("cloudsql_instance") == restore_cloudsql.get("instance") + and parity_connectivity_proof.get("private_network") == restore_cloudsql.get("private_network") + and parity_connectivity_proof.get("source_compute") == reasoning.get("source_compute") + and parity_connectivity_proof.get("source_compute_instance_id") == restore_compute.get("instance_id") + and parity_connectivity_proof.get("source_compute_zone") == restore_compute.get("zone") + and is_rfc1918(parity_connectivity_proof.get("source_compute_private_ip")) + and parity_connectivity_proof.get("server_address") == restore_connectivity.get("server_address") + and required_true_checks( + parity_connectivity_proof.get("cloudsql_control_plane_checks"), + { + "instance", + "state", + "postgres_16", + "public_ip_disabled", + "private_network", + "expected_private_host", + "no_non_private_address", + }, + ) + and required_true_checks( + parity_connectivity_proof.get("compute_identity_checks"), + { + "project", + "project_number", + "expected_network_project", + "instance", + "instance_id", + "zone", + "network", + "private_ip", + "service_account", + }, + ), + "reasoning_schema": reasoning.get("schema") == BLIND_REASONING_SCHEMA, + "reasoning_pass": reasoning.get("status") == "pass" and not reasoning.get("errors"), + "reasoning_target_bound": reasoning.get("target_database") == target_db, + "reasoning_compute_bound": reasoning.get("source_compute") == restore_compute.get("instance"), + "reasoning_parity_hash_bound": reasoning_parity.get("sha256") == sha256_file(parity_path), + "reasoning_required_checks": all(reasoning_checks.get(name) is True for name in required_reasoning_checks), + "reasoning_producer_evidence": all(producer_reasoning_checks.values()), + "reasoning_compute_attestation": reasoning_compute_attestation.get("artifact") + == "gcp_reasoning_compute_attestation" + and reasoning_compute_attestation.get("schema") == REASONING_COMPUTE_SCHEMA + and reasoning_compute_attestation.get("status") == "pass" + and reasoning_compute_attestation.get("method") == "gce_metadata_server_v1" + and nonempty_all_true(reasoning_compute_checks) + and reasoning_compute_attestation.get("restore_run_id") == run_id + and reasoning_compute_attestation.get("target_database") == target_db + and all(value is False for value in reasoning_compute_mutation.values()) + and { + "cloudsql_changed", + "compute_changed", + "database_changed", + "telegram_message_sent", + } + <= set(reasoning_compute_mutation), + "reasoning_compute_receipt_bound": (reasoning_compute_attestation.get("reasoning_receipt") or {}).get("sha256") + == sha256_file(reasoning_path), + "reasoning_compute_identity_bound": reasoning_compute.get("project") == restore_compute.get("project") + and reasoning_compute.get("project_number") == restore_compute.get("project_number") + and reasoning_compute.get("instance") == restore_compute.get("instance") + and reasoning_compute.get("instance_id") == restore_compute.get("instance_id") + and reasoning_compute.get("zone") == restore_compute.get("zone") + and reasoning_compute.get("expected_private_network") == restore_compute.get("expected_private_network") + and reasoning_compute.get("network_resource") == restore_compute.get("network_resource") + and reasoning_compute.get("private_ip") == restore_compute.get("private_ip") + and reasoning_compute.get("service_account") == restore_compute.get("service_account") + and required_true_checks( + reasoning_compute.get("checks"), + { + "project", + "project_number", + "expected_network_project", + "instance", + "instance_id", + "zone", + "network", + "private_ip", + "service_account", + }, + ), + "reasoning_no_send_no_write": reasoning.get("posted_to_telegram") is False + and reasoning.get("database_write_attempted") is False, + "reasoning_fingerprint_unchanged": bool(before_fingerprint.get("sha256")) + and before_fingerprint.get("sha256") == after_fingerprint.get("sha256"), + "cleanup_pass": cleanup.get("artifact") == "gcp_generated_postgres_snapshot_cleanup" + and cleanup.get("status") == "pass", + "cleanup_run_bound": cleanup.get("run_id") == run_id and cleanup.get("target_database") == target_db, + "cleanup_restore_receipt_bound": (cleanup.get("restore_receipt") or {}).get("sha256") + == sha256_file(restore_path), + "cleanup_execution_contract_bound": bool(restore_contract) and cleanup_contract == restore_contract, + "cleanup_cloudsql_bound": cleanup_cloudsql.get("project") == restore_cloudsql.get("project") + and cleanup_cloudsql.get("instance") == restore_cloudsql.get("instance") + and cleanup_cloudsql.get("expected_private_host") == restore_cloudsql.get("expected_private_host") + and cleanup_cloudsql.get("expected_private_network") == restore_cloudsql.get("expected_private_network") + and cleanup_cloudsql.get("public_ip_disabled") is True + and required_true_checks( + cleanup_cloudsql.get("checks"), + { + "instance", + "state", + "postgres_16", + "public_ip_disabled", + "private_network", + "expected_private_host", + "no_non_private_address", + }, + ), + "cleanup_compute_bound": cleanup_compute.get("instance_id") == restore_compute.get("instance_id") + and cleanup_compute.get("project") == restore_compute.get("project") + and cleanup_compute.get("project_number") == restore_compute.get("project_number") + and cleanup_compute.get("instance") == restore_compute.get("instance") + and cleanup_compute.get("zone") == restore_compute.get("zone") + and cleanup_compute.get("expected_private_network") == restore_compute.get("expected_private_network") + and cleanup_compute.get("network_resource") == restore_compute.get("network_resource") + and cleanup_compute.get("private_ip") == restore_compute.get("private_ip") + and cleanup_compute.get("service_account") == restore_compute.get("service_account") + and required_true_checks( + cleanup_compute.get("checks"), + { + "project", + "project_number", + "expected_network_project", + "instance", + "instance_id", + "zone", + "network", + "private_ip", + "service_account", + }, + ), + "clone_absent": cleanup_database.get("clone_database_remaining") == 0, + "cleanup_service_unchanged": cleanup_service.get("unchanged") is True + and service_healthy(cleanup_service.get("before")) + and cleanup_service.get("before") == cleanup_service.get("after"), + } + failed = sorted(name for name, passed in checks.items() if not passed) + return { + "artifact": "gcp_canonical_snapshot_lifecycle_verification", + "generated_at_utc": verification_time.isoformat(), + "status": "pass" if not failed else "fail", + "required_tier": "T3_live_private_gcp_staging", + "target_database": target_db, + "run_id": run_id, + "checks": checks, + "reasoning_producer_checks": producer_reasoning_checks, + "restore_inline_parity_checks": restore_inline_parity_checks, + "parity_evidence_checks": parity_evidence_checks, + "post_snapshot_source_delta": delta_details, + "timeline": timeline_values, + "postflight_age_seconds": postflight_age_seconds, + "max_postflight_age_seconds": max_postflight_age_seconds, + "lifecycle_age_seconds": lifecycle_age_seconds, + "lifecycle_span_seconds": lifecycle_span_seconds, + "max_lifecycle_age_seconds": max_lifecycle_age_seconds, + "failed_checks": failed, + "receipt_sha256": { + "source": sha256_file(source_path), + "current_source": sha256_file(current_source_path), + "source_delta": sha256_file(source_delta_path), + "restore": sha256_file(restore_path), + "parity": sha256_file(parity_path), + "reasoning": sha256_file(reasoning_path), + "reasoning_compute": sha256_file(reasoning_compute_path), + "cleanup": sha256_file(cleanup_path), + }, + "strongest_claim_allowed": ( + ( + "current private GCP staging parity, ID-free Leo reasoning, cleanup, and zero post-snapshot VPS delta verified" + if delta_details.get("delta_detected") is False + else "private GCP staging parity at the declared snapshot boundary, ID-free Leo reasoning, cleanup, and explicit post-snapshot VPS delta verified" + ) + if not failed + else "lifecycle incomplete; failed checks must be repaired" + ), + "not_proven": ["production cutover", "ongoing replication", "Telegram delivery"], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-receipt", required=True, type=Path) + parser.add_argument("--current-source-receipt", required=True, type=Path) + parser.add_argument("--source-delta-receipt", required=True, type=Path) + parser.add_argument("--restore-receipt", required=True, type=Path) + parser.add_argument("--parity-receipt", required=True, type=Path) + parser.add_argument("--reasoning-receipt", required=True, type=Path) + parser.add_argument("--reasoning-compute-receipt", required=True, type=Path) + parser.add_argument("--cleanup-receipt", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--max-postflight-age-seconds", default=900.0, type=float) + parser.add_argument("--max-lifecycle-age-seconds", default=3600.0, type=float) + args = parser.parse_args() + try: + payload = verify_lifecycle( + source_path=args.source_receipt, + current_source_path=args.current_source_receipt, + source_delta_path=args.source_delta_receipt, + restore_path=args.restore_receipt, + parity_path=args.parity_receipt, + reasoning_path=args.reasoning_receipt, + reasoning_compute_path=args.reasoning_compute_receipt, + cleanup_path=args.cleanup_receipt, + max_postflight_age_seconds=args.max_postflight_age_seconds, + max_lifecycle_age_seconds=args.max_lifecycle_age_seconds, + ) + except (OSError, ValueError, KeyError, TypeError) as exc: + payload = { + "artifact": "gcp_canonical_snapshot_lifecycle_verification", + "generated_at_utc": datetime.now(UTC).isoformat(), + "status": "fail", + "required_tier": "T3_live_private_gcp_staging", + "failed_checks": ["receipt_load_or_validation_error"], + "error": str(exc), + "strongest_claim_allowed": "lifecycle incomplete; receipts could not be validated", + } + write_private_json(args.output, payload) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 if payload.get("status") == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/verify_postgres_parity_manifest.py b/ops/verify_postgres_parity_manifest.py index b88ca50..fb533d4 100755 --- a/ops/verify_postgres_parity_manifest.py +++ b/ops/verify_postgres_parity_manifest.py @@ -11,6 +11,11 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +try: + from .private_receipt_io import write_private_json +except ImportError: # pragma: no cover - direct script execution + from private_receipt_io import write_private_json + SINGLETON_KINDS = { "identity", "schemas", @@ -33,6 +38,14 @@ def canonical_hash(value: Any) -> str: return hashlib.sha256(payload.encode()).hexdigest() +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def load_manifest(path: Path) -> dict[str, Any]: singleton: dict[str, dict[str, Any]] = {} tables: dict[str, dict[str, Any]] = {} @@ -295,6 +308,9 @@ def main() -> int: args = parser.parse_args() try: + source_manifest_sha256 = file_sha256(args.source) + target_manifest_sha256 = file_sha256(args.target) + connectivity_proof_sha256 = file_sha256(args.connectivity_proof) if args.connectivity_proof else None source = load_manifest(args.source) target = load_manifest(args.target) problems, details = compare_manifests( @@ -316,18 +332,26 @@ def main() -> int: problems = [f"manifest_error={exc}"] details = {} connectivity = {"required": args.scope == "gcp_staging", "status": "not_checked"} + source_manifest_sha256 = None + target_manifest_sha256 = None + connectivity_proof_sha256 = None error = str(exc) + completed_at_utc = datetime.now(UTC).isoformat() payload = { "artifact": "canonical_postgres_parity_verification", - "generated_at_utc": datetime.now(UTC).isoformat(), + "generated_at_utc": completed_at_utc, + "completed_at_utc": completed_at_utc, "scope": args.scope, "source_manifest": str(args.source), + "source_manifest_sha256": source_manifest_sha256, "target_manifest": str(args.target), + "target_manifest_sha256": target_manifest_sha256, "status": status, "problems": problems, "details": details, "private_connectivity": connectivity, + "connectivity_proof_sha256": connectivity_proof_sha256, "error": error, "not_proven_by_this_artifact": [ "GCP staging Leo composition replay", @@ -335,8 +359,7 @@ def main() -> int: "production cutover", ], } - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + write_private_json(args.output, payload) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 if status == "pass" else 1 diff --git a/ops/verify_vps_canonical_snapshot_delta.py b/ops/verify_vps_canonical_snapshot_delta.py new file mode 100644 index 0000000..b4dbe7a --- /dev/null +++ b/ops/verify_vps_canonical_snapshot_delta.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Bind a postflight VPS manifest to a captured snapshot and report exact deltas.""" + +from __future__ import annotations + +import argparse +import json +import re +import stat +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +try: + from .capture_vps_canonical_postgres_snapshot import ( + SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + build_provenance_binding, + ) + from .private_receipt_io import write_private_json + from .verify_postgres_parity_manifest import SINGLETON_KINDS, canonical_hash, file_sha256, load_manifest +except ImportError: # pragma: no cover - direct script execution + from capture_vps_canonical_postgres_snapshot import SOURCE_SNAPSHOT_RECEIPT_SCHEMA, build_provenance_binding + from private_receipt_io import write_private_json + from verify_postgres_parity_manifest import SINGLETON_KINDS, canonical_hash, file_sha256, load_manifest + +DELTA_SCHEMA = "livingip.vpsCanonicalSnapshotDelta.v1" +SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") + + +def service_healthy(value: Any) -> bool: + if not isinstance(value, dict): + return False + try: + main_pid = int(value.get("MainPID") or 0) + restarts = int(value.get("NRestarts") or 0) + except (TypeError, ValueError): + return False + return ( + value.get("ActiveState") == "active" and value.get("SubState") == "running" and main_pid > 0 and restarts >= 0 + ) + + +def regular_file(path: Path, label: str) -> Path: + try: + mode = path.lstat().st_mode + except OSError as exc: + raise ValueError(f"{label} is unavailable: {exc}") from exc + if not stat.S_ISREG(mode) or path.is_symlink(): + raise ValueError(f"{label} must be a regular non-symlink file") + return path.resolve() + + +def load_json(path: Path, label: str) -> dict[str, Any]: + path = regular_file(path, label) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"{label} is invalid JSON") from exc + if not isinstance(payload, dict): + raise ValueError(f"{label} must be a JSON object") + return payload + + +def parse_timestamp(value: Any, label: str) -> datetime: + text = str(value or "") + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError as exc: + raise ValueError(f"{label} is not an ISO-8601 timestamp") from exc + if parsed.tzinfo is None: + raise ValueError(f"{label} must include a timezone") + return parsed.astimezone(UTC) + + +def validate_capture( + receipt_path: Path, + manifest_path: Path, + *, + expected_authorization_ref: str, + label: str, +) -> tuple[dict[str, Any], dict[str, Any]]: + receipt = load_json(receipt_path, f"{label} receipt") + manifest_path = regular_file(manifest_path, f"{label} manifest") + manifest = load_manifest(manifest_path) + snapshot = receipt.get("snapshot") or {} + source = receipt.get("source") or {} + dump = receipt.get("dump") or {} + manifest_receipt = receipt.get("manifest") or {} + source_context = receipt.get("source_context") or {} + source_service = receipt.get("source_service") or {} + binding = receipt.get("provenance_binding") or {} + expected_binding = build_provenance_binding( + run_id=str(receipt.get("run_id") or ""), + authorization_ref=str(receipt.get("capture_authorization_ref") or ""), + source=source, + snapshot=snapshot, + dump_sha256=str(dump.get("sha256") or ""), + manifest_sql_sha256=str(manifest_receipt.get("manifest_sql_sha256") or ""), + source_manifest_sha256=file_sha256(manifest_path), + source_context_sha256=str(source_context.get("sha256") or ""), + ) + total_rows = sum(int(row["row_count"]) for row in manifest["tables"].values()) + checks = { + "schema": receipt.get("schema") == SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "receipt_version": receipt.get("receipt_version") == 2, + "status": receipt.get("status") == "pass", + "authorization": receipt.get("capture_authorization_ref") == expected_authorization_ref, + "snapshot_exported": receipt.get("snapshot_exported") is True, + "service_unchanged": receipt.get("service_unchanged") is True, + "source_service_healthy": source_service.get("unchanged") is True + and service_healthy(source_service.get("before")) + and source_service.get("before") == source_service.get("after"), + "source_not_mutated": receipt.get("production_db_mutated") is False, + "telegram_not_sent": receipt.get("telegram_send_attempted") is False, + "manifest_hash": manifest_receipt.get("sha256") == file_sha256(manifest_path), + "manifest_table_count": manifest_receipt.get("table_count") == len(manifest["tables"]), + "manifest_total_rows": manifest_receipt.get("total_rows") == total_rows, + "manifest_read_only": manifest["singleton"]["identity"].get("transaction_read_only") == "on", + "manifest_database": manifest["singleton"]["identity"].get("database") == source.get("database"), + "binding_payload": binding.get("payload") == expected_binding["payload"], + "binding_sha256": binding.get("sha256") == expected_binding["sha256"], + "binding_algorithm": binding.get("algorithm") == "sha256", + "dump_sha256": bool(SHA256_RE.fullmatch(str(dump.get("sha256") or ""))), + "manifest_sql_sha256": bool(SHA256_RE.fullmatch(str(manifest_receipt.get("manifest_sql_sha256") or ""))), + "source_context_sha256": bool(SHA256_RE.fullmatch(str(source_context.get("sha256") or ""))), + "snapshot_identity": all( + bool(snapshot.get(field)) + for field in ("captured_at_utc", "exported_snapshot_id", "txid_snapshot", "wal_lsn", "system_identifier") + ), + } + failed = sorted(name for name, passed in checks.items() if not passed) + if failed: + raise ValueError(f"{label} capture failed validation: " + ", ".join(failed)) + return receipt, manifest + + +def manifest_fingerprint(manifest: dict[str, Any]) -> dict[str, Any]: + tables = { + name: {"row_count": int(row["row_count"]), "rowset_md5": row.get("rowset_md5")} + for name, row in sorted(manifest["tables"].items()) + } + structures = { + kind: canonical_hash(manifest["singleton"][kind].get("items", [])) + for kind in sorted(SINGLETON_KINDS - {"identity"}) + } + return { + "table_count": len(tables), + "total_rows": sum(row["row_count"] for row in tables.values()), + "table_fingerprint_sha256": canonical_hash(tables), + "structure_fingerprint_sha256": canonical_hash(structures), + "tables": tables, + "structures": structures, + } + + +def verify_delta( + *, + baseline_receipt_path: Path, + baseline_manifest_path: Path, + baseline_authorization_ref: str, + current_receipt_path: Path, + current_manifest_path: Path, + current_authorization_ref: str, +) -> dict[str, Any]: + baseline_receipt, baseline_manifest = validate_capture( + baseline_receipt_path, + baseline_manifest_path, + expected_authorization_ref=baseline_authorization_ref, + label="baseline", + ) + current_receipt, current_manifest = validate_capture( + current_receipt_path, + current_manifest_path, + expected_authorization_ref=current_authorization_ref, + label="current", + ) + baseline_snapshot = baseline_receipt["snapshot"] + current_snapshot = current_receipt["snapshot"] + baseline_time = parse_timestamp(baseline_snapshot["captured_at_utc"], "baseline snapshot time") + current_time = parse_timestamp(current_snapshot["captured_at_utc"], "current snapshot time") + if current_time <= baseline_time: + raise ValueError("current source capture must be newer than the restored baseline capture") + if current_receipt.get("source") != baseline_receipt.get("source"): + raise ValueError("current source identity does not match the restored baseline source") + if current_snapshot.get("system_identifier") != baseline_snapshot.get("system_identifier"): + raise ValueError("current source PostgreSQL system identifier changed") + + baseline = manifest_fingerprint(baseline_manifest) + current = manifest_fingerprint(current_manifest) + baseline_tables = baseline.pop("tables") + current_tables = current.pop("tables") + baseline_structures = baseline.pop("structures") + current_structures = current.pop("structures") + added_tables = sorted(set(current_tables) - set(baseline_tables)) + removed_tables = sorted(set(baseline_tables) - set(current_tables)) + changed_tables = [ + {"table": table, "baseline": baseline_tables[table], "current": current_tables[table]} + for table in sorted(set(baseline_tables) & set(current_tables)) + if baseline_tables[table] != current_tables[table] + ] + changed_structures = [ + { + "kind": kind, + "baseline_sha256": baseline_structures.get(kind), + "current_sha256": current_structures.get(kind), + } + for kind in sorted(set(baseline_structures) | set(current_structures)) + if baseline_structures.get(kind) != current_structures.get(kind) + ] + delta_detected = bool(added_tables or removed_tables or changed_tables or changed_structures) + return { + "artifact": "vps_canonical_snapshot_postflight_delta", + "schema": DELTA_SCHEMA, + "generated_at_utc": datetime.now(UTC).isoformat(), + "status": "pass", + "baseline": { + "run_id": baseline_receipt["run_id"], + "capture_authorization_ref": baseline_authorization_ref, + "captured_at_utc": baseline_snapshot["captured_at_utc"], + "capture_receipt_sha256": file_sha256(baseline_receipt_path), + "manifest_sha256": file_sha256(baseline_manifest_path), + **baseline, + }, + "current": { + "run_id": current_receipt["run_id"], + "capture_authorization_ref": current_authorization_ref, + "captured_at_utc": current_snapshot["captured_at_utc"], + "capture_receipt_sha256": file_sha256(current_receipt_path), + "manifest_sha256": file_sha256(current_manifest_path), + **current, + }, + "delta": { + "delta_detected": delta_detected, + "added_tables": added_tables, + "removed_tables": removed_tables, + "changed_tables": changed_tables, + "changed_structures": changed_structures, + "total_row_delta": current["total_rows"] - baseline["total_rows"], + }, + "strongest_claim_allowed": ( + "restored snapshot still matches the newest postflight VPS capture" + if not delta_detected + else "restored snapshot parity is bounded to its capture; post-snapshot VPS deltas are explicit" + ), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline-receipt", required=True, type=Path) + parser.add_argument("--baseline-manifest", required=True, type=Path) + parser.add_argument("--baseline-authorization-ref", required=True) + parser.add_argument("--current-receipt", required=True, type=Path) + parser.add_argument("--current-manifest", required=True, type=Path) + parser.add_argument("--current-authorization-ref", required=True) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + try: + payload = verify_delta( + baseline_receipt_path=args.baseline_receipt, + baseline_manifest_path=args.baseline_manifest, + baseline_authorization_ref=args.baseline_authorization_ref, + current_receipt_path=args.current_receipt, + current_manifest_path=args.current_manifest, + current_authorization_ref=args.current_authorization_ref, + ) + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: + payload = { + "artifact": "vps_canonical_snapshot_postflight_delta", + "schema": DELTA_SCHEMA, + "generated_at_utc": datetime.now(UTC).isoformat(), + "status": "fail", + "error": str(exc), + "strongest_claim_allowed": "post-snapshot VPS delta is not proven", + } + write_private_json(args.output, payload) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 if payload.get("status") == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_attest_gcp_reasoning_compute.py b/tests/test_attest_gcp_reasoning_compute.py new file mode 100644 index 0000000..b3906fc --- /dev/null +++ b/tests/test_attest_gcp_reasoning_compute.py @@ -0,0 +1,128 @@ +import json +from pathlib import Path + +import pytest + +from ops import attest_gcp_reasoning_compute as attestation + + +def reasoning_receipt(path: Path) -> Path: + path.write_text( + json.dumps( + { + "schema": attestation.REASONING_SCHEMA, + "status": "pass", + "generated_at_utc": "2026-07-15T01:00:00+00:00", + "completed_at_utc": "2026-07-15T01:01:00+00:00", + "target_database": "teleo_clone_current_test", + "source_compute": "teleo-prod-1", + "errors": [], + "posted_to_telegram": False, + "database_write_attempted": False, + }, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return path + + +def compute_identity() -> dict: + return { + "project": "teleo-501523", + "project_number": "785938879453", + "instance": "teleo-prod-1", + "instance_id": "123456789", + "zone": "europe-west6-a", + "zone_resource": "projects/123456789/zones/europe-west6-a", + "network_resource": "projects/123456789/networks/teleo-staging-net", + "private_ip": "10.61.0.2", + "service_account": "teleo-runtime@teleo-501523.iam.gserviceaccount.com", + "expected_private_network": "projects/teleo-501523/global/networks/teleo-staging-net", + "checks": { + "project": True, + "project_number": True, + "expected_network_project": True, + "instance": True, + "instance_id": True, + "zone": True, + "network": True, + "private_ip": True, + "service_account": True, + }, + } + + +def test_attestation_binds_reasoning_hash_to_live_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + receipt_path = reasoning_receipt(tmp_path / "reasoning.json") + observed = {} + + def fake_identity(project, instance, zone, network, *, timeout): + observed.update( + { + "project": project, + "instance": instance, + "zone": zone, + "network": network, + "timeout": timeout, + } + ) + return compute_identity() + + monkeypatch.setattr(attestation, "gce_compute_identity", fake_identity) + + result = attestation.attest_reasoning_compute( + reasoning_receipt_path=receipt_path, + restore_run_id="gcp-restore-current-test123", + target_database="teleo_clone_current_test", + generated_at_utc="2026-07-15T01:01:05+00:00", + ) + + assert result["status"] == "pass" + assert result["checks"]["gce_metadata_identity"] is True + assert result["reasoning_receipt"]["sha256"] == attestation.sha256_file(receipt_path) + assert result["compute"]["instance_id"] == "123456789" + assert observed == { + "project": "teleo-501523", + "instance": "teleo-prod-1", + "zone": "europe-west6-a", + "network": "projects/teleo-501523/global/networks/teleo-staging-net", + "timeout": 10.0, + } + + +def test_attestation_rejects_claimed_wrong_compute_before_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + receipt_path = reasoning_receipt(tmp_path / "reasoning.json") + payload = json.loads(receipt_path.read_text()) + payload["source_compute"] = "untrusted-vm" + receipt_path.write_text(json.dumps(payload) + "\n", encoding="utf-8") + monkeypatch.setattr( + attestation, + "gce_compute_identity", + lambda *args, **kwargs: pytest.fail("metadata must not be queried"), + ) + + with pytest.raises(ValueError, match="source_compute"): + attestation.attest_reasoning_compute( + reasoning_receipt_path=receipt_path, + restore_run_id="gcp-restore-current-test123", + target_database="teleo_clone_current_test", + ) + + +def test_attestation_rejects_send_or_write_receipt(tmp_path: Path) -> None: + receipt_path = reasoning_receipt(tmp_path / "reasoning.json") + payload = json.loads(receipt_path.read_text()) + payload["posted_to_telegram"] = True + payload["database_write_attempted"] = True + receipt_path.write_text(json.dumps(payload) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match=r"no_database_write.*no_telegram_send"): + attestation.attest_reasoning_compute( + reasoning_receipt_path=receipt_path, + restore_run_id="gcp-restore-current-test123", + target_database="teleo_clone_current_test", + ) diff --git a/tests/test_capture_vps_canonical_postgres_snapshot.py b/tests/test_capture_vps_canonical_postgres_snapshot.py index ee1f07e..762d14e 100644 --- a/tests/test_capture_vps_canonical_postgres_snapshot.py +++ b/tests/test_capture_vps_canonical_postgres_snapshot.py @@ -1,8 +1,18 @@ +import json from pathlib import Path +import pytest + from ops.capture_vps_canonical_postgres_snapshot import ( + SAFE_AUTHORIZATION_REF_RE, SAFE_SSH_TARGET_RE, + SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + build_provenance_binding, + canonical_sha256, cleanup_failed_output, + load_service_state, + load_snapshot_metadata, + parse_args, remote_capture_script, toc_counts, ) @@ -21,8 +31,14 @@ def test_remote_capture_uses_one_exported_read_only_snapshot() -> None: assert "[0-9A-F]+-[0-9A-F]+-[0-9]+" in script assert '--snapshot="$snapshot_id"' in script assert '-v "snapshot_id=$snapshot_id"' in script + assert "txid_current_snapshot()::text" in script + assert "pg_current_wal_lsn()::text" in script + assert "system_identifier::text from pg_control_system()" in script + assert "source-snapshot.json" in script assert "--no-owner --no-acl" in script assert "cmp -s" in script + assert 'assert_service_healthy "$remote_root/service-before.txt"' in script + assert 'assert_service_healthy "$remote_root/service-after.txt"' in script assert "systemctl restart" not in script assert "docker restart" not in script @@ -62,8 +78,129 @@ def test_failure_cleanup_never_removes_preexisting_output(tmp_path: Path) -> Non assert not created_by_run.exists() +def test_source_service_state_requires_healthy_unchanged_capable_fields(tmp_path: Path) -> None: + state_path = tmp_path / "service.txt" + state_path.write_text("ActiveState=active\nSubState=running\nMainPID=347406\nNRestarts=0\n") + + assert load_service_state(state_path)["MainPID"] == "347406" + + state_path.write_text("ActiveState=failed\nSubState=dead\nMainPID=0\nNRestarts=0\n") + with pytest.raises(RuntimeError, match="not active/running"): + load_service_state(state_path) + + def test_ssh_target_rejects_open_ssh_option_injection() -> None: assert SAFE_SSH_TARGET_RE.fullmatch("root@77.42.65.182") assert SAFE_SSH_TARGET_RE.fullmatch("teleo-staging-1.internal") assert not SAFE_SSH_TARGET_RE.fullmatch("-oProxyCommand=malicious") assert not SAFE_SSH_TARGET_RE.fullmatch("root@host command") + + +def test_source_snapshot_metadata_and_provenance_binding(tmp_path: Path) -> None: + snapshot_path = tmp_path / "source-snapshot.json" + snapshot_path.write_text( + json.dumps( + { + "exported_snapshot_id": "00000003-000001A2-1", + "txid_snapshot": "100:120:105,111", + "wal_lsn": "0/16B6C50", + "system_identifier": "7649789040005668902", + "captured_at_utc": "2026-07-15T01:02:03.456789+00:00", + } + ) + ) + + snapshot = load_snapshot_metadata(snapshot_path) + binding = build_provenance_binding( + run_id="canonical-20260715", + authorization_ref="codex-delegation:019f47af-f2cc-7c10-b928-a9283afa2621", + source={ + "ssh_target": "root@77.42.65.182", + "container": "teleo-pg", + "database": "teleo", + "service": "leoclean-gateway.service", + }, + snapshot=snapshot, + dump_sha256="1" * 64, + manifest_sql_sha256="2" * 64, + source_manifest_sha256="3" * 64, + source_context_sha256="4" * 64, + ) + + assert binding["algorithm"] == "sha256" + assert binding["payload"] == { + "receipt_schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "run_id": "canonical-20260715", + "capture_authorization_ref": "codex-delegation:019f47af-f2cc-7c10-b928-a9283afa2621", + "source": { + "ssh_target": "root@77.42.65.182", + "container": "teleo-pg", + "database": "teleo", + "service": "leoclean-gateway.service", + }, + "exported_snapshot_id": "00000003-000001A2-1", + "txid_snapshot": "100:120:105,111", + "wal_lsn": "0/16B6C50", + "source_system_identifier": "7649789040005668902", + "dump_sha256": "1" * 64, + "manifest_sql_sha256": "2" * 64, + "source_manifest_sha256": "3" * 64, + "source_context_sha256": "4" * 64, + } + assert binding["sha256"] == canonical_sha256(binding["payload"]) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("exported_snapshot_id", "unsafe snapshot"), + ("txid_snapshot", "100:120:not-a-txid"), + ("wal_lsn", "not-an-lsn"), + ("system_identifier", "system-1"), + ], +) +def test_source_snapshot_metadata_rejects_invalid_identity(tmp_path: Path, field: str, value: str) -> None: + payload = { + "exported_snapshot_id": "00000003-000001A2-1", + "txid_snapshot": "100:120:", + "wal_lsn": "0/16B6C50", + "system_identifier": "7649789040005668902", + "captured_at_utc": "2026-07-15T01:02:03+00:00", + } + payload[field] = value + path = tmp_path / "source-snapshot.json" + path.write_text(json.dumps(payload)) + + with pytest.raises(RuntimeError, match=field): + load_snapshot_metadata(path) + + +def test_authorization_ref_is_required_and_safe_metadata(tmp_path: Path) -> None: + ssh_key = tmp_path / "id_ed25519" + ssh_key.write_text("test-only") + manifest = tmp_path / "manifest.sql" + manifest.write_text("select 1;\n") + base = [ + "--execute", + "--ssh-target", + "root@77.42.65.182", + "--ssh-key", + str(ssh_key), + "--manifest", + str(manifest), + "--run-id", + "canonical-20260715", + "--output-dir", + str(tmp_path / "capture"), + ] + + with pytest.raises(SystemExit): + parse_args(base) + with pytest.raises(SystemExit): + parse_args([*base, "--authorization-ref", "unsafe authorization ref"]) + + authorization_ref = "codex-delegation:019f47af-f2cc-7c10-b928-a9283afa2621" + args = parse_args([*base, "--authorization-ref", authorization_ref]) + assert args.authorization_ref == authorization_ref + assert SAFE_AUTHORIZATION_REF_RE.fullmatch(authorization_ref) + assert not SAFE_AUTHORIZATION_REF_RE.fullmatch("authorization\nforged") diff --git a/tests/test_restore_gcp_generated_postgres_snapshot.py b/tests/test_restore_gcp_generated_postgres_snapshot.py index 77f3a1c..5c152b9 100644 --- a/tests/test_restore_gcp_generated_postgres_snapshot.py +++ b/tests/test_restore_gcp_generated_postgres_snapshot.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest from ops import restore_gcp_generated_postgres_snapshot as restore +from ops.capture_vps_canonical_postgres_snapshot import build_provenance_binding SERVICE_STATE = { "ActiveState": "active", @@ -15,6 +16,7 @@ SERVICE_STATE = { "NRestarts": "0", "ExecMainStartTimestamp": "Thu 2026-07-09 07:00:04 UTC", } +AUTHORIZATION_REF = "codex-delegation:019f47af-f2cc-7c10-b928-a9283afa2621" def manifest_rows(database: str, *, target: bool, row_count: int = 1) -> list[dict]: @@ -94,6 +96,65 @@ def args_for(tmp_path: Path, **overrides) -> argparse.Namespace: dump.write_bytes(b"PGDMPfixture") source_manifest = tmp_path / "source-manifest.jsonl" source_manifest.write_text(manifest_text("teleo", target=False), encoding="utf-8") + manifest_sql = Path("ops/postgres_parity_manifest.sql").resolve() + source_context_sha256 = "4" * 64 + snapshot = { + "exported_snapshot_id": "00000003-000001A2-1", + "txid_snapshot": "100:120:105,111", + "wal_lsn": "0/16B6C50", + "system_identifier": "7649789040005668902", + "captured_at_utc": "2026-07-15T01:02:03.456789+00:00", + } + source = { + "ssh_target": "root@77.42.65.182", + "container": "teleo-pg", + "database": "teleo", + "service": "leoclean-gateway.service", + } + dump_sha256 = hashlib.sha256(dump.read_bytes()).hexdigest() + source_manifest_sha256 = hashlib.sha256(source_manifest.read_bytes()).hexdigest() + manifest_sql_sha256 = hashlib.sha256(manifest_sql.read_bytes()).hexdigest() + source_receipt = tmp_path / "source-receipt.json" + source_receipt.write_text( + json.dumps( + { + "artifact": "vps_canonical_postgres_exported_snapshot", + "schema": restore.SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "receipt_version": 2, + "generated_at_utc": "2026-07-15T01:02:04+00:00", + "status": "pass", + "run_id": "canonical-20260715", + "capture_authorization_ref": AUTHORIZATION_REF, + "source": source, + "snapshot_exported": True, + "snapshot": snapshot, + "dump": {"path": "/private/source.dump", "bytes": dump.stat().st_size, "sha256": dump_sha256}, + "manifest": { + "path": "/private/source-manifest.jsonl", + "sha256": source_manifest_sha256, + "manifest_sql_sha256": manifest_sql_sha256, + "table_count": 1, + "total_rows": 1, + }, + "source_context": {"path": "/private/source-context.txt", "sha256": source_context_sha256}, + "source_service": {"before": SERVICE_STATE, "after": SERVICE_STATE, "unchanged": True}, + "provenance_binding": build_provenance_binding( + run_id="canonical-20260715", + authorization_ref=AUTHORIZATION_REF, + source=source, + snapshot=snapshot, + dump_sha256=dump_sha256, + manifest_sql_sha256=manifest_sql_sha256, + source_manifest_sha256=source_manifest_sha256, + source_context_sha256=source_context_sha256, + ), + "service_unchanged": True, + "production_db_mutated": False, + "telegram_send_attempted": False, + } + ), + encoding="utf-8", + ) values = { "operation": "restore", "execute": True, @@ -101,15 +162,26 @@ def args_for(tmp_path: Path, **overrides) -> argparse.Namespace: "run_id": "gcp-restore-test12345", "host": restore.DEFAULT_HOST, "project": restore.DEFAULT_PROJECT, + "cloudsql_instance": restore.DEFAULT_INSTANCE, + "expected_private_network": restore.DEFAULT_PRIVATE_NETWORK, + "expected_compute_instance": restore.DEFAULT_COMPUTE_INSTANCE, + "expected_compute_zone": restore.DEFAULT_COMPUTE_ZONE, + "expected_source_ssh_target": restore.DEFAULT_SOURCE_SSH_TARGET, + "expected_source_container": restore.DEFAULT_SOURCE_CONTAINER, + "expected_source_database": restore.DEFAULT_SOURCE_DATABASE, + "expected_source_service": restore.DEFAULT_SOURCE_SERVICE, "password_secret": restore.DEFAULT_SECRET, "service": restore.DEFAULT_SERVICE, "command_timeout": 2.0, "psql_bin": "psql", "run_root": tmp_path / "runs", "dump": dump, - "expected_dump_sha256": hashlib.sha256(dump.read_bytes()).hexdigest(), + "expected_dump_sha256": dump_sha256, "source_manifest": source_manifest, - "manifest_sql": Path("ops/postgres_parity_manifest.sql").resolve(), + "source_receipt": source_receipt, + "expected_source_receipt_sha256": hashlib.sha256(source_receipt.read_bytes()).hexdigest(), + "expected_capture_authorization_ref": AUTHORIZATION_REF, + "manifest_sql": manifest_sql, "pg_restore_bin": "pg_restore", "restore_timeout": 5.0, "manifest_timeout": 5.0, @@ -128,6 +200,54 @@ def install_restore_fakes( state = {"exists": False, "dropped": False} monkeypatch.setattr(restore.os, "geteuid", lambda: 0) monkeypatch.setattr(restore, "service_state", lambda *_args, **_kwargs: dict(SERVICE_STATE)) + monkeypatch.setattr( + restore, + "gce_compute_identity", + lambda project, instance, zone, expected_private_network, **_kwargs: { + "project": project, + "project_number": "785938879453", + "instance": instance, + "instance_id": "123456789", + "zone": zone, + "expected_private_network": expected_private_network, + "private_ip": "10.61.0.2", + "network_resource": "projects/123456789/networks/teleo-staging-net", + "service_account": "teleo-runtime@teleo-501523.iam.gserviceaccount.com", + "checks": { + "project": True, + "project_number": True, + "expected_network_project": True, + "instance": True, + "instance_id": True, + "zone": True, + "network": True, + "private_ip": True, + "service_account": True, + }, + }, + ) + monkeypatch.setattr( + restore, + "cloudsql_control_plane_state", + lambda project, instance, host, expected_private_network, **_kwargs: { + "project": project, + "instance": instance, + "private_addresses": [host], + "expected_private_host": host, + "private_network": expected_private_network, + "expected_private_network": expected_private_network, + "public_ip_disabled": True, + "checks": { + "instance": True, + "state": True, + "postgres_16": True, + "public_ip_disabled": True, + "private_network": True, + "expected_private_host": True, + "no_non_private_address": True, + }, + }, + ) monkeypatch.setattr(restore, "resolve_password", lambda *_args, **_kwargs: "private-password") monkeypatch.setattr( restore, @@ -179,7 +299,10 @@ def test_restore_success_retains_only_exact_parity_clone( assert result["phase"] == "retained_for_no_send_replay" assert result["parity"]["problems"] == [] assert result["parity"]["details"]["source_total_rows"] == 1 + assert result["source"]["capture_receipt"]["schema"] == restore.SOURCE_SNAPSHOT_RECEIPT_SCHEMA + assert result["source"]["capture_receipt"]["provenance_binding_sha256"] assert result["target"]["connectivity"]["private_connectivity"] is True + assert result["target"]["cloudsql"]["public_ip_disabled"] is True assert result["target"]["connectivity"]["ssl"] is True assert result["live_service"]["unchanged"] is True assert result["cleanup"]["clone_database_remaining"] == 1 @@ -187,6 +310,15 @@ def test_restore_success_retains_only_exact_parity_clone( run_dir = args.run_root / args.run_id assert (run_dir / "target-manifest.jsonl").stat().st_mode & 0o777 == 0o600 assert (run_dir / "restore-receipt.json").stat().st_mode & 0o777 == 0o600 + connectivity_path = run_dir / "gcp-private-connectivity.json" + assert connectivity_path.stat().st_mode & 0o777 == 0o600 + connectivity = json.loads(connectivity_path.read_text()) + assert connectivity["target_database"] == args.target_db + assert connectivity["source_compute"] == restore.DEFAULT_COMPUTE_INSTANCE + assert connectivity["public_ip_disabled"] is True + assert ( + result["target"]["connectivity_proof"]["sha256"] == hashlib.sha256(connectivity_path.read_bytes()).hexdigest() + ) def test_parity_failure_drops_generated_clone_and_receipts_both_failures( @@ -234,6 +366,79 @@ def test_create_command_failure_still_drops_a_committed_clone( assert state == {"exists": False, "dropped": True} +def test_restore_rejects_manifest_not_bound_to_source_receipt_before_credentials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args = args_for(tmp_path) + install_restore_fakes(monkeypatch, target_manifest=manifest_text(args.target_db, target=True)) + receipt = json.loads(args.source_receipt.read_text()) + receipt["manifest"]["sha256"] = "0" * 64 + args.source_receipt.write_text(json.dumps(receipt)) + args.expected_source_receipt_sha256 = hashlib.sha256(args.source_receipt.read_bytes()).hexdigest() + credential_called = False + + def unexpected_credential(*_args, **_kwargs): + nonlocal credential_called + credential_called = True + return "private-password" + + monkeypatch.setattr(restore, "resolve_password", unexpected_credential) + + result = restore.run_restore(args) + + assert result["status"] == "fail" + assert result["error"]["phase"] == "validation" + assert "source_manifest_sha256" in result["error"]["message"] + assert credential_called is False + + +def test_restore_rejects_unexpected_capture_authorization_before_credentials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args = args_for(tmp_path, expected_capture_authorization_ref="codex-delegation:another-authorized-run") + install_restore_fakes(monkeypatch, target_manifest=manifest_text(args.target_db, target=True)) + credential_called = False + + def unexpected_credential(*_args, **_kwargs): + nonlocal credential_called + credential_called = True + return "private-password" + + monkeypatch.setattr(restore, "resolve_password", unexpected_credential) + + result = restore.run_restore(args) + + assert result["status"] == "fail" + assert result["error"]["phase"] == "validation" + assert "capture_authorization_ref" in result["error"]["message"] + assert credential_called is False + + +def test_restore_rejects_noncanonical_source_identity_before_credentials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args = args_for(tmp_path, expected_source_container="different-postgres") + install_restore_fakes(monkeypatch, target_manifest=manifest_text(args.target_db, target=True)) + credential_called = False + + def unexpected_credential(*_args, **_kwargs): + nonlocal credential_called + credential_called = True + return "private-password" + + monkeypatch.setattr(restore, "resolve_password", unexpected_credential) + + result = restore.run_restore(args) + + assert result["status"] == "fail" + assert result["error"]["phase"] == "validation" + assert "source_identity" in result["error"]["message"] + assert credential_called is False + + def test_changed_live_service_turns_successful_restore_into_cleanup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -283,6 +488,14 @@ def test_cleanup_is_bound_to_passing_restore_receipt( "run_id", "host", "project", + "cloudsql_instance", + "expected_private_network", + "expected_compute_instance", + "expected_compute_zone", + "expected_source_ssh_target", + "expected_source_container", + "expected_source_database", + "expected_source_service", "password_secret", "service", "command_timeout", @@ -302,6 +515,106 @@ def test_cleanup_is_bound_to_passing_restore_receipt( assert state == {"exists": False, "dropped": True} +def test_cleanup_rejects_target_topology_drift_before_credentials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args = args_for(tmp_path) + state = install_restore_fakes(monkeypatch, target_manifest=manifest_text(args.target_db, target=True)) + assert restore.run_restore(args)["status"] == "pass" + cleanup_args = argparse.Namespace( + **{ + key: getattr(args, key) + for key in ( + "target_db", + "run_id", + "host", + "project", + "cloudsql_instance", + "expected_private_network", + "expected_compute_instance", + "expected_compute_zone", + "expected_source_ssh_target", + "expected_source_container", + "expected_source_database", + "expected_source_service", + "password_secret", + "service", + "command_timeout", + "psql_bin", + "run_root", + ) + }, + operation="cleanup", + execute=True, + ) + cleanup_args.host = "10.61.0.4" + credential_called = False + + def unexpected_credential(*_args, **_kwargs): + nonlocal credential_called + credential_called = True + return "private-password" + + monkeypatch.setattr(restore, "resolve_password", unexpected_credential) + + with pytest.raises(restore.RestoreError, match="execution contract"): + restore.run_cleanup(cleanup_args) + + assert credential_called is False + assert state == {"exists": True, "dropped": False} + + +def test_cleanup_retains_failure_receipt_when_post_drop_readback_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args = args_for(tmp_path) + state = install_restore_fakes(monkeypatch, target_manifest=manifest_text(args.target_db, target=True)) + assert restore.run_restore(args)["status"] == "pass" + cleanup_args = argparse.Namespace( + **{ + key: getattr(args, key) + for key in ( + "target_db", + "run_id", + "host", + "project", + "cloudsql_instance", + "expected_private_network", + "expected_compute_instance", + "expected_compute_zone", + "expected_source_ssh_target", + "expected_source_container", + "expected_source_database", + "expected_source_service", + "password_secret", + "service", + "command_timeout", + "psql_bin", + "run_root", + ) + }, + operation="cleanup", + execute=True, + ) + monkeypatch.setattr( + restore, + "rollback_database_state", + lambda *_args, **_kwargs: (_ for _ in ()).throw(restore.RestoreError("rollback readback unavailable")), + ) + + result = restore.run_cleanup(cleanup_args) + + assert result["status"] == "fail" + assert result["error"]["phase"] == "rollback_readback" + assert result["database"]["clone_database_remaining"] == 0 + assert state == {"exists": False, "dropped": True} + receipt_path = args.run_root / args.run_id / "cleanup-receipt.json" + assert receipt_path.stat().st_mode & 0o777 == 0o600 + assert json.loads(receipt_path.read_text())["error"]["phase"] == "rollback_readback" + + def test_restore_dump_uses_tls_no_owner_no_acl_and_keeps_secret_out_of_argv( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -326,6 +639,110 @@ def test_restore_dump_uses_tls_no_owner_no_acl_and_keeps_secret_out_of_argv( assert captured["env"]["PGPASSWORD"] == "private-password" +def test_cloudsql_control_plane_preflight_requires_private_only_expected_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = { + "name": "teleo-pgvector-standby", + "state": "RUNNABLE", + "databaseVersion": "POSTGRES_16", + "region": "europe-west6", + "settings": { + "tier": "db-custom-1-3840", + "ipConfiguration": { + "ipv4Enabled": False, + "privateNetwork": "projects/teleo-501523/global/networks/teleo-staging-net", + }, + }, + "ipAddresses": [{"type": "PRIVATE", "ipAddress": "10.61.0.3"}], + } + + monkeypatch.setattr( + restore, + "_run", + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, json.dumps(payload), ""), + ) + result = restore.cloudsql_control_plane_state( + "teleo-501523", + "teleo-pgvector-standby", + "10.61.0.3", + restore.DEFAULT_PRIVATE_NETWORK, + timeout=2.0, + ) + assert result["public_ip_disabled"] is True + assert result["private_addresses"] == ["10.61.0.3"] + + payload["settings"]["ipConfiguration"]["ipv4Enabled"] = True + payload["ipAddresses"].append({"type": "PRIMARY", "ipAddress": "34.1.2.3"}) + with pytest.raises(restore.RestoreError, match="no_non_private_address, public_ip_disabled"): + restore.cloudsql_control_plane_state( + "teleo-501523", + "teleo-pgvector-standby", + "10.61.0.3", + restore.DEFAULT_PRIVATE_NETWORK, + timeout=2.0, + ) + + +def test_gce_compute_identity_is_bound_to_metadata_instance_zone_and_network( + monkeypatch: pytest.MonkeyPatch, +) -> None: + metadata = { + "project/project-id": "teleo-501523", + "project/numeric-project-id": "785938879453", + "instance/name": "teleo-prod-1", + "instance/id": "123456789", + "instance/zone": "projects/123456789/zones/europe-west6-a", + "instance/network-interfaces/0/network": "projects/785938879453/networks/teleo-staging-net", + "instance/network-interfaces/0/ip": "10.61.0.2", + "instance/service-accounts/default/email": "teleo-runtime@teleo-501523.iam.gserviceaccount.com", + } + monkeypatch.setattr(restore, "gce_metadata_value", lambda path, **_kwargs: metadata[path]) + + result = restore.gce_compute_identity( + "teleo-501523", + "teleo-prod-1", + "europe-west6-a", + restore.DEFAULT_PRIVATE_NETWORK, + timeout=2.0, + ) + + assert result["instance_id"] == "123456789" + assert result["project_number"] == "785938879453" + assert result["zone"] == "europe-west6-a" + assert all(result["checks"].values()) + + with pytest.raises(restore.RestoreError, match="instance"): + restore.gce_compute_identity( + "teleo-501523", + "different-instance", + "europe-west6-a", + restore.DEFAULT_PRIVATE_NETWORK, + timeout=2.0, + ) + + metadata["instance/name"] = "teleo-prod-1" + metadata["instance/network-interfaces/0/network"] = "projects/999999999999/networks/teleo-staging-net" + with pytest.raises(restore.RestoreError, match="network"): + restore.gce_compute_identity( + "teleo-501523", + "teleo-prod-1", + "europe-west6-a", + restore.DEFAULT_PRIVATE_NETWORK, + timeout=2.0, + ) + + metadata["instance/network-interfaces/0/network"] = "projects/785938879453/networks/teleo-staging-net" + with pytest.raises(restore.RestoreError, match="expected_network_project"): + restore.gce_compute_identity( + "teleo-501523", + "teleo-prod-1", + "europe-west6-a", + "projects/other-project/global/networks/teleo-staging-net", + timeout=2.0, + ) + + def test_pg_restore_preflight_rejects_client_older_than_snapshot( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_verify_gcp_canonical_lifecycle.py b/tests/test_verify_gcp_canonical_lifecycle.py new file mode 100644 index 0000000..28df37b --- /dev/null +++ b/tests/test_verify_gcp_canonical_lifecycle.py @@ -0,0 +1,703 @@ +import hashlib +import json +from datetime import UTC, datetime +from pathlib import Path + +from ops.verify_gcp_canonical_lifecycle import EXPECTED_BLIND_ROW_IDS +from ops.verify_gcp_canonical_lifecycle import verify_lifecycle as _verify_lifecycle + +VERIFY_NOW = datetime(2026, 7, 15, 1, 3, tzinfo=UTC) + + +def verify_lifecycle(**kwargs): + return _verify_lifecycle(**kwargs, now=VERIFY_NOW) + + +def write_json(path: Path, payload: dict) -> Path: + path.write_text(json.dumps(payload, sort_keys=True) + "\n", encoding="utf-8") + return path + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: + target_db = "teleo_clone_current_test" + run_id = "gcp-restore-current-test123" + healthy_service = { + "ActiveState": "active", + "SubState": "running", + "MainPID": "148735", + "NRestarts": "0", + } + structural_hashes = { + kind: {"source": format(index, "x") * 64, "target": format(index, "x") * 64} + for index, kind in enumerate( + ( + "schemas", + "columns", + "constraints", + "indexes", + "sequences", + "views", + "functions", + "triggers", + "types", + "policies", + ), + 1, + ) + } + exact_parity_details = { + "source_database": "teleo", + "target_database": target_db, + "source_table_count": 39, + "target_table_count": 39, + "source_total_rows": 52167, + "target_total_rows": 52167, + "table_mismatches": [], + "structural_hashes": structural_hashes, + "required_extension_mismatches": {}, + "application_role_mismatches": {}, + "target_extra_application_roles": [], + "performance": [{"query": "count_claims", "source_ms": 1.0, "target_ms": 2.0, "source_ratio": 0.4}], + "performance_problems": [], + } + source = { + "artifact": "vps_canonical_postgres_exported_snapshot", + "schema": "livingip.sourceSnapshotReceipt.v2", + "receipt_version": 2, + "status": "pass", + "run_id": "canonical-current-test", + "capture_authorization_ref": "codex-delegation:test1234", + "source": { + "ssh_target": "root@77.42.65.182", + "container": "teleo-pg", + "database": "teleo", + "service": "leoclean-gateway.service", + }, + "snapshot_exported": True, + "snapshot": { + "captured_at_utc": "2026-07-15T00:59:00+00:00", + "exported_snapshot_id": "00000003-000001A2-1", + "txid_snapshot": "100:120:", + "wal_lsn": "0/16B6C50", + "system_identifier": "7649789040005668902", + }, + "service_unchanged": True, + "source_service": {"before": healthy_service, "after": healthy_service, "unchanged": True}, + "production_db_mutated": False, + "telegram_send_attempted": False, + "dump": {"sha256": "a" * 64}, + "manifest": {"sha256": "b" * 64, "table_count": 39, "total_rows": 52167}, + "provenance_binding": {"algorithm": "sha256", "payload": {"bound": True}, "sha256": "c" * 64}, + } + source_path = write_json(tmp_path / "source.json", source) + restore = { + "artifact": "gcp_generated_postgres_snapshot_restore", + "status": "pass", + "generated_at_utc": "2026-07-15T00:59:10+00:00", + "completed_at_utc": "2026-07-15T00:59:30+00:00", + "run_id": run_id, + "target_database": target_db, + "source": { + "capture_receipt": { + "sha256": sha256(source_path), + "capture_authorization_ref": "codex-delegation:test1234", + "provenance_binding_sha256": "c" * 64, + "dump_sha256": "a" * 64, + "manifest_sha256": "b" * 64, + "checks": { + name: True + for name in ( + "artifact", + "schema", + "receipt_version", + "status", + "snapshot_exported", + "service_unchanged", + "source_service_healthy", + "source_not_mutated", + "telegram_not_sent", + "run_id", + "capture_authorization_ref", + "source_identity", + "source_database", + "snapshot_identity", + "dump_sha256", + "dump_bytes", + "source_manifest_sha256", + "manifest_sql_sha256", + "source_context_sha256", + "table_count", + "total_rows", + "binding_algorithm", + "binding_payload", + "binding_sha256", + ) + }, + } + }, + "execution_contract": { + "project": "teleo-501523", + "cloudsql_instance": "teleo-pgvector-standby", + "host": "10.61.0.3", + "private_network": "projects/teleo-501523/global/networks/teleo-staging-net", + "compute_instance": "teleo-prod-1", + "compute_zone": "europe-west6-a", + "source": source["source"], + "password_secret": "gcp-teleo-pgvector-standby-postgres-password", + "service": "leoclean-gcp-prod-parallel.service", + }, + "target": { + "manifest_sha256": "d" * 64, + "connectivity": { + "private_connectivity": True, + "ssl": True, + "server_address": "10.61.0.3", + "source_compute": "teleo-prod-1", + }, + "cloudsql": { + "project": "teleo-501523", + "instance": "teleo-pgvector-standby", + "expected_private_host": "10.61.0.3", + "private_network": "projects/teleo-501523/global/networks/teleo-staging-net", + "expected_private_network": "projects/teleo-501523/global/networks/teleo-staging-net", + "public_ip_disabled": True, + "checks": { + "instance": True, + "state": True, + "postgres_16": True, + "public_ip_disabled": True, + "private_network": True, + "expected_private_host": True, + "no_non_private_address": True, + }, + }, + "compute": { + "project": "teleo-501523", + "project_number": "785938879453", + "instance": "teleo-prod-1", + "instance_id": "123456789", + "zone": "europe-west6-a", + "expected_private_network": "projects/teleo-501523/global/networks/teleo-staging-net", + "network_resource": "projects/123456789/networks/teleo-staging-net", + "private_ip": "10.61.0.2", + "service_account": "teleo-runtime@teleo-501523.iam.gserviceaccount.com", + "checks": { + "project": True, + "project_number": True, + "expected_network_project": True, + "instance": True, + "instance_id": True, + "zone": True, + "network": True, + "private_ip": True, + "service_account": True, + }, + }, + "connectivity_proof": { + "sha256": "f" * 64, + "schema": "livingip.gcpPrivatePostgresConnectivity.v2", + }, + }, + "parity": {"status": "pass", "problems": [], "details": exact_parity_details}, + "live_service": {"before": healthy_service, "after": healthy_service, "unchanged": True}, + } + restore_path = write_json(tmp_path / "restore.json", restore) + parity = { + "artifact": "canonical_postgres_parity_verification", + "status": "pass", + "completed_at_utc": "2026-07-15T00:59:45+00:00", + "scope": "gcp_staging", + "problems": [], + "source_manifest_sha256": "b" * 64, + "target_manifest_sha256": "d" * 64, + "connectivity_proof_sha256": "f" * 64, + "details": exact_parity_details, + "private_connectivity": { + "required": True, + "proof": { + "artifact": "gcp_private_postgres_connectivity", + "schema": "livingip.gcpPrivatePostgresConnectivity.v2", + "status": "pass", + "restore_run_id": run_id, + "private_connectivity": True, + "public_ip_disabled": True, + "target_database": target_db, + "source_compute": "teleo-prod-1", + "source_compute_instance_id": "123456789", + "source_compute_zone": "europe-west6-a", + "source_compute_private_ip": "10.61.0.2", + "server_address": "10.61.0.3", + "project": "teleo-501523", + "cloudsql_instance": "teleo-pgvector-standby", + "private_network": "projects/teleo-501523/global/networks/teleo-staging-net", + "cloudsql_control_plane_checks": restore["target"]["cloudsql"]["checks"], + "compute_identity_checks": restore["target"]["compute"]["checks"], + }, + }, + } + parity_path = write_json(tmp_path / "parity.json", parity) + required_checks = { + name: True + for name in ( + "exact_blind_prompt_without_ids", + "claim_and_sources_retrieved", + "discovery_show_evidence_completed", + "retrieval_receipts_proven", + "private_tls_read_only_target", + "generated_database_unchanged", + "canonical_counts_unchanged", + "database_calls_read_only", + "no_database_write", + "no_telegram_send", + "live_service_unchanged", + "live_profile_unchanged", + "temporary_profile_removed", + ) + } + reasoning = { + "schema": "livingip.gcpGeneratedDbBlindClaimCanary.v1", + "status": "pass", + "generated_at_utc": "2026-07-15T01:00:00+00:00", + "completed_at_utc": "2026-07-15T01:01:00+00:00", + "mode": "gcp_generated_db_gatewayrunner_blind_claim_no_send", + "source_compute": "teleo-prod-1", + "prompt": "Inspect the live claim without me giving you an ID. Do not change the database.", + "errors": [], + "target_database": target_db, + "posted_to_telegram": False, + "database_write_attempted": False, + "parity_receipt": {"sha256": sha256(parity_path)}, + "checks": required_checks, + "database_fingerprint_before": {"sha256": "e" * 64}, + "database_fingerprint_after": {"sha256": "e" * 64}, + "canonical_status_before": {"claims": 1}, + "canonical_status_after": {"claims": 1}, + "database_identity_before": {"database": target_db}, + "database_identity_after": {"database": target_db}, + "service_before": healthy_service, + "service_after": healthy_service, + "temp_profile_absent": True, + "temp_profile_removed": True, + "outcomes": { + "asks_for_user_iteration_before_live": True, + "challenges_weak_support": True, + "decomposes_the_bundled_legal_claim": True, + "does_not_claim_a_database_change": True, + "proposes_candidate_claims": True, + "uses_only_m3taversal_handle": True, + }, + "result": { + "reply": "The claim is weak; here are grounded, review-first alternatives.", + "database_tool_trace": { + "schema": "livingip.leoKbToolTrace.v1", + "database_tool_call_proven": True, + "database_retrieval_receipt_proven": True, + "database_tool_calls_read_only": True, + "database_tool_call_count": 3, + "database_tool_completed_count": 3, + "calls": [ + { + "database_invocations": [{"access_mode": "read_only", "subcommand": subcommand}], + "result": { + "nonempty": True, + "error_detected": False, + "row_ids": sorted(EXPECTED_BLIND_ROW_IDS), + "row_id_count": len(EXPECTED_BLIND_ROW_IDS), + "retrieval_receipt": { + "schema": "livingip.teleoKbRetrievalReceipt.v1", + "semantic_context_sha256": "1" * 64, + "artifact_state_sha256": "2" * 64, + "read_consistency_status": "stable_wal_marker", + }, + }, + } + for subcommand in ("search", "show", "evidence") + ], + }, + "gateway": { + "authorized": True, + "handler_invoked": True, + "model_free_fallback_used": False, + "posted_to_telegram": False, + "child_process": { + "exitcode": 0, + "alive_after_readback": False, + "process_group_alive_after_readback": False, + "timed_out": False, + }, + "tool_surface": {"send_message_tool_enabled": False}, + }, + }, + "wrapper_tool_proof": { + "all_bound_to_supplied_target": True, + "all_completed_successfully": True, + "complete_start_end_pairing": True, + "database_read_only_required": True, + "default_read_only_required": True, + "invocations": [ + { + "database": target_db, + "returncode": 0, + "subcommand": subcommand, + "database_identity": { + "transaction_read_only": "on", + "default_transaction_read_only": "on", + "ssl": True, + "server_address": "10.61.0.3", + }, + } + for subcommand in ("search", "show", "evidence") + ], + }, + } + reasoning_path = write_json(tmp_path / "reasoning.json", reasoning) + reasoning_compute = { + "artifact": "gcp_reasoning_compute_attestation", + "schema": "livingip.gcpReasoningComputeAttestation.v1", + "status": "pass", + "generated_at_utc": "2026-07-15T01:01:05+00:00", + "restore_run_id": run_id, + "target_database": target_db, + "reasoning_receipt": { + "sha256": sha256(reasoning_path), + "schema": reasoning["schema"], + "generated_at_utc": reasoning["generated_at_utc"], + "completed_at_utc": reasoning["completed_at_utc"], + "claimed_source_compute": reasoning["source_compute"], + }, + "compute": restore["target"]["compute"], + "checks": { + "schema": True, + "status": True, + "target_database": True, + "source_compute": True, + "chronology": True, + "no_telegram_send": True, + "no_database_write": True, + "gce_metadata_identity": True, + }, + "method": "gce_metadata_server_v1", + "mutation": { + "cloudsql_changed": False, + "compute_changed": False, + "database_changed": False, + "telegram_message_sent": False, + }, + } + reasoning_compute_path = write_json(tmp_path / "reasoning-compute.json", reasoning_compute) + current_source = json.loads(json.dumps(source)) + current_source["run_id"] = "canonical-current-postflight" + current_source["capture_authorization_ref"] = "codex-delegation:postflight1234" + current_source["snapshot"]["captured_at_utc"] = "2026-07-15T01:02:00+00:00" + current_source["snapshot"]["exported_snapshot_id"] = "00000003-000001B2-1" + current_source_path = write_json(tmp_path / "current-source.json", current_source) + source_delta = { + "artifact": "vps_canonical_snapshot_postflight_delta", + "schema": "livingip.vpsCanonicalSnapshotDelta.v1", + "status": "pass", + "baseline": { + "run_id": "canonical-current-test", + "capture_receipt_sha256": sha256(source_path), + "manifest_sha256": "b" * 64, + "capture_authorization_ref": "codex-delegation:test1234", + "captured_at_utc": "2026-07-15T00:59:00+00:00", + "table_count": 39, + "total_rows": 52167, + "table_fingerprint_sha256": "d" * 64, + "structure_fingerprint_sha256": "e" * 64, + }, + "current": { + "run_id": "canonical-current-postflight", + "capture_receipt_sha256": sha256(current_source_path), + "manifest_sha256": "b" * 64, + "capture_authorization_ref": "codex-delegation:postflight1234", + "captured_at_utc": "2026-07-15T01:02:00+00:00", + "table_count": 39, + "total_rows": 52167, + "table_fingerprint_sha256": "d" * 64, + "structure_fingerprint_sha256": "e" * 64, + }, + "delta": { + "delta_detected": False, + "added_tables": [], + "removed_tables": [], + "changed_tables": [], + "changed_structures": [], + "total_row_delta": 0, + }, + } + source_delta_path = write_json(tmp_path / "source-delta.json", source_delta) + cleanup = { + "artifact": "gcp_generated_postgres_snapshot_cleanup", + "status": "pass", + "generated_at_utc": "2026-07-15T01:01:10+00:00", + "completed_at_utc": "2026-07-15T01:01:30+00:00", + "run_id": run_id, + "target_database": target_db, + "database": {"clone_database_remaining": 0}, + "live_service": {"before": healthy_service, "after": healthy_service, "unchanged": True}, + "restore_receipt": {"sha256": sha256(restore_path)}, + "execution_contract": restore["execution_contract"], + "target": { + "cloudsql": restore["target"]["cloudsql"], + "compute": restore["target"]["compute"], + }, + } + cleanup_path = write_json(tmp_path / "cleanup.json", cleanup) + return { + "source_path": source_path, + "current_source_path": current_source_path, + "source_delta_path": source_delta_path, + "restore_path": restore_path, + "parity_path": parity_path, + "reasoning_path": reasoning_path, + "reasoning_compute_path": reasoning_compute_path, + "cleanup_path": cleanup_path, + } + + +def test_bound_live_lifecycle_passes(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + + result = verify_lifecycle(**paths) + + assert result["status"] == "pass" + assert result["failed_checks"] == [] + assert result["checks"]["clone_absent"] is True + + +def test_local_parity_or_missing_cleanup_fails_closed(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + parity = json.loads(paths["parity_path"].read_text()) + parity["scope"] = "local_restore" + write_json(paths["parity_path"], parity) + reasoning = json.loads(paths["reasoning_path"].read_text()) + reasoning["parity_receipt"]["sha256"] = sha256(paths["parity_path"]) + write_json(paths["reasoning_path"], reasoning) + cleanup = json.loads(paths["cleanup_path"].read_text()) + cleanup["database"]["clone_database_remaining"] = 1 + write_json(paths["cleanup_path"], cleanup) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert {"parity_pass", "clone_absent"} <= set(result["failed_checks"]) + + +def test_mismatched_source_or_reasoning_receipt_fails_closed(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + restore = json.loads(paths["restore_path"].read_text()) + restore["source"]["capture_receipt"]["sha256"] = "0" * 64 + write_json(paths["restore_path"], restore) + reasoning = json.loads(paths["reasoning_path"].read_text()) + reasoning["checks"]["claim_and_sources_retrieved"] = False + write_json(paths["reasoning_path"], reasoning) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert {"source_receipt_hash_bound", "reasoning_required_checks"} <= set(result["failed_checks"]) + + +def test_hand_authored_reasoning_boole_without_producer_evidence_fail(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + reasoning = json.loads(paths["reasoning_path"].read_text()) + reasoning.pop("result") + reasoning.pop("wrapper_tool_proof") + write_json(paths["reasoning_path"], reasoning) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "reasoning_producer_evidence" in result["failed_checks"] + assert result["reasoning_producer_checks"]["tool_trace"] is False + + +def test_reasoning_compute_attestation_must_bind_live_metadata(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + attestation = json.loads(paths["reasoning_compute_path"].read_text()) + attestation["compute"]["instance_id"] = "999999999" + write_json(paths["reasoning_compute_path"], attestation) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "reasoning_compute_identity_bound" in result["failed_checks"] + + +def test_source_postflight_must_be_newer_than_cleanup_completion(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + cleanup = json.loads(paths["cleanup_path"].read_text()) + cleanup["completed_at_utc"] = "2026-07-15T02:00:00+00:00" + write_json(paths["cleanup_path"], cleanup) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "source_postflight_after_lifecycle" in result["failed_checks"] + + +def test_parity_must_retain_exact_structure_roles_rows_and_performance(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + parity = json.loads(paths["parity_path"].read_text()) + parity["details"]["structural_hashes"].pop("constraints") + parity["details"]["application_role_mismatches"] = {"kb_apply": {"missing": True}} + parity["details"]["performance"] = [] + write_json(paths["parity_path"], parity) + reasoning = json.loads(paths["reasoning_path"].read_text()) + reasoning["parity_receipt"]["sha256"] = sha256(paths["parity_path"]) + write_json(paths["reasoning_path"], reasoning) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "parity_exact_evidence" in result["failed_checks"] + assert result["parity_evidence_checks"]["structural_hashes"] is False + assert result["parity_evidence_checks"]["roles"] is False + assert result["parity_evidence_checks"]["performance"] is False + + +def test_reasoning_must_retrieve_expected_claim_sources_and_preserve_handle(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + reasoning = json.loads(paths["reasoning_path"].read_text()) + for call in reasoning["result"]["database_tool_trace"]["calls"]: + call["result"]["row_ids"] = [] + call["result"]["row_id_count"] = 0 + reasoning["result"]["reply"] = "Cory found a weak claim and two sources." + write_json(paths["reasoning_path"], reasoning) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "reasoning_producer_evidence" in result["failed_checks"] + assert result["reasoning_producer_checks"]["expected_claim_and_sources_retrieved"] is False + assert result["reasoning_producer_checks"]["reply_uses_only_m3taversal_handle"] is False + + +def test_lifecycle_requires_restore_parity_reasoning_cleanup_chronology(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + cleanup = json.loads(paths["cleanup_path"].read_text()) + cleanup["generated_at_utc"] = "2026-07-15T00:58:00+00:00" + write_json(paths["cleanup_path"], cleanup) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "lifecycle_chronology" in result["failed_checks"] + + +def test_source_delta_requires_real_fingerprints_and_consistent_no_delta(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + delta = json.loads(paths["source_delta_path"].read_text()) + delta["baseline"].pop("table_fingerprint_sha256") + delta["current"]["total_rows"] += 1 + write_json(paths["source_delta_path"], delta) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert {"source_delta_baseline_shape", "source_delta_current_shape", "source_delta_consistent"} <= set( + result["failed_checks"] + ) + + +def test_detected_source_delta_requires_exact_row_arithmetic(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + delta = json.loads(paths["source_delta_path"].read_text()) + delta["delta"]["delta_detected"] = True + delta["delta"]["changed_tables"] = [{"table": "public.claims"}] + delta["delta"]["total_row_delta"] = 99 + write_json(paths["source_delta_path"], delta) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "source_delta_consistent" in result["failed_checks"] + + +def test_postflight_source_requires_structured_healthy_service(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + current = json.loads(paths["current_source_path"].read_text()) + current["source_service"]["before"]["ActiveState"] = "failed" + current["source_service"]["after"]["ActiveState"] = "failed" + write_json(paths["current_source_path"], current) + delta = json.loads(paths["source_delta_path"].read_text()) + delta["current"]["capture_receipt_sha256"] = sha256(paths["current_source_path"]) + write_json(paths["source_delta_path"], delta) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "current_source_pass" in result["failed_checks"] + + +def test_unchanged_but_inactive_service_cannot_pass_t3(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + restore = json.loads(paths["restore_path"].read_text()) + restore["live_service"]["before"]["ActiveState"] = "failed" + restore["live_service"]["after"]["ActiveState"] = "failed" + write_json(paths["restore_path"], restore) + cleanup = json.loads(paths["cleanup_path"].read_text()) + cleanup["restore_receipt"]["sha256"] = sha256(paths["restore_path"]) + write_json(paths["cleanup_path"], cleanup) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "restore_service_healthy_unchanged" in result["failed_checks"] + + +def test_historical_but_monotonic_receipts_cannot_claim_current_t3(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + + result = _verify_lifecycle(**paths, now=datetime(2026, 7, 15, 3, 0, tzinfo=UTC)) + + assert result["status"] == "fail" + assert "source_postflight_fresh" in result["failed_checks"] + assert result["postflight_age_seconds"] > result["max_postflight_age_seconds"] + + +def test_fresh_postflight_cannot_mask_historical_gcp_lifecycle(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + source = json.loads(paths["source_path"].read_text()) + source["snapshot"]["captured_at_utc"] = "2026-07-12T00:59:00+00:00" + write_json(paths["source_path"], source) + restore = json.loads(paths["restore_path"].read_text()) + restore["generated_at_utc"] = "2026-07-12T00:59:10+00:00" + restore["completed_at_utc"] = "2026-07-12T00:59:30+00:00" + restore["source"]["capture_receipt"]["sha256"] = sha256(paths["source_path"]) + write_json(paths["restore_path"], restore) + parity = json.loads(paths["parity_path"].read_text()) + parity["completed_at_utc"] = "2026-07-12T00:59:45+00:00" + write_json(paths["parity_path"], parity) + reasoning = json.loads(paths["reasoning_path"].read_text()) + reasoning["generated_at_utc"] = "2026-07-12T01:00:00+00:00" + reasoning["completed_at_utc"] = "2026-07-12T01:01:00+00:00" + reasoning["parity_receipt"]["sha256"] = sha256(paths["parity_path"]) + write_json(paths["reasoning_path"], reasoning) + attestation = json.loads(paths["reasoning_compute_path"].read_text()) + attestation["generated_at_utc"] = "2026-07-12T01:01:05+00:00" + attestation["reasoning_receipt"]["sha256"] = sha256(paths["reasoning_path"]) + attestation["reasoning_receipt"]["generated_at_utc"] = reasoning["generated_at_utc"] + attestation["reasoning_receipt"]["completed_at_utc"] = reasoning["completed_at_utc"] + write_json(paths["reasoning_compute_path"], attestation) + cleanup = json.loads(paths["cleanup_path"].read_text()) + cleanup["generated_at_utc"] = "2026-07-12T01:01:10+00:00" + cleanup["completed_at_utc"] = "2026-07-12T01:01:30+00:00" + cleanup["restore_receipt"]["sha256"] = sha256(paths["restore_path"]) + write_json(paths["cleanup_path"], cleanup) + delta = json.loads(paths["source_delta_path"].read_text()) + delta["baseline"]["captured_at_utc"] = "2026-07-12T00:59:00+00:00" + delta["baseline"]["capture_receipt_sha256"] = sha256(paths["source_path"]) + write_json(paths["source_delta_path"], delta) + + result = verify_lifecycle(**paths) + + assert result["checks"]["source_postflight_fresh"] is True + assert result["checks"]["lifecycle_chronology"] is True + assert result["status"] == "fail" + assert {"gcp_lifecycle_fresh", "lifecycle_span_bounded"} <= set(result["failed_checks"]) diff --git a/tests/test_verify_postgres_parity_manifest.py b/tests/test_verify_postgres_parity_manifest.py index 7199a3a..d016aee 100644 --- a/tests/test_verify_postgres_parity_manifest.py +++ b/tests/test_verify_postgres_parity_manifest.py @@ -90,7 +90,10 @@ def test_matching_local_manifests_pass(tmp_path: Path) -> None: assert completed.returncode == 0, completed.stderr assert payload["status"] == "pass" assert payload["details"]["source_total_rows"] == 2 + assert len(payload["source_manifest_sha256"]) == 64 + assert len(payload["target_manifest_sha256"]) == 64 assert payload["private_connectivity"]["status"] == "not_applicable_local_restore" + assert (tmp_path / "result.json").stat().st_mode & 0o777 == 0o600 def test_row_hash_and_role_mismatches_fail(tmp_path: Path) -> None: @@ -135,6 +138,7 @@ def test_gcp_scope_accepts_matching_private_connectivity_receipt(tmp_path: Path) assert completed.returncode == 0, completed.stderr assert payload["status"] == "pass" assert payload["private_connectivity"]["proof"]["source_compute"] == "teleo-staging-1" + assert len(payload["connectivity_proof_sha256"]) == 64 def test_gcp_scope_rejects_receipt_for_a_different_server(tmp_path: Path) -> None: diff --git a/tests/test_verify_vps_canonical_snapshot_delta.py b/tests/test_verify_vps_canonical_snapshot_delta.py new file mode 100644 index 0000000..451c6f6 --- /dev/null +++ b/tests/test_verify_vps_canonical_snapshot_delta.py @@ -0,0 +1,259 @@ +import hashlib +import json +from pathlib import Path + +import pytest + +from ops.capture_vps_canonical_postgres_snapshot import ( + SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + build_provenance_binding, +) +from ops.verify_vps_canonical_snapshot_delta import verify_delta, write_private_json + + +def manifest_text(*, claims: int = 1, constraint_name: str = "claims_pkey") -> str: + rows = [ + { + "kind": "identity", + "database": "teleo", + "server_version_num": 160014, + "transaction_read_only": "on", + }, + {"kind": "schemas", "items": ["kb_stage", "public"]}, + {"kind": "extensions", "items": [{"name": "pgcrypto", "version": "1.3"}]}, + {"kind": "application_roles", "items": [{"name": "kb_apply", "can_login": True}]}, + {"kind": "columns", "items": []}, + {"kind": "constraints", "items": [{"name": constraint_name}]}, + {"kind": "indexes", "items": []}, + {"kind": "sequences", "items": []}, + {"kind": "views", "items": []}, + {"kind": "functions", "items": []}, + {"kind": "triggers", "items": []}, + {"kind": "types", "items": []}, + {"kind": "policies", "items": []}, + { + "kind": "table", + "schema": "public", + "table": "claims", + "row_count": claims, + "rowset_md5": hashlib.md5(str(claims).encode(), usedforsecurity=False).hexdigest(), + }, + {"kind": "performance", "query": "count_claims", "elapsed_ms": 1.0, "observed_rows": claims}, + ] + return "".join(json.dumps(row, sort_keys=True) + "\n" for row in rows) + + +def capture_fixture( + root: Path, + *, + name: str, + captured_at: str, + authorization_ref: str, + claims: int = 1, + constraint_name: str = "claims_pkey", +) -> tuple[Path, Path]: + manifest = root / f"{name}-manifest.jsonl" + manifest.write_text(manifest_text(claims=claims, constraint_name=constraint_name), encoding="utf-8") + dump_sha256 = hashlib.sha256(f"{name}-dump".encode()).hexdigest() + manifest_sha256 = hashlib.sha256(manifest.read_bytes()).hexdigest() + manifest_sql_sha256 = "2" * 64 + source_context_sha256 = "3" * 64 + source = { + "ssh_target": "root@77.42.65.182", + "container": "teleo-pg", + "database": "teleo", + "service": "leoclean-gateway.service", + } + healthy_service = { + "ActiveState": "active", + "SubState": "running", + "MainPID": "148735", + "NRestarts": "0", + } + snapshot = { + "exported_snapshot_id": "00000003-000001A2-1", + "txid_snapshot": "100:120:105,111", + "wal_lsn": "0/16B6C50", + "system_identifier": "7649789040005668902", + "captured_at_utc": captured_at, + } + receipt = { + "artifact": "vps_canonical_postgres_exported_snapshot", + "schema": SOURCE_SNAPSHOT_RECEIPT_SCHEMA, + "receipt_version": 2, + "status": "pass", + "run_id": name, + "capture_authorization_ref": authorization_ref, + "source": source, + "snapshot_exported": True, + "snapshot": snapshot, + "dump": {"sha256": dump_sha256}, + "manifest": { + "sha256": manifest_sha256, + "manifest_sql_sha256": manifest_sql_sha256, + "table_count": 1, + "total_rows": claims, + }, + "source_context": {"sha256": source_context_sha256}, + "provenance_binding": build_provenance_binding( + run_id=name, + authorization_ref=authorization_ref, + source=source, + snapshot=snapshot, + dump_sha256=dump_sha256, + manifest_sql_sha256=manifest_sql_sha256, + source_manifest_sha256=manifest_sha256, + source_context_sha256=source_context_sha256, + ), + "service_unchanged": True, + "source_service": {"before": healthy_service, "after": healthy_service, "unchanged": True}, + "production_db_mutated": False, + "telegram_send_attempted": False, + } + receipt_path = root / f"{name}-receipt.json" + receipt_path.write_text(json.dumps(receipt, sort_keys=True) + "\n", encoding="utf-8") + return receipt_path, manifest + + +def test_postflight_no_delta_is_bound_and_explicit(tmp_path: Path) -> None: + baseline_receipt, baseline_manifest = capture_fixture( + tmp_path, + name="canonical-baseline", + captured_at="2026-07-15T01:00:00+00:00", + authorization_ref="codex-delegation:baseline123", + ) + current_receipt, current_manifest = capture_fixture( + tmp_path, + name="canonical-postflight", + captured_at="2026-07-15T02:00:00+00:00", + authorization_ref="codex-delegation:postflight123", + ) + + result = verify_delta( + baseline_receipt_path=baseline_receipt, + baseline_manifest_path=baseline_manifest, + baseline_authorization_ref="codex-delegation:baseline123", + current_receipt_path=current_receipt, + current_manifest_path=current_manifest, + current_authorization_ref="codex-delegation:postflight123", + ) + + assert result["status"] == "pass" + assert result["delta"]["delta_detected"] is False + assert result["delta"]["total_row_delta"] == 0 + + +def test_postflight_data_and_structure_delta_is_reported_not_hidden(tmp_path: Path) -> None: + baseline_receipt, baseline_manifest = capture_fixture( + tmp_path, + name="canonical-baseline", + captured_at="2026-07-15T01:00:00+00:00", + authorization_ref="codex-delegation:baseline123", + ) + current_receipt, current_manifest = capture_fixture( + tmp_path, + name="canonical-postflight", + captured_at="2026-07-15T02:00:00+00:00", + authorization_ref="codex-delegation:postflight123", + claims=2, + constraint_name="claims_pkey_v2", + ) + + result = verify_delta( + baseline_receipt_path=baseline_receipt, + baseline_manifest_path=baseline_manifest, + baseline_authorization_ref="codex-delegation:baseline123", + current_receipt_path=current_receipt, + current_manifest_path=current_manifest, + current_authorization_ref="codex-delegation:postflight123", + ) + + assert result["status"] == "pass" + assert result["delta"]["delta_detected"] is True + assert result["delta"]["total_row_delta"] == 1 + assert result["delta"]["changed_tables"][0]["table"] == "public.claims" + assert result["delta"]["changed_structures"][0]["kind"] == "constraints" + + +def test_postflight_rejects_wrong_authorization_or_stale_capture(tmp_path: Path) -> None: + baseline_receipt, baseline_manifest = capture_fixture( + tmp_path, + name="canonical-baseline", + captured_at="2026-07-15T02:00:00+00:00", + authorization_ref="codex-delegation:baseline123", + ) + current_receipt, current_manifest = capture_fixture( + tmp_path, + name="canonical-postflight", + captured_at="2026-07-15T01:00:00+00:00", + authorization_ref="codex-delegation:postflight123", + ) + + with pytest.raises(ValueError, match="newer than"): + verify_delta( + baseline_receipt_path=baseline_receipt, + baseline_manifest_path=baseline_manifest, + baseline_authorization_ref="codex-delegation:baseline123", + current_receipt_path=current_receipt, + current_manifest_path=current_manifest, + current_authorization_ref="codex-delegation:postflight123", + ) + + with pytest.raises(ValueError, match="authorization"): + verify_delta( + baseline_receipt_path=baseline_receipt, + baseline_manifest_path=baseline_manifest, + baseline_authorization_ref="codex-delegation:wrong123", + current_receipt_path=current_receipt, + current_manifest_path=current_manifest, + current_authorization_ref="codex-delegation:postflight123", + ) + + +def test_postflight_rejects_unchanged_but_unhealthy_service(tmp_path: Path) -> None: + baseline_receipt, baseline_manifest = capture_fixture( + tmp_path, + name="canonical-baseline", + captured_at="2026-07-15T01:00:00+00:00", + authorization_ref="codex-delegation:baseline123", + ) + current_receipt, current_manifest = capture_fixture( + tmp_path, + name="canonical-postflight", + captured_at="2026-07-15T02:00:00+00:00", + authorization_ref="codex-delegation:postflight123", + ) + current = json.loads(current_receipt.read_text()) + current["source_service"]["before"]["ActiveState"] = "failed" + current["source_service"]["after"]["ActiveState"] = "failed" + current_receipt.write_text(json.dumps(current, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="source_service_healthy"): + verify_delta( + baseline_receipt_path=baseline_receipt, + baseline_manifest_path=baseline_manifest, + baseline_authorization_ref="codex-delegation:baseline123", + current_receipt_path=current_receipt, + current_manifest_path=current_manifest, + current_authorization_ref="codex-delegation:postflight123", + ) + + +def test_delta_receipt_writer_keeps_private_mode(tmp_path: Path) -> None: + output = tmp_path / "private" / "delta.json" + + write_private_json(output, {"status": "pass"}) + + assert output.stat().st_mode & 0o777 == 0o600 + assert output.parent.stat().st_mode & 0o777 == 0o700 + + +def test_delta_receipt_writer_does_not_chmod_an_existing_parent(tmp_path: Path) -> None: + parent = tmp_path / "existing" + parent.mkdir(mode=0o755) + output = parent / "delta.json" + + write_private_json(output, {"status": "pass"}) + + assert parent.stat().st_mode & 0o777 == 0o755 + assert output.stat().st_mode & 0o777 == 0o600 From 22cf8a96986086a58aeb8c0fcb9250b6b7ed10fa Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 04:16:54 +0200 Subject: [PATCH 2/5] fix(docs): mark private receipt paths as runtime variables --- .agents/skills/teleo-gcp-parity-ops/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.agents/skills/teleo-gcp-parity-ops/SKILL.md b/.agents/skills/teleo-gcp-parity-ops/SKILL.md index 2a0b41f..eef09a4 100644 --- a/.agents/skills/teleo-gcp-parity-ops/SKILL.md +++ b/.agents/skills/teleo-gcp-parity-ops/SKILL.md @@ -158,14 +158,14 @@ before the receipt-bound cleanup: ```bash sudo python3 ops/attest_gcp_reasoning_compute.py \ - --reasoning-receipt "$PRIVATE_RUN_DIR/blind-reasoning-receipt.json" \ + --reasoning-receipt "${PRIVATE_RUN_DIR}/blind-reasoning-receipt.json" \ --restore-run-id "$RESTORE_RUN_ID" \ --target-db "$TARGET_DB" \ --project teleo-501523 \ --expected-compute-instance teleo-prod-1 \ --expected-compute-zone europe-west6-a \ --expected-private-network projects/teleo-501523/global/networks/teleo-staging-net \ - --output "$PRIVATE_RUN_DIR/reasoning-compute-attestation.json" + --output "${PRIVATE_RUN_DIR}/reasoning-compute-attestation.json" ``` The replay must use the Hermes virtualenv, not system Python: From 1a043d36a16558a137199b5793f73ec271d16706 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 10:20:27 +0200 Subject: [PATCH 3/5] Harden GCP parity proof against forged lifecycle claims --- .agents/skills/teleo-gcp-parity-ops/SKILL.md | 43 ++- .github/workflows/ci.yml | 4 + docs/gcp-kb-cloudsql-restore-runbook.md | 46 ++- .../gcp-canonical-parity-live-20260715.json | 28 +- ops/attest_gcp_reasoning_compute.py | 12 +- ...capture_vps_canonical_postgres_snapshot.py | 15 +- ops/postgres_parity_manifest.sql | 288 +++++++++++++++++- ops/private_receipt_io.py | 29 ++ ...restore_gcp_generated_postgres_snapshot.py | 86 +++++- ops/run_local_canonical_postgres_rebuild.py | 2 +- ops/verify_gcp_canonical_lifecycle.py | 59 +++- ops/verify_postgres_parity_manifest.py | 59 +++- ops/verify_vps_canonical_snapshot_delta.py | 29 +- ...run_gcp_generated_db_direct_claim_suite.py | 5 +- tests/test_attest_gcp_reasoning_compute.py | 6 +- ...capture_vps_canonical_postgres_snapshot.py | 5 + tests/test_private_receipt_io.py | 29 ++ ...restore_gcp_generated_postgres_snapshot.py | 66 +++- ...st_run_local_canonical_postgres_rebuild.py | 3 + .../test_run_local_genesis_ledger_rebuild.py | 3 + tests/test_verify_gcp_canonical_lifecycle.py | 93 +++++- tests/test_verify_postgres_parity_manifest.py | 57 +++- ...est_verify_vps_canonical_snapshot_delta.py | 53 +++- 23 files changed, 939 insertions(+), 81 deletions(-) create mode 100644 tests/test_private_receipt_io.py diff --git a/.agents/skills/teleo-gcp-parity-ops/SKILL.md b/.agents/skills/teleo-gcp-parity-ops/SKILL.md index eef09a4..5ae4069 100644 --- a/.agents/skills/teleo-gcp-parity-ops/SKILL.md +++ b/.agents/skills/teleo-gcp-parity-ops/SKILL.md @@ -8,7 +8,8 @@ description: Use for passwordless Teleo GCP VM access, private Cloud SQL canonic ## Working Target Restore a copy of the VPS canonical Leo database to GCP, prove exact schema, -row, role, extension, performance, and private-connectivity parity, then run the +row, role membership, ownership, ACL, extension, performance, and +private-connectivity parity, then run the real GCP Leo read/reasoning path without Telegram sends or DB mutation. The required tier is T3: a live private GCP staging restore, readback, @@ -45,8 +46,14 @@ Historically verified and intended target; reverify before any new lifecycle: - VM `teleo-prod-1` in `europe-west6-a`; - Cloud SQL `teleo-pgvector-standby`, PostgreSQL 16.14; - private endpoint `10.61.0.3:5432`, public IP disabled, TLS required; +- restore identity `sa-teleo-restore-drill@teleo-501523.iam.gserviceaccount.com`; - service `leoclean-gcp-prod-parallel.service`. +Direct libpq connections must use `sslmode=verify-full`, a reviewed DNS name +covered by the server certificate, and a private regular CA file. The private +IP is `hostaddr`, not the certificate identity. Never downgrade to +`sslmode=require` or accept an arbitrary `*.iam.gserviceaccount.com` identity. + Never print the Cloud SQL or Google password. On the VM, use the attached service account and Secret Manager through reviewed wrappers or a short-lived environment variable, then unset it. @@ -92,6 +99,10 @@ Do not describe a passing Hermes memory sync as canonical KB parity. - Persistent GCP `teleo_canonical` remains the older staging copy measured at `52,164` rows and `26` proposals. It has not been promoted or cut over. +That historical clone predates the expanded ownership/ACL and independently +authenticated receipt contract. It remains useful historical row/schema proof, +but it is not current T3 proof under this skill. + Primary retained proof: - `docs/reports/leo-working-state-20260709/gcp-db-first-working-leo-20260714.md` @@ -107,7 +118,8 @@ Track these independently: 1. control-plane project, VM, Cloud SQL, private-IP, and TLS identity; 2. canonical database schema, counts, row hashes, constraints, indexes, - functions, extensions, roles, and performance; + functions, extensions, exact roles and memberships, owners, normalized + schema/relation/column/function/type ACLs, and performance; 3. GCP service PID/restarts plus live profile and tool hashes; 4. `DC-01` through `DC-06` DB-read readiness; 5. real no-send model replies, strict score, and exact count consistency; @@ -127,6 +139,8 @@ Use: receipt-bound private-TLS restore into a bounded `teleo_clone_*` database; pass the exact capture `--source-receipt`, and require its live Cloud SQL control-plane check to prove public IP disabled before clone creation; it + requires the exact restore service account plus `--ssl-server-name` and + `--ssl-root-cert` for certificate-authenticated `verify-full` TLS; it writes `target-manifest.jsonl`, `restore-receipt.json`, and the generated `gcp-private-connectivity.json` under `/var/lib/teleo-gcp-restore-runs//`; @@ -147,8 +161,11 @@ Use: post-snapshot VPS delta in `source-delta-receipt.json`; - `ops/verify_gcp_canonical_lifecycle.py` after cleanup to bind the exact source, current source, source delta, restore, `gcp_staging` parity, ID-free reasoning, - reasoning-compute attestation, and cleanup receipts into one fail-closed T3 - lifecycle verdict. Pass `--max-postflight-age-seconds 900` and + reasoning-compute attestation, and cleanup receipts into one fail-closed + preparation verdict. Caller-writable JSON can never emit T3 from this API; + even an internally valid bundle returns + `prepared_external_attestation_required`, `current_tier=T2`, and + `accepted_by_this_api=false`. Pass `--max-postflight-age-seconds 900` and `--max-lifecycle-age-seconds 3600`; the latter bounds both cleanup-to-verdict age and restore-to-postflight span. @@ -165,6 +182,7 @@ sudo python3 ops/attest_gcp_reasoning_compute.py \ --expected-compute-instance teleo-prod-1 \ --expected-compute-zone europe-west6-a \ --expected-private-network projects/teleo-501523/global/networks/teleo-staging-net \ + --expected-restore-service-account sa-teleo-restore-drill@teleo-501523.iam.gserviceaccount.com \ --output "${PRIVATE_RUN_DIR}/reasoning-compute-attestation.json" ``` @@ -182,9 +200,17 @@ input, supporting receipt, and verifier output; do not treat that evidence as transient cleanup. The canonical restore streams through `pg_restore` and creates no GCS import object. +The current snapshot and restore commands intentionally use `--no-owner` and +`--no-acl`/`--no-privileges`. The expanded manifest therefore fails GCP parity +until a separately reviewed Cloud-SQL-compatible authorization replay restores +the captured owner/grant semantics. Do not bypass this by dropping the new +manifest rows or by running the Docker/superuser gate bootstrap blindly on +shared Cloud SQL. + Static inspection or a dry run is insufficient for these helpers: verify a new -capture against the live VPS, run the focused tests, and require a passing live -lifecycle receipt before describing the workflow as current parity. +capture against the live VPS and run the focused tests. A current T3 claim also +requires a separate independently authenticated provider/platform receipt that +the bundle author cannot mint. ## Safety And Claim Ceiling @@ -195,8 +221,9 @@ lifecycle receipt before describing the workflow as current parity. - Do not call control-plane inventory, memory sync, route readiness, or a nominal scorer pass full m3taversal parity. -The strongest accepted claim requires exact DB parity plus real no-send model -replies with truthful counts and cleanup. It still does not prove Telegram +The strongest claim emitted by the repository lifecycle verifier is T2 +structural consistency. Exact DB parity plus real no-send model replies with +truthful counts and cleanup still requires independent T3 attestation. Neither proves Telegram delivery, GCP canonical mutation, ongoing replication, or production cutover. ## Current Access And Next Action diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccb4925..c069e36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,7 @@ jobs: ops/check_gcp_service_communications.py \ ops/plan_gcp_iam_split.py \ ops/private_receipt_io.py \ + ops/run_local_canonical_postgres_rebuild.py \ ops/redact_sqlite_postgres_restore_canary.py \ ops/restore_gcp_generated_postgres_snapshot.py \ ops/sqlite_to_postgres_dump.py \ @@ -106,7 +107,10 @@ jobs: tests/test_working_leo_m3taversal_oos_benchmark.py \ tests/test_working_leo_open_ended_benchmark.py \ tests/test_phase1b_end_to_end.py \ + tests/test_private_receipt_io.py \ tests/test_restore_gcp_generated_postgres_snapshot.py \ + tests/test_run_local_canonical_postgres_rebuild.py \ + tests/test_run_local_genesis_ledger_rebuild.py \ tests/test_sqlite_to_postgres_dump.py \ tests/test_sqlite_postgres_restore_canary_capsule.py \ tests/test_eval_parse.py \ diff --git a/docs/gcp-kb-cloudsql-restore-runbook.md b/docs/gcp-kb-cloudsql-restore-runbook.md index f22aa21..2a0132b 100644 --- a/docs/gcp-kb-cloudsql-restore-runbook.md +++ b/docs/gcp-kb-cloudsql-restore-runbook.md @@ -86,7 +86,10 @@ network mode `none` and tmpfs-only database storage. It waits for an actual --no-owner --no-privileges --exit-on-error`, compares the full parity manifest, then removes the container and proves it is absent. A passing local receipt is the exact-recovery preflight; it is not semantic recompilation from raw source -documents. +documents. Full parity now includes `kb_gate_owner`, `kb_apply`, and +`kb_review`, their memberships, object ownership, and normalized database, +schema, relation, column, function, and type ACLs. A restore that recovers rows +but not those authorization semantics fails closed. Choose one bounded suffix and use it consistently on the GCP replay VM. The restore helper retains its mode-0700 evidence directory at the fixed private @@ -110,7 +113,11 @@ creating the clone, and fails closed unless the exact instance is runnable, PostgreSQL 16, private-networked, bound to the expected RFC1918 host, and has public IP disabled. A GCE metadata preflight independently binds the operator runtime to `teleo-prod-1` in `europe-west6-a` on `teleo-staging-net`; a -hard-coded compute label is not accepted: +hard-coded compute label is not accepted. It also requires the exact +least-privilege restore identity and libpq `verify-full` using a reviewed server +name and private CA file. Determine the certificate DNS SAN from the live Cloud +SQL certificate before filling `--ssl-server-name`; do not substitute the +private IP or downgrade TLS verification: ```bash sudo python3 ops/restore_gcp_generated_postgres_snapshot.py restore \ @@ -128,6 +135,9 @@ sudo python3 ops/restore_gcp_generated_postgres_snapshot.py restore \ --expected-private-network projects/teleo-501523/global/networks/teleo-staging-net \ --expected-compute-instance teleo-prod-1 \ --expected-compute-zone europe-west6-a \ + --expected-restore-service-account sa-teleo-restore-drill@teleo-501523.iam.gserviceaccount.com \ + --ssl-server-name \ + --ssl-root-cert /etc/teleo/cloudsql/server-ca.pem \ --expected-source-ssh-target root@77.42.65.182 \ --expected-source-container teleo-pg \ --expected-source-database teleo \ @@ -140,7 +150,8 @@ directory. The latter is composed from live GCE metadata, Cloud SQL control-plane checks, and the private TLS database identity; do not hand-compose the connectivity receipt. The restore also records identical healthy before/after state for `leoclean-gcp-prod-parallel.service`, including its PID -and restart count. +and restart count. Receipt payloads remain in mode-0600 files; command stdout +contains only a status, output path, and receipt hash. Compare the captured target manifest and generated connectivity receipt to the authorized source: @@ -157,9 +168,20 @@ sudo python3 ops/verify_postgres_parity_manifest.py \ The verifier checks all table row counts and collation-independent row hashes, plus schemas, columns/defaults, constraints, indexes, sequences, views, functions, triggers, enum/domain types, policies, required extensions, -password-free application-role attributes, and bounded query timings. In GCP -scope it also requires a receipt proving a staging compute source, a private -server address, TLS, and public-IP-disabled instance metadata. +password-free application-role attributes, exact role memberships, object +owners, normalized ACLs, and bounded query timings. Role and extension sets are +symmetric: an unexpected extra role, superuser, or extension is a mismatch. +Every table must have a nonempty row hash. In GCP scope it also requires a +receipt proving a staging compute source, a private server address, +certificate-authenticated `verify-full` TLS, and public-IP-disabled instance +metadata. + +The snapshot dump intentionally strips owners and ACLs. Therefore the GCP +restore cannot pass the expanded parity gate until a separately reviewed, +Cloud-SQL-compatible authorization replay reconstructs the captured ownership +and grants. Do not run the Docker/superuser bootstrap blindly on shared Cloud +SQL. Until that replay exists, this lane remains T2 preparation and must fail +before a T3 claim. After the parity verifier passes, run the no-send m3taversal blind reasoning replay from staging compute against that generated database, attest the replay @@ -193,6 +215,7 @@ sudo python3 ops/attest_gcp_reasoning_compute.py \ --expected-compute-instance teleo-prod-1 \ --expected-compute-zone europe-west6-a \ --expected-private-network projects/teleo-501523/global/networks/teleo-staging-net \ + --expected-restore-service-account sa-teleo-restore-drill@teleo-501523.iam.gserviceaccount.com \ --output "$PRIVATE_RUN_DIR/reasoning-compute-attestation.json" ``` @@ -230,7 +253,8 @@ verifier rejects local-scope parity, stale or mismatched source hashes, a postflight source capture that predates reasoning or cleanup, hand-authored reasoning booleans without real retrieval/tool receipts, an unattested replay host, compute or Cloud SQL topology drift, and any cleanup receipt that leaves -the clone present. The postflight capture may be at most 900 seconds old at +the clone present or claims cleanup when the named clone never existed. Both +source captures must use the same reviewed manifest SQL. The postflight capture may be at most 900 seconds old at verification; the GCP lifecycle age and restore-to-postflight span must each be at most 3600 seconds: @@ -249,6 +273,14 @@ sudo python3 ops/verify_gcp_canonical_lifecycle.py \ --output "$PRIVATE_RUN_DIR/lifecycle-verification.json" ``` +This command validates only the internal consistency of caller-supplied +receipts. A clean result is emitted as +`prepared_external_attestation_required`, with +`current_tier=T2_structural_receipt_bundle`, `t3_receipt_ready=false`, and +`accepted_by_this_api=false`. It cannot emit a live-GCP/T3 claim. T3 requires a +separate independently authenticated provider or platform receipt that the +bundle author cannot mint. + Retain the private receipt run directory and every useful lifecycle input, supporting receipt, and verifier output. Remove only transient upload/bundle directories, temporary Hermes profiles, helper clients, and child processes diff --git a/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json b/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json index 258e20a..0e7a214 100644 --- a/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json +++ b/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json @@ -4,7 +4,7 @@ "generated_at_utc": "2026-07-15T01:50:20Z", "status": "gated_external", "required_tier": "T3_live_private_gcp_staging", - "current_tier": "T2_current_source_and_isolated_restore_plus_live_access_gate", + "current_tier": "T2_historical_source_and_isolated_restore_under_superseded_parity_contract", "current_canary": "Restore the newest authorized canonical snapshot into one private generated Cloud SQL database, prove exact parity and ID-free Leo reasoning from staging compute, then drop the generated database and verify absence.", "current_source": { "run_id": "canonical-20260715t014930z", @@ -57,7 +57,24 @@ "performance_problems": [], "network_mode": "none", "persistent_postgres_storage_used": false, - "container_absent_after_test": true + "container_absent_after_test": true, + "authorization_state_verified": false, + "prior_manifest_contract_only": true + }, + "contract_reassessment": { + "status": "changes_required", + "reviewed_manifest_sql_sha256": "430390129bc8980c4ece9939f2aee18e8c40e9da4b226782579349d5c45c5301", + "prior_receipts_recomputed_under_current_contract": false, + "newly_required_parity": [ + "kb_gate_owner plus exact application-role attributes", + "role memberships", + "database, schema, relation, function, and type ownership", + "normalized database, schema, relation, column, function, and type ACLs", + "symmetric role and extension sets", + "nonempty row hash for every table" + ], + "t3_receipt_accepted_by_repository_api": false, + "independent_platform_attestation_required": true }, "permission_profile": { "approval_policy": "never", @@ -90,7 +107,7 @@ ], "exact_gate": "The privileged local Google OAuth session requires human reauthentication, the fixed IAP workflow names an unavailable teleo-iap-operator WIF provider, and no existing noninteractive service-account route can obtain a Cloud SQL/Compute-capable token. Current Cloud SQL topology and generated-database inventory cannot be read, so the private restore cannot start.", "why_autonomous_repair_stops": "Google reauthentication may require a password, MFA, passkey, or consent that Codex must not retrieve or enter. Recreating or rebinding the WIF/IAP identities requires an already-authorized GCP administrator, and no safe noninteractive identity currently has that authority.", - "next_non_user_action": "After operator-controlled Google reauthentication or administrator convergence of the reviewed least-privilege IAP operator, read the current Cloud SQL and GCE topology, stream the receipt-bound dump to a teleo_clone_* database from teleo-prod-1, run exact parity, blind reasoning, live GCE metadata attestation, cleanup, postflight source capture, and the eight-receipt lifecycle verifier.", + "next_non_user_action": "After the reviewed operator route and exact restore service account are available, capture a new source snapshot with the expanded manifest, implement and review a Cloud-SQL-compatible authorization replay, use verify-full TLS with the reviewed server name and CA, run the disposable clone lifecycle, then obtain an independently authenticated platform receipt. The repository eight-receipt verifier remains preparation-only.", "gcp_effects_from_this_attempt": { "cloudsql_database_created": false, "gcp_resource_created": false, @@ -122,12 +139,15 @@ "sha256": "0fda5e6d58d72dcb084dc277bc59312bf62c002c728e249a9f16e79ae096037b" } }, - "strongest_claim_allowed": "The newest authorized VPS snapshot is stable and exactly reconstructs in an isolated PostgreSQL 16 canary; current private GCP restore, reasoning, and cleanup are not proven because the live operator route is externally gated.", + "strongest_claim_allowed": "The retained snapshot was stable and reconstructed data and schema under the prior parity contract. It does not prove current authorization parity or live GCP operation under the expanded contract.", "not_proven": [ "current Cloud SQL topology or public-IP-disabled state", "current generated-database inventory", "private staging-to-Cloud-SQL TLS connectivity", "current Cloud SQL restore parity", + "current role membership, ownership, and ACL parity", + "certificate-authenticated verify-full database access", + "independently authenticated T3 lifecycle attestation", "blind Leo reasoning against the generated Cloud SQL database", "generated Cloud SQL database teardown" ] diff --git a/ops/attest_gcp_reasoning_compute.py b/ops/attest_gcp_reasoning_compute.py index e9d5124..42a1105 100644 --- a/ops/attest_gcp_reasoning_compute.py +++ b/ops/attest_gcp_reasoning_compute.py @@ -11,24 +11,26 @@ from pathlib import Path from typing import Any try: - from .private_receipt_io import write_private_json + from .private_receipt_io import print_private_receipt_summary, write_private_json from .restore_gcp_generated_postgres_snapshot import ( DEFAULT_COMPUTE_INSTANCE, DEFAULT_COMPUTE_ZONE, DEFAULT_PRIVATE_NETWORK, DEFAULT_PROJECT, + DEFAULT_RESTORE_SERVICE_ACCOUNT, RUN_ID_RE, TARGET_DATABASE_RE, gce_compute_identity, sha256_file, ) except ImportError: # pragma: no cover - direct script execution on the GCP VM - from private_receipt_io import write_private_json + from private_receipt_io import print_private_receipt_summary, write_private_json from restore_gcp_generated_postgres_snapshot import ( DEFAULT_COMPUTE_INSTANCE, DEFAULT_COMPUTE_ZONE, DEFAULT_PRIVATE_NETWORK, DEFAULT_PROJECT, + DEFAULT_RESTORE_SERVICE_ACCOUNT, RUN_ID_RE, TARGET_DATABASE_RE, gce_compute_identity, @@ -81,6 +83,7 @@ def attest_reasoning_compute( expected_compute_instance: str = DEFAULT_COMPUTE_INSTANCE, expected_compute_zone: str = DEFAULT_COMPUTE_ZONE, expected_private_network: str = DEFAULT_PRIVATE_NETWORK, + expected_restore_service_account: str = DEFAULT_RESTORE_SERVICE_ACCOUNT, metadata_timeout: float = 10.0, generated_at_utc: str | None = None, ) -> dict[str, Any]: @@ -110,6 +113,7 @@ def attest_reasoning_compute( expected_compute_instance, expected_compute_zone, expected_private_network, + expected_restore_service_account, timeout=metadata_timeout, ) return { @@ -147,6 +151,7 @@ def main() -> int: parser.add_argument("--expected-compute-instance", default=DEFAULT_COMPUTE_INSTANCE) parser.add_argument("--expected-compute-zone", default=DEFAULT_COMPUTE_ZONE) parser.add_argument("--expected-private-network", default=DEFAULT_PRIVATE_NETWORK) + parser.add_argument("--expected-restore-service-account", default=DEFAULT_RESTORE_SERVICE_ACCOUNT) parser.add_argument("--metadata-timeout", default=10.0, type=float) parser.add_argument("--output", required=True, type=Path) args = parser.parse_args() @@ -159,12 +164,13 @@ def main() -> int: expected_compute_instance=args.expected_compute_instance, expected_compute_zone=args.expected_compute_zone, expected_private_network=args.expected_private_network, + expected_restore_service_account=args.expected_restore_service_account, metadata_timeout=args.metadata_timeout, ) write_private_json(args.output, receipt) except (OSError, ValueError, RuntimeError) as exc: parser.error(str(exc)) - print(json.dumps(receipt, indent=2, sort_keys=True)) + print_private_receipt_summary(args.output, receipt) return 0 diff --git a/ops/capture_vps_canonical_postgres_snapshot.py b/ops/capture_vps_canonical_postgres_snapshot.py index 9e02dea..7cde893 100755 --- a/ops/capture_vps_canonical_postgres_snapshot.py +++ b/ops/capture_vps_canonical_postgres_snapshot.py @@ -18,8 +18,10 @@ from pathlib import Path from typing import Any try: + from .private_receipt_io import print_private_receipt_summary, write_private_json from .verify_postgres_parity_manifest import load_manifest except ImportError: # pragma: no cover - direct script execution + from private_receipt_io import print_private_receipt_summary, write_private_json from verify_postgres_parity_manifest import load_manifest @@ -471,12 +473,17 @@ def main() -> int: "capture_authorization_ref": args.authorization_ref, "error": str(exc), } - print(json.dumps(payload, indent=2, sort_keys=True)) + receipt = ( + output_dir.parent / f".{output_dir.name}-{args.run_id}-failure.json" + if output_preexisted + else output_dir / "receipt.json" + ) + write_private_json(receipt, payload) + print_private_receipt_summary(receipt, payload) return 1 receipt = args.output_dir.resolve() / "receipt.json" - receipt.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - os.chmod(receipt, 0o600) - print(json.dumps(payload, indent=2, sort_keys=True)) + write_private_json(receipt, payload) + print_private_receipt_summary(receipt, payload) return 0 diff --git a/ops/postgres_parity_manifest.sql b/ops/postgres_parity_manifest.sql index 3166eba..8f5ebc8 100644 --- a/ops/postgres_parity_manifest.sql +++ b/ops/postgres_parity_manifest.sql @@ -55,11 +55,297 @@ select jsonb_build_object( 'bypass_rls', rolbypassrls ) order by rolname) from pg_roles - where rolname in ('kb_apply', 'kb_review')), + where rolname in ('kb_gate_owner', 'kb_apply', 'kb_review')), '[]'::jsonb ) )::text; +select jsonb_build_object( + 'kind', 'role_memberships', + 'items', coalesce( + (select jsonb_agg( + jsonb_build_object( + 'role', role_row.rolname, + 'member', member_row.rolname, + 'grantor', grantor_row.rolname, + 'admin_option', membership.admin_option, + 'inherit_option', membership.inherit_option, + 'set_option', membership.set_option + ) order by role_row.rolname, member_row.rolname, grantor_row.rolname) + from pg_auth_members membership + join pg_roles role_row on role_row.oid = membership.roleid + join pg_roles member_row on member_row.oid = membership.member + join pg_roles grantor_row on grantor_row.oid = membership.grantor + where role_row.rolname in ('kb_gate_owner', 'kb_apply', 'kb_review') + or member_row.rolname in ('kb_gate_owner', 'kb_apply', 'kb_review')), + '[]'::jsonb + ) +)::text; + +with ownership as ( + select + format('database:%s', database_row.datname) as sort_key, + 'database'::text as object_type, + null::text as schema, + database_row.datname as name, + null::text as identity_arguments, + pg_get_userbyid(database_row.datdba) as owner + from pg_database database_row + where database_row.datname = current_database() + + union all + + select + format('schema:%s', namespace.nspname), + 'schema', + namespace.nspname, + namespace.nspname, + null::text, + pg_get_userbyid(namespace.nspowner) + from pg_namespace namespace + where namespace.nspname in ('public', 'kb_stage') + + union all + + select + format('relation:%s.%s', namespace.nspname, relation.relname), + case relation.relkind + when 'r' then 'table' + when 'p' then 'partitioned_table' + when 'v' then 'view' + when 'm' then 'materialized_view' + when 'S' then 'sequence' + when 'f' then 'foreign_table' + end, + namespace.nspname, + relation.relname, + null::text, + pg_get_userbyid(relation.relowner) + from pg_class relation + join pg_namespace namespace on namespace.oid = relation.relnamespace + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'S', 'f') + + union all + + select + format( + 'function:%s.%s(%s)', + namespace.nspname, + procedure.proname, + pg_get_function_identity_arguments(procedure.oid) + ), + 'function', + namespace.nspname, + procedure.proname, + pg_get_function_identity_arguments(procedure.oid), + pg_get_userbyid(procedure.proowner) + from pg_proc procedure + join pg_namespace namespace on namespace.oid = procedure.pronamespace + where namespace.nspname in ('public', 'kb_stage') + + union all + + select + format('type:%s.%s', namespace.nspname, type_row.typname), + case type_row.typtype when 'd' then 'domain' else 'enum' end, + namespace.nspname, + type_row.typname, + null::text, + pg_get_userbyid(type_row.typowner) + from pg_type type_row + join pg_namespace namespace on namespace.oid = type_row.typnamespace + where namespace.nspname in ('public', 'kb_stage') + and type_row.typtype in ('d', 'e') +) +select jsonb_build_object( + 'kind', 'object_ownership', + 'items', coalesce( + jsonb_agg(to_jsonb(ownership) - 'sort_key' order by sort_key), + '[]'::jsonb + ) +)::text +from ownership; + +with acl_rows as ( + select + format( + 'database:%s:%s:%s:%s', + database_row.datname, + acl.grantee, + acl.privilege_type, + acl.is_grantable + ) as sort_key, + 'database'::text as object_type, + null::text as schema, + database_row.datname as name, + null::text as column_name, + null::text as identity_arguments, + pg_get_userbyid(acl.grantor) as grantor, + case when acl.grantee = 0 then 'PUBLIC' else pg_get_userbyid(acl.grantee) end as grantee, + acl.privilege_type, + acl.is_grantable + from pg_database database_row + cross join lateral aclexplode(coalesce(database_row.datacl, acldefault('d', database_row.datdba))) acl + where database_row.datname = current_database() + + union all + + select + format( + 'schema:%s:%s:%s:%s', + namespace.nspname, + acl.grantee, + acl.privilege_type, + acl.is_grantable + ), + 'schema', + namespace.nspname, + namespace.nspname, + null::text, + null::text, + pg_get_userbyid(acl.grantor), + case when acl.grantee = 0 then 'PUBLIC' else pg_get_userbyid(acl.grantee) end, + acl.privilege_type, + acl.is_grantable + from pg_namespace namespace + cross join lateral aclexplode(coalesce(namespace.nspacl, acldefault('n', namespace.nspowner))) acl + where namespace.nspname in ('public', 'kb_stage') + + union all + + select + format( + 'relation:%s.%s:%s:%s:%s', + namespace.nspname, + relation.relname, + acl.grantee, + acl.privilege_type, + acl.is_grantable + ), + case when relation.relkind = 'S' then 'sequence' else 'relation' end, + namespace.nspname, + relation.relname, + null::text, + null::text, + pg_get_userbyid(acl.grantor), + case when acl.grantee = 0 then 'PUBLIC' else pg_get_userbyid(acl.grantee) end, + acl.privilege_type, + acl.is_grantable + from pg_class relation + join pg_namespace namespace on namespace.oid = relation.relnamespace + cross join lateral aclexplode( + coalesce( + relation.relacl, + acldefault(case when relation.relkind = 'S' then 'S'::"char" else 'r'::"char" end, relation.relowner) + ) + ) acl + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'S', 'f') + + union all + + select + format( + 'column:%s.%s.%s:%s:%s:%s', + namespace.nspname, + relation.relname, + attribute.attname, + acl.grantee, + acl.privilege_type, + acl.is_grantable + ), + 'column', + namespace.nspname, + relation.relname, + attribute.attname, + null::text, + pg_get_userbyid(acl.grantor), + case when acl.grantee = 0 then 'PUBLIC' else pg_get_userbyid(acl.grantee) end, + acl.privilege_type, + acl.is_grantable + from pg_attribute attribute + join pg_class relation on relation.oid = attribute.attrelid + join pg_namespace namespace on namespace.oid = relation.relnamespace + cross join lateral aclexplode(attribute.attacl) acl + where namespace.nspname in ('public', 'kb_stage') + and relation.relkind in ('r', 'p', 'v', 'm', 'f') + and attribute.attnum > 0 + and not attribute.attisdropped + + union all + + select + format( + 'function:%s.%s(%s):%s:%s:%s', + namespace.nspname, + procedure.proname, + pg_get_function_identity_arguments(procedure.oid), + acl.grantee, + acl.privilege_type, + acl.is_grantable + ), + 'function', + namespace.nspname, + procedure.proname, + null::text, + pg_get_function_identity_arguments(procedure.oid), + pg_get_userbyid(acl.grantor), + case when acl.grantee = 0 then 'PUBLIC' else pg_get_userbyid(acl.grantee) end, + acl.privilege_type, + acl.is_grantable + from pg_proc procedure + join pg_namespace namespace on namespace.oid = procedure.pronamespace + cross join lateral aclexplode(coalesce(procedure.proacl, acldefault('f', procedure.proowner))) acl + where namespace.nspname in ('public', 'kb_stage') + + union all + + select + format( + 'type:%s.%s:%s:%s:%s', + namespace.nspname, + type_row.typname, + acl.grantee, + acl.privilege_type, + acl.is_grantable + ), + 'type', + namespace.nspname, + type_row.typname, + null::text, + null::text, + pg_get_userbyid(acl.grantor), + case when acl.grantee = 0 then 'PUBLIC' else pg_get_userbyid(acl.grantee) end, + acl.privilege_type, + acl.is_grantable + from pg_type type_row + join pg_namespace namespace on namespace.oid = type_row.typnamespace + cross join lateral aclexplode(coalesce(type_row.typacl, acldefault('T', type_row.typowner))) acl + where namespace.nspname in ('public', 'kb_stage') + and type_row.typtype in ('d', 'e') +) +select jsonb_build_object( + 'kind', 'object_acl', + 'items', coalesce( + jsonb_agg( + to_jsonb(acl_rows) - 'sort_key' + order by + object_type, + schema nulls first, + name, + column_name nulls first, + identity_arguments nulls first, + grantor, + grantee, + privilege_type, + is_grantable + ), + '[]'::jsonb + ) +)::text +from acl_rows; + select jsonb_build_object( 'kind', 'columns', 'items', coalesce( diff --git a/ops/private_receipt_io.py b/ops/private_receipt_io.py index 13dc3d6..1655006 100644 --- a/ops/private_receipt_io.py +++ b/ops/private_receipt_io.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import os import uuid @@ -9,6 +10,14 @@ from pathlib import Path from typing import Any +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def write_private_json(path: Path, payload: dict[str, Any]) -> None: """Write one JSON receipt as mode 0600 without chmodding existing parents.""" parent_preexisting = path.parent.exists() @@ -32,3 +41,23 @@ def write_private_json(path: Path, payload: dict[str, Any]) -> None: temporary.unlink() except FileNotFoundError: pass + + +def private_receipt_summary(path: Path, payload: dict[str, Any]) -> dict[str, Any]: + """Return a stdout-safe pointer without replaying private receipt contents.""" + failed_checks = payload.get("failed_checks") + problems = payload.get("problems") + error = payload.get("error") + return { + "artifact": payload.get("artifact"), + "status": payload.get("status"), + "output_name": path.name, + "output_sha256": file_sha256(path), + "failed_check_count": len(failed_checks) if isinstance(failed_checks, list) else None, + "problem_count": len(problems) if isinstance(problems, list) else None, + "error_type": error.get("type") if isinstance(error, dict) else (type(error).__name__ if error else None), + } + + +def print_private_receipt_summary(path: Path, payload: dict[str, Any]) -> None: + print(json.dumps(private_receipt_summary(path, payload), indent=2, sort_keys=True)) diff --git a/ops/restore_gcp_generated_postgres_snapshot.py b/ops/restore_gcp_generated_postgres_snapshot.py index 7e09f4c..c14537a 100644 --- a/ops/restore_gcp_generated_postgres_snapshot.py +++ b/ops/restore_gcp_generated_postgres_snapshot.py @@ -29,9 +29,11 @@ from pathlib import Path from typing import Any try: - from .verify_postgres_parity_manifest import compare_manifests, load_manifest + from .private_receipt_io import print_private_receipt_summary + from .verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256, compare_manifests, load_manifest except ImportError: # pragma: no cover - direct script execution on the GCP VM - from verify_postgres_parity_manifest import compare_manifests, load_manifest + from private_receipt_io import print_private_receipt_summary + from verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256, compare_manifests, load_manifest DEFAULT_HOST = "10.61.0.3" @@ -40,6 +42,7 @@ DEFAULT_INSTANCE = "teleo-pgvector-standby" DEFAULT_PRIVATE_NETWORK = "projects/teleo-501523/global/networks/teleo-staging-net" DEFAULT_COMPUTE_INSTANCE = "teleo-prod-1" DEFAULT_COMPUTE_ZONE = "europe-west6-a" +DEFAULT_RESTORE_SERVICE_ACCOUNT = "sa-teleo-restore-drill@teleo-501523.iam.gserviceaccount.com" DEFAULT_SOURCE_SSH_TARGET = "root@77.42.65.182" DEFAULT_SOURCE_CONTAINER = "teleo-pg" DEFAULT_SOURCE_DATABASE = "teleo" @@ -48,7 +51,6 @@ DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service" DEFAULT_RUN_ROOT = Path("/var/lib/teleo-gcp-restore-runs") DEFAULT_MANIFEST_SQL = Path(__file__).with_name("postgres_parity_manifest.sql") -REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" SOURCE_SNAPSHOT_RECEIPT_SCHEMA = "livingip.sourceSnapshotReceipt.v2" TARGET_DATABASE_RE = re.compile(r"teleo_clone_[a-z0-9][a-z0-9_]{0,50}\Z") RUN_ID_RE = re.compile(r"gcp-restore-[a-z0-9][a-z0-9-]{7,47}\Z") @@ -61,6 +63,8 @@ GCE_INSTANCE_RE = re.compile(r"[a-z](?:[a-z0-9-]{0,61}[a-z0-9])?\Z") GCE_ZONE_RE = re.compile(r"[a-z]+-[a-z]+[0-9]-[a-z]\Z") SAFE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z") SAFE_SSH_TARGET_RE = re.compile(r"(?:[A-Za-z0-9][A-Za-z0-9._-]{0,63}@)?[A-Za-z0-9][A-Za-z0-9.-]{0,252}\Z") +SAFE_DNS_NAME_RE = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?\Z") +SERVICE_ACCOUNT_RE = re.compile(r"[a-z][a-z0-9-]{0,62}@[a-z][a-z0-9-]{4,28}[a-z0-9]\.iam\.gserviceaccount\.com\Z") PG_RESTORE_VERSION_RE = re.compile(r"pg_restore \(PostgreSQL\) (?P[0-9]+)(?:\.[0-9]+)*") ROLLBACK_DATABASE = "teleo_canonical_pre_20260712t1905z" GCE_METADATA_ROOT = "http://metadata.google.internal/computeMetadata/v1" @@ -387,6 +391,7 @@ def gce_compute_identity( expected_instance: str, expected_zone: str, expected_private_network: str, + expected_service_account: str, *, timeout: float, ) -> dict[str, Any]: @@ -423,7 +428,7 @@ def gce_compute_identity( ipaddress.ip_network("192.168.0.0/16"), ) ), - "service_account": metadata["service_account"].endswith(".iam.gserviceaccount.com"), + "service_account": metadata["service_account"] == expected_service_account, } failed = sorted(name for name, passed in checks.items() if not passed) if failed: @@ -434,6 +439,7 @@ def gce_compute_identity( "expected_zone": expected_zone, "expected_private_network": expected_private_network, "expected_network_resource": expected_network_resource, + "expected_service_account": expected_service_account, "checks": checks, } @@ -492,10 +498,29 @@ def postgres_env(password: str, *, read_only: bool) -> dict[str, str]: return env +def conninfo_value(value: Any) -> str: + return "'" + str(value).replace("\\", "\\\\").replace("'", "\\'") + "'" + + +def postgres_conninfo(args: argparse.Namespace, database: str) -> str: + return " ".join( + ( + f"host={conninfo_value(args.ssl_server_name)}", + f"hostaddr={conninfo_value(args.host)}", + "port=5432", + f"dbname={conninfo_value(database)}", + "user=postgres", + "sslmode=verify-full", + f"sslrootcert={conninfo_value(args.ssl_root_cert)}", + "connect_timeout=8", + ) + ) + + def psql_command(args: argparse.Namespace, database: str) -> list[str]: return [ args.psql_bin, - f"host={args.host} port=5432 dbname={database} user=postgres sslmode=require connect_timeout=8", + postgres_conninfo(args, database), "-X", "-Atq", "-v", @@ -566,7 +591,7 @@ def restore_dump(args: argparse.Namespace, password: str) -> None: command = [ args.pg_restore_bin, "-d", - f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8", + postgres_conninfo(args, args.target_db), "--no-owner", "--no-acl", "--exit-on-error", @@ -594,6 +619,8 @@ def validate_private_identity( target_manifest: dict[str, Any], target_db: str, source_compute: str, + ssl_server_name: str, + ssl_root_cert_sha256: str, ) -> dict[str, Any]: identity = target_manifest["singleton"]["identity"] if identity.get("database") != target_db: @@ -622,6 +649,10 @@ def validate_private_identity( "server_port": identity.get("server_port"), "ssl": True, "ssl_version": identity.get("ssl_version"), + "sslmode": "verify-full", + "ssl_server_name": ssl_server_name, + "ssl_root_cert_sha256": ssl_root_cert_sha256, + "server_identity_verified": True, "private_connectivity": True, "transaction_read_only": "on", "source_compute": source_compute, @@ -718,6 +749,8 @@ def write_private(path: Path, content: str | bytes) -> None: def run_restore(args: argparse.Namespace) -> dict[str, Any]: started = time.monotonic() + args.ssl_root_cert = validate_regular_file(args.ssl_root_cert, "Cloud SQL server CA") + ssl_root_cert_sha256 = sha256_file(args.ssl_root_cert) run_dir = secure_run_dir(args.run_root, args.run_id, create=True) receipt: dict[str, Any] = { "artifact": "gcp_generated_postgres_snapshot_restore", @@ -731,6 +764,9 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: "private_network": args.expected_private_network, "compute_instance": args.expected_compute_instance, "compute_zone": args.expected_compute_zone, + "restore_service_account": args.expected_restore_service_account, + "ssl_server_name": args.ssl_server_name, + "ssl_root_cert_sha256": ssl_root_cert_sha256, "source": { "ssh_target": args.expected_source_ssh_target, "container": args.expected_source_container, @@ -795,6 +831,7 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: args.expected_compute_instance, args.expected_compute_zone, args.expected_private_network, + args.expected_restore_service_account, timeout=args.command_timeout, ) receipt["live_service"]["before"] = service_state(args.service, timeout=args.command_timeout) @@ -837,6 +874,8 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: target_manifest, args.target_db, receipt["target"]["compute"]["instance"], + args.ssl_server_name, + ssl_root_cert_sha256, ), } ) @@ -861,6 +900,10 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: "server_port": connectivity["server_port"], "ssl": connectivity["ssl"], "ssl_version": connectivity["ssl_version"], + "sslmode": connectivity["sslmode"], + "ssl_server_name": connectivity["ssl_server_name"], + "ssl_root_cert_sha256": connectivity["ssl_root_cert_sha256"], + "server_identity_verified": connectivity["server_identity_verified"], "private_connectivity": connectivity["private_connectivity"], "public_ip_disabled": cloudsql["public_ip_disabled"], "cloudsql_control_plane_checks": cloudsql["checks"], @@ -927,6 +970,12 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: args.expected_compute_instance, "--expected-compute-zone", args.expected_compute_zone, + "--expected-restore-service-account", + args.expected_restore_service_account, + "--ssl-server-name", + args.ssl_server_name, + "--ssl-root-cert", + str(args.ssl_root_cert), "--expected-source-ssh-target", args.expected_source_ssh_target, "--expected-source-container", @@ -988,6 +1037,8 @@ def run_restore(args: argparse.Namespace) -> dict[str, Any]: def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: + args.ssl_root_cert = validate_regular_file(args.ssl_root_cert, "Cloud SQL server CA") + ssl_root_cert_sha256 = sha256_file(args.ssl_root_cert) run_dir = secure_run_dir(args.run_root, args.run_id, create=False) restore_receipt_path = run_dir / "restore-receipt.json" restore_receipt_path = validate_regular_file(restore_receipt_path, "restore receipt") @@ -1007,6 +1058,9 @@ def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: "private_network": args.expected_private_network, "compute_instance": args.expected_compute_instance, "compute_zone": args.expected_compute_zone, + "restore_service_account": args.expected_restore_service_account, + "ssl_server_name": args.ssl_server_name, + "ssl_root_cert_sha256": ssl_root_cert_sha256, "source": { "ssh_target": args.expected_source_ssh_target, "container": args.expected_source_container, @@ -1037,6 +1091,7 @@ def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: "instance": restore_compute.get("instance") == args.expected_compute_instance, "zone": restore_compute.get("zone") == args.expected_compute_zone, "private_network": restore_compute.get("expected_private_network") == args.expected_private_network, + "service_account": restore_compute.get("service_account") == args.expected_restore_service_account, "checks": bool(restore_compute.get("checks")) and all(value is True for value in restore_compute["checks"].values()), } @@ -1083,6 +1138,7 @@ def run_cleanup(args: argparse.Namespace) -> dict[str, Any]: args.expected_compute_instance, args.expected_compute_zone, args.expected_private_network, + args.expected_restore_service_account, timeout=args.command_timeout, ) cleanup_compute_mismatches = sorted( @@ -1169,6 +1225,9 @@ def add_common_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--expected-private-network", default=DEFAULT_PRIVATE_NETWORK) parser.add_argument("--expected-compute-instance", default=DEFAULT_COMPUTE_INSTANCE) parser.add_argument("--expected-compute-zone", default=DEFAULT_COMPUTE_ZONE) + parser.add_argument("--expected-restore-service-account", default=DEFAULT_RESTORE_SERVICE_ACCOUNT) + parser.add_argument("--ssl-server-name", required=True) + parser.add_argument("--ssl-root-cert", required=True, type=Path) parser.add_argument("--expected-source-ssh-target", default=DEFAULT_SOURCE_SSH_TARGET) parser.add_argument("--expected-source-container", default=DEFAULT_SOURCE_CONTAINER) parser.add_argument("--expected-source-database", default=DEFAULT_SOURCE_DATABASE) @@ -1218,6 +1277,17 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--expected-compute-instance must be a safe lowercase GCE instance name") if not GCE_ZONE_RE.fullmatch(args.expected_compute_zone): parser.error("--expected-compute-zone must be a safe GCE zone") + if not SERVICE_ACCOUNT_RE.fullmatch(args.expected_restore_service_account): + parser.error("--expected-restore-service-account must be an exact GCP service-account email") + if not SAFE_DNS_NAME_RE.fullmatch(args.ssl_server_name): + parser.error("--ssl-server-name must be a safe DNS name covered by the Cloud SQL server certificate") + try: + ssl_root_cert_mode = args.ssl_root_cert.lstat().st_mode + except OSError: + parser.error("--ssl-root-cert must be an existing regular file") + if not stat.S_ISREG(ssl_root_cert_mode) or args.ssl_root_cert.is_symlink(): + parser.error("--ssl-root-cert must be a regular non-symlink file") + args.ssl_root_cert = args.ssl_root_cert.resolve() if not SAFE_SSH_TARGET_RE.fullmatch(args.expected_source_ssh_target): parser.error("--expected-source-ssh-target must be a host or user@host without options") for value, flag in ( @@ -1273,7 +1343,9 @@ def main(argv: list[str] | None = None) -> int: finally: for watched_signal, previous in previous_handlers.items(): signal.signal(watched_signal, previous) - print(json.dumps(payload, indent=2, sort_keys=True)) + receipt_name = "restore-receipt.json" if args.operation == "restore" else "cleanup-receipt.json" + receipt_path = args.run_root.resolve() / args.run_id / receipt_name + print_private_receipt_summary(receipt_path, payload) return 0 if payload.get("status") == "pass" else 1 diff --git a/ops/run_local_canonical_postgres_rebuild.py b/ops/run_local_canonical_postgres_rebuild.py index a5d3d10..3b6363d 100755 --- a/ops/run_local_canonical_postgres_rebuild.py +++ b/ops/run_local_canonical_postgres_rebuild.py @@ -31,7 +31,7 @@ MANIFEST_DESTINATION = "/tmp/postgres-parity-manifest.sql" CANARY_LABEL = "com.livingip.teleo.canonical-rebuild-canary" DATABASE_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,62}\Z") CONTAINER_PREFIX_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,39}\Z") -ALLOWED_APPLICATION_ROLES = frozenset({"kb_apply", "kb_review"}) +ALLOWED_APPLICATION_ROLES = frozenset({"kb_gate_owner", "kb_apply", "kb_review"}) ROLE_BOOLEAN_FIELDS = ( "can_login", "superuser", diff --git a/ops/verify_gcp_canonical_lifecycle.py b/ops/verify_gcp_canonical_lifecycle.py index aeb7760..b70aa3c 100644 --- a/ops/verify_gcp_canonical_lifecycle.py +++ b/ops/verify_gcp_canonical_lifecycle.py @@ -14,8 +14,12 @@ from pathlib import Path from typing import Any try: + from .private_receipt_io import print_private_receipt_summary + from .restore_gcp_generated_postgres_snapshot import DEFAULT_RESTORE_SERVICE_ACCOUNT from .verify_vps_canonical_snapshot_delta import DELTA_SCHEMA, parse_timestamp, write_private_json except ImportError: # pragma: no cover - direct script execution + from private_receipt_io import print_private_receipt_summary + from restore_gcp_generated_postgres_snapshot import DEFAULT_RESTORE_SERVICE_ACCOUNT from verify_vps_canonical_snapshot_delta import DELTA_SCHEMA, parse_timestamp, write_private_json SOURCE_SCHEMA = "livingip.sourceSnapshotReceipt.v2" @@ -99,6 +103,9 @@ def parity_evidence(details: dict[str, Any]) -> dict[str, bool]: "triggers", "types", "policies", + "role_memberships", + "object_ownership", + "object_acl", } structural_valid = required_structures <= set(structural) and all( bool(SHA256_RE.fullmatch(str((structural.get(kind) or {}).get("source") or ""))) @@ -124,11 +131,12 @@ def parity_evidence(details: dict[str, Any]) -> dict[str, bool]: "row_counts": isinstance(source_total_rows, int) and source_total_rows > 0 and source_total_rows == target_total_rows, - "table_hashes": details.get("table_mismatches") == [], + "table_hashes": details.get("table_mismatches") == [] and details.get("table_hash_problems") == [], "structural_hashes": structural_valid, "roles": details.get("application_role_mismatches") == {} and details.get("target_extra_application_roles") == [], - "extensions": details.get("required_extension_mismatches") == {}, + "extensions": details.get("required_extension_mismatches") == {} + and details.get("target_extra_extensions") == [], "performance": performance_valid and details.get("performance_problems") == [], } @@ -192,6 +200,12 @@ def blind_reasoning_evidence(reasoning: dict[str, Any], target_db: Any) -> dict[ and ((invocation.get("database_identity") or {}).get("transaction_read_only") == "on") and ((invocation.get("database_identity") or {}).get("default_transaction_read_only") == "on") and ((invocation.get("database_identity") or {}).get("ssl") is True) + and ((invocation.get("database_identity") or {}).get("sslmode") == "verify-full") + and ((invocation.get("database_identity") or {}).get("server_identity_verified") is True) + and bool((invocation.get("database_identity") or {}).get("ssl_server_name")) + and bool( + SHA256_RE.fullmatch(str((invocation.get("database_identity") or {}).get("ssl_root_cert_sha256") or "")) + ) and is_rfc1918((invocation.get("database_identity") or {}).get("server_address")) for invocation in invocations ) @@ -386,6 +400,11 @@ def verify_lifecycle( "source_delta_manifests_bound": delta_baseline.get("manifest_sha256") == (source.get("manifest") or {}).get("sha256") and delta_current.get("manifest_sha256") == (current_source.get("manifest") or {}).get("sha256"), + "source_delta_manifest_sql_bound": delta_baseline.get("manifest_sql_sha256") + == (source.get("manifest") or {}).get("manifest_sql_sha256") + and delta_current.get("manifest_sql_sha256") + == (current_source.get("manifest") or {}).get("manifest_sql_sha256") + and delta_baseline.get("manifest_sql_sha256") == delta_current.get("manifest_sql_sha256"), "source_delta_authorizations_bound": delta_baseline.get("capture_authorization_ref") == source.get("capture_authorization_ref") and delta_current.get("capture_authorization_ref") == current_source.get("capture_authorization_ref"), @@ -470,6 +489,10 @@ def verify_lifecycle( == parity.get("target_manifest_sha256"), "restore_private_tls": restore_connectivity.get("private_connectivity") is True and restore_connectivity.get("ssl") is True + and restore_connectivity.get("sslmode") == "verify-full" + and restore_connectivity.get("server_identity_verified") is True + and restore_connectivity.get("ssl_server_name") == restore_contract.get("ssl_server_name") + and restore_connectivity.get("ssl_root_cert_sha256") == restore_contract.get("ssl_root_cert_sha256") and is_rfc1918(restore_connectivity.get("server_address")), "restore_public_ip_disabled": restore_cloudsql.get("public_ip_disabled") is True, "restore_cloudsql_checks": required_true_checks( @@ -504,7 +527,10 @@ def verify_lifecycle( and restore_compute.get("expected_private_network") == restore_contract.get("private_network") and bool(restore_compute.get("network_resource")) and is_rfc1918(restore_compute.get("private_ip")) - and str(restore_compute.get("service_account") or "").endswith(".iam.gserviceaccount.com"), + and restore_contract.get("restore_service_account") == DEFAULT_RESTORE_SERVICE_ACCOUNT + and restore_compute.get("service_account") == DEFAULT_RESTORE_SERVICE_ACCOUNT, + "restore_tls_identity_contract": bool(restore_contract.get("ssl_server_name")) + and bool(SHA256_RE.fullmatch(str(restore_contract.get("ssl_root_cert_sha256") or ""))), "restore_connectivity_compute_bound": restore_connectivity.get("source_compute") == restore_compute.get("instance"), "restore_inline_parity": restore_parity.get("status") == "pass" and restore_parity.get("problems") == [], @@ -536,6 +562,10 @@ def verify_lifecycle( and parity_connectivity_proof.get("source_compute_zone") == restore_compute.get("zone") and is_rfc1918(parity_connectivity_proof.get("source_compute_private_ip")) and parity_connectivity_proof.get("server_address") == restore_connectivity.get("server_address") + and parity_connectivity_proof.get("sslmode") == "verify-full" + and parity_connectivity_proof.get("server_identity_verified") is True + and parity_connectivity_proof.get("ssl_server_name") == restore_contract.get("ssl_server_name") + and parity_connectivity_proof.get("ssl_root_cert_sha256") == restore_contract.get("ssl_root_cert_sha256") and required_true_checks( parity_connectivity_proof.get("cloudsql_control_plane_checks"), { @@ -660,17 +690,22 @@ def verify_lifecycle( "service_account", }, ), - "clone_absent": cleanup_database.get("clone_database_remaining") == 0, + "clone_absent": cleanup_database.get("existed_before") is True + and cleanup_database.get("clone_database_remaining") == 0, "cleanup_service_unchanged": cleanup_service.get("unchanged") is True and service_healthy(cleanup_service.get("before")) and cleanup_service.get("before") == cleanup_service.get("after"), } failed = sorted(name for name, passed in checks.items() if not passed) + structurally_valid = not failed return { "artifact": "gcp_canonical_snapshot_lifecycle_verification", "generated_at_utc": verification_time.isoformat(), - "status": "pass" if not failed else "fail", + "status": "prepared_external_attestation_required" if structurally_valid else "fail", + "current_tier": "T2_structural_receipt_bundle", "required_tier": "T3_live_private_gcp_staging", + "t3_receipt_ready": False, + "accepted_by_this_api": False, "target_database": target_db, "run_id": run_id, "checks": checks, @@ -696,13 +731,9 @@ def verify_lifecycle( "cleanup": sha256_file(cleanup_path), }, "strongest_claim_allowed": ( - ( - "current private GCP staging parity, ID-free Leo reasoning, cleanup, and zero post-snapshot VPS delta verified" - if delta_details.get("delta_detected") is False - else "private GCP staging parity at the declared snapshot boundary, ID-free Leo reasoning, cleanup, and explicit post-snapshot VPS delta verified" - ) - if not failed - else "lifecycle incomplete; failed checks must be repaired" + "caller-supplied lifecycle receipts are internally consistent; an independently authenticated platform receipt is still required" + if structurally_valid + else "lifecycle preparation is incomplete; failed checks must be repaired" ), "not_proven": ["production cutover", "ongoing replication", "Telegram delivery"], } @@ -746,8 +777,8 @@ def main() -> int: "strongest_claim_allowed": "lifecycle incomplete; receipts could not be validated", } write_private_json(args.output, payload) - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 if payload.get("status") == "pass" else 1 + print_private_receipt_summary(args.output, payload) + return 0 if payload.get("status") == "prepared_external_attestation_required" else 1 if __name__ == "__main__": diff --git a/ops/verify_postgres_parity_manifest.py b/ops/verify_postgres_parity_manifest.py index fb533d4..6e4eeb0 100755 --- a/ops/verify_postgres_parity_manifest.py +++ b/ops/verify_postgres_parity_manifest.py @@ -7,20 +7,26 @@ import argparse import hashlib import ipaddress import json +import re from datetime import UTC, datetime from pathlib import Path from typing import Any try: - from .private_receipt_io import write_private_json + from .private_receipt_io import print_private_receipt_summary, write_private_json except ImportError: # pragma: no cover - direct script execution - from private_receipt_io import write_private_json + from private_receipt_io import print_private_receipt_summary, write_private_json + +REVIEWED_MANIFEST_SQL_SHA256 = "430390129bc8980c4ece9939f2aee18e8c40e9da4b226782579349d5c45c5301" SINGLETON_KINDS = { "identity", "schemas", "extensions", "application_roles", + "role_memberships", + "object_ownership", + "object_acl", "columns", "constraints", "indexes", @@ -85,7 +91,18 @@ def load_manifest(path: Path) -> dict[str, Any]: def item_map(row: dict[str, Any], key: str) -> dict[str, dict[str, Any]]: - return {str(item[key]): item for item in row.get("items", [])} + items = row.get("items", []) + if not isinstance(items, list): + raise ValueError(f"manifest {row.get('kind')} items must be a list") + mapped: dict[str, dict[str, Any]] = {} + for item in items: + if not isinstance(item, dict) or key not in item: + raise ValueError(f"manifest {row.get('kind')} item is missing {key}") + item_key = str(item[key]) + if item_key in mapped: + raise ValueError(f"manifest {row.get('kind')} contains duplicate {key}={item_key}") + mapped[item_key] = item + return mapped def compare_manifests( @@ -120,9 +137,14 @@ def compare_manifests( + json.dumps(sorted(set(target_tables) - set(source_tables))) ) table_mismatches = [] + table_hash_problems = [] for table in sorted(set(source_tables) & set(target_tables)): source_row = source_tables[table] target_row = target_tables[table] + for side, row in (("source", source_row), ("target", target_row)): + rowset_md5 = row.get("rowset_md5") + if not isinstance(rowset_md5, str) or not rowset_md5: + table_hash_problems.append(f"{table}:{side}:rowset_md5_missing") mismatch = { field: {"source": source_row.get(field), "target": target_row.get(field)} for field in ("row_count", "rowset_md5") @@ -132,6 +154,8 @@ def compare_manifests( table_mismatches.append({"table": table, "mismatch": mismatch}) if table_mismatches: problems.append(f"table_content_mismatches={len(table_mismatches)}") + if table_hash_problems: + problems.append(f"table_hash_problems={len(table_hash_problems)}") structural_hashes: dict[str, dict[str, str]] = {} for kind in ( @@ -145,6 +169,9 @@ def compare_manifests( "triggers", "types", "policies", + "role_memberships", + "object_ownership", + "object_acl", ): source_hash = canonical_hash(source_singleton[kind].get("items", [])) target_hash = canonical_hash(target_singleton[kind].get("items", [])) @@ -155,19 +182,19 @@ def compare_manifests( source_extensions = item_map(source_singleton["extensions"], "name") target_extensions = item_map(target_singleton["extensions"], "name") extension_mismatches = { - name: {"source": item, "target": target_extensions.get(name)} - for name, item in source_extensions.items() - if target_extensions.get(name) != item + name: {"source": source_extensions.get(name), "target": target_extensions.get(name)} + for name in sorted(set(source_extensions) | set(target_extensions)) + if source_extensions.get(name) != target_extensions.get(name) } if extension_mismatches: - problems.append(f"required_extension_mismatches={len(extension_mismatches)}") + problems.append(f"extension_mismatches={len(extension_mismatches)}") source_roles = item_map(source_singleton["application_roles"], "name") target_roles = item_map(target_singleton["application_roles"], "name") role_mismatches = { - name: {"source": item, "target": target_roles.get(name)} - for name, item in source_roles.items() - if target_roles.get(name) != item + name: {"source": source_roles.get(name), "target": target_roles.get(name)} + for name in sorted(set(source_roles) | set(target_roles)) + if source_roles.get(name) != target_roles.get(name) } if role_mismatches: problems.append(f"application_role_mismatches={len(role_mismatches)}") @@ -217,8 +244,10 @@ def compare_manifests( "source_total_rows": sum(int(row["row_count"]) for row in source_tables.values()), "target_total_rows": sum(int(row["row_count"]) for row in target_tables.values()), "table_mismatches": table_mismatches, + "table_hash_problems": table_hash_problems, "structural_hashes": structural_hashes, "required_extension_mismatches": extension_mismatches, + "target_extra_extensions": sorted(set(target_extensions) - set(source_extensions)), "application_role_mismatches": role_mismatches, "target_extra_application_roles": sorted(set(target_roles) - set(source_roles)), "performance": performance_rows, @@ -293,6 +322,14 @@ def load_connectivity( problems.append("gcp_target_manifest_tls_not_proven") if proof.get("ssl") is not True: problems.append("gcp_connectivity_tls_not_proven") + if proof.get("sslmode") != "verify-full": + problems.append("gcp_connectivity_server_identity_mode_not_proven") + if proof.get("server_identity_verified") is not True: + problems.append("gcp_connectivity_server_identity_not_proven") + if not proof.get("ssl_server_name"): + problems.append("gcp_connectivity_ssl_server_name_missing") + if not re.fullmatch(r"[0-9a-f]{64}", str(proof.get("ssl_root_cert_sha256") or "")): + problems.append("gcp_connectivity_ssl_root_cert_hash_missing") return problems, {"required": True, "path": str(path), "proof": proof} @@ -360,7 +397,7 @@ def main() -> int: ], } write_private_json(args.output, payload) - print(json.dumps(payload, indent=2, sort_keys=True)) + print_private_receipt_summary(args.output, payload) return 0 if status == "pass" else 1 diff --git a/ops/verify_vps_canonical_snapshot_delta.py b/ops/verify_vps_canonical_snapshot_delta.py index b4dbe7a..4f388f4 100644 --- a/ops/verify_vps_canonical_snapshot_delta.py +++ b/ops/verify_vps_canonical_snapshot_delta.py @@ -16,12 +16,24 @@ try: SOURCE_SNAPSHOT_RECEIPT_SCHEMA, build_provenance_binding, ) - from .private_receipt_io import write_private_json - from .verify_postgres_parity_manifest import SINGLETON_KINDS, canonical_hash, file_sha256, load_manifest + from .private_receipt_io import print_private_receipt_summary, write_private_json + from .verify_postgres_parity_manifest import ( + REVIEWED_MANIFEST_SQL_SHA256, + SINGLETON_KINDS, + canonical_hash, + file_sha256, + load_manifest, + ) except ImportError: # pragma: no cover - direct script execution from capture_vps_canonical_postgres_snapshot import SOURCE_SNAPSHOT_RECEIPT_SCHEMA, build_provenance_binding - from private_receipt_io import write_private_json - from verify_postgres_parity_manifest import SINGLETON_KINDS, canonical_hash, file_sha256, load_manifest + from private_receipt_io import print_private_receipt_summary, write_private_json + from verify_postgres_parity_manifest import ( + REVIEWED_MANIFEST_SQL_SHA256, + SINGLETON_KINDS, + canonical_hash, + file_sha256, + load_manifest, + ) DELTA_SCHEMA = "livingip.vpsCanonicalSnapshotDelta.v1" SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") @@ -124,6 +136,7 @@ def validate_capture( "binding_algorithm": binding.get("algorithm") == "sha256", "dump_sha256": bool(SHA256_RE.fullmatch(str(dump.get("sha256") or ""))), "manifest_sql_sha256": bool(SHA256_RE.fullmatch(str(manifest_receipt.get("manifest_sql_sha256") or ""))), + "manifest_sql_reviewed": manifest_receipt.get("manifest_sql_sha256") == REVIEWED_MANIFEST_SQL_SHA256, "source_context_sha256": bool(SHA256_RE.fullmatch(str(source_context.get("sha256") or ""))), "snapshot_identity": all( bool(snapshot.get(field)) @@ -186,6 +199,10 @@ def verify_delta( raise ValueError("current source identity does not match the restored baseline source") if current_snapshot.get("system_identifier") != baseline_snapshot.get("system_identifier"): raise ValueError("current source PostgreSQL system identifier changed") + baseline_manifest_sql_sha256 = str((baseline_receipt.get("manifest") or {}).get("manifest_sql_sha256") or "") + current_manifest_sql_sha256 = str((current_receipt.get("manifest") or {}).get("manifest_sql_sha256") or "") + if baseline_manifest_sql_sha256 != current_manifest_sql_sha256: + raise ValueError("baseline and current captures used different manifest SQL algorithms") baseline = manifest_fingerprint(baseline_manifest) current = manifest_fingerprint(current_manifest) @@ -221,6 +238,7 @@ def verify_delta( "captured_at_utc": baseline_snapshot["captured_at_utc"], "capture_receipt_sha256": file_sha256(baseline_receipt_path), "manifest_sha256": file_sha256(baseline_manifest_path), + "manifest_sql_sha256": baseline_manifest_sql_sha256, **baseline, }, "current": { @@ -229,6 +247,7 @@ def verify_delta( "captured_at_utc": current_snapshot["captured_at_utc"], "capture_receipt_sha256": file_sha256(current_receipt_path), "manifest_sha256": file_sha256(current_manifest_path), + "manifest_sql_sha256": current_manifest_sql_sha256, **current, }, "delta": { @@ -276,7 +295,7 @@ def main() -> int: "strongest_claim_allowed": "post-snapshot VPS delta is not proven", } write_private_json(args.output, payload) - print(json.dumps(payload, indent=2, sort_keys=True)) + print_private_receipt_summary(args.output, payload) return 0 if payload.get("status") == "pass" else 1 diff --git a/scripts/run_gcp_generated_db_direct_claim_suite.py b/scripts/run_gcp_generated_db_direct_claim_suite.py index 79b0704..dc5f465 100644 --- a/scripts/run_gcp_generated_db_direct_claim_suite.py +++ b/scripts/run_gcp_generated_db_direct_claim_suite.py @@ -35,11 +35,15 @@ if __name__ == "__main__" and should_reexec_in_hermes(Path(sys.executable)): ) HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parent +sys.path.insert(0, str(REPO_ROOT)) sys.path.insert(0, str(HERE)) import run_leo_clone_bound_handler_checkpoint as bound # noqa: E402 import working_leo_open_ended_benchmark as benchmark # noqa: E402 +from ops.verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256 # noqa: E402 + SCHEMA = "livingip.gcpGeneratedDbDirectClaimSuite.v1" SOURCE_COMPUTE = "teleo-prod-1" DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service" @@ -48,7 +52,6 @@ DEFAULT_PROJECT = "teleo-501523" DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password" DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql" REVIEWED_CLOUDSQL_TOOL_SHA256 = "7d2074a0fc5f0d48fbf8f7905a72ead8f2696c86a041fa43e078fff5500baa51" -REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a" COUNT_READBACK_RE = re.compile( r"DB readback:\s*claims:\s*`?(?P\d+)`?;\s*" r"sources:\s*`?(?P\d+)`?;\s*" diff --git a/tests/test_attest_gcp_reasoning_compute.py b/tests/test_attest_gcp_reasoning_compute.py index b3906fc..49190f7 100644 --- a/tests/test_attest_gcp_reasoning_compute.py +++ b/tests/test_attest_gcp_reasoning_compute.py @@ -38,7 +38,7 @@ def compute_identity() -> dict: "zone_resource": "projects/123456789/zones/europe-west6-a", "network_resource": "projects/123456789/networks/teleo-staging-net", "private_ip": "10.61.0.2", - "service_account": "teleo-runtime@teleo-501523.iam.gserviceaccount.com", + "service_account": attestation.DEFAULT_RESTORE_SERVICE_ACCOUNT, "expected_private_network": "projects/teleo-501523/global/networks/teleo-staging-net", "checks": { "project": True, @@ -58,13 +58,14 @@ def test_attestation_binds_reasoning_hash_to_live_metadata(tmp_path: Path, monke receipt_path = reasoning_receipt(tmp_path / "reasoning.json") observed = {} - def fake_identity(project, instance, zone, network, *, timeout): + def fake_identity(project, instance, zone, network, service_account, *, timeout): observed.update( { "project": project, "instance": instance, "zone": zone, "network": network, + "service_account": service_account, "timeout": timeout, } ) @@ -88,6 +89,7 @@ def test_attestation_binds_reasoning_hash_to_live_metadata(tmp_path: Path, monke "instance": "teleo-prod-1", "zone": "europe-west6-a", "network": "projects/teleo-501523/global/networks/teleo-staging-net", + "service_account": attestation.DEFAULT_RESTORE_SERVICE_ACCOUNT, "timeout": 10.0, } diff --git a/tests/test_capture_vps_canonical_postgres_snapshot.py b/tests/test_capture_vps_canonical_postgres_snapshot.py index 762d14e..fed31b8 100644 --- a/tests/test_capture_vps_canonical_postgres_snapshot.py +++ b/tests/test_capture_vps_canonical_postgres_snapshot.py @@ -48,6 +48,11 @@ def test_manifest_hash_order_is_collation_independent() -> None: assert 'collate "C"' in manifest assert "'server_address', host(inet_server_addr())" in manifest + assert "'kb_gate_owner', 'kb_apply', 'kb_review'" in manifest + assert "'kind', 'role_memberships'" in manifest + assert "'kind', 'object_ownership'" in manifest + assert "'kind', 'object_acl'" in manifest + assert "aclexplode(attribute.attacl)" in manifest def test_toc_counts_database_objects(tmp_path: Path) -> None: diff --git a/tests/test_private_receipt_io.py b/tests/test_private_receipt_io.py new file mode 100644 index 0000000..4df55cb --- /dev/null +++ b/tests/test_private_receipt_io.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from ops.private_receipt_io import print_private_receipt_summary, write_private_json + + +def test_stdout_summary_never_replays_private_receipt_payload(tmp_path: Path, capsys) -> None: + output = tmp_path / "private" / "receipt.json" + payload = { + "artifact": "private_test_receipt", + "status": "pass", + "private_claim_body": "must-not-appear-on-stdout", + "database_password": "must-not-appear-on-stdout-either", + } + + write_private_json(output, payload) + print_private_receipt_summary(output, payload) + + stdout = capsys.readouterr().out + summary = json.loads(stdout) + assert "must-not-appear" not in stdout + assert summary["artifact"] == "private_test_receipt" + assert summary["status"] == "pass" + assert summary["output_name"] == output.name + assert str(output.parent) not in stdout + assert len(summary["output_sha256"]) == 64 + assert output.stat().st_mode & 0o777 == 0o600 diff --git a/tests/test_restore_gcp_generated_postgres_snapshot.py b/tests/test_restore_gcp_generated_postgres_snapshot.py index 5c152b9..e1c9535 100644 --- a/tests/test_restore_gcp_generated_postgres_snapshot.py +++ b/tests/test_restore_gcp_generated_postgres_snapshot.py @@ -49,7 +49,27 @@ def manifest_rows(database: str, *, target: bool, row_count: int = 1) -> list[di "inherit": True, "replication": False, "bypass_rls": False, - } + }, + { + "name": "kb_gate_owner", + "can_login": False, + "superuser": False, + "create_db": False, + "create_role": False, + "inherit": True, + "replication": False, + "bypass_rls": False, + }, + { + "name": "kb_review", + "can_login": True, + "superuser": False, + "create_db": False, + "create_role": False, + "inherit": True, + "replication": False, + "bypass_rls": False, + }, ], }, ] @@ -65,6 +85,9 @@ def manifest_rows(database: str, *, target: bool, row_count: int = 1) -> list[di "triggers", "types", "policies", + "role_memberships", + "object_ownership", + "object_acl", ) ) rows.extend( @@ -97,6 +120,8 @@ def args_for(tmp_path: Path, **overrides) -> argparse.Namespace: source_manifest = tmp_path / "source-manifest.jsonl" source_manifest.write_text(manifest_text("teleo", target=False), encoding="utf-8") manifest_sql = Path("ops/postgres_parity_manifest.sql").resolve() + ssl_root_cert = tmp_path / "cloudsql-server-ca.pem" + ssl_root_cert.write_text("test-only-ca\n", encoding="utf-8") source_context_sha256 = "4" * 64 snapshot = { "exported_snapshot_id": "00000003-000001A2-1", @@ -166,6 +191,9 @@ def args_for(tmp_path: Path, **overrides) -> argparse.Namespace: "expected_private_network": restore.DEFAULT_PRIVATE_NETWORK, "expected_compute_instance": restore.DEFAULT_COMPUTE_INSTANCE, "expected_compute_zone": restore.DEFAULT_COMPUTE_ZONE, + "expected_restore_service_account": restore.DEFAULT_RESTORE_SERVICE_ACCOUNT, + "ssl_server_name": "teleo-pgvector-standby.private.teleo.internal", + "ssl_root_cert": ssl_root_cert, "expected_source_ssh_target": restore.DEFAULT_SOURCE_SSH_TARGET, "expected_source_container": restore.DEFAULT_SOURCE_CONTAINER, "expected_source_database": restore.DEFAULT_SOURCE_DATABASE, @@ -203,7 +231,7 @@ def install_restore_fakes( monkeypatch.setattr( restore, "gce_compute_identity", - lambda project, instance, zone, expected_private_network, **_kwargs: { + lambda project, instance, zone, expected_private_network, expected_service_account, **_kwargs: { "project": project, "project_number": "785938879453", "instance": instance, @@ -212,7 +240,7 @@ def install_restore_fakes( "expected_private_network": expected_private_network, "private_ip": "10.61.0.2", "network_resource": "projects/123456789/networks/teleo-staging-net", - "service_account": "teleo-runtime@teleo-501523.iam.gserviceaccount.com", + "service_account": expected_service_account, "checks": { "project": True, "project_number": True, @@ -492,6 +520,9 @@ def test_cleanup_is_bound_to_passing_restore_receipt( "expected_private_network", "expected_compute_instance", "expected_compute_zone", + "expected_restore_service_account", + "ssl_server_name", + "ssl_root_cert", "expected_source_ssh_target", "expected_source_container", "expected_source_database", @@ -534,6 +565,9 @@ def test_cleanup_rejects_target_topology_drift_before_credentials( "expected_private_network", "expected_compute_instance", "expected_compute_zone", + "expected_restore_service_account", + "ssl_server_name", + "ssl_root_cert", "expected_source_ssh_target", "expected_source_container", "expected_source_database", @@ -584,6 +618,9 @@ def test_cleanup_retains_failure_receipt_when_post_drop_readback_fails( "expected_private_network", "expected_compute_instance", "expected_compute_zone", + "expected_restore_service_account", + "ssl_server_name", + "ssl_root_cert", "expected_source_ssh_target", "expected_source_container", "expected_source_database", @@ -631,7 +668,10 @@ def test_restore_dump_uses_tls_no_owner_no_acl_and_keeps_secret_out_of_argv( assert captured["command"][0] == "pg_restore" assert captured["command"][1] == "-d" - assert "sslmode=require" in captured["command"][2] + assert "sslmode=verify-full" in captured["command"][2] + assert "hostaddr='10.61.0.3'" in captured["command"][2] + assert "host='teleo-pgvector-standby.private.teleo.internal'" in captured["command"][2] + assert f"sslrootcert='{args.ssl_root_cert}'" in captured["command"][2] assert "--no-owner" in captured["command"] assert "--no-acl" in captured["command"] assert "--exit-on-error" in captured["command"] @@ -695,7 +735,7 @@ def test_gce_compute_identity_is_bound_to_metadata_instance_zone_and_network( "instance/zone": "projects/123456789/zones/europe-west6-a", "instance/network-interfaces/0/network": "projects/785938879453/networks/teleo-staging-net", "instance/network-interfaces/0/ip": "10.61.0.2", - "instance/service-accounts/default/email": "teleo-runtime@teleo-501523.iam.gserviceaccount.com", + "instance/service-accounts/default/email": restore.DEFAULT_RESTORE_SERVICE_ACCOUNT, } monkeypatch.setattr(restore, "gce_metadata_value", lambda path, **_kwargs: metadata[path]) @@ -704,6 +744,7 @@ def test_gce_compute_identity_is_bound_to_metadata_instance_zone_and_network( "teleo-prod-1", "europe-west6-a", restore.DEFAULT_PRIVATE_NETWORK, + restore.DEFAULT_RESTORE_SERVICE_ACCOUNT, timeout=2.0, ) @@ -718,6 +759,7 @@ def test_gce_compute_identity_is_bound_to_metadata_instance_zone_and_network( "different-instance", "europe-west6-a", restore.DEFAULT_PRIVATE_NETWORK, + restore.DEFAULT_RESTORE_SERVICE_ACCOUNT, timeout=2.0, ) @@ -729,6 +771,7 @@ def test_gce_compute_identity_is_bound_to_metadata_instance_zone_and_network( "teleo-prod-1", "europe-west6-a", restore.DEFAULT_PRIVATE_NETWORK, + restore.DEFAULT_RESTORE_SERVICE_ACCOUNT, timeout=2.0, ) @@ -739,6 +782,19 @@ def test_gce_compute_identity_is_bound_to_metadata_instance_zone_and_network( "teleo-prod-1", "europe-west6-a", "projects/other-project/global/networks/teleo-staging-net", + restore.DEFAULT_RESTORE_SERVICE_ACCOUNT, + timeout=2.0, + ) + + metadata["instance/network-interfaces/0/network"] = "projects/785938879453/networks/teleo-staging-net" + metadata["instance/service-accounts/default/email"] = "project-owner@teleo-501523.iam.gserviceaccount.com" + with pytest.raises(restore.RestoreError, match="service_account"): + restore.gce_compute_identity( + "teleo-501523", + "teleo-prod-1", + "europe-west6-a", + restore.DEFAULT_PRIVATE_NETWORK, + restore.DEFAULT_RESTORE_SERVICE_ACCOUNT, timeout=2.0, ) diff --git a/tests/test_run_local_canonical_postgres_rebuild.py b/tests/test_run_local_canonical_postgres_rebuild.py index 4bd4860..b03cb69 100644 --- a/tests/test_run_local_canonical_postgres_rebuild.py +++ b/tests/test_run_local_canonical_postgres_rebuild.py @@ -137,6 +137,9 @@ def manifest_rows() -> list[dict]: {"kind": "schemas", "items": ["kb_stage", "public"]}, {"kind": "extensions", "items": []}, {"kind": "application_roles", "items": [role]}, + {"kind": "role_memberships", "items": []}, + {"kind": "object_ownership", "items": []}, + {"kind": "object_acl", "items": []}, ] rows.extend( {"kind": kind, "items": []} diff --git a/tests/test_run_local_genesis_ledger_rebuild.py b/tests/test_run_local_genesis_ledger_rebuild.py index 4919b1c..865ed98 100644 --- a/tests/test_run_local_genesis_ledger_rebuild.py +++ b/tests/test_run_local_genesis_ledger_rebuild.py @@ -64,6 +64,9 @@ def manifest_rows(database: str = "teleo_fixture") -> list[dict]: {"kind": "schemas", "items": ["kb_stage", "public"]}, {"kind": "extensions", "items": []}, {"kind": "application_roles", "items": [role]}, + {"kind": "role_memberships", "items": []}, + {"kind": "object_ownership", "items": []}, + {"kind": "object_acl", "items": []}, ] rows.extend( {"kind": kind, "items": []} diff --git a/tests/test_verify_gcp_canonical_lifecycle.py b/tests/test_verify_gcp_canonical_lifecycle.py index 28df37b..b2a26da 100644 --- a/tests/test_verify_gcp_canonical_lifecycle.py +++ b/tests/test_verify_gcp_canonical_lifecycle.py @@ -3,8 +3,10 @@ import json from datetime import UTC, datetime from pathlib import Path +from ops.restore_gcp_generated_postgres_snapshot import DEFAULT_RESTORE_SERVICE_ACCOUNT from ops.verify_gcp_canonical_lifecycle import EXPECTED_BLIND_ROW_IDS from ops.verify_gcp_canonical_lifecycle import verify_lifecycle as _verify_lifecycle +from ops.verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256 VERIFY_NOW = datetime(2026, 7, 15, 1, 3, tzinfo=UTC) @@ -45,6 +47,9 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "triggers", "types", "policies", + "role_memberships", + "object_ownership", + "object_acl", ), 1, ) @@ -57,8 +62,10 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "source_total_rows": 52167, "target_total_rows": 52167, "table_mismatches": [], + "table_hash_problems": [], "structural_hashes": structural_hashes, "required_extension_mismatches": {}, + "target_extra_extensions": [], "application_role_mismatches": {}, "target_extra_application_roles": [], "performance": [{"query": "count_claims", "source_ms": 1.0, "target_ms": 2.0, "source_ratio": 0.4}], @@ -90,7 +97,12 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "production_db_mutated": False, "telegram_send_attempted": False, "dump": {"sha256": "a" * 64}, - "manifest": {"sha256": "b" * 64, "table_count": 39, "total_rows": 52167}, + "manifest": { + "sha256": "b" * 64, + "manifest_sql_sha256": REVIEWED_MANIFEST_SQL_SHA256, + "table_count": 39, + "total_rows": 52167, + }, "provenance_binding": {"algorithm": "sha256", "payload": {"bound": True}, "sha256": "c" * 64}, } source_path = write_json(tmp_path / "source.json", source) @@ -146,6 +158,9 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "private_network": "projects/teleo-501523/global/networks/teleo-staging-net", "compute_instance": "teleo-prod-1", "compute_zone": "europe-west6-a", + "restore_service_account": DEFAULT_RESTORE_SERVICE_ACCOUNT, + "ssl_server_name": "teleo-pgvector-standby.private.teleo.internal", + "ssl_root_cert_sha256": "9" * 64, "source": source["source"], "password_secret": "gcp-teleo-pgvector-standby-postgres-password", "service": "leoclean-gcp-prod-parallel.service", @@ -155,6 +170,10 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "connectivity": { "private_connectivity": True, "ssl": True, + "sslmode": "verify-full", + "ssl_server_name": "teleo-pgvector-standby.private.teleo.internal", + "ssl_root_cert_sha256": "9" * 64, + "server_identity_verified": True, "server_address": "10.61.0.3", "source_compute": "teleo-prod-1", }, @@ -184,7 +203,7 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "expected_private_network": "projects/teleo-501523/global/networks/teleo-staging-net", "network_resource": "projects/123456789/networks/teleo-staging-net", "private_ip": "10.61.0.2", - "service_account": "teleo-runtime@teleo-501523.iam.gserviceaccount.com", + "service_account": DEFAULT_RESTORE_SERVICE_ACCOUNT, "checks": { "project": True, "project_number": True, @@ -231,6 +250,10 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "source_compute_zone": "europe-west6-a", "source_compute_private_ip": "10.61.0.2", "server_address": "10.61.0.3", + "sslmode": "verify-full", + "ssl_server_name": "teleo-pgvector-standby.private.teleo.internal", + "ssl_root_cert_sha256": "9" * 64, + "server_identity_verified": True, "project": "teleo-501523", "cloudsql_instance": "teleo-pgvector-standby", "private_network": "projects/teleo-501523/global/networks/teleo-staging-net", @@ -347,6 +370,10 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "transaction_read_only": "on", "default_transaction_read_only": "on", "ssl": True, + "sslmode": "verify-full", + "ssl_server_name": "teleo-pgvector-standby.private.teleo.internal", + "ssl_root_cert_sha256": "9" * 64, + "server_identity_verified": True, "server_address": "10.61.0.3", }, } @@ -403,6 +430,7 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "run_id": "canonical-current-test", "capture_receipt_sha256": sha256(source_path), "manifest_sha256": "b" * 64, + "manifest_sql_sha256": REVIEWED_MANIFEST_SQL_SHA256, "capture_authorization_ref": "codex-delegation:test1234", "captured_at_utc": "2026-07-15T00:59:00+00:00", "table_count": 39, @@ -414,6 +442,7 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "run_id": "canonical-current-postflight", "capture_receipt_sha256": sha256(current_source_path), "manifest_sha256": "b" * 64, + "manifest_sql_sha256": REVIEWED_MANIFEST_SQL_SHA256, "capture_authorization_ref": "codex-delegation:postflight1234", "captured_at_utc": "2026-07-15T01:02:00+00:00", "table_count": 39, @@ -438,7 +467,7 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "completed_at_utc": "2026-07-15T01:01:30+00:00", "run_id": run_id, "target_database": target_db, - "database": {"clone_database_remaining": 0}, + "database": {"existed_before": True, "clone_database_remaining": 0}, "live_service": {"before": healthy_service, "after": healthy_service, "unchanged": True}, "restore_receipt": {"sha256": sha256(restore_path)}, "execution_contract": restore["execution_contract"], @@ -460,14 +489,19 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: } -def test_bound_live_lifecycle_passes(tmp_path: Path) -> None: +def test_caller_authored_bundle_is_preparation_only_even_when_all_checks_pass(tmp_path: Path) -> None: paths = lifecycle_receipts(tmp_path) result = verify_lifecycle(**paths) - assert result["status"] == "pass" + assert result["status"] == "prepared_external_attestation_required" assert result["failed_checks"] == [] assert result["checks"]["clone_absent"] is True + assert result["current_tier"] == "T2_structural_receipt_bundle" + assert result["required_tier"] == "T3_live_private_gcp_staging" + assert result["t3_receipt_ready"] is False + assert result["accepted_by_this_api"] is False + assert "current private GCP staging" not in result["strongest_claim_allowed"] def test_local_parity_or_missing_cleanup_fails_closed(tmp_path: Path) -> None: @@ -488,6 +522,18 @@ def test_local_parity_or_missing_cleanup_fails_closed(tmp_path: Path) -> None: assert {"parity_pass", "clone_absent"} <= set(result["failed_checks"]) +def test_cleanup_cannot_pass_when_the_named_clone_never_existed(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + cleanup = json.loads(paths["cleanup_path"].read_text()) + cleanup["database"]["existed_before"] = False + write_json(paths["cleanup_path"], cleanup) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "clone_absent" in result["failed_checks"] + + def test_mismatched_source_or_reasoning_receipt_fails_closed(tmp_path: Path) -> None: paths = lifecycle_receipts(tmp_path) restore = json.loads(paths["restore_path"].read_text()) @@ -529,6 +575,28 @@ def test_reasoning_compute_attestation_must_bind_live_metadata(tmp_path: Path) - assert "reasoning_compute_identity_bound" in result["failed_checks"] +def test_project_owner_shaped_service_account_cannot_pass_preparation(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + project_owner = "project-owner@teleo-501523.iam.gserviceaccount.com" + restore = json.loads(paths["restore_path"].read_text()) + restore["target"]["compute"]["service_account"] = project_owner + restore["execution_contract"]["restore_service_account"] = project_owner + write_json(paths["restore_path"], restore) + attestation = json.loads(paths["reasoning_compute_path"].read_text()) + attestation["compute"]["service_account"] = project_owner + write_json(paths["reasoning_compute_path"], attestation) + cleanup = json.loads(paths["cleanup_path"].read_text()) + cleanup["target"]["compute"]["service_account"] = project_owner + cleanup["execution_contract"]["restore_service_account"] = project_owner + cleanup["restore_receipt"]["sha256"] = sha256(paths["restore_path"]) + write_json(paths["cleanup_path"], cleanup) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "restore_compute_contract" in result["failed_checks"] + + def test_source_postflight_must_be_newer_than_cleanup_completion(tmp_path: Path) -> None: paths = lifecycle_receipts(tmp_path) cleanup = json.loads(paths["cleanup_path"].read_text()) @@ -578,6 +646,21 @@ def test_reasoning_must_retrieve_expected_claim_sources_and_preserve_handle(tmp_ assert result["reasoning_producer_checks"]["reply_uses_only_m3taversal_handle"] is False +def test_reasoning_wrapper_must_authenticate_the_cloudsql_server(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + reasoning = json.loads(paths["reasoning_path"].read_text()) + for invocation in reasoning["wrapper_tool_proof"]["invocations"]: + invocation["database_identity"]["sslmode"] = "require" + invocation["database_identity"]["server_identity_verified"] = False + write_json(paths["reasoning_path"], reasoning) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "reasoning_producer_evidence" in result["failed_checks"] + assert result["reasoning_producer_checks"]["wrapper_invocations"] is False + + def test_lifecycle_requires_restore_parity_reasoning_cleanup_chronology(tmp_path: Path) -> None: paths = lifecycle_receipts(tmp_path) cleanup = json.loads(paths["cleanup_path"].read_text()) diff --git a/tests/test_verify_postgres_parity_manifest.py b/tests/test_verify_postgres_parity_manifest.py index d016aee..37f5045 100644 --- a/tests/test_verify_postgres_parity_manifest.py +++ b/tests/test_verify_postgres_parity_manifest.py @@ -28,7 +28,23 @@ def manifest_rows( {"kind": "extensions", "items": [{"name": "plpgsql", "version": "1.0"}]}, { "kind": "application_roles", - "items": [{"name": "kb_apply", "can_login": True, "superuser": False}], + "items": [ + {"name": "kb_apply", "can_login": True, "superuser": False}, + {"name": "kb_gate_owner", "can_login": False, "superuser": False}, + {"name": "kb_review", "can_login": True, "superuser": False}, + ], + }, + { + "kind": "role_memberships", + "items": [{"role": "kb_gate_owner", "member": "kb_apply", "admin_option": False}], + }, + { + "kind": "object_ownership", + "items": [{"object_type": "schema", "schema": "public", "name": "public", "owner": "kb_gate_owner"}], + }, + { + "kind": "object_acl", + "items": [{"object_type": "schema", "name": "public", "grantee": "kb_apply", "privilege_type": "USAGE"}], }, {"kind": "columns", "items": [{"schema": "public", "table": "claims", "name": "id"}]}, {"kind": "constraints", "items": [{"schema": "public", "table": "claims", "name": "claims_pkey"}]}, @@ -94,6 +110,8 @@ def test_matching_local_manifests_pass(tmp_path: Path) -> None: assert len(payload["target_manifest_sha256"]) == 64 assert payload["private_connectivity"]["status"] == "not_applicable_local_restore" assert (tmp_path / "result.json").stat().st_mode & 0o777 == 0o600 + assert '"source_database"' not in completed.stdout + assert '"output_sha256"' in completed.stdout def test_row_hash_and_role_mismatches_fail(tmp_path: Path) -> None: @@ -103,7 +121,32 @@ def test_row_hash_and_role_mismatches_fail(tmp_path: Path) -> None: assert completed.returncode == 1 assert "table_content_mismatches=1" in payload["problems"] + assert "application_role_mismatches=3" in payload["problems"] + + +def test_missing_row_hash_and_extra_roles_or_extensions_fail(tmp_path: Path) -> None: + target = manifest_rows(database="teleo_copy", rowset_md5="") + target[2]["items"].append({"name": "unreviewed", "version": "1.0"}) + target[3]["items"].append({"name": "project_owner", "can_login": True, "superuser": True}) + + completed, payload = run_verifier(tmp_path, manifest_rows(database="teleo"), target) + + assert completed.returncode == 1 + assert "table_hash_problems=1" in payload["problems"] + assert "extension_mismatches=1" in payload["problems"] assert "application_role_mismatches=1" in payload["problems"] + assert payload["details"]["target_extra_extensions"] == ["unreviewed"] + assert payload["details"]["target_extra_application_roles"] == ["project_owner"] + + +def test_authorization_ownership_or_acl_drift_fails(tmp_path: Path) -> None: + target = manifest_rows(database="teleo_copy") + target[5]["items"][0]["owner"] = "postgres" + + completed, payload = run_verifier(tmp_path, manifest_rows(database="teleo"), target) + + assert completed.returncode == 1 + assert "object_ownership_hash_mismatch" in payload["problems"] def test_gcp_scope_requires_private_connectivity_receipt(tmp_path: Path) -> None: @@ -132,6 +175,10 @@ def test_gcp_scope_accepts_matching_private_connectivity_receipt(tmp_path: Path) "target_database": "teleo_gcp_copy", "server_address": "10.61.0.3", "ssl": True, + "sslmode": "verify-full", + "ssl_server_name": "teleo-pgvector-standby.private.teleo.internal", + "ssl_root_cert_sha256": "a" * 64, + "server_identity_verified": True, }, ) @@ -155,6 +202,10 @@ def test_gcp_scope_rejects_receipt_for_a_different_server(tmp_path: Path) -> Non "target_database": "teleo_gcp_copy", "server_address": "10.61.0.4", "ssl": True, + "sslmode": "verify-full", + "ssl_server_name": "teleo-pgvector-standby.private.teleo.internal", + "ssl_root_cert_sha256": "a" * 64, + "server_identity_verified": True, }, ) @@ -176,6 +227,10 @@ def test_gcp_scope_rejects_public_or_non_tls_target_manifest(tmp_path: Path) -> "target_database": "teleo_gcp_copy", "server_address": "34.1.2.3", "ssl": False, + "sslmode": "verify-full", + "ssl_server_name": "teleo-pgvector-standby.private.teleo.internal", + "ssl_root_cert_sha256": "a" * 64, + "server_identity_verified": True, }, ) diff --git a/tests/test_verify_vps_canonical_snapshot_delta.py b/tests/test_verify_vps_canonical_snapshot_delta.py index 451c6f6..61e264c 100644 --- a/tests/test_verify_vps_canonical_snapshot_delta.py +++ b/tests/test_verify_vps_canonical_snapshot_delta.py @@ -8,6 +8,7 @@ from ops.capture_vps_canonical_postgres_snapshot import ( SOURCE_SNAPSHOT_RECEIPT_SCHEMA, build_provenance_binding, ) +from ops.verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256 from ops.verify_vps_canonical_snapshot_delta import verify_delta, write_private_json @@ -21,7 +22,17 @@ def manifest_text(*, claims: int = 1, constraint_name: str = "claims_pkey") -> s }, {"kind": "schemas", "items": ["kb_stage", "public"]}, {"kind": "extensions", "items": [{"name": "pgcrypto", "version": "1.3"}]}, - {"kind": "application_roles", "items": [{"name": "kb_apply", "can_login": True}]}, + { + "kind": "application_roles", + "items": [ + {"name": "kb_apply", "can_login": True}, + {"name": "kb_gate_owner", "can_login": False}, + {"name": "kb_review", "can_login": True}, + ], + }, + {"kind": "role_memberships", "items": []}, + {"kind": "object_ownership", "items": []}, + {"kind": "object_acl", "items": []}, {"kind": "columns", "items": []}, {"kind": "constraints", "items": [{"name": constraint_name}]}, {"kind": "indexes", "items": []}, @@ -56,7 +67,7 @@ def capture_fixture( manifest.write_text(manifest_text(claims=claims, constraint_name=constraint_name), encoding="utf-8") dump_sha256 = hashlib.sha256(f"{name}-dump".encode()).hexdigest() manifest_sha256 = hashlib.sha256(manifest.read_bytes()).hexdigest() - manifest_sql_sha256 = "2" * 64 + manifest_sql_sha256 = REVIEWED_MANIFEST_SQL_SHA256 source_context_sha256 = "3" * 64 source = { "ssh_target": "root@77.42.65.182", @@ -239,6 +250,44 @@ def test_postflight_rejects_unchanged_but_unhealthy_service(tmp_path: Path) -> N ) +def test_postflight_rejects_a_different_manifest_algorithm_even_when_rebound(tmp_path: Path) -> None: + baseline_receipt, baseline_manifest = capture_fixture( + tmp_path, + name="canonical-baseline", + captured_at="2026-07-15T01:00:00+00:00", + authorization_ref="codex-delegation:baseline123", + ) + current_receipt, current_manifest = capture_fixture( + tmp_path, + name="canonical-postflight", + captured_at="2026-07-15T02:00:00+00:00", + authorization_ref="codex-delegation:postflight123", + ) + current = json.loads(current_receipt.read_text()) + current["manifest"]["manifest_sql_sha256"] = "f" * 64 + current["provenance_binding"] = build_provenance_binding( + run_id=current["run_id"], + authorization_ref=current["capture_authorization_ref"], + source=current["source"], + snapshot=current["snapshot"], + dump_sha256=current["dump"]["sha256"], + manifest_sql_sha256="f" * 64, + source_manifest_sha256=current["manifest"]["sha256"], + source_context_sha256=current["source_context"]["sha256"], + ) + current_receipt.write_text(json.dumps(current, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="manifest_sql_reviewed"): + verify_delta( + baseline_receipt_path=baseline_receipt, + baseline_manifest_path=baseline_manifest, + baseline_authorization_ref="codex-delegation:baseline123", + current_receipt_path=current_receipt, + current_manifest_path=current_manifest, + current_authorization_ref="codex-delegation:postflight123", + ) + + def test_delta_receipt_writer_keeps_private_mode(tmp_path: Path) -> None: output = tmp_path / "private" / "delta.json" From 52fc0a88fe93aadde5ce575c073cc1d13bdab40e Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 10:49:55 +0200 Subject: [PATCH 4/5] Normalize database authorization parity and validate hashes --- .../gcp-canonical-parity-live-20260715.json | 2 +- ops/postgres_parity_manifest.sql | 9 ++-- ops/verify_postgres_parity_manifest.py | 7 +-- ...est_gcp_generated_db_direct_claim_suite.py | 4 +- ...restore_gcp_generated_postgres_snapshot.py | 2 +- ...st_run_local_canonical_postgres_rebuild.py | 2 +- .../test_run_local_genesis_ledger_rebuild.py | 48 ++++++++++++++++++- tests/test_verify_postgres_parity_manifest.py | 16 ++++++- 8 files changed, 74 insertions(+), 16 deletions(-) diff --git a/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json b/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json index 0e7a214..74489b4 100644 --- a/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json +++ b/docs/reports/leo-working-state-20260709/gcp-canonical-parity-live-20260715.json @@ -63,7 +63,7 @@ }, "contract_reassessment": { "status": "changes_required", - "reviewed_manifest_sql_sha256": "430390129bc8980c4ece9939f2aee18e8c40e9da4b226782579349d5c45c5301", + "reviewed_manifest_sql_sha256": "2bd3e353627d44d4a292949952c06b1897d1a9a9e98243905c7a834d007db188", "prior_receipts_recomputed_under_current_contract": false, "newly_required_parity": [ "kb_gate_owner plus exact application-role attributes", diff --git a/ops/postgres_parity_manifest.sql b/ops/postgres_parity_manifest.sql index 8f5ebc8..ebeea10 100644 --- a/ops/postgres_parity_manifest.sql +++ b/ops/postgres_parity_manifest.sql @@ -84,10 +84,10 @@ select jsonb_build_object( with ownership as ( select - format('database:%s', database_row.datname) as sort_key, + 'database:canonical_database'::text as sort_key, 'database'::text as object_type, null::text as schema, - database_row.datname as name, + 'canonical_database'::text as name, null::text as identity_arguments, pg_get_userbyid(database_row.datdba) as owner from pg_database database_row @@ -170,15 +170,14 @@ from ownership; with acl_rows as ( select format( - 'database:%s:%s:%s:%s', - database_row.datname, + 'database:canonical_database:%s:%s:%s', acl.grantee, acl.privilege_type, acl.is_grantable ) as sort_key, 'database'::text as object_type, null::text as schema, - database_row.datname as name, + 'canonical_database'::text as name, null::text as column_name, null::text as identity_arguments, pg_get_userbyid(acl.grantor) as grantor, diff --git a/ops/verify_postgres_parity_manifest.py b/ops/verify_postgres_parity_manifest.py index 6e4eeb0..bf5a373 100755 --- a/ops/verify_postgres_parity_manifest.py +++ b/ops/verify_postgres_parity_manifest.py @@ -17,7 +17,8 @@ try: except ImportError: # pragma: no cover - direct script execution from private_receipt_io import print_private_receipt_summary, write_private_json -REVIEWED_MANIFEST_SQL_SHA256 = "430390129bc8980c4ece9939f2aee18e8c40e9da4b226782579349d5c45c5301" +REVIEWED_MANIFEST_SQL_SHA256 = "2bd3e353627d44d4a292949952c06b1897d1a9a9e98243905c7a834d007db188" +ROWSET_MD5_RE = re.compile(r"[0-9a-f]{32}\Z") SINGLETON_KINDS = { "identity", @@ -143,8 +144,8 @@ def compare_manifests( target_row = target_tables[table] for side, row in (("source", source_row), ("target", target_row)): rowset_md5 = row.get("rowset_md5") - if not isinstance(rowset_md5, str) or not rowset_md5: - table_hash_problems.append(f"{table}:{side}:rowset_md5_missing") + if not isinstance(rowset_md5, str) or not ROWSET_MD5_RE.fullmatch(rowset_md5): + table_hash_problems.append(f"{table}:{side}:rowset_md5_invalid") mismatch = { field: {"source": source_row.get(field), "target": target_row.get(field)} for field in ("row_count", "rowset_md5") diff --git a/tests/test_gcp_generated_db_direct_claim_suite.py b/tests/test_gcp_generated_db_direct_claim_suite.py index 18ce5d0..5aa40ea 100644 --- a/tests/test_gcp_generated_db_direct_claim_suite.py +++ b/tests/test_gcp_generated_db_direct_claim_suite.py @@ -160,7 +160,7 @@ def test_manifest_normalization_excludes_nondeterministic_timing() -> None: "BEGIN", '{"kind":"identity","database":"teleo_clone_test","captured_at":"2026-07-12T00:00:00Z"}', '{"kind":"schemas","items":["kb_stage","public"]}', - '{"kind":"table","schema":"public","table":"claims","row_count":2,"rowset_md5":"abc"}', + '{"kind":"table","schema":"public","table":"claims","row_count":2,"rowset_md5":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}', '{"kind":"performance","query":"count_claims","elapsed_ms":1.2,"observed_rows":2}', "ROLLBACK", ] @@ -189,7 +189,7 @@ def test_database_fingerprint_uses_full_normalized_manifest(monkeypatch: pytest. '{"kind":"identity","database":"teleo_clone_test","server_address":"10.61.0.3",' '"ssl":true,"transaction_read_only":"on","captured_at":"one"}', '{"kind":"schemas","items":["kb_stage","public"]}', - '{"kind":"table","schema":"public","table":"claims","row_count":2,"rowset_md5":"abc"}', + '{"kind":"table","schema":"public","table":"claims","row_count":2,"rowset_md5":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}', ] ) monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.database_password", lambda _args: "secret") diff --git a/tests/test_restore_gcp_generated_postgres_snapshot.py b/tests/test_restore_gcp_generated_postgres_snapshot.py index e1c9535..baf6893 100644 --- a/tests/test_restore_gcp_generated_postgres_snapshot.py +++ b/tests/test_restore_gcp_generated_postgres_snapshot.py @@ -97,7 +97,7 @@ def manifest_rows(database: str, *, target: bool, row_count: int = 1) -> list[di "schema": "public", "table": "claims", "row_count": row_count, - "rowset_md5": "same" if row_count == 1 else "different", + "rowset_md5": ("a" if row_count == 1 else "b") * 32, }, { "kind": "performance", diff --git a/tests/test_run_local_canonical_postgres_rebuild.py b/tests/test_run_local_canonical_postgres_rebuild.py index b03cb69..da42fe4 100644 --- a/tests/test_run_local_canonical_postgres_rebuild.py +++ b/tests/test_run_local_canonical_postgres_rebuild.py @@ -162,7 +162,7 @@ def manifest_rows() -> list[dict]: "schema": "public", "table": "claims", "row_count": 1837, - "rowset_md5": "same-row-hash", + "rowset_md5": "c" * 32, }, { "kind": "performance", diff --git a/tests/test_run_local_genesis_ledger_rebuild.py b/tests/test_run_local_genesis_ledger_rebuild.py index 4daefb7..63ca9c1 100644 --- a/tests/test_run_local_genesis_ledger_rebuild.py +++ b/tests/test_run_local_genesis_ledger_rebuild.py @@ -16,6 +16,7 @@ from pathlib import Path import pytest from ops import run_local_genesis_ledger_rebuild as rebuild +from ops.verify_postgres_parity_manifest import compare_manifests, load_manifest CLAIM_A = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" CLAIM_B = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" @@ -101,7 +102,7 @@ def manifest_rows(database: str = "teleo_fixture") -> list[dict]: "schema": "public", "table": "claims", "row_count": 2, - "rowset_md5": "fixture", + "rowset_md5": "f" * 32, }, { "kind": "performance", @@ -1086,6 +1087,51 @@ def source_postgres() -> Iterator[tuple[str, str]]: pytest.fail(f"fixture PostgreSQL container was not removed: {container}") +def test_live_manifest_normalizes_physical_database_authorization_identity( + source_postgres: tuple[str, str], + tmp_path: Path, +) -> None: + container, database = source_postgres + clone_database = f"teleo_clone_{uuid.uuid4().hex[:12]}" + docker_psql( + container, + "postgres", + f'create database "{clone_database}" template "{database}";', + ) + try: + source_manifest_path = tmp_path / "source-database-manifest.jsonl" + clone_manifest_path = tmp_path / "clone-database-manifest.jsonl" + capture_manifest(container, database, source_manifest_path) + capture_manifest(container, clone_database, clone_manifest_path) + + source_manifest = load_manifest(source_manifest_path) + clone_manifest = load_manifest(clone_manifest_path) + problems, _details = compare_manifests( + source_manifest, + clone_manifest, + max_target_query_ms=1_000_000, + max_target_source_ratio=1_000_000, + ) + + assert problems == [] + for manifest in (source_manifest, clone_manifest): + ownership = manifest["singleton"]["object_ownership"]["items"] + database_owner = next(row for row in ownership if row["object_type"] == "database") + assert database_owner["name"] == "canonical_database" + database_acls = [ + row for row in manifest["singleton"]["object_acl"]["items"] if row["object_type"] == "database" + ] + assert database_acls + assert {row["name"] for row in database_acls} == {"canonical_database"} + finally: + docker_psql( + container, + "postgres", + f'drop database if exists "{clone_database}" with (force);', + check=False, + ) + + def capture_source_snapshot( source_container: str, database: str, diff --git a/tests/test_verify_postgres_parity_manifest.py b/tests/test_verify_postgres_parity_manifest.py index 37f5045..ff41d86 100644 --- a/tests/test_verify_postgres_parity_manifest.py +++ b/tests/test_verify_postgres_parity_manifest.py @@ -9,7 +9,7 @@ def manifest_rows( *, database: str, row_count: int = 2, - rowset_md5: str = "abc", + rowset_md5: str = "a" * 32, elapsed_ms: float = 4.0, server_address: str | None = None, ssl: bool = False, @@ -115,7 +115,7 @@ def test_matching_local_manifests_pass(tmp_path: Path) -> None: def test_row_hash_and_role_mismatches_fail(tmp_path: Path) -> None: - target = manifest_rows(database="teleo_copy", row_count=1, rowset_md5="different") + target = manifest_rows(database="teleo_copy", row_count=1, rowset_md5="b" * 32) target[3]["items"] = [] completed, payload = run_verifier(tmp_path, manifest_rows(database="teleo"), target) @@ -139,6 +139,18 @@ def test_missing_row_hash_and_extra_roles_or_extensions_fail(tmp_path: Path) -> assert payload["details"]["target_extra_application_roles"] == ["project_owner"] +def test_matching_malformed_row_hashes_fail(tmp_path: Path) -> None: + completed, payload = run_verifier( + tmp_path, + manifest_rows(database="teleo", rowset_md5="not-a-hash"), + manifest_rows(database="teleo_copy", rowset_md5="not-a-hash"), + ) + + assert completed.returncode == 1 + assert "table_hash_problems=2" in payload["problems"] + assert payload["details"]["table_mismatches"] == [] + + def test_authorization_ownership_or_acl_drift_fails(tmp_path: Path) -> None: target = manifest_rows(database="teleo_copy") target[5]["items"][0]["owner"] = "postgres" From 1c6b7b182e343a1710b8c8561335b151dabc0c4a Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 10:59:43 +0200 Subject: [PATCH 5/5] Pin lifecycle receipts to the reviewed parity manifest --- ops/verify_gcp_canonical_lifecycle.py | 7 ++++ tests/test_verify_gcp_canonical_lifecycle.py | 34 ++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/ops/verify_gcp_canonical_lifecycle.py b/ops/verify_gcp_canonical_lifecycle.py index b70aa3c..5d182b5 100644 --- a/ops/verify_gcp_canonical_lifecycle.py +++ b/ops/verify_gcp_canonical_lifecycle.py @@ -16,10 +16,12 @@ from typing import Any try: from .private_receipt_io import print_private_receipt_summary from .restore_gcp_generated_postgres_snapshot import DEFAULT_RESTORE_SERVICE_ACCOUNT + from .verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256 from .verify_vps_canonical_snapshot_delta import DELTA_SCHEMA, parse_timestamp, write_private_json except ImportError: # pragma: no cover - direct script execution from private_receipt_io import print_private_receipt_summary from restore_gcp_generated_postgres_snapshot import DEFAULT_RESTORE_SERVICE_ACCOUNT + from verify_postgres_parity_manifest import REVIEWED_MANIFEST_SQL_SHA256 from verify_vps_canonical_snapshot_delta import DELTA_SCHEMA, parse_timestamp, write_private_json SOURCE_SCHEMA = "livingip.sourceSnapshotReceipt.v2" @@ -405,6 +407,11 @@ def verify_lifecycle( and delta_current.get("manifest_sql_sha256") == (current_source.get("manifest") or {}).get("manifest_sql_sha256") and delta_baseline.get("manifest_sql_sha256") == delta_current.get("manifest_sql_sha256"), + "source_delta_manifest_sql_reviewed": delta_baseline.get("manifest_sql_sha256") == REVIEWED_MANIFEST_SQL_SHA256 + and delta_current.get("manifest_sql_sha256") == REVIEWED_MANIFEST_SQL_SHA256 + and (source.get("manifest") or {}).get("manifest_sql_sha256") == REVIEWED_MANIFEST_SQL_SHA256 + and (current_source.get("manifest") or {}).get("manifest_sql_sha256") == REVIEWED_MANIFEST_SQL_SHA256 + and capture_binding.get("manifest_sql_sha256") == REVIEWED_MANIFEST_SQL_SHA256, "source_delta_authorizations_bound": delta_baseline.get("capture_authorization_ref") == source.get("capture_authorization_ref") and delta_current.get("capture_authorization_ref") == current_source.get("capture_authorization_ref"), diff --git a/tests/test_verify_gcp_canonical_lifecycle.py b/tests/test_verify_gcp_canonical_lifecycle.py index b2a26da..c0da1f0 100644 --- a/tests/test_verify_gcp_canonical_lifecycle.py +++ b/tests/test_verify_gcp_canonical_lifecycle.py @@ -120,6 +120,7 @@ def lifecycle_receipts(tmp_path: Path) -> dict[str, Path]: "provenance_binding_sha256": "c" * 64, "dump_sha256": "a" * 64, "manifest_sha256": "b" * 64, + "manifest_sql_sha256": REVIEWED_MANIFEST_SQL_SHA256, "checks": { name: True for name in ( @@ -504,6 +505,39 @@ def test_caller_authored_bundle_is_preparation_only_even_when_all_checks_pass(tm assert "current private GCP staging" not in result["strongest_claim_allowed"] +def test_rebound_unreviewed_manifest_sql_digest_fails_closed(tmp_path: Path) -> None: + paths = lifecycle_receipts(tmp_path) + unreviewed_digest = "f" * 64 + + source = json.loads(paths["source_path"].read_text()) + source["manifest"]["manifest_sql_sha256"] = unreviewed_digest + write_json(paths["source_path"], source) + current_source = json.loads(paths["current_source_path"].read_text()) + current_source["manifest"]["manifest_sql_sha256"] = unreviewed_digest + write_json(paths["current_source_path"], current_source) + + restore = json.loads(paths["restore_path"].read_text()) + restore["source"]["capture_receipt"]["sha256"] = sha256(paths["source_path"]) + restore["source"]["capture_receipt"]["manifest_sql_sha256"] = unreviewed_digest + write_json(paths["restore_path"], restore) + + delta = json.loads(paths["source_delta_path"].read_text()) + delta["baseline"]["capture_receipt_sha256"] = sha256(paths["source_path"]) + delta["baseline"]["manifest_sql_sha256"] = unreviewed_digest + delta["current"]["capture_receipt_sha256"] = sha256(paths["current_source_path"]) + delta["current"]["manifest_sql_sha256"] = unreviewed_digest + write_json(paths["source_delta_path"], delta) + + cleanup = json.loads(paths["cleanup_path"].read_text()) + cleanup["restore_receipt"]["sha256"] = sha256(paths["restore_path"]) + write_json(paths["cleanup_path"], cleanup) + + result = verify_lifecycle(**paths) + + assert result["status"] == "fail" + assert "source_delta_manifest_sql_reviewed" in result["failed_checks"] + + def test_local_parity_or_missing_cleanup_fails_closed(tmp_path: Path) -> None: paths = lifecycle_receipts(tmp_path) parity = json.loads(paths["parity_path"].read_text())