Harden GCP clone-bound Cory replay

This commit is contained in:
twentyOne2x 2026-07-12 02:04:58 +02:00
parent 3b534abd8e
commit 6f0dc0ffb0
12 changed files with 1317 additions and 27 deletions

View file

@ -55,9 +55,15 @@ jobs:
telegram/approvals.py \ telegram/approvals.py \
scripts/check_crabbox_ci_contract.py \ scripts/check_crabbox_ci_contract.py \
scripts/check_llm_refinement_contract.py \ scripts/check_llm_refinement_contract.py \
scripts/build_working_leo_cory_outcome_sandbox.py \
scripts/replay_decision_engine_eval.py \ scripts/replay_decision_engine_eval.py \
scripts/prove_phase1b_local.py \ scripts/prove_phase1b_local.py \
scripts/run_gcp_generated_db_direct_claim_suite.py \
scripts/run_leo_clone_bound_handler_checkpoint.py \
scripts/working_leo_open_ended_benchmark.py \
tests/test_agent_routing.py \ tests/test_agent_routing.py \
tests/test_assemble_telegram_visible_direct_claim_capture_receipt.py \
tests/test_build_working_leo_cory_outcome_sandbox.py \
tests/test_decision_engine_replay.py \ tests/test_decision_engine_replay.py \
tests/test_evaluate_agent_routing.py \ tests/test_evaluate_agent_routing.py \
tests/test_gcp_artifact_workflow.py \ tests/test_gcp_artifact_workflow.py \
@ -71,7 +77,9 @@ jobs:
tests/test_gcp_iam_split_apply.py \ tests/test_gcp_iam_split_apply.py \
tests/test_gcp_iam_split_plan.py \ tests/test_gcp_iam_split_plan.py \
tests/test_gcp_readiness_workflow.py \ tests/test_gcp_readiness_workflow.py \
tests/test_gcp_generated_db_direct_claim_suite.py \
tests/test_verify_postgres_parity_manifest.py \ tests/test_verify_postgres_parity_manifest.py \
tests/test_working_leo_open_ended_benchmark.py \
tests/test_phase1b_end_to_end.py \ tests/test_phase1b_end_to_end.py \
tests/test_sqlite_to_postgres_dump.py \ tests/test_sqlite_to_postgres_dump.py \
tests/test_sqlite_postgres_restore_canary_capsule.py \ tests/test_sqlite_postgres_restore_canary_capsule.py \

View file

@ -14,7 +14,7 @@ select jsonb_build_object(
'database_collation', (select datcollate from pg_database where datname = current_database()), 'database_collation', (select datcollate from pg_database where datname = current_database()),
'database_ctype', (select datctype from pg_database where datname = current_database()), 'database_ctype', (select datctype from pg_database where datname = current_database()),
'transaction_read_only', current_setting('transaction_read_only'), 'transaction_read_only', current_setting('transaction_read_only'),
'server_address', inet_server_addr()::text, 'server_address', host(inet_server_addr()),
'server_port', inet_server_port(), 'server_port', inet_server_port(),
'ssl', coalesce((select ssl from pg_stat_ssl where pid = pg_backend_pid()), false), 'ssl', coalesce((select ssl from pg_stat_ssl where pid = pg_backend_pid()), false),
'ssl_version', (select version from pg_stat_ssl where pid = pg_backend_pid()), 'ssl_version', (select version from pg_stat_ssl where pid = pg_backend_pid()),

View file

@ -214,8 +214,12 @@ def compare_manifests(
return problems, details return problems, details
def canonical_ip(address: str) -> str:
return str(ipaddress.ip_interface(address).ip)
def is_rfc1918(address: str) -> bool: def is_rfc1918(address: str) -> bool:
parsed = ipaddress.ip_address(address) parsed = ipaddress.ip_address(canonical_ip(address))
return any( return any(
parsed in network parsed in network
for network in ( for network in (
@ -248,6 +252,8 @@ def load_connectivity(
target_database = target_identity.get("database") target_database = target_identity.get("database")
target_address = target_identity.get("server_address") target_address = target_identity.get("server_address")
target_ssl = target_identity.get("ssl") target_ssl = target_identity.get("ssl")
canonical_target_address = None
canonical_proof_address = None
if not target_database: if not target_database:
problems.append("gcp_target_manifest_database_missing") problems.append("gcp_target_manifest_database_missing")
if proof.get("target_database") != target_database: if proof.get("target_database") != target_database:
@ -256,11 +262,19 @@ def load_connectivity(
problems.append("gcp_target_manifest_server_address_missing") problems.append("gcp_target_manifest_server_address_missing")
else: else:
try: try:
canonical_target_address = canonical_ip(str(target_address))
if not is_rfc1918(str(target_address)): if not is_rfc1918(str(target_address)):
problems.append("gcp_target_manifest_server_address_not_rfc1918") problems.append("gcp_target_manifest_server_address_not_rfc1918")
except ValueError: except ValueError:
problems.append("gcp_target_manifest_server_address_invalid") problems.append("gcp_target_manifest_server_address_invalid")
if proof.get("server_address") != target_address: if proof.get("server_address"):
try:
canonical_proof_address = canonical_ip(str(proof["server_address"]))
except ValueError:
problems.append("gcp_connectivity_server_address_invalid")
else:
problems.append("gcp_connectivity_server_address_missing")
if canonical_proof_address != canonical_target_address:
problems.append("gcp_connectivity_server_address_mismatch") problems.append("gcp_connectivity_server_address_mismatch")
if target_ssl is not True: if target_ssl is not True:
problems.append("gcp_target_manifest_tls_not_proven") problems.append("gcp_target_manifest_tls_not_proven")

View file

@ -115,7 +115,8 @@ DC_REPLIES: dict[str, str] = {
"public.claim_edges, and public.claim_evidence, plus kb_stage.kb_proposals applied_at/readback. I need " "public.claim_edges, and public.claim_evidence, plus kb_stage.kb_proposals applied_at/readback. I need "
"row-level before/after counts, row ids, and postflight proof; I cannot claim a DB update without proof. " "row-level before/after counts, row ids, and postflight proof; I cannot claim a DB update without proof. "
"Follow-up: I would ask which proposal or time window Cory means, then query kb_stage and public.* and " "Follow-up: I would ask which proposal or time window Cory means, then query kb_stage and public.* and "
"either show the pending gap or prepare the next guarded apply packet." "either show the pending gap or prepare the next guarded apply packet. DB readback: Proposal: "
"`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`."
), ),
"DC-02": ( "DC-02": (
"Helmer's 7 Powers can be packet-ready or clone-proven without being canonical production state. I would " "Helmer's 7 Powers can be packet-ready or clone-proven without being canonical production state. I would "
@ -123,7 +124,8 @@ DC_REPLIES: dict[str, str] = {
"and the proposal ledger with postflight row counts before saying it is in Leo now. If production_apply " "and the proposal ledger with postflight row counts before saying it is in Leo now. If production_apply "
"is false, I cannot claim it is production-applied. Follow-up: I would show the readiness packet, ask for " "is false, I cannot claim it is production-applied. Follow-up: I would show the readiness packet, ask for "
"explicit authorization if Cory wants it applied, run the integrated order, then postflight and Telegram " "explicit authorization if Cory wants it applied, run the integrated order, then postflight and Telegram "
"regression." "regression. DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; "
"Applied at: `none`."
), ),
"DC-03": ( "DC-03": (
"I would check the schema before claiming a decision-matrix approved anything. If matrix_voters, " "I would check the schema before claiming a decision-matrix approved anything. If matrix_voters, "
@ -131,14 +133,17 @@ DC_REPLIES: dict[str, str] = {
"and no tally approved this. The current proof should come from kb_stage.kb_proposals review/apply columns " "and no tally approved this. The current proof should come from kb_stage.kb_proposals review/apply columns "
"and canonical public.* row readback. I cannot claim a matrix result without proof. Follow-up: I would run a " "and canonical public.* row readback. I cannot claim a matrix result without proof. Follow-up: I would run a "
"fresh schema readback, then show the proposal ledger and canonical rows instead of overclaiming a matrix " "fresh schema readback, then show the proposal ledger and canonical rows instead of overclaiming a matrix "
"result." "result. DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; "
"Applied at: `none`."
), ),
"DC-04": ( "DC-04": (
"Telegram file refs, document_evaluations, proposal source_ref, raw PDFs, and public.sources are separate. " "Telegram file refs, document_evaluations, proposal source_ref, raw PDFs, and public.sources are separate. "
"A pending proposal can have staging rows and file pointers but still not have canonical source rows, " "A pending proposal can have staging rows and file pointers but still not have canonical source rows, "
"claim_evidence, or an apply contract. I will not claim document pointers are the only blocker without " "claim_evidence, or an apply contract. I will not claim document pointers are the only blocker without "
"row-link proof. Follow-up: I would check and audit each proposal source_ref against public.sources, identify " "row-link proof. Follow-up: I would check and audit each proposal source_ref against public.sources, identify "
"missing canonical rows, then stage a guarded source/evidence apply proposal for review." "missing canonical rows, then stage a guarded source/evidence apply proposal for review. DB readback: Proposal: "
"`e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; "
"Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`."
), ),
"DC-05": ( "DC-05": (
"Demo answer: Leo can show live staging and strict existing-ID apply proof, but proposed, approved, applied, " "Demo answer: Leo can show live staging and strict existing-ID apply proof, but proposed, approved, applied, "
@ -146,6 +151,7 @@ DC_REPLIES: dict[str, str] = {
"without being production-applied. I cannot claim production KB changes without authorization, postflight " "without being production-applied. I cannot claim production KB changes without authorization, postflight "
"counts, row ids, and an artifact or receipt. Follow-up: I would ask which demo tier Cory wants: show the " "counts, row ids, and an artifact or receipt. Follow-up: I would ask which demo tier Cory wants: show the "
"existing live proof, run a safe staging canary, or authorize the prepared apply packet plus regression. " "existing live proof, run a safe staging canary, or authorize the prepared apply packet plus regression. "
"DB readback: kb_proposals: 26; canary proposal status: `pending_review`; canonical rows were not changed."
), ),
"DC-06": ( "DC-06": (
"A direct SOUL.md edit changes the runtime/profile artifact, not canonical identity by itself. Canonical " "A direct SOUL.md edit changes the runtime/profile artifact, not canonical identity by itself. Canonical "
@ -153,7 +159,8 @@ DC_REPLIES: dict[str, str] = {
"and public.strategy_node_anchors must have row-level proof, row ids, and verify/read back evidence before " "and public.strategy_node_anchors must have row-level proof, row ids, and verify/read back evidence before "
"they render or sync into SOUL.md. Until readback and renderer proof exist, I cannot claim canonical identity " "they render or sync into SOUL.md. Until readback and renderer proof exist, I cannot claim canonical identity "
"changed. Follow-up: I would ask whether to audit the canonical identity rows, stage the missing DB proposal, " "changed. Follow-up: I would ask whether to audit the canonical identity rows, stage the missing DB proposal, "
"or run the renderer/sync readback." "or run the renderer/sync readback. DB readback: Proposal: "
"`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`."
), ),
} }

View file

@ -0,0 +1,769 @@
#!/usr/bin/env python3
"""Run the six no-send Cory direct-claim prompts against a generated GCP DB."""
from __future__ import annotations
import argparse
import asyncio
import hashlib
import ipaddress
import json
import os
import shlex
import shutil
import subprocess
import sys
import uuid
from pathlib import Path
from types import SimpleNamespace
from typing import Any
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import run_leo_clone_bound_handler_checkpoint as bound # noqa: E402
import working_leo_open_ended_benchmark as benchmark # noqa: E402
SCHEMA = "livingip.gcpGeneratedDbDirectClaimSuite.v1"
SOURCE_COMPUTE = "teleo-prod-1"
DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service"
DEFAULT_HOST = "10.61.0.3"
DEFAULT_PROJECT = "teleo-501523"
DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql"
REVIEWED_CLOUDSQL_TOOL_SHA256 = "f54040141551a22a9a39ab020abfd897963bc2a53205f00cd49d7212cbaffff1"
REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a"
def run(command: list[str], *, env: dict[str, str] | None = None, input_text: str | None = None) -> str:
completed = subprocess.run(
command,
env=env,
input=input_text,
text=True,
capture_output=True,
check=False,
)
if completed.returncode != 0:
detail = bound.redact_text((completed.stderr or completed.stdout).strip())
raise bound.CheckpointError(f"command failed ({completed.returncode}): {detail}")
return completed.stdout
def service_state(service: str) -> dict[str, Any]:
completed = subprocess.run(
[
"systemctl",
"show",
service,
"-p",
"ActiveState",
"-p",
"SubState",
"-p",
"MainPID",
"-p",
"NRestarts",
"-p",
"ExecMainStartTimestamp",
"--no-pager",
],
text=True,
capture_output=True,
check=False,
)
result: dict[str, Any] = {"returncode": completed.returncode}
for line in completed.stdout.splitlines():
key, separator, value = line.partition("=")
if separator:
result[key] = value
if completed.returncode != 0:
result["error"] = bound.redact_text(completed.stderr or completed.stdout)
return result
def profile_hashes(profile: Path) -> dict[str, str | None]:
paths = {
"wrapper": profile / "bin" / "teleo-kb",
"cloudsql_tool": profile / "bin" / "cloudsql_memory_tool.py",
"bridge_skill": profile / "skills" / "teleo-kb-bridge" / "SKILL.md",
}
result = {}
for name, path in paths.items():
try:
result[name] = bound.sha256_bytes(path.read_bytes())
except OSError:
result[name] = None
return result
def database_password(args: argparse.Namespace) -> str:
value = run(
[
"gcloud",
"secrets",
"versions",
"access",
"latest",
f"--secret={args.password_secret}",
f"--project={args.project}",
]
).strip()
if not value:
raise bound.CheckpointError("Cloud SQL password secret resolved to an empty value")
return value
def validate_reviewed_file(path: Path, expected_sha256: str, label: str) -> str:
actual = bound.sha256_bytes(path.read_bytes())
if actual != expected_sha256:
raise bound.CheckpointError(f"{label} does not match the reviewed SHA-256")
return actual
def validate_parity_receipt(path: Path, target_db: str) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise bound.CheckpointError("canonical parity receipt is unreadable") from exc
details = payload.get("details") or {}
structural = details.get("structural_hashes") or {}
checks = {
"artifact": payload.get("artifact") == "canonical_postgres_parity_verification",
"status": payload.get("status") == "pass",
"problems_empty": payload.get("problems") == [],
"target_database": details.get("target_database") == target_db,
"nonempty_table_set": int(details.get("target_table_count") or 0) > 0,
"nonempty_row_set": int(details.get("target_total_rows") or 0) > 0,
"table_counts_equal": details.get("source_table_count") == details.get("target_table_count"),
"row_counts_equal": details.get("source_total_rows") == details.get("target_total_rows"),
"table_mismatches_empty": details.get("table_mismatches") == [],
"role_mismatches_empty": details.get("application_role_mismatches") == {},
"extension_mismatches_empty": details.get("required_extension_mismatches") == {},
"performance_problems_empty": details.get("performance_problems") == [],
"structural_hashes_equal": bool(structural)
and all(item.get("source") == item.get("target") for item in structural.values()),
}
failed = sorted(name for name, ok in checks.items() if not ok)
if failed:
raise bound.CheckpointError(f"canonical parity receipt failed validation: {', '.join(failed)}")
return {
"sha256": bound.sha256_bytes(path.read_bytes()),
"scope": payload.get("scope"),
"target_database": details.get("target_database"),
"target_table_count": details.get("target_table_count"),
"target_total_rows": details.get("target_total_rows"),
"checks": checks,
}
def run_target_psql(
args: argparse.Namespace,
*,
input_text: str | None = None,
sql_file: Path | None = None,
) -> str:
password = database_password(args)
env = {
**os.environ,
"PGPASSWORD": password,
"PGOPTIONS": "-c default_transaction_read_only=on",
}
command = [
"psql",
f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8",
"-X",
"-Atq",
"-v",
"ON_ERROR_STOP=1",
]
if sql_file is not None:
command.extend(["-f", str(sql_file)])
try:
return run(command, env=env, input_text=input_text)
finally:
env.pop("PGPASSWORD", None)
env.pop("PGOPTIONS", None)
password = ""
def database_identity(args: argparse.Namespace) -> dict[str, Any]:
sql = """
begin transaction read only;
select jsonb_build_object(
'current_database', current_database(),
'system_identifier', (select system_identifier::text from pg_control_system()),
'server_address', host(inet_server_addr()),
'server_port', inet_server_port(),
'ssl', coalesce((select ssl from pg_stat_ssl where pid=pg_backend_pid()), false),
'transaction_read_only', current_setting('transaction_read_only'),
'default_transaction_read_only', current_setting('default_transaction_read_only')
)::text;
rollback;
"""
output = run_target_psql(args, input_text=sql)
rows = [line for line in output.splitlines() if line.strip().startswith("{")]
if not rows:
raise bound.CheckpointError("Cloud SQL identity query returned no JSON")
return json.loads(rows[-1])
def validate_database_identity(identity: dict[str, Any], target_db: str) -> None:
if identity.get("current_database") != target_db:
raise bound.CheckpointError("Cloud SQL identity does not match the generated target database")
if identity.get("ssl") is not True:
raise bound.CheckpointError("Cloud SQL target session is not protected by TLS")
if identity.get("transaction_read_only") != "on":
raise bound.CheckpointError("Cloud SQL identity receipt was not captured in a read-only transaction")
if identity.get("default_transaction_read_only") != "on":
raise bound.CheckpointError("Cloud SQL target session does not default every transaction to read-only")
try:
address = ipaddress.ip_address(str(identity.get("server_address") or ""))
except ValueError as exc:
raise bound.CheckpointError("Cloud SQL target did not expose a valid server address") from exc
private_networks = (
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("172.16.0.0/12"),
ipaddress.ip_network("192.168.0.0/16"),
)
if not any(address in network for network in private_networks):
raise bound.CheckpointError("Cloud SQL target address is not RFC1918-private")
if not identity.get("system_identifier"):
raise bound.CheckpointError("Cloud SQL target system identifier is missing")
def canonical_status(args: argparse.Namespace) -> dict[str, Any]:
sql = """
begin transaction read only;
select jsonb_build_object(
'db_identity', current_database() || '|' || current_user,
'schema_tables', (
select coalesce(jsonb_object_agg(table_schema, table_count order by table_schema), '{}'::jsonb)
from (
select table_schema, count(*)::integer as table_count
from information_schema.tables
where table_schema in ('public', 'kb_stage')
group by table_schema
) counts
),
'high_signal_rows', jsonb_build_object(
'agents', (select count(*) from public.agents),
'personas', (select count(*) from public.personas),
'strategies', (select count(*) from public.strategies),
'beliefs', (select count(*) from public.beliefs),
'claims', (select count(*) from public.claims),
'claim_edges', (select count(*) from public.claim_edges),
'claim_evidence', (select count(*) from public.claim_evidence),
'sources', (select count(*) from public.sources),
'kb_proposals', (select count(*) from kb_stage.kb_proposals)
)
)::text;
rollback;
"""
output = run_target_psql(args, input_text=sql)
rows = [line for line in output.splitlines() if line.strip().startswith("{")]
if not rows:
raise bound.CheckpointError("canonical status query returned no JSON")
return json.loads(rows[-1])
def normalize_database_manifest_rows(output: str) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for line_number, raw_line in enumerate(output.splitlines(), 1):
line = raw_line.strip()
if not line or line in {"BEGIN", "SET", "ROLLBACK"}:
continue
try:
row = json.loads(line)
except json.JSONDecodeError as exc:
raise bound.CheckpointError(f"database manifest line {line_number} is not JSON") from exc
kind = row.get("kind")
if kind == "performance":
continue
if kind == "identity":
row.pop("captured_at", None)
rows.append(row)
if not rows or not any(row.get("kind") == "identity" for row in rows):
raise bound.CheckpointError("database manifest is missing its identity row")
table_rows = [row for row in rows if row.get("kind") == "table"]
if not table_rows:
raise bound.CheckpointError("database manifest contains no table rows")
return sorted(
rows,
key=lambda row: (
str(row.get("kind") or ""),
str(row.get("schema") or ""),
str(row.get("table") or ""),
),
)
def database_fingerprint(args: argparse.Namespace) -> dict[str, Any]:
output = run_target_psql(args, sql_file=args.manifest_sql)
rows = normalize_database_manifest_rows(output)
payload = json.dumps(rows, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
table_rows = [row for row in rows if row.get("kind") == "table"]
identity = next(row for row in rows if row.get("kind") == "identity")
return {
"sha256": hashlib.sha256(payload.encode()).hexdigest(),
"table_count": len(table_rows),
"total_rows": sum(int(row.get("row_count") or 0) for row in table_rows),
"database": identity.get("database"),
"server_address": identity.get("server_address"),
"ssl": identity.get("ssl"),
"transaction_read_only": identity.get("transaction_read_only"),
}
def build_cloudsql_wrapper(
*,
cloudsql_tool: Path,
tool_log: Path,
target_db: str,
host: str,
project: str,
password_secret: str,
run_nonce: str,
system_exec_path: str = bound.SYSTEM_EXEC_PATH,
) -> str:
python = str(Path(sys.executable).resolve())
return f"""#!/bin/bash
set -uo pipefail
CLOUDSQL_TOOL={shlex.quote(str(cloudsql_tool))}
TOOL_LOG={shlex.quote(str(tool_log))}
TARGET_DATABASE={shlex.quote(target_db)}
TARGET_HOST={shlex.quote(host)}
SOURCE_COMPUTE={shlex.quote(SOURCE_COMPUTE)}
PROJECT={shlex.quote(project)}
PASSWORD_SECRET={shlex.quote(password_secret)}
RUN_NONCE={shlex.quote(run_nonce)}
PYTHON={shlex.quote(python)}
export PATH={shlex.quote(system_exec_path)}
export CLOUDSDK_CONFIG=/home/teleo/.config/gcloud
export PGOPTIONS="-c default_transaction_read_only=on"
INVOCATION_ID="$($PYTHON -c 'import uuid; print(uuid.uuid4())')"
if ! PGPASSWORD="$(gcloud secrets versions access latest --secret="$PASSWORD_SECRET" --project="$PROJECT")"; then
exit 124
fi
export PGPASSWORD
trap 'unset PGPASSWORD PGOPTIONS' EXIT
DB_RECEIPT="$(psql "host=$TARGET_HOST port=5432 dbname=$TARGET_DATABASE user=postgres sslmode=require connect_timeout=8" -X -Atq -v ON_ERROR_STOP=1 <<'SQL'
begin transaction read only;
select jsonb_build_object(
'current_database', current_database(),
'system_identifier', (select system_identifier::text from pg_control_system()),
'server_address', host(inet_server_addr()),
'server_port', inet_server_port(),
'ssl', coalesce((select ssl from pg_stat_ssl where pid=pg_backend_pid()), false),
'transaction_read_only', current_setting('transaction_read_only'),
'default_transaction_read_only', current_setting('default_transaction_read_only')
)::text;
rollback;
SQL
)"
if [[ -z "$DB_RECEIPT" ]]; then
unset PGPASSWORD
exit 125
fi
$PYTHON - "$TOOL_LOG" "$RUN_NONCE" "$INVOCATION_ID" "$SOURCE_COMPUTE" "$TARGET_DATABASE" "$DB_RECEIPT" "$@" <<'PY'
import json
import sys
from datetime import datetime, timezone
path, run_nonce, invocation_id, compute, database, db_receipt, *argv = sys.argv[1:]
event = {{
"phase": "start",
"at_utc": datetime.now(timezone.utc).isoformat(),
"invocation_id": invocation_id,
"run_nonce": run_nonce,
"container": compute,
"database": database,
"database_identity": json.loads(db_receipt),
"argv": argv,
}}
with open(path, "a", encoding="utf-8") as handle:
handle.write(json.dumps(event, sort_keys=True) + "\\n")
PY
$PYTHON "$CLOUDSQL_TOOL" \
--host "$TARGET_HOST" \
--db "$TARGET_DATABASE" \
--canonical-db "$TARGET_DATABASE" \
--project "$PROJECT" \
--password-secret "$PASSWORD_SECRET" \
"$@"
STATUS=$?
$PYTHON - "$TOOL_LOG" "$RUN_NONCE" "$INVOCATION_ID" "$SOURCE_COMPUTE" "$TARGET_DATABASE" "$STATUS" <<'PY'
import json
import sys
from datetime import datetime, timezone
path, run_nonce, invocation_id, compute, database, status = sys.argv[1:]
event = {{
"phase": "end",
"at_utc": datetime.now(timezone.utc).isoformat(),
"invocation_id": invocation_id,
"run_nonce": run_nonce,
"container": compute,
"database": database,
"returncode": int(status),
}}
with open(path, "a", encoding="utf-8") as handle:
handle.write(json.dumps(event, sort_keys=True) + "\\n")
PY
exit "$STATUS"
"""
def patch_temp_bridge(args: argparse.Namespace, temp_profile: Path) -> dict[str, Any]:
wrapper = temp_profile / "bin" / "teleo-kb"
target_tool = temp_profile / "bin" / "cloudsql_memory_tool.py"
skill = temp_profile / "skills" / "teleo-kb-bridge" / "SKILL.md"
tool_log = temp_profile / "gcp-checkpoint-tool-calls.jsonl"
if not skill.is_file():
raise bound.CheckpointError("temporary teleo-kb bridge skill is missing")
shutil.copyfile(args.cloudsql_tool, target_tool)
target_tool.chmod(0o700)
run_nonce = uuid.uuid4().hex
wrapper_text = build_cloudsql_wrapper(
cloudsql_tool=target_tool,
tool_log=tool_log,
target_db=args.target_db,
host=args.host,
project=args.project,
password_secret=args.password_secret,
run_nonce=run_nonce,
)
binding = f"""
## Generated GCP Database Checkpoint
This temporary no-send profile is bound only to private-TLS database
`{args.target_db}` on `{args.host}` from `{SOURCE_COMPUTE}`. Every terminal call
must start directly with the bare command `teleo-kb` and contain exactly one
read command. Never use an absolute path, Python, shell variables, `which`,
`env`, pipes, redirects, heredocs, cache files, or a second shell command. Use
only search, context, show, evidence, edges, list-proposals,
search-proposals, show-proposal, and decision-matrix-status. Keep list/search
limits at 30 or below. For aggregate proposal status counts, call
`teleo-kb decision-matrix-status`; do not parse cached output. Do not stage,
approve, apply, write, send, or use live-profile paths. Distinguish pending,
approved, applied, and canonical rows from database readback; never infer
decision-matrix approval when its tables are absent. `status` and the restored
audit fallback are unavailable in this clone-only profile. Every helper session
is server-enforced with `default_transaction_read_only=on`.
For every direct-claim answer, use this sequence: direct yes/no/partly answer;
exact proposal/canonical-row readback; what is not proven; then a final line
beginning exactly `Next Cory-style follow-up:` with one concrete query, review,
apply-authorization, renderer, or demo-tier action. When a proposal is approved
but has no applied timestamp, say that approval is not application and that
explicit operator authorization is required before apply. When discussing the
matrix, say the conclusion comes from the fresh schema readback. When discussing
identity, name the DB rows/readback needed in addition to the rendered file.
"""
skill_text = skill.read_text(encoding="utf-8")
for live_wrapper in {
"/home/teleo/.hermes/profiles/leoclean/bin/teleo-kb",
str(args.live_profile / "bin" / "teleo-kb"),
}:
skill_text = skill_text.replace(live_wrapper, "teleo-kb")
skill_text = bound.inject_skill_binding(skill_text, binding.lstrip())
descriptor = os.open(tool_log, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
os.close(descriptor)
bound._detach_and_write(wrapper, wrapper_text, mode=0o700)
bound._detach_and_write(skill, skill_text, mode=0o600)
return {
"wrapper_path": str(wrapper),
"tool_log_path": str(tool_log),
"run_nonce": run_nonce,
"wrapper_sha256": bound.sha256_bytes(wrapper_text.encode()),
"cloudsql_tool_sha256": bound.sha256_bytes(target_tool.read_bytes()),
"reviewed_cloudsql_tool_sha256": REVIEWED_CLOUDSQL_TOOL_SHA256,
"bridge_skill_sha256": bound.sha256_bytes(skill_text.encode()),
"source_compute": SOURCE_COMPUTE,
"target_database": args.target_db,
}
def turn_args(args: argparse.Namespace, prompt: dict[str, Any]) -> argparse.Namespace:
return SimpleNamespace(
chat_id=args.chat_id,
user_id=args.user_id,
user_name=args.user_name,
prompt=prompt["message"],
prompt_id=prompt["id"],
run_id=args.run_id,
turn_timeout=args.turn_timeout,
)
def direct_prompts() -> list[dict[str, Any]]:
return [
{"id": row["id"], "dimension": row["dimension"], "message": row["message"]}
for row in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
]
async def run_suite(args: argparse.Namespace) -> dict[str, Any]:
report: dict[str, Any] = {
"schema": SCHEMA,
"generated_at_utc": bound.utc_now(),
"mode": "gcp_generated_db_gatewayrunner_no_send",
"source_compute": SOURCE_COMPUTE,
"target_database": args.target_db,
"private_tls": False,
"posted_to_telegram": False,
"production_service_restart_attempted": False,
"database_write_attempted": False,
"results": [],
"errors": [],
}
before_service: dict[str, Any] = {}
before_hashes: dict[str, str | None] = {}
before_identity: dict[str, Any] = {}
before_fingerprint: dict[str, Any] = {}
before_status: dict[str, Any] = {}
temp_profile: Path | None = None
provider_environment: dict[str, str] = {}
provider_secrets: tuple[str, ...] = ()
bridge: dict[str, Any] = {}
termination: bound.TerminationRequested | None = None
try:
tool_sha = validate_reviewed_file(
args.cloudsql_tool,
REVIEWED_CLOUDSQL_TOOL_SHA256,
"Cloud SQL bridge helper",
)
manifest_sha = validate_reviewed_file(
args.manifest_sql,
REVIEWED_MANIFEST_SQL_SHA256,
"Postgres parity manifest",
)
report["reviewed_inputs"] = {
"cloudsql_tool_sha256": tool_sha,
"manifest_sql_sha256": manifest_sha,
}
report["parity_receipt"] = validate_parity_receipt(args.parity_receipt, args.target_db)
before_service = service_state(args.service)
before_hashes = profile_hashes(args.live_profile)
before_identity = database_identity(args)
validate_database_identity(before_identity, args.target_db)
report["private_tls"] = before_identity.get("ssl") is True
before_fingerprint = database_fingerprint(args)
before_status = canonical_status(args)
report["service_before"] = before_service
report["live_profile_hashes_before"] = before_hashes
report["database_identity_before"] = before_identity
report["database_fingerprint_before"] = before_fingerprint
report["canonical_status_before"] = before_status
temp_profile, copy_audit = bound.copy_profile(
run_id=args.run_id,
prompt_id="gcp-direct-claim",
live_profile=args.live_profile,
)
report["temporary_profile_copy_audit"] = copy_audit
report["model_auth_binding"] = bound.copy_ephemeral_model_auth(
temp_profile,
live_profile=args.live_profile,
)
provider_environment, environment_audit = bound.load_ephemeral_provider_environment(temp_profile)
provider_secrets = tuple(provider_environment.values())
report["model_auth_binding"]["environment_binding"] = environment_audit
bridge = patch_temp_bridge(args, temp_profile)
report["temporary_bridge"] = bridge
for index, prompt in enumerate(direct_prompts(), 1):
gateway = await bound.invoke_gateway_subprocess(
turn_args(args, prompt),
temp_profile,
provider_environment=provider_environment,
)
report["results"].append(
{
"turn": index,
"prompt_id": prompt["id"],
"dimension": prompt["dimension"],
"prompt": prompt["message"],
"reply": gateway.get("reply") or "",
"ok": bool(str(gateway.get("reply") or "").strip()),
"mutates_kb": False,
"evidence_tier": "gcp_generated_db_gatewayrunner_no_send",
"gateway": gateway,
}
)
expected_ids = [prompt["id"] for prompt in direct_prompts()]
report["score"] = benchmark.score_result_subset(
report["results"],
catalog=benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS,
expected_prompt_ids=expected_ids,
)
report["tool_proof"] = bound.read_tool_proof(
Path(bridge["tool_log_path"]),
SOURCE_COMPUTE,
args.target_db,
run_nonce=bridge["run_nonce"],
database_identity=before_identity,
require_database_read_only=True,
require_default_read_only=True,
)
except bound.TerminationRequested as exc:
termination = exc
report["errors"].append(
{"phase": "execution", "type": type(exc).__name__, "message": f"signal={exc.signum}"}
)
except Exception as exc:
report["errors"].append(
{"phase": "execution", "type": type(exc).__name__, "message": bound.redact_text(str(exc))}
)
finally:
provider_environment.clear()
report["gateway_child_cleanup"] = bound.cleanup_active_gateway_children()
temp_root = temp_profile.parent if temp_profile else None
cleanup_clear = not bound._ACTIVE_GATEWAY_CHILDREN
report["temp_profile_removed"] = bound.remove_tree(temp_root) if cleanup_clear else False
report["temp_profile_absent"] = bound.temp_path_absent(temp_root)
report["temp_root_registry_clear"] = temp_root not in bound._ACTIVE_TEMP_ROOTS if temp_root else True
postflight_steps = {
"service_after": lambda: service_state(args.service),
"live_profile_hashes_after": lambda: profile_hashes(args.live_profile),
}
if before_identity:
postflight_steps["database_identity_after"] = lambda: database_identity(args)
postflight_steps["canonical_status_after"] = lambda: canonical_status(args)
if before_fingerprint and report.get("reviewed_inputs"):
postflight_steps["database_fingerprint_after"] = lambda: database_fingerprint(args)
for key, capture in postflight_steps.items():
try:
report[key] = capture()
if key == "database_identity_after":
validate_database_identity(report[key], args.target_db)
except BaseException as exc:
if isinstance(exc, bound.TerminationRequested) and termination is None:
termination = exc
report["errors"].append(
{"phase": key, "type": type(exc).__name__, "message": bound.redact_text(str(exc))}
)
report[key] = None
after_service = report.get("service_after") or {}
after_hashes = report.get("live_profile_hashes_after") or {}
after_identity = report.get("database_identity_after") or {}
after_fingerprint = report.get("database_fingerprint_after") or {}
gateways = [row.get("gateway") or {} for row in report["results"]]
allowed_tools = {"skills_list", "skill_view", "terminal"}
tool_proof = report.get("tool_proof") or {}
report["checks"] = {
"reviewed_inputs_pinned": bool(report.get("reviewed_inputs")),
"parity_receipt_validated": bool(report.get("parity_receipt")),
"six_replies_returned": len(report["results"]) == 6 and all(row.get("ok") for row in report["results"]),
"strict_score_passed": report.get("score", {}).get("pass") is True,
"tool_calls_bound_read_only_and_successful": tool_proof.get("all_bound_to_supplied_target") is True
and tool_proof.get("all_completed_successfully") is True
and tool_proof.get("default_read_only_required") is True,
"gateway_adapters_absent": bool(gateways)
and all((gateway.get("tool_surface") or {}).get("gateway_adapter_count") == 0 for gateway in gateways),
"send_message_tool_absent": bool(gateways)
and all((gateway.get("tool_surface") or {}).get("send_message_tool_enabled") is False for gateway in gateways),
"tool_registry_allowlisted": bool(gateways)
and all(
set((gateway.get("tool_surface") or {}).get("actual_registry_tools") or []) == allowed_tools
for gateway in gateways
),
"all_children_exited": bool(gateways)
and all(
(gateway.get("child_process") or {}).get("alive_after_readback") is False
and (gateway.get("child_process") or {}).get("process_group_alive_after_readback") is False
for gateway in gateways
),
"generated_database_unchanged": bool(before_fingerprint)
and before_fingerprint == after_fingerprint,
"database_identity_unchanged": bool(before_identity) and before_identity == after_identity,
"live_service_unchanged": bool(before_service) and before_service == after_service,
"live_profile_unchanged": any(before_hashes.values()) and before_hashes == after_hashes,
"temporary_profile_removed": report.get("temp_profile_removed") is True
and report.get("temp_profile_absent") is True
and report.get("temp_root_registry_clear") is True,
"no_telegram_send": report.get("posted_to_telegram") is False,
}
report["status"] = "pass" if not report["errors"] and all(report["checks"].values()) else "fail"
report["claim_ceiling"] = (
"A pass proves six no-send direct-claim prompts through an adapter-free GatewayRunner temporary profile "
"bound to one generated Cloud SQL database over private TLS. Actual helper sessions default to read-only, "
"and the public/kb_stage catalog plus table-row fingerprints remain unchanged. It validates a supplied "
"point-in-time parity receipt; it does not independently prove database creation provenance, Telegram "
"delivery, separate-VM staging isolation, source extraction/apply composition, ongoing replication, or "
"production cutover."
)
report["completed_at_utc"] = bound.utc_now()
bound.write_report(args.output, report, secret_values=provider_secrets)
if termination is not None:
raise termination
return report
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--execute", action="store_true")
parser.add_argument("--target-db", required=True)
parser.add_argument("--cloudsql-tool", required=True, type=Path)
parser.add_argument("--manifest-sql", default=DEFAULT_MANIFEST_SQL, type=Path)
parser.add_argument("--parity-receipt", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--project", default=DEFAULT_PROJECT)
parser.add_argument("--password-secret", default=DEFAULT_SECRET)
parser.add_argument("--service", default=DEFAULT_SERVICE)
parser.add_argument("--live-profile", default=bound.LIVE_PROFILE, type=Path)
parser.add_argument("--chat-id", default=bound.DEFAULT_CHAT_ID)
parser.add_argument("--user-id", default=bound.DEFAULT_USER_ID)
parser.add_argument("--user-name", default="codex GCP generated DB checkpoint")
parser.add_argument("--run-id", default="gcp-direct-claim-" + uuid.uuid4().hex[:12])
parser.add_argument("--turn-timeout", default=300, type=int)
args = parser.parse_args(argv)
if not args.execute:
parser.error("--execute is required because this runs paid model calls")
if not bound.SAFE_DB_NAME_RE.fullmatch(args.target_db):
parser.error("--target-db must be a safe Postgres database name")
if not args.target_db.startswith("teleo_clone_"):
parser.error("--target-db must start with teleo_clone_ and cannot name a live database")
if not args.cloudsql_tool.is_file():
parser.error("--cloudsql-tool must exist")
if not args.manifest_sql.is_file():
parser.error("--manifest-sql must exist")
if not args.parity_receipt.is_file():
parser.error("--parity-receipt must exist")
if not args.live_profile.is_dir():
parser.error("--live-profile must exist")
if args.turn_timeout <= 0:
parser.error("--turn-timeout must be positive")
try:
args.output = bound.validate_private_output_path(args.output)
except bound.CheckpointError as exc:
parser.error(str(exc))
return args
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
with bound.termination_cleanup_handlers():
report = asyncio.run(run_suite(args))
print(
json.dumps(
{
"status": report.get("status"),
"score": (report.get("score") or {}).get("coverage"),
"checks_passed": sum(bool(value) for value in report.get("checks", {}).values()),
"checks_total": len(report.get("checks", {})),
"output": str(args.output),
},
sort_keys=True,
)
)
return 0 if report.get("status") == "pass" else 1
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -1062,6 +1062,7 @@ def validate_read_only_bridge_argv(argv: list[str]) -> None:
def command(name: str) -> argparse.ArgumentParser: def command(name: str) -> argparse.ArgumentParser:
item = subparsers.add_parser(name, add_help=False) item = subparsers.add_parser(name, add_help=False)
item.add_argument("--help", action="store_true")
item.add_argument("--format", choices=("markdown", "json"), default="markdown") item.add_argument("--format", choices=("markdown", "json"), default="markdown")
return item return item
@ -1082,7 +1083,7 @@ def validate_read_only_bridge_argv(argv: list[str]) -> None:
list_proposals = command("list-proposals") list_proposals = command("list-proposals")
list_proposals.add_argument( list_proposals.add_argument(
"--status", "--status",
choices=("pending_review", "approved", "applied", "canceled", "all"), choices=("pending_review", "approved", "applied", "canceled", "rejected", "all"),
default="pending_review", default="pending_review",
) )
list_proposals.add_argument("--limit", type=int, default=10) list_proposals.add_argument("--limit", type=int, default=10)
@ -1090,7 +1091,7 @@ def validate_read_only_bridge_argv(argv: list[str]) -> None:
search_proposals.add_argument("query") search_proposals.add_argument("query")
search_proposals.add_argument( search_proposals.add_argument(
"--status", "--status",
choices=("pending_review", "approved", "applied", "canceled", "all"), choices=("pending_review", "approved", "applied", "canceled", "rejected", "all"),
default="all", default="all",
) )
search_proposals.add_argument("--limit", type=int, default=10) search_proposals.add_argument("--limit", type=int, default=10)
@ -1548,6 +1549,7 @@ def read_tool_proof(
run_nonce: str, run_nonce: str,
database_identity: dict[str, Any], database_identity: dict[str, Any],
require_database_read_only: bool = True, require_database_read_only: bool = True,
require_default_read_only: bool = False,
) -> dict[str, Any]: ) -> dict[str, Any]:
events: list[dict[str, Any]] = [] events: list[dict[str, Any]] = []
parse_errors: list[str] = [] parse_errors: list[str] = []
@ -1604,6 +1606,10 @@ def read_tool_proof(
not require_database_read_only not require_database_read_only
or str((item["database_identity"] or {}).get("transaction_read_only")) == "on" or str((item["database_identity"] or {}).get("transaction_read_only")) == "on"
) )
and (
not require_default_read_only
or str((item["database_identity"] or {}).get("default_transaction_read_only")) == "on"
)
for item in invocations for item in invocations
) )
return { return {
@ -1615,6 +1621,7 @@ def read_tool_proof(
"invocation_count": len(invocations), "invocation_count": len(invocations),
"all_bound_to_supplied_target": complete_pairing and all_bound, "all_bound_to_supplied_target": complete_pairing and all_bound,
"database_read_only_required": require_database_read_only, "database_read_only_required": require_database_read_only,
"default_read_only_required": require_default_read_only,
"all_completed_successfully": complete_pairing "all_completed_successfully": complete_pairing
and bool(invocations) and bool(invocations)
and all(item["returncode"] == 0 for item in invocations), and all(item["returncode"] == 0 for item in invocations),

View file

@ -261,7 +261,14 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
"Ask or run the row-level readback: which proposal/time window to inspect, kb_stage status, public.* " "Ask or run the row-level readback: which proposal/time window to inspect, kb_stage status, public.* "
"before/after counts, and the next guarded apply packet if approved rows are still pending." "before/after counts, and the next guarded apply packet if approved rows are still pending."
), ),
"required_signals": ["state_semantics", "canonical_db", "row_level_proof", "cory_followup", "no_overclaim"], "required_signals": [
"state_semantics",
"canonical_db",
"row_level_proof",
"structured_db_readback",
"cory_followup",
"no_overclaim",
],
}, },
{ {
"id": "DC-02", "id": "DC-02",
@ -276,7 +283,14 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
"Offer the next proof-changing action: show the packet/readiness artifact, get explicit apply " "Offer the next proof-changing action: show the packet/readiness artifact, get explicit apply "
"authorization, run apply in integrated order, then postflight and Telegram regression." "authorization, run apply in integrated order, then postflight and Telegram regression."
), ),
"required_signals": ["canonical_db", "row_level_proof", "authorization", "cory_followup", "no_overclaim"], "required_signals": [
"canonical_db",
"row_level_proof",
"structured_db_readback",
"authorization",
"cory_followup",
"no_overclaim",
],
}, },
{ {
"id": "DC-03", "id": "DC-03",
@ -291,7 +305,14 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
"Follow up with the schema/readback request and then fall back to kb_stage.kb_proposals review/apply " "Follow up with the schema/readback request and then fall back to kb_stage.kb_proposals review/apply "
"columns plus canonical row readback." "columns plus canonical row readback."
), ),
"required_signals": ["decision_matrix", "canonical_db", "fresh_readback", "cory_followup", "no_overclaim"], "required_signals": [
"decision_matrix",
"canonical_db",
"fresh_readback",
"structured_db_readback",
"cory_followup",
"no_overclaim",
],
}, },
{ {
"id": "DC-04", "id": "DC-04",
@ -310,6 +331,7 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
"document_artifact_linking", "document_artifact_linking",
"staging_or_review", "staging_or_review",
"canonical_db", "canonical_db",
"structured_db_readback",
"cory_followup", "cory_followup",
"no_overclaim", "no_overclaim",
], ],
@ -330,6 +352,7 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
"required_signals": [ "required_signals": [
"state_semantics", "state_semantics",
"row_level_proof", "row_level_proof",
"structured_db_readback",
"authorization", "authorization",
"artifact", "artifact",
"cory_followup", "cory_followup",
@ -349,7 +372,14 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
"Follow up by asking whether to audit canonical identity rows, stage the missing DB proposal, or run the " "Follow up by asking whether to audit canonical identity rows, stage the missing DB proposal, or run the "
"renderer/sync readback if an approved identity row already exists." "renderer/sync readback if an approved identity row already exists."
), ),
"required_signals": ["identity_rendering", "canonical_db", "row_level_proof", "cory_followup", "no_overclaim"], "required_signals": [
"identity_rendering",
"canonical_db",
"row_level_proof",
"structured_db_readback",
"cory_followup",
"no_overclaim",
],
}, },
] ]
@ -361,6 +391,9 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
r"approved (?:is not|is not the same as|does not mean|!=|not the same as) applied|" r"approved (?:is not|is not the same as|does not mean|!=|not the same as) applied|"
r"not assume approval changed|approval changed .* canonical|approved[- ]not[- ]applied gap|" r"not assume approval changed|approval changed .* canonical|approved[- ]not[- ]applied gap|"
r"approval is a human gate, not a write|approved with zero canonical effect|" r"approval is a human gate, not a write|approved with zero canonical effect|"
r"`?applied`? status (?:alone )?is not sufficient proof|status alone does not prove|"
r"approved[\s\S]{0,120}applied at:\s*(?:none|-)|canonical truth still requires review/apply|"
r"proposal staging plus reviewer/apply workflow|"
r"applied_at:\s*NULL|no applied_at|NOTHING was applied|not applied|payload-only|" r"applied_at:\s*NULL|no applied_at|NOTHING was applied|not applied|payload-only|"
r"neither row has been promoted|canonical (?:layer|tables?|public\.\*) (?:is )?locked|" r"neither row has been promoted|canonical (?:layer|tables?|public\.\*) (?:is )?locked|"
r"reviewable proposal|canonical apply step|demo tier|staging write|stages? a real write", r"reviewable proposal|canonical apply step|demo tier|staging write|stages? a real write",
@ -383,12 +416,29 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
r"\bapplied_at\b|superseded_by:\s*NULL|public\.[a-z_]+ rows?|" r"\bapplied_at\b|superseded_by:\s*NULL|public\.[a-z_]+ rows?|"
r"\bsingle canonical row\b|\bcanonical rows?\b|" r"\bsingle canonical row\b|\bcanonical rows?\b|"
r"\b[a-f0-9]{8}\b.*\b(?:approved|applied|pending|canceled)|" r"\b[a-f0-9]{8}\b.*\b(?:approved|applied|pending|canceled)|"
r"\b(?:approved|applied|pending|canceled)\b.*\b[a-f0-9]{8}\b", r"\b(?:approved|applied|pending|canceled)\b.*\b[a-f0-9]{8}\b|"
r"\bproposal\b[\s\S]{0,180}\bstatus:\s*`?(?:approved|applied|pending_review|canceled)`?\b",
re.I, re.I,
), ),
re.compile( re.compile(
r"\bquery|read ?back|verify|postflight|preflight|canonical state|applied_at|superseded_by|" r"\bquery|read ?back|readback|verify|postflight|preflight|canonical state|applied_at|superseded_by|"
r"canonical `?public\.\*`?|proposal ledger|verifiable readback|row appears|ledger\b", r"canonical `?public\.\*`?|proposal ledger|verifiable readback|row appears|ledger\b|"
r"\bproposal\b[\s\S]{0,220}\b[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}\b"
r"[\s\S]{0,140}\bstatus:\s*`?(?:approved|applied|pending_review|canceled)`?\b",
re.I,
),
],
"structured_db_readback": [
re.compile(
r"\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\b|"
r"\b(?:claims|sources|claim_edges|claim_evidence|kb_proposals)\b[^\d\n]{0,24}\d+",
re.I,
),
re.compile(
r"\bstatus:\s*`?(?:approved|applied|pending_review|canceled)`?|"
r"\bapplied[_ ]at\s*:?\s*`?(?:none|-|\d{4}-\d{2}-\d{2})|"
r"\bsource(?:_ref)?\s*:\s*`?[^\s`]+|"
r"\b(?:claims|sources|claim_edges|claim_evidence|kb_proposals)\b[^\d\n]{0,24}\d+",
re.I, re.I,
), ),
], ],
@ -399,6 +449,8 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
"no_overclaim": [ "no_overclaim": [
re.compile( re.compile(
r"\bnot (?:claim|assume|overclaim)|cannot (?:say|claim|overclaim)|do not know|without proof|if .* only\b|" r"\bnot (?:claim|assume|overclaim)|cannot (?:say|claim|overclaim)|do not know|without proof|if .* only\b|"
r"\bhave not (?:yet )?verified\b|\bnot (?:fully )?landed\b|\bnot sufficient proof\b|"
r"\bwould not demo it as\b|\bnot by itself\b|\bnot the whole cause\b|"
r"\bonly ground truth\b|\bauthoritative test\b|candidate only|zero canonical effect|" r"\bonly ground truth\b|\bauthoritative test\b|candidate only|zero canonical effect|"
r"\bno mutations? made\b|\bNOTHING was applied\b|\bout of scope\b|\bnot bundled\b|" r"\bno mutations? made\b|\bNOTHING was applied\b|\bout of scope\b|\bnot bundled\b|"
r"\bcannot (?:approve|publish|move|sign)|\bzero unilateral action\b|" r"\bcannot (?:approve|publish|move|sign)|\bzero unilateral action\b|"
@ -412,6 +464,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
r"\bsource rows? (?:themselves )?(?:do not|don't) exist\b|" r"\bsource rows? (?:themselves )?(?:do not|don't) exist\b|"
r"\bnothing has been inserted\b|\bcanonical apply step is the missing link\b|" r"\bnothing has been inserted\b|\bcanonical apply step is the missing link\b|"
r"\bnot shipped\b|\bnot approved by decision[- ]matrix\b|" r"\bnot shipped\b|\bnot approved by decision[- ]matrix\b|"
r"\bdo not infer (?:proper )?matrix approval\b|\bno decision[- ]matrix approval record\b|"
r"\bnot just (?:a )?pointer mismatch\b|\bdoes not prove\b|" r"\bnot just (?:a )?pointer mismatch\b|\bdoes not prove\b|"
r"\bdoes NOT show\b|\bdoes not show\b|" r"\bdoes NOT show\b|\bdoes not show\b|"
r"\bnot a canonical commit\b|\bnot collective truth\b|" r"\bnot a canonical commit\b|\bnot collective truth\b|"
@ -428,6 +481,8 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
), ),
re.compile( re.compile(
r"\buntil\b.*\b(readback|proof|query|postflight|canonical)\b|" r"\buntil\b.*\b(readback|proof|query|postflight|canonical)\b|"
r"\bhave not (?:yet )?verified\b|\bnot (?:fully )?landed\b|\bnot sufficient proof\b|"
r"\bwould not demo it as\b|\bnot by itself\b|\bnot the whole cause\b|"
r"\bwithout\b.*\b(readback|proof|query|postflight|canonical)\b|" r"\bwithout\b.*\b(readback|proof|query|postflight|canonical)\b|"
r"only ground truth.*canonical|canonical row.*authoritative test|readback confirms|zero canonical effect|" r"only ground truth.*canonical|canonical row.*authoritative test|readback confirms|zero canonical effect|"
r"\bgovernance gap\b|\bno audit trail\b|\bwithout authorization\b|outside the PR-everything rule|" r"\bgovernance gap\b|\bno audit trail\b|\bwithout authorization\b|outside the PR-everything rule|"
@ -443,6 +498,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
r"\bsource rows? (?:themselves )?(?:do not|don't) exist\b|" r"\bsource rows? (?:themselves )?(?:do not|don't) exist\b|"
r"\bnothing has been inserted\b|\bcanonical apply step is the missing link\b|" r"\bnothing has been inserted\b|\bcanonical apply step is the missing link\b|"
r"\bnot shipped\b|\bnot approved by decision[- ]matrix\b|" r"\bnot shipped\b|\bnot approved by decision[- ]matrix\b|"
r"\bdo not infer (?:proper )?matrix approval\b|\bno decision[- ]matrix approval record\b|"
r"\bnot just (?:a )?pointer mismatch\b|\bdoes not prove\b|" r"\bnot just (?:a )?pointer mismatch\b|\bdoes not prove\b|"
r"\bdoes NOT show\b|\bdoes not show\b|" r"\bdoes NOT show\b|\bdoes not show\b|"
r"\bnot a canonical commit\b|\bnot collective truth\b|" r"\bnot a canonical commit\b|\bnot collective truth\b|"
@ -464,6 +520,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
r"\bapply\b.*\bguard|guarded\b.*\bapply|narrow\b.*\bapply\b|" r"\bapply\b.*\bguard|guarded\b.*\bapply|narrow\b.*\bapply\b|"
r"\bstaged separately|out of scope|proposal containing|source evidence required|" r"\bstaged separately|out of scope|proposal containing|source evidence required|"
r"\bmanual apply\b|operator tooling|admin tooling|admin review|" r"\bmanual apply\b|operator tooling|admin tooling|admin review|"
r"\breviewer/apply step\b|\bhuman/operator resolution\b|"
r"\bnot just (?:a )?pointer mismatch\b|\brow-link audit\b|\bguarded apply contract\b|" r"\bnot just (?:a )?pointer mismatch\b|\brow-link audit\b|\bguarded apply contract\b|"
r"\breviewer/operator\b|\bpending backlog\b", r"\breviewer/operator\b|\bpending backlog\b",
re.I, re.I,
@ -483,13 +540,13 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
], ],
"fresh_readback": [ "fresh_readback": [
re.compile( re.compile(
r"\bcheck|read back|inspect|git status|rev-parse|systemctl|service|no \.git|not a git repo\b|" r"\bcheck|read ?back|readback|inspect|git status|rev-parse|systemctl|service|no \.git|not a git repo\b|"
r"\bschema\b|\btables?\b", r"\bschema\b|\btables?\b",
re.I, re.I,
), ),
re.compile( re.compile(
r"\bfresh|current|before claiming|last modified|latest mirrored|v0\.7\.0|MainPID|" r"\bfresh|current|before claiming|last modified|latest mirrored|v0\.7\.0|MainPID|"
r"schema (?:absent|doesn't exist|does not exist)|tables?.*absent|do not exist|" r"schema (?:is )?(?:absent|missing|doesn't exist|does not exist)|tables?.*absent|do not exist|"
r"\bjust (?:ran|did)\b|\breadback we just ran\b", r"\bjust (?:ran|did)\b|\breadback we just ran\b",
re.I, re.I,
), ),
@ -497,7 +554,8 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
"authorization": [ "authorization": [
re.compile(r"\bauthori[sz]ation|approval|explicit|permission|allowed|not allowed|admin|operator\b", re.I), re.compile(r"\bauthori[sz]ation|approval|explicit|permission|allowed|not allowed|admin|operator\b", re.I),
re.compile( re.compile(
r"\bpublic update|post|capital|funds|sign|broadcast|admin review|operator pass|operator action|apply tool\b", r"\bpublic update|post|capital|funds|sign|broadcast|admin review|operator pass|operator action|apply tool|"
r"operator authorization|operator-approved apply path|reviewer/apply workflow\b",
re.I, re.I,
), ),
], ],
@ -515,7 +573,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
re.compile(r"\bdecision[- ]matrix|matrix_voters|proposal_votes|proposal_decisions|weighted voters?|tally\b", re.I), re.compile(r"\bdecision[- ]matrix|matrix_voters|proposal_votes|proposal_decisions|weighted voters?|tally\b", re.I),
re.compile( re.compile(
r"\bnot (?:yet )?shipped|schema lacks|tables? (?:do not|does not|don't) exist|" r"\bnot (?:yet )?shipped|schema lacks|tables? (?:do not|does not|don't) exist|"
r"tables?.*absent|schema absent|doesn't exist yet|check the schema|kb_stage\.kb_proposals\b", r"tables?.*absent|schema (?:is )?absent|doesn't exist yet|check the schema|kb_stage\.kb_proposals\b",
re.I, re.I,
), ),
], ],

View file

@ -2,7 +2,6 @@
from __future__ import annotations from __future__ import annotations
import argparse
import json import json
import sys import sys
from pathlib import Path from pathlib import Path
@ -12,6 +11,18 @@ sys.path.insert(0, str(REPO_ROOT / "scripts"))
import assemble_telegram_visible_direct_claim_capture_receipt as receipt # noqa: E402 import assemble_telegram_visible_direct_claim_capture_receipt as receipt # noqa: E402
STRUCTURED_FIXTURE_READBACKS = {
"DC-01": " DB readback: Proposal: f004bbb2-ac9a-481f-b7b8-74319373ba6a; Status: applied; Applied at: 2026-07-05.",
"DC-02": " DB readback: Proposal: a64df080-8502-42e2-98f4-9bbdecb8da73; Status: approved; Applied at: none.",
"DC-03": " DB readback: Proposal: a64df080-8502-42e2-98f4-9bbdecb8da73; Status: approved; Applied at: none.",
"DC-04": (
" DB readback: Proposal: e987b07c-9c23-45ba-b95c-d6d04c6536c7; Status: pending_review; "
"Source_ref: telegram-resolved-attachment:doc_996e2e86cd94."
),
"DC-05": " DB readback: kb_proposals: 26; canary proposal status: pending_review; canonical rows unchanged.",
"DC-06": " DB readback: Proposal: f004bbb2-ac9a-481f-b7b8-74319373ba6a; Status: applied; Applied at: 2026-07-05.",
}
def load_packet() -> dict: def load_packet() -> dict:
return json.loads(receipt.DEFAULT_PACKET.read_text(encoding="utf-8")) return json.loads(receipt.DEFAULT_PACKET.read_text(encoding="utf-8"))
@ -30,7 +41,7 @@ def good_capture(packet: dict) -> dict:
{ {
"prompt_id": item["prompt_id"], "prompt_id": item["prompt_id"],
"sent_text": item["message"], "sent_text": item["message"],
"reply": replies[item["prompt_id"]], "reply": replies[item["prompt_id"]] + STRUCTURED_FIXTURE_READBACKS[item["prompt_id"]],
"sent_at_utc": f"2026-07-10T03:0{index}:00Z", "sent_at_utc": f"2026-07-10T03:0{index}:00Z",
"reply_at_utc": f"2026-07-10T03:0{index}:30Z", "reply_at_utc": f"2026-07-10T03:0{index}:30Z",
"telegram_message_id": f"sent-{index}", "telegram_message_id": f"sent-{index}",

View file

@ -31,6 +31,7 @@ def test_manifest_hash_order_is_collation_independent() -> None:
manifest = Path("ops/postgres_parity_manifest.sql").read_text() manifest = Path("ops/postgres_parity_manifest.sql").read_text()
assert 'collate "C"' in manifest assert 'collate "C"' in manifest
assert "'server_address', host(inet_server_addr())" in manifest
def test_toc_counts_database_objects(tmp_path: Path) -> None: def test_toc_counts_database_objects(tmp_path: Path) -> None:

View file

@ -0,0 +1,335 @@
import argparse
import asyncio
import json
import os
import subprocess
from pathlib import Path
import pytest
from scripts.run_gcp_generated_db_direct_claim_suite import (
REVIEWED_CLOUDSQL_TOOL_SHA256,
REVIEWED_MANIFEST_SQL_SHA256,
build_cloudsql_wrapper,
database_fingerprint,
direct_prompts,
normalize_database_manifest_rows,
run_suite,
validate_database_identity,
validate_parity_receipt,
)
from scripts.run_leo_clone_bound_handler_checkpoint import (
READ_ONLY_BRIDGE_COMMANDS,
read_tool_proof,
validate_read_only_bridge_argv,
)
def test_direct_prompt_catalog_is_the_exact_six_case_suite() -> None:
prompts = direct_prompts()
assert [prompt["id"] for prompt in prompts] == [f"DC-{index:02d}" for index in range(1, 7)]
assert "status" not in READ_ONLY_BRIDGE_COMMANDS
validate_read_only_bridge_argv(["list-proposals", "--help"])
validate_read_only_bridge_argv(["list-proposals", "--status", "rejected"])
def test_wrapper_binds_private_read_only_database_and_records_calls(tmp_path: Path) -> None:
wrapper = build_cloudsql_wrapper(
cloudsql_tool=tmp_path / "cloudsql_memory_tool.py",
tool_log=tmp_path / "calls.jsonl",
target_db="teleo_clone_test",
host="10.61.0.3",
project="teleo-501523",
password_secret="test-secret-name",
run_nonce="nonce",
)
assert "dbname=$TARGET_DATABASE" in wrapper
assert '--db "$TARGET_DATABASE"' in wrapper
assert '--canonical-db "$TARGET_DATABASE"' in wrapper
assert "sslmode=require" in wrapper
assert "begin transaction read only" in wrapper
assert 'export PGOPTIONS="-c default_transaction_read_only=on"' in wrapper
assert "default_transaction_read_only" in wrapper
assert "host(inet_server_addr())" in wrapper
assert '"phase": "start"' in wrapper
assert '"phase": "end"' in wrapper
assert "unset PGPASSWORD" in wrapper
assert "systemctl restart" not in wrapper
assert "send_message" not in wrapper
def test_database_identity_requires_private_tls_and_read_only_target() -> None:
validate_database_identity(
{
"current_database": "teleo_clone_test",
"system_identifier": "1234",
"server_address": "10.61.0.3",
"ssl": True,
"transaction_read_only": "on",
"default_transaction_read_only": "on",
},
"teleo_clone_test",
)
with pytest.raises(RuntimeError, match="not RFC1918-private"):
validate_database_identity(
{
"current_database": "teleo_clone_test",
"system_identifier": "1234",
"server_address": "34.1.2.3",
"ssl": True,
"transaction_read_only": "on",
"default_transaction_read_only": "on",
},
"teleo_clone_test",
)
def test_manifest_normalization_excludes_nondeterministic_timing() -> None:
output = "\n".join(
[
'BEGIN',
'{"kind":"identity","database":"teleo_clone_test","captured_at":"2026-07-12T00:00:00Z"}',
'{"kind":"schemas","items":["kb_stage","public"]}',
'{"kind":"table","schema":"public","table":"claims","row_count":2,"rowset_md5":"abc"}',
'{"kind":"performance","query":"count_claims","elapsed_ms":1.2,"observed_rows":2}',
'ROLLBACK',
]
)
rows = normalize_database_manifest_rows(output)
assert all(row.get("kind") != "performance" for row in rows)
assert next(row for row in rows if row["kind"] == "identity").get("captured_at") is None
def test_database_fingerprint_uses_full_normalized_manifest(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
manifest_sql = tmp_path / "manifest.sql"
manifest_sql.write_text("select 1;\n")
args = type(
"Args",
(),
{
"host": "10.61.0.3",
"target_db": "teleo_clone_test",
"manifest_sql": manifest_sql,
},
)()
output = "\n".join(
[
'{"kind":"identity","database":"teleo_clone_test","server_address":"10.61.0.3",'
'"ssl":true,"transaction_read_only":"on","captured_at":"one"}',
'{"kind":"schemas","items":["kb_stage","public"]}',
'{"kind":"table","schema":"public","table":"claims","row_count":2,"rowset_md5":"abc"}',
]
)
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.database_password", lambda _args: "secret")
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.run", lambda *_args, **_kwargs: output)
receipt = database_fingerprint(args)
assert receipt["table_count"] == 1
assert receipt["total_rows"] == 2
assert receipt["database"] == "teleo_clone_test"
assert len(receipt["sha256"]) == 64
def test_reviewed_helper_hashes_match_repo_files() -> None:
import hashlib
assert (
hashlib.sha256(Path("hermes-agent/leoclean-bin/cloudsql_memory_tool.py").read_bytes()).hexdigest()
== REVIEWED_CLOUDSQL_TOOL_SHA256
)
assert hashlib.sha256(Path("ops/postgres_parity_manifest.sql").read_bytes()).hexdigest() == REVIEWED_MANIFEST_SQL_SHA256
def parity_payload(target_db: str = "teleo_clone_test") -> dict[str, object]:
return {
"artifact": "canonical_postgres_parity_verification",
"status": "pass",
"scope": "local_restore",
"problems": [],
"details": {
"target_database": target_db,
"source_table_count": 2,
"target_table_count": 2,
"source_total_rows": 3,
"target_total_rows": 3,
"table_mismatches": [],
"application_role_mismatches": {},
"required_extension_mismatches": {},
"performance_problems": [],
"structural_hashes": {"columns": {"source": "same", "target": "same"}},
},
}
def test_parity_receipt_is_bound_to_generated_target(tmp_path: Path) -> None:
receipt_path = tmp_path / "parity.json"
receipt_path.write_text(json.dumps(parity_payload()))
receipt = validate_parity_receipt(receipt_path, "teleo_clone_test")
assert receipt["target_database"] == "teleo_clone_test"
assert receipt["target_table_count"] == 2
with pytest.raises(RuntimeError, match="target_database"):
validate_parity_receipt(receipt_path, "teleo_clone_other")
def test_generated_wrapper_executes_only_clone_bound_default_read_only_tool(tmp_path: Path) -> None:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
gcloud = fake_bin / "gcloud"
gcloud.write_text("#!/bin/sh\nprintf 'fake-password\\n'\n")
gcloud.chmod(0o700)
psql = fake_bin / "psql"
psql.write_text(
"#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' "
"'{\"current_database\":\"teleo_clone_test\",\"system_identifier\":\"system-1\","
"\"server_address\":\"10.61.0.3\",\"ssl\":true,\"transaction_read_only\":\"on\","
"\"default_transaction_read_only\":\"on\"}'\n"
)
psql.chmod(0o700)
tool = tmp_path / "tool.py"
tool.write_text(
"import json, os, sys\n"
"print(json.dumps({'argv': sys.argv[1:], 'pgoptions': os.environ.get('PGOPTIONS'), "
"'has_password': bool(os.environ.get('PGPASSWORD'))}))\n"
)
log = tmp_path / "calls.jsonl"
wrapper = tmp_path / "teleo-kb"
wrapper.write_text(
build_cloudsql_wrapper(
cloudsql_tool=tool,
tool_log=log,
target_db="teleo_clone_test",
host="10.61.0.3",
project="teleo-501523",
password_secret="test-secret",
run_nonce="test-nonce",
system_exec_path=f"{fake_bin}:/usr/bin:/bin",
)
)
wrapper.chmod(0o700)
completed = subprocess.run(
[str(wrapper), "decision-matrix-status"],
text=True,
capture_output=True,
env={**os.environ, "HOME": str(tmp_path)},
check=False,
)
assert completed.returncode == 0, completed.stderr
tool_receipt = json.loads(completed.stdout)
assert tool_receipt["pgoptions"] == "-c default_transaction_read_only=on"
assert tool_receipt["has_password"] is True
assert tool_receipt["argv"][:6] == [
"--host",
"10.61.0.3",
"--db",
"teleo_clone_test",
"--canonical-db",
"teleo_clone_test",
]
events = [json.loads(line) for line in log.read_text().splitlines()]
assert [event["phase"] for event in events] == ["start", "end"]
assert events[0]["database_identity"]["default_transaction_read_only"] == "on"
assert "fake-password" not in log.read_text()
proof = read_tool_proof(
log,
"teleo-prod-1",
"teleo_clone_test",
run_nonce="test-nonce",
database_identity={"system_identifier": "system-1"},
require_database_read_only=True,
require_default_read_only=True,
)
assert proof["all_bound_to_supplied_target"] is True
events[0]["database_identity"]["default_transaction_read_only"] = "off"
unsafe_log = tmp_path / "unsafe-calls.jsonl"
unsafe_log.write_text("".join(json.dumps(event) + "\n" for event in events))
unsafe_proof = read_tool_proof(
unsafe_log,
"teleo-prod-1",
"teleo_clone_test",
run_nonce="test-nonce",
database_identity={"system_identifier": "system-1"},
require_database_read_only=True,
require_default_read_only=True,
)
assert unsafe_proof["all_bound_to_supplied_target"] is False
def test_run_suite_retains_failure_report_when_postflight_fails(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
output = tmp_path / "report.json"
placeholder = tmp_path / "placeholder"
placeholder.write_text("x")
args = argparse.Namespace(
target_db="teleo_clone_test",
cloudsql_tool=placeholder,
manifest_sql=placeholder,
parity_receipt=placeholder,
service="leo.service",
live_profile=tmp_path,
output=output,
chat_id="1",
user_id="2",
user_name="test",
run_id="test-run",
turn_timeout=1,
host="10.61.0.3",
project="teleo-501523",
password_secret="secret-name",
)
identity_calls = 0
def fake_identity(_args: argparse.Namespace) -> dict[str, object]:
nonlocal identity_calls
identity_calls += 1
if identity_calls > 1:
raise RuntimeError("postflight unavailable")
return {
"current_database": "teleo_clone_test",
"system_identifier": "system-1",
"server_address": "10.61.0.3",
"ssl": True,
"transaction_read_only": "on",
"default_transaction_read_only": "on",
}
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.validate_reviewed_file", lambda *args: "hash")
monkeypatch.setattr(
"scripts.run_gcp_generated_db_direct_claim_suite.validate_parity_receipt",
lambda *args: {"target_database": "teleo_clone_test"},
)
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.service_state", lambda *args: {"MainPID": "1"})
monkeypatch.setattr(
"scripts.run_gcp_generated_db_direct_claim_suite.profile_hashes",
lambda *args: {"wrapper": "hash"},
)
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.database_identity", fake_identity)
monkeypatch.setattr(
"scripts.run_gcp_generated_db_direct_claim_suite.database_fingerprint",
lambda *args: {"sha256": "fingerprint"},
)
monkeypatch.setattr(
"scripts.run_gcp_generated_db_direct_claim_suite.canonical_status",
lambda *args: {"high_signal_rows": {"claims": 1}},
)
monkeypatch.setattr(
"scripts.run_gcp_generated_db_direct_claim_suite.bound.copy_profile",
lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("runner setup failed")),
)
report = asyncio.run(run_suite(args))
retained = json.loads(output.read_text())
assert report["status"] == "fail"
assert retained["status"] == "fail"
assert {error["phase"] for error in retained["errors"]} >= {"execution", "database_identity_after"}
assert "gateway_child_cleanup" in retained

View file

@ -119,7 +119,7 @@ def test_gcp_scope_accepts_matching_private_connectivity_receipt(tmp_path: Path)
completed, payload = run_verifier( completed, payload = run_verifier(
tmp_path, tmp_path,
manifest_rows(database="teleo"), manifest_rows(database="teleo"),
manifest_rows(database="teleo_gcp_copy", server_address="10.61.0.3", ssl=True), manifest_rows(database="teleo_gcp_copy", server_address="10.61.0.3/32", ssl=True),
scope="gcp_staging", scope="gcp_staging",
connectivity={ connectivity={
"status": "pass", "status": "pass",

View file

@ -10,6 +10,21 @@ sys.path.insert(0, str(REPO_ROOT / "scripts"))
import working_leo_open_ended_benchmark as bench # noqa: E402 import working_leo_open_ended_benchmark as bench # noqa: E402
APPLIED_READBACK = (
" DB readback: Proposal: `f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; "
"Applied at: `2026-07-05`."
)
HELMER_READBACK = (
" DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; Applied at: `none`."
)
PENDING_SOURCE_READBACK = (
" DB readback: Proposal: `e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; "
"Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`."
)
DEMO_READBACK = (
" DB readback: kb_proposals: 26; canary proposal status: `pending_review`; canonical rows were not changed."
)
def test_prompts_are_open_ended_and_not_id_led(): def test_prompts_are_open_ended_and_not_id_led():
assert len(bench.OPEN_ENDED_PROMPTS) >= 5 assert len(bench.OPEN_ENDED_PROMPTS) >= 5
@ -168,6 +183,7 @@ def test_scores_direct_claim_followup_answer_as_pass():
"row ids, applied_at, and postflight proof. Until readback confirms canonical rows changed, I cannot claim " "row ids, applied_at, and postflight proof. Until readback confirms canonical rows changed, I cannot claim "
"the database updated. Follow-up: I would ask which proposal or time window Cory means, then show the pending " "the database updated. Follow-up: I would ask which proposal or time window Cory means, then show the pending "
"gap or the next guarded apply packet." "gap or the next guarded apply packet."
+ APPLIED_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -180,11 +196,59 @@ def test_scores_direct_claim_natural_db_split_answer_as_pass():
"applied_at empty. Everything else is pending_review or canceled. The substantive work is staged and " "applied_at empty. Everything else is pending_review or canceled. The substantive work is staged and "
"approved but not applied. Next admin action: run the apply tool or draft the review packet. Want me to " "approved but not applied. Next admin action: run the apply tool or draft the review packet. Want me to "
"pull the full packet?" "pull the full packet?"
+ APPLIED_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
def test_gcp_readback_phrasings_count_as_semantic_proof() -> None:
assert bench.matched_signal(
"Approved proposal a64df080 has Applied at: none; `applied` status is not sufficient proof of "
"canonical truth, which still requires review/apply.",
"state_semantics",
)
assert bench.matched_signal(
"Readback from the current tables: the decision-matrix schema is absent and proposal_votes do not exist.",
"fresh_readback",
)
assert bench.matched_signal(
"The pending proposal needs a reviewer/apply step and human/operator resolution before guarded apply.",
"staging_or_review",
)
assert bench.matched_signal(
"Explicit operator authorization is required before the operator-approved apply path.",
"authorization",
)
assert bench.matched_signal(
"For the current situation, I see one relevant proposal:\n"
"`f004bbb2-ac9a-481f-b7b8-74319373ba6a`\nStatus: `applied`",
"row_level_proof",
)
assert bench.matched_signal(
"Do not infer proper matrix approval from status alone; there is no decision-matrix approval record.",
"no_overclaim",
)
assert bench.matched_signal(APPLIED_READBACK, "structured_db_readback")
assert bench.matched_signal(
"Proposal f004bbb2-ac9a-481f-b7b8-74319373ba6a was applied at `2026-07-05 00:14:36 UTC`.",
"structured_db_readback",
)
def test_direct_claim_keyword_bundle_without_structured_readback_fails() -> None:
prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-01")
reply = (
"Proposed, approved, and applied are distinct. I would query canonical public.claims and kb_stage proposals "
"for counts and readback, avoid overclaiming, then ask Cory which guarded apply packet to review next."
)
score = bench.score_reply(prompt, reply)
assert score["pass"] is False
assert score["signals"]["structured_db_readback"] is False
def test_scores_direct_claim_natural_helmer_answer_as_pass(): def test_scores_direct_claim_natural_helmer_answer_as_pass():
prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-02") prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-02")
reply = ( reply = (
@ -192,6 +256,7 @@ def test_scores_direct_claim_natural_helmer_answer_as_pass():
"cea989fc is canceled. Proof-changing action: apply a64df080 through the admin review panel or apply tool. " "cea989fc is canceled. Proof-changing action: apply a64df080 through the admin review panel or apply tool. "
"Until applied_at is set and public.claims/public.claim_edges rows appear, it is only in the proposal ledger. " "Until applied_at is set and public.claims/public.claim_edges rows appear, it is only in the proposal ledger. "
"Want me to pull the full packet for review?" "Want me to pull the full packet for review?"
+ HELMER_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -204,6 +269,7 @@ def test_scores_direct_claim_natural_matrix_absent_answer_as_pass():
"tables are absent in both kb_stage and public. a64df080 has manual m3ta approval in kb_stage.kb_proposals, " "tables are absent in both kb_stage and public. a64df080 has manual m3ta approval in kb_stage.kb_proposals, "
"not a matrix vote, and it is not applied to canonical public.*. Next action: use the admin apply tool or " "not a matrix vote, and it is not applied to canonical public.*. Next action: use the admin apply tool or "
"pull the packet; want me to show it?" "pull the packet; want me to show it?"
+ HELMER_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -216,6 +282,7 @@ def test_scores_deployed_matrix_readback_answer_as_pass():
"kb_stage.matrix_voters, proposal_votes, and proposal_decisions are absent, and the public.* equivalents " "kb_stage.matrix_voters, proposal_votes, and proposal_decisions are absent, and the public.* equivalents "
"are absent too. a64df080 is m3ta reviewer sign-off in kb_stage.kb_proposals, not approved by " "are absent too. a64df080 is m3ta reviewer sign-off in kb_stage.kb_proposals, not approved by "
"decision-matrix vote. Next Cory-style follow-up: scope the schema or treat m3ta sign-off as sufficient." "decision-matrix vote. Next Cory-style follow-up: scope the schema or treat m3ta sign-off as sufficient."
+ HELMER_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -229,6 +296,7 @@ def test_scores_deployed_document_pointer_answer_as_pass():
"and some will need source rows during apply. The proof-changing action is a reviewer/operator row-link audit " "and some will need source rows during apply. The proof-changing action is a reviewer/operator row-link audit "
"and guarded apply contract for the pending backlog. Next Cory-style follow-up: draft the prioritized review " "and guarded apply contract for the pending backlog. Next Cory-style follow-up: draft the prioritized review "
"packet." "packet."
+ PENDING_SOURCE_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -242,6 +310,7 @@ def test_scores_deployed_demo_tier_answer_as_pass():
"public.claim_evidence, or public.claim_edges and requires explicit operator authorization plus an apply tool " "public.claim_evidence, or public.claim_edges and requires explicit operator authorization plus an apply tool "
"and postflight readback. Tier 1 does not prove canonical public.* changed. Next Cory-style follow-up: choose " "and postflight readback. Tier 1 does not prove canonical public.* changed. Next Cory-style follow-up: choose "
"a safe staging canary or authorize a Tier 2 apply." "a safe staging canary or authorize a Tier 2 apply."
+ DEMO_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -256,6 +325,7 @@ def test_scores_deployed_demo_tier_does_not_show_claim_change_answer_as_pass():
"public.claim_edges. What it does NOT show: the claim actually changing, because the proposal sits in " "public.claim_edges. What it does NOT show: the claim actually changing, because the proposal sits in "
"pending_review and is not applied until a reviewer applies it. Next Cory-style follow-up: confirm whether " "pending_review and is not applied until a reviewer applies it. Next Cory-style follow-up: confirm whether "
"you want the demo staged now, and whether you want a canary edge or a real pending proposal." "you want the demo staged now, and whether you want a canary edge or a real pending proposal."
+ DEMO_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -269,6 +339,7 @@ def test_scores_deployed_soul_identity_answer_as_pass():
"identity changed requires new or updated rows in public.* plus a render/sync and postflight proof. A direct " "identity changed requires new or updated rows in public.* plus a render/sync and postflight proof. A direct "
"SOUL.md edit is not a canonical commit and not collective truth. Next Cory-style follow-up: inspect the " "SOUL.md edit is not a canonical commit and not collective truth. Next Cory-style follow-up: inspect the "
"SOUL.md diff and draft the corresponding KB proposal." "SOUL.md diff and draft the corresponding KB proposal."
+ APPLIED_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -283,6 +354,7 @@ def test_scores_deployed_soul_identity_does_not_touch_db_answer_as_pass():
"requires a proposal staged to kb_stage.kb_proposals, reviewer approval, applied to public.* with postflight " "requires a proposal staged to kb_stage.kb_proposals, reviewer approval, applied to public.* with postflight "
"proof, and SOUL.md regenerated from the updated DB rows. Next Cory-style follow-up: stage the specific edit " "proof, and SOUL.md regenerated from the updated DB rows. Next Cory-style follow-up: stage the specific edit "
"as a proposal and run the apply sequence with authorization." "as a proposal and run the apply sequence with authorization."
+ APPLIED_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -296,6 +368,7 @@ def test_scores_fresh_live_document_source_layer_answer_as_pass():
"maps. The blocker is layered: reviewer attention, source rows, proposal type mismatch, and missing apply " "maps. The blocker is layered: reviewer attention, source rows, proposal type mismatch, and missing apply "
"tooling. Next Cory-style follow-up: draft the apply-tooling spec or prep a review packet for the pending " "tooling. Next Cory-style follow-up: draft the apply-tooling spec or prep a review packet for the pending "
"queue." "queue."
+ PENDING_SOURCE_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -309,6 +382,7 @@ def test_scores_fresh_live_demo_apply_tooling_answer_as_pass():
"or public.claim_edges is not safe to demo from chat; it requires explicit operator/admin authorization, " "or public.claim_edges is not safe to demo from chat; it requires explicit operator/admin authorization, "
"apply tooling, and retained before/after postflight readback. Full loop is not yet, blocked on apply tooling. " "apply tooling, and retained before/after postflight readback. Full loop is not yet, blocked on apply tooling. "
"Next Cory-style follow-up: run the staging demo or draft teleo-kb apply-proposal." "Next Cory-style follow-up: run the staging demo or draft teleo-kb apply-proposal."
+ DEMO_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -323,6 +397,7 @@ def test_scores_fresh_live_demo_not_provable_from_chat_answer_as_pass():
"operator/admin authorization plus before/after postflight readback. Canonical KB change is not provable " "operator/admin authorization plus before/after postflight readback. Canonical KB change is not provable "
"from chat, and that apply path doesn't exist as a chat command. Next Cory-style follow-up: choose a safe " "from chat, and that apply path doesn't exist as a chat command. Next Cory-style follow-up: choose a safe "
"staging canary or authorize the prepared apply packet." "staging canary or authorize the prepared apply packet."
+ DEMO_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -336,6 +411,7 @@ def test_scores_fresh_live_soul_runtime_patch_answer_as_pass():
"public.claim_edges. A SOUL.md edit without a proposal, review, apply, and postflight sequence is a runtime " "public.claim_edges. A SOUL.md edit without a proposal, review, apply, and postflight sequence is a runtime "
"patch that cannot be audited and is at risk of being lost. Next Cory-style follow-up: stage the affected " "patch that cannot be audited and is at risk of being lost. Next Cory-style follow-up: stage the affected "
"claim IDs as a proposal and apply after review." "claim IDs as a proposal and apply after review."
+ APPLIED_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -349,6 +425,7 @@ def test_scores_fresh_live_soul_did_not_change_identity_answer_as_pass():
"public.personas, public.beliefs, or behavioral_rules. A direct SOUL.md edit stays local/runtime until a " "public.personas, public.beliefs, or behavioral_rules. A direct SOUL.md edit stays local/runtime until a "
"sync writes DB rows with before/after readback. Next Cory-style follow-up: audit the canonical identity " "sync writes DB rows with before/after readback. Next Cory-style follow-up: audit the canonical identity "
"rows, stage the missing proposal, or run the renderer/sync readback if an approved identity row exists." "rows, stage the missing proposal, or run the renderer/sync readback if an approved identity row exists."
+ APPLIED_READBACK
) )
assert bench.score_reply(prompt, reply)["pass"] is True assert bench.score_reply(prompt, reply)["pass"] is True
@ -428,7 +505,7 @@ def test_score_results_can_require_full_cory_style_catalog():
assert score["pass"] is True assert score["pass"] is True
def test_score_results_can_require_full_direct_claim_followup_catalog(): def test_generic_keyword_bundle_cannot_pass_full_direct_claim_catalog():
broad_reply = ( broad_reply = (
"I will separate proposed, pending_review, approved, applied, and not applied. Approved is not the same as " "I will separate proposed, pending_review, approved, applied, and not applied. Approved is not the same as "
"applied, and I will not claim the database changed until canonical readback confirms it. I will query " "applied, and I will not claim the database changed until canonical readback confirms it. I will query "
@ -460,7 +537,10 @@ def test_score_results_can_require_full_direct_claim_followup_catalog():
include_cory_style=True, include_cory_style=True,
include_direct_claim_followups=True, include_direct_claim_followups=True,
) )
assert score["pass"] is True direct_scores = [item for item in score["scores"] if item["prompt_id"].startswith("DC-")]
assert score["pass"] is False
assert len(direct_scores) == len(bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS)
assert all(item["signals"]["structured_db_readback"] is False for item in direct_scores)
def test_score_result_subset_labels_retained_single_prompt_as_partial_pass(): def test_score_result_subset_labels_retained_single_prompt_as_partial_pass():