Merge pull request #77 from living-ip/codex/gcp-canonical-clone-proof-20260711
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
Harden GCP clone-bound Cory replay
This commit is contained in:
commit
1a0abd882d
12 changed files with 1317 additions and 27 deletions
8
.github/workflows/ci.yml
vendored
8
.github/workflows/ci.yml
vendored
|
|
@ -55,9 +55,15 @@ jobs:
|
|||
telegram/approvals.py \
|
||||
scripts/check_crabbox_ci_contract.py \
|
||||
scripts/check_llm_refinement_contract.py \
|
||||
scripts/build_working_leo_cory_outcome_sandbox.py \
|
||||
scripts/replay_decision_engine_eval.py \
|
||||
scripts/prove_phase1b_local.py \
|
||||
scripts/run_gcp_generated_db_direct_claim_suite.py \
|
||||
scripts/run_leo_clone_bound_handler_checkpoint.py \
|
||||
scripts/working_leo_open_ended_benchmark.py \
|
||||
tests/test_agent_routing.py \
|
||||
tests/test_assemble_telegram_visible_direct_claim_capture_receipt.py \
|
||||
tests/test_build_working_leo_cory_outcome_sandbox.py \
|
||||
tests/test_decision_engine_replay.py \
|
||||
tests/test_evaluate_agent_routing.py \
|
||||
tests/test_gcp_artifact_workflow.py \
|
||||
|
|
@ -71,7 +77,9 @@ jobs:
|
|||
tests/test_gcp_iam_split_apply.py \
|
||||
tests/test_gcp_iam_split_plan.py \
|
||||
tests/test_gcp_readiness_workflow.py \
|
||||
tests/test_gcp_generated_db_direct_claim_suite.py \
|
||||
tests/test_verify_postgres_parity_manifest.py \
|
||||
tests/test_working_leo_open_ended_benchmark.py \
|
||||
tests/test_phase1b_end_to_end.py \
|
||||
tests/test_sqlite_to_postgres_dump.py \
|
||||
tests/test_sqlite_postgres_restore_canary_capsule.py \
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ select jsonb_build_object(
|
|||
'database_collation', (select datcollate from pg_database where datname = current_database()),
|
||||
'database_ctype', (select datctype from pg_database where datname = current_database()),
|
||||
'transaction_read_only', current_setting('transaction_read_only'),
|
||||
'server_address', inet_server_addr()::text,
|
||||
'server_address', host(inet_server_addr()),
|
||||
'server_port', inet_server_port(),
|
||||
'ssl', coalesce((select ssl from pg_stat_ssl where pid = pg_backend_pid()), false),
|
||||
'ssl_version', (select version from pg_stat_ssl where pid = pg_backend_pid()),
|
||||
|
|
|
|||
|
|
@ -214,8 +214,12 @@ def compare_manifests(
|
|||
return problems, details
|
||||
|
||||
|
||||
def canonical_ip(address: str) -> str:
|
||||
return str(ipaddress.ip_interface(address).ip)
|
||||
|
||||
|
||||
def is_rfc1918(address: str) -> bool:
|
||||
parsed = ipaddress.ip_address(address)
|
||||
parsed = ipaddress.ip_address(canonical_ip(address))
|
||||
return any(
|
||||
parsed in network
|
||||
for network in (
|
||||
|
|
@ -248,6 +252,8 @@ def load_connectivity(
|
|||
target_database = target_identity.get("database")
|
||||
target_address = target_identity.get("server_address")
|
||||
target_ssl = target_identity.get("ssl")
|
||||
canonical_target_address = None
|
||||
canonical_proof_address = None
|
||||
if not target_database:
|
||||
problems.append("gcp_target_manifest_database_missing")
|
||||
if proof.get("target_database") != target_database:
|
||||
|
|
@ -256,11 +262,19 @@ def load_connectivity(
|
|||
problems.append("gcp_target_manifest_server_address_missing")
|
||||
else:
|
||||
try:
|
||||
canonical_target_address = canonical_ip(str(target_address))
|
||||
if not is_rfc1918(str(target_address)):
|
||||
problems.append("gcp_target_manifest_server_address_not_rfc1918")
|
||||
except ValueError:
|
||||
problems.append("gcp_target_manifest_server_address_invalid")
|
||||
if proof.get("server_address") != target_address:
|
||||
if proof.get("server_address"):
|
||||
try:
|
||||
canonical_proof_address = canonical_ip(str(proof["server_address"]))
|
||||
except ValueError:
|
||||
problems.append("gcp_connectivity_server_address_invalid")
|
||||
else:
|
||||
problems.append("gcp_connectivity_server_address_missing")
|
||||
if canonical_proof_address != canonical_target_address:
|
||||
problems.append("gcp_connectivity_server_address_mismatch")
|
||||
if target_ssl is not True:
|
||||
problems.append("gcp_target_manifest_tls_not_proven")
|
||||
|
|
|
|||
|
|
@ -115,7 +115,8 @@ DC_REPLIES: dict[str, str] = {
|
|||
"public.claim_edges, and public.claim_evidence, plus kb_stage.kb_proposals applied_at/readback. I need "
|
||||
"row-level before/after counts, row ids, and postflight proof; I cannot claim a DB update without proof. "
|
||||
"Follow-up: I would ask which proposal or time window Cory means, then query kb_stage and public.* and "
|
||||
"either show the pending gap or prepare the next guarded apply packet."
|
||||
"either show the pending gap or prepare the next guarded apply packet. DB readback: Proposal: "
|
||||
"`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`."
|
||||
),
|
||||
"DC-02": (
|
||||
"Helmer's 7 Powers can be packet-ready or clone-proven without being canonical production state. I would "
|
||||
|
|
@ -123,7 +124,8 @@ DC_REPLIES: dict[str, str] = {
|
|||
"and the proposal ledger with postflight row counts before saying it is in Leo now. If production_apply "
|
||||
"is false, I cannot claim it is production-applied. Follow-up: I would show the readiness packet, ask for "
|
||||
"explicit authorization if Cory wants it applied, run the integrated order, then postflight and Telegram "
|
||||
"regression."
|
||||
"regression. DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; "
|
||||
"Applied at: `none`."
|
||||
),
|
||||
"DC-03": (
|
||||
"I would check the schema before claiming a decision-matrix approved anything. If matrix_voters, "
|
||||
|
|
@ -131,14 +133,17 @@ DC_REPLIES: dict[str, str] = {
|
|||
"and no tally approved this. The current proof should come from kb_stage.kb_proposals review/apply columns "
|
||||
"and canonical public.* row readback. I cannot claim a matrix result without proof. Follow-up: I would run a "
|
||||
"fresh schema readback, then show the proposal ledger and canonical rows instead of overclaiming a matrix "
|
||||
"result."
|
||||
"result. DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; "
|
||||
"Applied at: `none`."
|
||||
),
|
||||
"DC-04": (
|
||||
"Telegram file refs, document_evaluations, proposal source_ref, raw PDFs, and public.sources are separate. "
|
||||
"A pending proposal can have staging rows and file pointers but still not have canonical source rows, "
|
||||
"claim_evidence, or an apply contract. I will not claim document pointers are the only blocker without "
|
||||
"row-link proof. Follow-up: I would check and audit each proposal source_ref against public.sources, identify "
|
||||
"missing canonical rows, then stage a guarded source/evidence apply proposal for review."
|
||||
"missing canonical rows, then stage a guarded source/evidence apply proposal for review. DB readback: Proposal: "
|
||||
"`e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; "
|
||||
"Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`."
|
||||
),
|
||||
"DC-05": (
|
||||
"Demo answer: Leo can show live staging and strict existing-ID apply proof, but proposed, approved, applied, "
|
||||
|
|
@ -146,6 +151,7 @@ DC_REPLIES: dict[str, str] = {
|
|||
"without being production-applied. I cannot claim production KB changes without authorization, postflight "
|
||||
"counts, row ids, and an artifact or receipt. Follow-up: I would ask which demo tier Cory wants: show the "
|
||||
"existing live proof, run a safe staging canary, or authorize the prepared apply packet plus regression. "
|
||||
"DB readback: kb_proposals: 26; canary proposal status: `pending_review`; canonical rows were not changed."
|
||||
),
|
||||
"DC-06": (
|
||||
"A direct SOUL.md edit changes the runtime/profile artifact, not canonical identity by itself. Canonical "
|
||||
|
|
@ -153,7 +159,8 @@ DC_REPLIES: dict[str, str] = {
|
|||
"and public.strategy_node_anchors must have row-level proof, row ids, and verify/read back evidence before "
|
||||
"they render or sync into SOUL.md. Until readback and renderer proof exist, I cannot claim canonical identity "
|
||||
"changed. Follow-up: I would ask whether to audit the canonical identity rows, stage the missing DB proposal, "
|
||||
"or run the renderer/sync readback."
|
||||
"or run the renderer/sync readback. DB readback: Proposal: "
|
||||
"`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`."
|
||||
),
|
||||
}
|
||||
|
||||
|
|
|
|||
769
scripts/run_gcp_generated_db_direct_claim_suite.py
Normal file
769
scripts/run_gcp_generated_db_direct_claim_suite.py
Normal file
|
|
@ -0,0 +1,769 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Run the six no-send Cory direct-claim prompts against a generated GCP DB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
import run_leo_clone_bound_handler_checkpoint as bound # noqa: E402
|
||||
import working_leo_open_ended_benchmark as benchmark # noqa: E402
|
||||
|
||||
SCHEMA = "livingip.gcpGeneratedDbDirectClaimSuite.v1"
|
||||
SOURCE_COMPUTE = "teleo-prod-1"
|
||||
DEFAULT_SERVICE = "leoclean-gcp-prod-parallel.service"
|
||||
DEFAULT_HOST = "10.61.0.3"
|
||||
DEFAULT_PROJECT = "teleo-501523"
|
||||
DEFAULT_SECRET = "gcp-teleo-pgvector-standby-postgres-password"
|
||||
DEFAULT_MANIFEST_SQL = HERE.parent / "ops" / "postgres_parity_manifest.sql"
|
||||
REVIEWED_CLOUDSQL_TOOL_SHA256 = "f54040141551a22a9a39ab020abfd897963bc2a53205f00cd49d7212cbaffff1"
|
||||
REVIEWED_MANIFEST_SQL_SHA256 = "8b8cdc25d54fdd8de05eb38c6e4423d2836953eb6012d4545f5c9c71b5f0150a"
|
||||
|
||||
|
||||
def run(command: list[str], *, env: dict[str, str] | None = None, input_text: str | None = None) -> str:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
env=env,
|
||||
input=input_text,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
detail = bound.redact_text((completed.stderr or completed.stdout).strip())
|
||||
raise bound.CheckpointError(f"command failed ({completed.returncode}): {detail}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def service_state(service: str) -> dict[str, Any]:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"systemctl",
|
||||
"show",
|
||||
service,
|
||||
"-p",
|
||||
"ActiveState",
|
||||
"-p",
|
||||
"SubState",
|
||||
"-p",
|
||||
"MainPID",
|
||||
"-p",
|
||||
"NRestarts",
|
||||
"-p",
|
||||
"ExecMainStartTimestamp",
|
||||
"--no-pager",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
result: dict[str, Any] = {"returncode": completed.returncode}
|
||||
for line in completed.stdout.splitlines():
|
||||
key, separator, value = line.partition("=")
|
||||
if separator:
|
||||
result[key] = value
|
||||
if completed.returncode != 0:
|
||||
result["error"] = bound.redact_text(completed.stderr or completed.stdout)
|
||||
return result
|
||||
|
||||
|
||||
def profile_hashes(profile: Path) -> dict[str, str | None]:
|
||||
paths = {
|
||||
"wrapper": profile / "bin" / "teleo-kb",
|
||||
"cloudsql_tool": profile / "bin" / "cloudsql_memory_tool.py",
|
||||
"bridge_skill": profile / "skills" / "teleo-kb-bridge" / "SKILL.md",
|
||||
}
|
||||
result = {}
|
||||
for name, path in paths.items():
|
||||
try:
|
||||
result[name] = bound.sha256_bytes(path.read_bytes())
|
||||
except OSError:
|
||||
result[name] = None
|
||||
return result
|
||||
|
||||
|
||||
def database_password(args: argparse.Namespace) -> str:
|
||||
value = run(
|
||||
[
|
||||
"gcloud",
|
||||
"secrets",
|
||||
"versions",
|
||||
"access",
|
||||
"latest",
|
||||
f"--secret={args.password_secret}",
|
||||
f"--project={args.project}",
|
||||
]
|
||||
).strip()
|
||||
if not value:
|
||||
raise bound.CheckpointError("Cloud SQL password secret resolved to an empty value")
|
||||
return value
|
||||
|
||||
|
||||
def validate_reviewed_file(path: Path, expected_sha256: str, label: str) -> str:
|
||||
actual = bound.sha256_bytes(path.read_bytes())
|
||||
if actual != expected_sha256:
|
||||
raise bound.CheckpointError(f"{label} does not match the reviewed SHA-256")
|
||||
return actual
|
||||
|
||||
|
||||
def validate_parity_receipt(path: Path, target_db: str) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise bound.CheckpointError("canonical parity receipt is unreadable") from exc
|
||||
details = payload.get("details") or {}
|
||||
structural = details.get("structural_hashes") or {}
|
||||
checks = {
|
||||
"artifact": payload.get("artifact") == "canonical_postgres_parity_verification",
|
||||
"status": payload.get("status") == "pass",
|
||||
"problems_empty": payload.get("problems") == [],
|
||||
"target_database": details.get("target_database") == target_db,
|
||||
"nonempty_table_set": int(details.get("target_table_count") or 0) > 0,
|
||||
"nonempty_row_set": int(details.get("target_total_rows") or 0) > 0,
|
||||
"table_counts_equal": details.get("source_table_count") == details.get("target_table_count"),
|
||||
"row_counts_equal": details.get("source_total_rows") == details.get("target_total_rows"),
|
||||
"table_mismatches_empty": details.get("table_mismatches") == [],
|
||||
"role_mismatches_empty": details.get("application_role_mismatches") == {},
|
||||
"extension_mismatches_empty": details.get("required_extension_mismatches") == {},
|
||||
"performance_problems_empty": details.get("performance_problems") == [],
|
||||
"structural_hashes_equal": bool(structural)
|
||||
and all(item.get("source") == item.get("target") for item in structural.values()),
|
||||
}
|
||||
failed = sorted(name for name, ok in checks.items() if not ok)
|
||||
if failed:
|
||||
raise bound.CheckpointError(f"canonical parity receipt failed validation: {', '.join(failed)}")
|
||||
return {
|
||||
"sha256": bound.sha256_bytes(path.read_bytes()),
|
||||
"scope": payload.get("scope"),
|
||||
"target_database": details.get("target_database"),
|
||||
"target_table_count": details.get("target_table_count"),
|
||||
"target_total_rows": details.get("target_total_rows"),
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def run_target_psql(
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
input_text: str | None = None,
|
||||
sql_file: Path | None = None,
|
||||
) -> str:
|
||||
password = database_password(args)
|
||||
env = {
|
||||
**os.environ,
|
||||
"PGPASSWORD": password,
|
||||
"PGOPTIONS": "-c default_transaction_read_only=on",
|
||||
}
|
||||
command = [
|
||||
"psql",
|
||||
f"host={args.host} port=5432 dbname={args.target_db} user=postgres sslmode=require connect_timeout=8",
|
||||
"-X",
|
||||
"-Atq",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
]
|
||||
if sql_file is not None:
|
||||
command.extend(["-f", str(sql_file)])
|
||||
try:
|
||||
return run(command, env=env, input_text=input_text)
|
||||
finally:
|
||||
env.pop("PGPASSWORD", None)
|
||||
env.pop("PGOPTIONS", None)
|
||||
password = ""
|
||||
|
||||
|
||||
def database_identity(args: argparse.Namespace) -> dict[str, Any]:
|
||||
sql = """
|
||||
begin transaction read only;
|
||||
select jsonb_build_object(
|
||||
'current_database', current_database(),
|
||||
'system_identifier', (select system_identifier::text from pg_control_system()),
|
||||
'server_address', host(inet_server_addr()),
|
||||
'server_port', inet_server_port(),
|
||||
'ssl', coalesce((select ssl from pg_stat_ssl where pid=pg_backend_pid()), false),
|
||||
'transaction_read_only', current_setting('transaction_read_only'),
|
||||
'default_transaction_read_only', current_setting('default_transaction_read_only')
|
||||
)::text;
|
||||
rollback;
|
||||
"""
|
||||
output = run_target_psql(args, input_text=sql)
|
||||
rows = [line for line in output.splitlines() if line.strip().startswith("{")]
|
||||
if not rows:
|
||||
raise bound.CheckpointError("Cloud SQL identity query returned no JSON")
|
||||
return json.loads(rows[-1])
|
||||
|
||||
|
||||
def validate_database_identity(identity: dict[str, Any], target_db: str) -> None:
|
||||
if identity.get("current_database") != target_db:
|
||||
raise bound.CheckpointError("Cloud SQL identity does not match the generated target database")
|
||||
if identity.get("ssl") is not True:
|
||||
raise bound.CheckpointError("Cloud SQL target session is not protected by TLS")
|
||||
if identity.get("transaction_read_only") != "on":
|
||||
raise bound.CheckpointError("Cloud SQL identity receipt was not captured in a read-only transaction")
|
||||
if identity.get("default_transaction_read_only") != "on":
|
||||
raise bound.CheckpointError("Cloud SQL target session does not default every transaction to read-only")
|
||||
try:
|
||||
address = ipaddress.ip_address(str(identity.get("server_address") or ""))
|
||||
except ValueError as exc:
|
||||
raise bound.CheckpointError("Cloud SQL target did not expose a valid server address") from exc
|
||||
private_networks = (
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
)
|
||||
if not any(address in network for network in private_networks):
|
||||
raise bound.CheckpointError("Cloud SQL target address is not RFC1918-private")
|
||||
if not identity.get("system_identifier"):
|
||||
raise bound.CheckpointError("Cloud SQL target system identifier is missing")
|
||||
|
||||
|
||||
def canonical_status(args: argparse.Namespace) -> dict[str, Any]:
|
||||
sql = """
|
||||
begin transaction read only;
|
||||
select jsonb_build_object(
|
||||
'db_identity', current_database() || '|' || current_user,
|
||||
'schema_tables', (
|
||||
select coalesce(jsonb_object_agg(table_schema, table_count order by table_schema), '{}'::jsonb)
|
||||
from (
|
||||
select table_schema, count(*)::integer as table_count
|
||||
from information_schema.tables
|
||||
where table_schema in ('public', 'kb_stage')
|
||||
group by table_schema
|
||||
) counts
|
||||
),
|
||||
'high_signal_rows', jsonb_build_object(
|
||||
'agents', (select count(*) from public.agents),
|
||||
'personas', (select count(*) from public.personas),
|
||||
'strategies', (select count(*) from public.strategies),
|
||||
'beliefs', (select count(*) from public.beliefs),
|
||||
'claims', (select count(*) from public.claims),
|
||||
'claim_edges', (select count(*) from public.claim_edges),
|
||||
'claim_evidence', (select count(*) from public.claim_evidence),
|
||||
'sources', (select count(*) from public.sources),
|
||||
'kb_proposals', (select count(*) from kb_stage.kb_proposals)
|
||||
)
|
||||
)::text;
|
||||
rollback;
|
||||
"""
|
||||
output = run_target_psql(args, input_text=sql)
|
||||
rows = [line for line in output.splitlines() if line.strip().startswith("{")]
|
||||
if not rows:
|
||||
raise bound.CheckpointError("canonical status query returned no JSON")
|
||||
return json.loads(rows[-1])
|
||||
|
||||
|
||||
def normalize_database_manifest_rows(output: str) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for line_number, raw_line in enumerate(output.splitlines(), 1):
|
||||
line = raw_line.strip()
|
||||
if not line or line in {"BEGIN", "SET", "ROLLBACK"}:
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise bound.CheckpointError(f"database manifest line {line_number} is not JSON") from exc
|
||||
kind = row.get("kind")
|
||||
if kind == "performance":
|
||||
continue
|
||||
if kind == "identity":
|
||||
row.pop("captured_at", None)
|
||||
rows.append(row)
|
||||
if not rows or not any(row.get("kind") == "identity" for row in rows):
|
||||
raise bound.CheckpointError("database manifest is missing its identity row")
|
||||
table_rows = [row for row in rows if row.get("kind") == "table"]
|
||||
if not table_rows:
|
||||
raise bound.CheckpointError("database manifest contains no table rows")
|
||||
return sorted(
|
||||
rows,
|
||||
key=lambda row: (
|
||||
str(row.get("kind") or ""),
|
||||
str(row.get("schema") or ""),
|
||||
str(row.get("table") or ""),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def database_fingerprint(args: argparse.Namespace) -> dict[str, Any]:
|
||||
output = run_target_psql(args, sql_file=args.manifest_sql)
|
||||
rows = normalize_database_manifest_rows(output)
|
||||
payload = json.dumps(rows, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
||||
table_rows = [row for row in rows if row.get("kind") == "table"]
|
||||
identity = next(row for row in rows if row.get("kind") == "identity")
|
||||
return {
|
||||
"sha256": hashlib.sha256(payload.encode()).hexdigest(),
|
||||
"table_count": len(table_rows),
|
||||
"total_rows": sum(int(row.get("row_count") or 0) for row in table_rows),
|
||||
"database": identity.get("database"),
|
||||
"server_address": identity.get("server_address"),
|
||||
"ssl": identity.get("ssl"),
|
||||
"transaction_read_only": identity.get("transaction_read_only"),
|
||||
}
|
||||
|
||||
|
||||
def build_cloudsql_wrapper(
|
||||
*,
|
||||
cloudsql_tool: Path,
|
||||
tool_log: Path,
|
||||
target_db: str,
|
||||
host: str,
|
||||
project: str,
|
||||
password_secret: str,
|
||||
run_nonce: str,
|
||||
system_exec_path: str = bound.SYSTEM_EXEC_PATH,
|
||||
) -> str:
|
||||
python = str(Path(sys.executable).resolve())
|
||||
return f"""#!/bin/bash
|
||||
set -uo pipefail
|
||||
|
||||
CLOUDSQL_TOOL={shlex.quote(str(cloudsql_tool))}
|
||||
TOOL_LOG={shlex.quote(str(tool_log))}
|
||||
TARGET_DATABASE={shlex.quote(target_db)}
|
||||
TARGET_HOST={shlex.quote(host)}
|
||||
SOURCE_COMPUTE={shlex.quote(SOURCE_COMPUTE)}
|
||||
PROJECT={shlex.quote(project)}
|
||||
PASSWORD_SECRET={shlex.quote(password_secret)}
|
||||
RUN_NONCE={shlex.quote(run_nonce)}
|
||||
PYTHON={shlex.quote(python)}
|
||||
export PATH={shlex.quote(system_exec_path)}
|
||||
export CLOUDSDK_CONFIG=/home/teleo/.config/gcloud
|
||||
export PGOPTIONS="-c default_transaction_read_only=on"
|
||||
INVOCATION_ID="$($PYTHON -c 'import uuid; print(uuid.uuid4())')"
|
||||
if ! PGPASSWORD="$(gcloud secrets versions access latest --secret="$PASSWORD_SECRET" --project="$PROJECT")"; then
|
||||
exit 124
|
||||
fi
|
||||
export PGPASSWORD
|
||||
trap 'unset PGPASSWORD PGOPTIONS' EXIT
|
||||
DB_RECEIPT="$(psql "host=$TARGET_HOST port=5432 dbname=$TARGET_DATABASE user=postgres sslmode=require connect_timeout=8" -X -Atq -v ON_ERROR_STOP=1 <<'SQL'
|
||||
begin transaction read only;
|
||||
select jsonb_build_object(
|
||||
'current_database', current_database(),
|
||||
'system_identifier', (select system_identifier::text from pg_control_system()),
|
||||
'server_address', host(inet_server_addr()),
|
||||
'server_port', inet_server_port(),
|
||||
'ssl', coalesce((select ssl from pg_stat_ssl where pid=pg_backend_pid()), false),
|
||||
'transaction_read_only', current_setting('transaction_read_only'),
|
||||
'default_transaction_read_only', current_setting('default_transaction_read_only')
|
||||
)::text;
|
||||
rollback;
|
||||
SQL
|
||||
)"
|
||||
if [[ -z "$DB_RECEIPT" ]]; then
|
||||
unset PGPASSWORD
|
||||
exit 125
|
||||
fi
|
||||
|
||||
$PYTHON - "$TOOL_LOG" "$RUN_NONCE" "$INVOCATION_ID" "$SOURCE_COMPUTE" "$TARGET_DATABASE" "$DB_RECEIPT" "$@" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
path, run_nonce, invocation_id, compute, database, db_receipt, *argv = sys.argv[1:]
|
||||
event = {{
|
||||
"phase": "start",
|
||||
"at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"invocation_id": invocation_id,
|
||||
"run_nonce": run_nonce,
|
||||
"container": compute,
|
||||
"database": database,
|
||||
"database_identity": json.loads(db_receipt),
|
||||
"argv": argv,
|
||||
}}
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(event, sort_keys=True) + "\\n")
|
||||
PY
|
||||
|
||||
$PYTHON "$CLOUDSQL_TOOL" \
|
||||
--host "$TARGET_HOST" \
|
||||
--db "$TARGET_DATABASE" \
|
||||
--canonical-db "$TARGET_DATABASE" \
|
||||
--project "$PROJECT" \
|
||||
--password-secret "$PASSWORD_SECRET" \
|
||||
"$@"
|
||||
STATUS=$?
|
||||
|
||||
$PYTHON - "$TOOL_LOG" "$RUN_NONCE" "$INVOCATION_ID" "$SOURCE_COMPUTE" "$TARGET_DATABASE" "$STATUS" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
path, run_nonce, invocation_id, compute, database, status = sys.argv[1:]
|
||||
event = {{
|
||||
"phase": "end",
|
||||
"at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"invocation_id": invocation_id,
|
||||
"run_nonce": run_nonce,
|
||||
"container": compute,
|
||||
"database": database,
|
||||
"returncode": int(status),
|
||||
}}
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(event, sort_keys=True) + "\\n")
|
||||
PY
|
||||
|
||||
exit "$STATUS"
|
||||
"""
|
||||
|
||||
|
||||
def patch_temp_bridge(args: argparse.Namespace, temp_profile: Path) -> dict[str, Any]:
|
||||
wrapper = temp_profile / "bin" / "teleo-kb"
|
||||
target_tool = temp_profile / "bin" / "cloudsql_memory_tool.py"
|
||||
skill = temp_profile / "skills" / "teleo-kb-bridge" / "SKILL.md"
|
||||
tool_log = temp_profile / "gcp-checkpoint-tool-calls.jsonl"
|
||||
if not skill.is_file():
|
||||
raise bound.CheckpointError("temporary teleo-kb bridge skill is missing")
|
||||
shutil.copyfile(args.cloudsql_tool, target_tool)
|
||||
target_tool.chmod(0o700)
|
||||
run_nonce = uuid.uuid4().hex
|
||||
wrapper_text = build_cloudsql_wrapper(
|
||||
cloudsql_tool=target_tool,
|
||||
tool_log=tool_log,
|
||||
target_db=args.target_db,
|
||||
host=args.host,
|
||||
project=args.project,
|
||||
password_secret=args.password_secret,
|
||||
run_nonce=run_nonce,
|
||||
)
|
||||
binding = f"""
|
||||
## Generated GCP Database Checkpoint
|
||||
|
||||
This temporary no-send profile is bound only to private-TLS database
|
||||
`{args.target_db}` on `{args.host}` from `{SOURCE_COMPUTE}`. Every terminal call
|
||||
must start directly with the bare command `teleo-kb` and contain exactly one
|
||||
read command. Never use an absolute path, Python, shell variables, `which`,
|
||||
`env`, pipes, redirects, heredocs, cache files, or a second shell command. Use
|
||||
only search, context, show, evidence, edges, list-proposals,
|
||||
search-proposals, show-proposal, and decision-matrix-status. Keep list/search
|
||||
limits at 30 or below. For aggregate proposal status counts, call
|
||||
`teleo-kb decision-matrix-status`; do not parse cached output. Do not stage,
|
||||
approve, apply, write, send, or use live-profile paths. Distinguish pending,
|
||||
approved, applied, and canonical rows from database readback; never infer
|
||||
decision-matrix approval when its tables are absent. `status` and the restored
|
||||
audit fallback are unavailable in this clone-only profile. Every helper session
|
||||
is server-enforced with `default_transaction_read_only=on`.
|
||||
|
||||
For every direct-claim answer, use this sequence: direct yes/no/partly answer;
|
||||
exact proposal/canonical-row readback; what is not proven; then a final line
|
||||
beginning exactly `Next Cory-style follow-up:` with one concrete query, review,
|
||||
apply-authorization, renderer, or demo-tier action. When a proposal is approved
|
||||
but has no applied timestamp, say that approval is not application and that
|
||||
explicit operator authorization is required before apply. When discussing the
|
||||
matrix, say the conclusion comes from the fresh schema readback. When discussing
|
||||
identity, name the DB rows/readback needed in addition to the rendered file.
|
||||
"""
|
||||
skill_text = skill.read_text(encoding="utf-8")
|
||||
for live_wrapper in {
|
||||
"/home/teleo/.hermes/profiles/leoclean/bin/teleo-kb",
|
||||
str(args.live_profile / "bin" / "teleo-kb"),
|
||||
}:
|
||||
skill_text = skill_text.replace(live_wrapper, "teleo-kb")
|
||||
skill_text = bound.inject_skill_binding(skill_text, binding.lstrip())
|
||||
descriptor = os.open(tool_log, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
os.close(descriptor)
|
||||
bound._detach_and_write(wrapper, wrapper_text, mode=0o700)
|
||||
bound._detach_and_write(skill, skill_text, mode=0o600)
|
||||
return {
|
||||
"wrapper_path": str(wrapper),
|
||||
"tool_log_path": str(tool_log),
|
||||
"run_nonce": run_nonce,
|
||||
"wrapper_sha256": bound.sha256_bytes(wrapper_text.encode()),
|
||||
"cloudsql_tool_sha256": bound.sha256_bytes(target_tool.read_bytes()),
|
||||
"reviewed_cloudsql_tool_sha256": REVIEWED_CLOUDSQL_TOOL_SHA256,
|
||||
"bridge_skill_sha256": bound.sha256_bytes(skill_text.encode()),
|
||||
"source_compute": SOURCE_COMPUTE,
|
||||
"target_database": args.target_db,
|
||||
}
|
||||
|
||||
|
||||
def turn_args(args: argparse.Namespace, prompt: dict[str, Any]) -> argparse.Namespace:
|
||||
return SimpleNamespace(
|
||||
chat_id=args.chat_id,
|
||||
user_id=args.user_id,
|
||||
user_name=args.user_name,
|
||||
prompt=prompt["message"],
|
||||
prompt_id=prompt["id"],
|
||||
run_id=args.run_id,
|
||||
turn_timeout=args.turn_timeout,
|
||||
)
|
||||
|
||||
|
||||
def direct_prompts() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"id": row["id"], "dimension": row["dimension"], "message": row["message"]}
|
||||
for row in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
|
||||
]
|
||||
|
||||
|
||||
async def run_suite(args: argparse.Namespace) -> dict[str, Any]:
|
||||
report: dict[str, Any] = {
|
||||
"schema": SCHEMA,
|
||||
"generated_at_utc": bound.utc_now(),
|
||||
"mode": "gcp_generated_db_gatewayrunner_no_send",
|
||||
"source_compute": SOURCE_COMPUTE,
|
||||
"target_database": args.target_db,
|
||||
"private_tls": False,
|
||||
"posted_to_telegram": False,
|
||||
"production_service_restart_attempted": False,
|
||||
"database_write_attempted": False,
|
||||
"results": [],
|
||||
"errors": [],
|
||||
}
|
||||
before_service: dict[str, Any] = {}
|
||||
before_hashes: dict[str, str | None] = {}
|
||||
before_identity: dict[str, Any] = {}
|
||||
before_fingerprint: dict[str, Any] = {}
|
||||
before_status: dict[str, Any] = {}
|
||||
temp_profile: Path | None = None
|
||||
provider_environment: dict[str, str] = {}
|
||||
provider_secrets: tuple[str, ...] = ()
|
||||
bridge: dict[str, Any] = {}
|
||||
termination: bound.TerminationRequested | None = None
|
||||
try:
|
||||
tool_sha = validate_reviewed_file(
|
||||
args.cloudsql_tool,
|
||||
REVIEWED_CLOUDSQL_TOOL_SHA256,
|
||||
"Cloud SQL bridge helper",
|
||||
)
|
||||
manifest_sha = validate_reviewed_file(
|
||||
args.manifest_sql,
|
||||
REVIEWED_MANIFEST_SQL_SHA256,
|
||||
"Postgres parity manifest",
|
||||
)
|
||||
report["reviewed_inputs"] = {
|
||||
"cloudsql_tool_sha256": tool_sha,
|
||||
"manifest_sql_sha256": manifest_sha,
|
||||
}
|
||||
report["parity_receipt"] = validate_parity_receipt(args.parity_receipt, args.target_db)
|
||||
before_service = service_state(args.service)
|
||||
before_hashes = profile_hashes(args.live_profile)
|
||||
before_identity = database_identity(args)
|
||||
validate_database_identity(before_identity, args.target_db)
|
||||
report["private_tls"] = before_identity.get("ssl") is True
|
||||
before_fingerprint = database_fingerprint(args)
|
||||
before_status = canonical_status(args)
|
||||
report["service_before"] = before_service
|
||||
report["live_profile_hashes_before"] = before_hashes
|
||||
report["database_identity_before"] = before_identity
|
||||
report["database_fingerprint_before"] = before_fingerprint
|
||||
report["canonical_status_before"] = before_status
|
||||
temp_profile, copy_audit = bound.copy_profile(
|
||||
run_id=args.run_id,
|
||||
prompt_id="gcp-direct-claim",
|
||||
live_profile=args.live_profile,
|
||||
)
|
||||
report["temporary_profile_copy_audit"] = copy_audit
|
||||
report["model_auth_binding"] = bound.copy_ephemeral_model_auth(
|
||||
temp_profile,
|
||||
live_profile=args.live_profile,
|
||||
)
|
||||
provider_environment, environment_audit = bound.load_ephemeral_provider_environment(temp_profile)
|
||||
provider_secrets = tuple(provider_environment.values())
|
||||
report["model_auth_binding"]["environment_binding"] = environment_audit
|
||||
bridge = patch_temp_bridge(args, temp_profile)
|
||||
report["temporary_bridge"] = bridge
|
||||
for index, prompt in enumerate(direct_prompts(), 1):
|
||||
gateway = await bound.invoke_gateway_subprocess(
|
||||
turn_args(args, prompt),
|
||||
temp_profile,
|
||||
provider_environment=provider_environment,
|
||||
)
|
||||
report["results"].append(
|
||||
{
|
||||
"turn": index,
|
||||
"prompt_id": prompt["id"],
|
||||
"dimension": prompt["dimension"],
|
||||
"prompt": prompt["message"],
|
||||
"reply": gateway.get("reply") or "",
|
||||
"ok": bool(str(gateway.get("reply") or "").strip()),
|
||||
"mutates_kb": False,
|
||||
"evidence_tier": "gcp_generated_db_gatewayrunner_no_send",
|
||||
"gateway": gateway,
|
||||
}
|
||||
)
|
||||
expected_ids = [prompt["id"] for prompt in direct_prompts()]
|
||||
report["score"] = benchmark.score_result_subset(
|
||||
report["results"],
|
||||
catalog=benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS,
|
||||
expected_prompt_ids=expected_ids,
|
||||
)
|
||||
report["tool_proof"] = bound.read_tool_proof(
|
||||
Path(bridge["tool_log_path"]),
|
||||
SOURCE_COMPUTE,
|
||||
args.target_db,
|
||||
run_nonce=bridge["run_nonce"],
|
||||
database_identity=before_identity,
|
||||
require_database_read_only=True,
|
||||
require_default_read_only=True,
|
||||
)
|
||||
except bound.TerminationRequested as exc:
|
||||
termination = exc
|
||||
report["errors"].append(
|
||||
{"phase": "execution", "type": type(exc).__name__, "message": f"signal={exc.signum}"}
|
||||
)
|
||||
except Exception as exc:
|
||||
report["errors"].append(
|
||||
{"phase": "execution", "type": type(exc).__name__, "message": bound.redact_text(str(exc))}
|
||||
)
|
||||
finally:
|
||||
provider_environment.clear()
|
||||
report["gateway_child_cleanup"] = bound.cleanup_active_gateway_children()
|
||||
temp_root = temp_profile.parent if temp_profile else None
|
||||
cleanup_clear = not bound._ACTIVE_GATEWAY_CHILDREN
|
||||
report["temp_profile_removed"] = bound.remove_tree(temp_root) if cleanup_clear else False
|
||||
report["temp_profile_absent"] = bound.temp_path_absent(temp_root)
|
||||
report["temp_root_registry_clear"] = temp_root not in bound._ACTIVE_TEMP_ROOTS if temp_root else True
|
||||
postflight_steps = {
|
||||
"service_after": lambda: service_state(args.service),
|
||||
"live_profile_hashes_after": lambda: profile_hashes(args.live_profile),
|
||||
}
|
||||
if before_identity:
|
||||
postflight_steps["database_identity_after"] = lambda: database_identity(args)
|
||||
postflight_steps["canonical_status_after"] = lambda: canonical_status(args)
|
||||
if before_fingerprint and report.get("reviewed_inputs"):
|
||||
postflight_steps["database_fingerprint_after"] = lambda: database_fingerprint(args)
|
||||
for key, capture in postflight_steps.items():
|
||||
try:
|
||||
report[key] = capture()
|
||||
if key == "database_identity_after":
|
||||
validate_database_identity(report[key], args.target_db)
|
||||
except BaseException as exc:
|
||||
if isinstance(exc, bound.TerminationRequested) and termination is None:
|
||||
termination = exc
|
||||
report["errors"].append(
|
||||
{"phase": key, "type": type(exc).__name__, "message": bound.redact_text(str(exc))}
|
||||
)
|
||||
report[key] = None
|
||||
|
||||
after_service = report.get("service_after") or {}
|
||||
after_hashes = report.get("live_profile_hashes_after") or {}
|
||||
after_identity = report.get("database_identity_after") or {}
|
||||
after_fingerprint = report.get("database_fingerprint_after") or {}
|
||||
gateways = [row.get("gateway") or {} for row in report["results"]]
|
||||
allowed_tools = {"skills_list", "skill_view", "terminal"}
|
||||
tool_proof = report.get("tool_proof") or {}
|
||||
report["checks"] = {
|
||||
"reviewed_inputs_pinned": bool(report.get("reviewed_inputs")),
|
||||
"parity_receipt_validated": bool(report.get("parity_receipt")),
|
||||
"six_replies_returned": len(report["results"]) == 6 and all(row.get("ok") for row in report["results"]),
|
||||
"strict_score_passed": report.get("score", {}).get("pass") is True,
|
||||
"tool_calls_bound_read_only_and_successful": tool_proof.get("all_bound_to_supplied_target") is True
|
||||
and tool_proof.get("all_completed_successfully") is True
|
||||
and tool_proof.get("default_read_only_required") is True,
|
||||
"gateway_adapters_absent": bool(gateways)
|
||||
and all((gateway.get("tool_surface") or {}).get("gateway_adapter_count") == 0 for gateway in gateways),
|
||||
"send_message_tool_absent": bool(gateways)
|
||||
and all((gateway.get("tool_surface") or {}).get("send_message_tool_enabled") is False for gateway in gateways),
|
||||
"tool_registry_allowlisted": bool(gateways)
|
||||
and all(
|
||||
set((gateway.get("tool_surface") or {}).get("actual_registry_tools") or []) == allowed_tools
|
||||
for gateway in gateways
|
||||
),
|
||||
"all_children_exited": bool(gateways)
|
||||
and all(
|
||||
(gateway.get("child_process") or {}).get("alive_after_readback") is False
|
||||
and (gateway.get("child_process") or {}).get("process_group_alive_after_readback") is False
|
||||
for gateway in gateways
|
||||
),
|
||||
"generated_database_unchanged": bool(before_fingerprint)
|
||||
and before_fingerprint == after_fingerprint,
|
||||
"database_identity_unchanged": bool(before_identity) and before_identity == after_identity,
|
||||
"live_service_unchanged": bool(before_service) and before_service == after_service,
|
||||
"live_profile_unchanged": any(before_hashes.values()) and before_hashes == after_hashes,
|
||||
"temporary_profile_removed": report.get("temp_profile_removed") is True
|
||||
and report.get("temp_profile_absent") is True
|
||||
and report.get("temp_root_registry_clear") is True,
|
||||
"no_telegram_send": report.get("posted_to_telegram") is False,
|
||||
}
|
||||
report["status"] = "pass" if not report["errors"] and all(report["checks"].values()) else "fail"
|
||||
report["claim_ceiling"] = (
|
||||
"A pass proves six no-send direct-claim prompts through an adapter-free GatewayRunner temporary profile "
|
||||
"bound to one generated Cloud SQL database over private TLS. Actual helper sessions default to read-only, "
|
||||
"and the public/kb_stage catalog plus table-row fingerprints remain unchanged. It validates a supplied "
|
||||
"point-in-time parity receipt; it does not independently prove database creation provenance, Telegram "
|
||||
"delivery, separate-VM staging isolation, source extraction/apply composition, ongoing replication, or "
|
||||
"production cutover."
|
||||
)
|
||||
report["completed_at_utc"] = bound.utc_now()
|
||||
bound.write_report(args.output, report, secret_values=provider_secrets)
|
||||
if termination is not None:
|
||||
raise termination
|
||||
return report
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--execute", action="store_true")
|
||||
parser.add_argument("--target-db", required=True)
|
||||
parser.add_argument("--cloudsql-tool", required=True, type=Path)
|
||||
parser.add_argument("--manifest-sql", default=DEFAULT_MANIFEST_SQL, type=Path)
|
||||
parser.add_argument("--parity-receipt", required=True, type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
||||
parser.add_argument("--project", default=DEFAULT_PROJECT)
|
||||
parser.add_argument("--password-secret", default=DEFAULT_SECRET)
|
||||
parser.add_argument("--service", default=DEFAULT_SERVICE)
|
||||
parser.add_argument("--live-profile", default=bound.LIVE_PROFILE, type=Path)
|
||||
parser.add_argument("--chat-id", default=bound.DEFAULT_CHAT_ID)
|
||||
parser.add_argument("--user-id", default=bound.DEFAULT_USER_ID)
|
||||
parser.add_argument("--user-name", default="codex GCP generated DB checkpoint")
|
||||
parser.add_argument("--run-id", default="gcp-direct-claim-" + uuid.uuid4().hex[:12])
|
||||
parser.add_argument("--turn-timeout", default=300, type=int)
|
||||
args = parser.parse_args(argv)
|
||||
if not args.execute:
|
||||
parser.error("--execute is required because this runs paid model calls")
|
||||
if not bound.SAFE_DB_NAME_RE.fullmatch(args.target_db):
|
||||
parser.error("--target-db must be a safe Postgres database name")
|
||||
if not args.target_db.startswith("teleo_clone_"):
|
||||
parser.error("--target-db must start with teleo_clone_ and cannot name a live database")
|
||||
if not args.cloudsql_tool.is_file():
|
||||
parser.error("--cloudsql-tool must exist")
|
||||
if not args.manifest_sql.is_file():
|
||||
parser.error("--manifest-sql must exist")
|
||||
if not args.parity_receipt.is_file():
|
||||
parser.error("--parity-receipt must exist")
|
||||
if not args.live_profile.is_dir():
|
||||
parser.error("--live-profile must exist")
|
||||
if args.turn_timeout <= 0:
|
||||
parser.error("--turn-timeout must be positive")
|
||||
try:
|
||||
args.output = bound.validate_private_output_path(args.output)
|
||||
except bound.CheckpointError as exc:
|
||||
parser.error(str(exc))
|
||||
return args
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
with bound.termination_cleanup_handlers():
|
||||
report = asyncio.run(run_suite(args))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": report.get("status"),
|
||||
"score": (report.get("score") or {}).get("coverage"),
|
||||
"checks_passed": sum(bool(value) for value in report.get("checks", {}).values()),
|
||||
"checks_total": len(report.get("checks", {})),
|
||||
"output": str(args.output),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0 if report.get("status") == "pass" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -1062,6 +1062,7 @@ def validate_read_only_bridge_argv(argv: list[str]) -> None:
|
|||
|
||||
def command(name: str) -> argparse.ArgumentParser:
|
||||
item = subparsers.add_parser(name, add_help=False)
|
||||
item.add_argument("--help", action="store_true")
|
||||
item.add_argument("--format", choices=("markdown", "json"), default="markdown")
|
||||
return item
|
||||
|
||||
|
|
@ -1082,7 +1083,7 @@ def validate_read_only_bridge_argv(argv: list[str]) -> None:
|
|||
list_proposals = command("list-proposals")
|
||||
list_proposals.add_argument(
|
||||
"--status",
|
||||
choices=("pending_review", "approved", "applied", "canceled", "all"),
|
||||
choices=("pending_review", "approved", "applied", "canceled", "rejected", "all"),
|
||||
default="pending_review",
|
||||
)
|
||||
list_proposals.add_argument("--limit", type=int, default=10)
|
||||
|
|
@ -1090,7 +1091,7 @@ def validate_read_only_bridge_argv(argv: list[str]) -> None:
|
|||
search_proposals.add_argument("query")
|
||||
search_proposals.add_argument(
|
||||
"--status",
|
||||
choices=("pending_review", "approved", "applied", "canceled", "all"),
|
||||
choices=("pending_review", "approved", "applied", "canceled", "rejected", "all"),
|
||||
default="all",
|
||||
)
|
||||
search_proposals.add_argument("--limit", type=int, default=10)
|
||||
|
|
@ -1548,6 +1549,7 @@ def read_tool_proof(
|
|||
run_nonce: str,
|
||||
database_identity: dict[str, Any],
|
||||
require_database_read_only: bool = True,
|
||||
require_default_read_only: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
events: list[dict[str, Any]] = []
|
||||
parse_errors: list[str] = []
|
||||
|
|
@ -1604,6 +1606,10 @@ def read_tool_proof(
|
|||
not require_database_read_only
|
||||
or str((item["database_identity"] or {}).get("transaction_read_only")) == "on"
|
||||
)
|
||||
and (
|
||||
not require_default_read_only
|
||||
or str((item["database_identity"] or {}).get("default_transaction_read_only")) == "on"
|
||||
)
|
||||
for item in invocations
|
||||
)
|
||||
return {
|
||||
|
|
@ -1615,6 +1621,7 @@ def read_tool_proof(
|
|||
"invocation_count": len(invocations),
|
||||
"all_bound_to_supplied_target": complete_pairing and all_bound,
|
||||
"database_read_only_required": require_database_read_only,
|
||||
"default_read_only_required": require_default_read_only,
|
||||
"all_completed_successfully": complete_pairing
|
||||
and bool(invocations)
|
||||
and all(item["returncode"] == 0 for item in invocations),
|
||||
|
|
|
|||
|
|
@ -261,7 +261,14 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
|
|||
"Ask or run the row-level readback: which proposal/time window to inspect, kb_stage status, public.* "
|
||||
"before/after counts, and the next guarded apply packet if approved rows are still pending."
|
||||
),
|
||||
"required_signals": ["state_semantics", "canonical_db", "row_level_proof", "cory_followup", "no_overclaim"],
|
||||
"required_signals": [
|
||||
"state_semantics",
|
||||
"canonical_db",
|
||||
"row_level_proof",
|
||||
"structured_db_readback",
|
||||
"cory_followup",
|
||||
"no_overclaim",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "DC-02",
|
||||
|
|
@ -276,7 +283,14 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
|
|||
"Offer the next proof-changing action: show the packet/readiness artifact, get explicit apply "
|
||||
"authorization, run apply in integrated order, then postflight and Telegram regression."
|
||||
),
|
||||
"required_signals": ["canonical_db", "row_level_proof", "authorization", "cory_followup", "no_overclaim"],
|
||||
"required_signals": [
|
||||
"canonical_db",
|
||||
"row_level_proof",
|
||||
"structured_db_readback",
|
||||
"authorization",
|
||||
"cory_followup",
|
||||
"no_overclaim",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "DC-03",
|
||||
|
|
@ -291,7 +305,14 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
|
|||
"Follow up with the schema/readback request and then fall back to kb_stage.kb_proposals review/apply "
|
||||
"columns plus canonical row readback."
|
||||
),
|
||||
"required_signals": ["decision_matrix", "canonical_db", "fresh_readback", "cory_followup", "no_overclaim"],
|
||||
"required_signals": [
|
||||
"decision_matrix",
|
||||
"canonical_db",
|
||||
"fresh_readback",
|
||||
"structured_db_readback",
|
||||
"cory_followup",
|
||||
"no_overclaim",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "DC-04",
|
||||
|
|
@ -310,6 +331,7 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
|
|||
"document_artifact_linking",
|
||||
"staging_or_review",
|
||||
"canonical_db",
|
||||
"structured_db_readback",
|
||||
"cory_followup",
|
||||
"no_overclaim",
|
||||
],
|
||||
|
|
@ -330,6 +352,7 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
|
|||
"required_signals": [
|
||||
"state_semantics",
|
||||
"row_level_proof",
|
||||
"structured_db_readback",
|
||||
"authorization",
|
||||
"artifact",
|
||||
"cory_followup",
|
||||
|
|
@ -349,7 +372,14 @@ CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS: list[dict[str, Any]] = [
|
|||
"Follow up by asking whether to audit canonical identity rows, stage the missing DB proposal, or run the "
|
||||
"renderer/sync readback if an approved identity row already exists."
|
||||
),
|
||||
"required_signals": ["identity_rendering", "canonical_db", "row_level_proof", "cory_followup", "no_overclaim"],
|
||||
"required_signals": [
|
||||
"identity_rendering",
|
||||
"canonical_db",
|
||||
"row_level_proof",
|
||||
"structured_db_readback",
|
||||
"cory_followup",
|
||||
"no_overclaim",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
|
@ -361,6 +391,9 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
r"approved (?:is not|is not the same as|does not mean|!=|not the same as) applied|"
|
||||
r"not assume approval changed|approval changed .* canonical|approved[- ]not[- ]applied gap|"
|
||||
r"approval is a human gate, not a write|approved with zero canonical effect|"
|
||||
r"`?applied`? status (?:alone )?is not sufficient proof|status alone does not prove|"
|
||||
r"approved[\s\S]{0,120}applied at:\s*(?:none|-)|canonical truth still requires review/apply|"
|
||||
r"proposal staging plus reviewer/apply workflow|"
|
||||
r"applied_at:\s*NULL|no applied_at|NOTHING was applied|not applied|payload-only|"
|
||||
r"neither row has been promoted|canonical (?:layer|tables?|public\.\*) (?:is )?locked|"
|
||||
r"reviewable proposal|canonical apply step|demo tier|staging write|stages? a real write",
|
||||
|
|
@ -383,12 +416,29 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
r"\bapplied_at\b|superseded_by:\s*NULL|public\.[a-z_]+ rows?|"
|
||||
r"\bsingle canonical row\b|\bcanonical rows?\b|"
|
||||
r"\b[a-f0-9]{8}\b.*\b(?:approved|applied|pending|canceled)|"
|
||||
r"\b(?:approved|applied|pending|canceled)\b.*\b[a-f0-9]{8}\b",
|
||||
r"\b(?:approved|applied|pending|canceled)\b.*\b[a-f0-9]{8}\b|"
|
||||
r"\bproposal\b[\s\S]{0,180}\bstatus:\s*`?(?:approved|applied|pending_review|canceled)`?\b",
|
||||
re.I,
|
||||
),
|
||||
re.compile(
|
||||
r"\bquery|read ?back|verify|postflight|preflight|canonical state|applied_at|superseded_by|"
|
||||
r"canonical `?public\.\*`?|proposal ledger|verifiable readback|row appears|ledger\b",
|
||||
r"\bquery|read ?back|readback|verify|postflight|preflight|canonical state|applied_at|superseded_by|"
|
||||
r"canonical `?public\.\*`?|proposal ledger|verifiable readback|row appears|ledger\b|"
|
||||
r"\bproposal\b[\s\S]{0,220}\b[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}\b"
|
||||
r"[\s\S]{0,140}\bstatus:\s*`?(?:approved|applied|pending_review|canceled)`?\b",
|
||||
re.I,
|
||||
),
|
||||
],
|
||||
"structured_db_readback": [
|
||||
re.compile(
|
||||
r"\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\b|"
|
||||
r"\b(?:claims|sources|claim_edges|claim_evidence|kb_proposals)\b[^\d\n]{0,24}\d+",
|
||||
re.I,
|
||||
),
|
||||
re.compile(
|
||||
r"\bstatus:\s*`?(?:approved|applied|pending_review|canceled)`?|"
|
||||
r"\bapplied[_ ]at\s*:?\s*`?(?:none|-|\d{4}-\d{2}-\d{2})|"
|
||||
r"\bsource(?:_ref)?\s*:\s*`?[^\s`]+|"
|
||||
r"\b(?:claims|sources|claim_edges|claim_evidence|kb_proposals)\b[^\d\n]{0,24}\d+",
|
||||
re.I,
|
||||
),
|
||||
],
|
||||
|
|
@ -399,6 +449,8 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
"no_overclaim": [
|
||||
re.compile(
|
||||
r"\bnot (?:claim|assume|overclaim)|cannot (?:say|claim|overclaim)|do not know|without proof|if .* only\b|"
|
||||
r"\bhave not (?:yet )?verified\b|\bnot (?:fully )?landed\b|\bnot sufficient proof\b|"
|
||||
r"\bwould not demo it as\b|\bnot by itself\b|\bnot the whole cause\b|"
|
||||
r"\bonly ground truth\b|\bauthoritative test\b|candidate only|zero canonical effect|"
|
||||
r"\bno mutations? made\b|\bNOTHING was applied\b|\bout of scope\b|\bnot bundled\b|"
|
||||
r"\bcannot (?:approve|publish|move|sign)|\bzero unilateral action\b|"
|
||||
|
|
@ -412,6 +464,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
r"\bsource rows? (?:themselves )?(?:do not|don't) exist\b|"
|
||||
r"\bnothing has been inserted\b|\bcanonical apply step is the missing link\b|"
|
||||
r"\bnot shipped\b|\bnot approved by decision[- ]matrix\b|"
|
||||
r"\bdo not infer (?:proper )?matrix approval\b|\bno decision[- ]matrix approval record\b|"
|
||||
r"\bnot just (?:a )?pointer mismatch\b|\bdoes not prove\b|"
|
||||
r"\bdoes NOT show\b|\bdoes not show\b|"
|
||||
r"\bnot a canonical commit\b|\bnot collective truth\b|"
|
||||
|
|
@ -428,6 +481,8 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
),
|
||||
re.compile(
|
||||
r"\buntil\b.*\b(readback|proof|query|postflight|canonical)\b|"
|
||||
r"\bhave not (?:yet )?verified\b|\bnot (?:fully )?landed\b|\bnot sufficient proof\b|"
|
||||
r"\bwould not demo it as\b|\bnot by itself\b|\bnot the whole cause\b|"
|
||||
r"\bwithout\b.*\b(readback|proof|query|postflight|canonical)\b|"
|
||||
r"only ground truth.*canonical|canonical row.*authoritative test|readback confirms|zero canonical effect|"
|
||||
r"\bgovernance gap\b|\bno audit trail\b|\bwithout authorization\b|outside the PR-everything rule|"
|
||||
|
|
@ -443,6 +498,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
r"\bsource rows? (?:themselves )?(?:do not|don't) exist\b|"
|
||||
r"\bnothing has been inserted\b|\bcanonical apply step is the missing link\b|"
|
||||
r"\bnot shipped\b|\bnot approved by decision[- ]matrix\b|"
|
||||
r"\bdo not infer (?:proper )?matrix approval\b|\bno decision[- ]matrix approval record\b|"
|
||||
r"\bnot just (?:a )?pointer mismatch\b|\bdoes not prove\b|"
|
||||
r"\bdoes NOT show\b|\bdoes not show\b|"
|
||||
r"\bnot a canonical commit\b|\bnot collective truth\b|"
|
||||
|
|
@ -464,6 +520,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
r"\bapply\b.*\bguard|guarded\b.*\bapply|narrow\b.*\bapply\b|"
|
||||
r"\bstaged separately|out of scope|proposal containing|source evidence required|"
|
||||
r"\bmanual apply\b|operator tooling|admin tooling|admin review|"
|
||||
r"\breviewer/apply step\b|\bhuman/operator resolution\b|"
|
||||
r"\bnot just (?:a )?pointer mismatch\b|\brow-link audit\b|\bguarded apply contract\b|"
|
||||
r"\breviewer/operator\b|\bpending backlog\b",
|
||||
re.I,
|
||||
|
|
@ -483,13 +540,13 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
],
|
||||
"fresh_readback": [
|
||||
re.compile(
|
||||
r"\bcheck|read back|inspect|git status|rev-parse|systemctl|service|no \.git|not a git repo\b|"
|
||||
r"\bcheck|read ?back|readback|inspect|git status|rev-parse|systemctl|service|no \.git|not a git repo\b|"
|
||||
r"\bschema\b|\btables?\b",
|
||||
re.I,
|
||||
),
|
||||
re.compile(
|
||||
r"\bfresh|current|before claiming|last modified|latest mirrored|v0\.7\.0|MainPID|"
|
||||
r"schema (?:absent|doesn't exist|does not exist)|tables?.*absent|do not exist|"
|
||||
r"schema (?:is )?(?:absent|missing|doesn't exist|does not exist)|tables?.*absent|do not exist|"
|
||||
r"\bjust (?:ran|did)\b|\breadback we just ran\b",
|
||||
re.I,
|
||||
),
|
||||
|
|
@ -497,7 +554,8 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
"authorization": [
|
||||
re.compile(r"\bauthori[sz]ation|approval|explicit|permission|allowed|not allowed|admin|operator\b", re.I),
|
||||
re.compile(
|
||||
r"\bpublic update|post|capital|funds|sign|broadcast|admin review|operator pass|operator action|apply tool\b",
|
||||
r"\bpublic update|post|capital|funds|sign|broadcast|admin review|operator pass|operator action|apply tool|"
|
||||
r"operator authorization|operator-approved apply path|reviewer/apply workflow\b",
|
||||
re.I,
|
||||
),
|
||||
],
|
||||
|
|
@ -515,7 +573,7 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = {
|
|||
re.compile(r"\bdecision[- ]matrix|matrix_voters|proposal_votes|proposal_decisions|weighted voters?|tally\b", re.I),
|
||||
re.compile(
|
||||
r"\bnot (?:yet )?shipped|schema lacks|tables? (?:do not|does not|don't) exist|"
|
||||
r"tables?.*absent|schema absent|doesn't exist yet|check the schema|kb_stage\.kb_proposals\b",
|
||||
r"tables?.*absent|schema (?:is )?absent|doesn't exist yet|check the schema|kb_stage\.kb_proposals\b",
|
||||
re.I,
|
||||
),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
|
@ -12,6 +11,18 @@ sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
|||
|
||||
import assemble_telegram_visible_direct_claim_capture_receipt as receipt # noqa: E402
|
||||
|
||||
STRUCTURED_FIXTURE_READBACKS = {
|
||||
"DC-01": " DB readback: Proposal: f004bbb2-ac9a-481f-b7b8-74319373ba6a; Status: applied; Applied at: 2026-07-05.",
|
||||
"DC-02": " DB readback: Proposal: a64df080-8502-42e2-98f4-9bbdecb8da73; Status: approved; Applied at: none.",
|
||||
"DC-03": " DB readback: Proposal: a64df080-8502-42e2-98f4-9bbdecb8da73; Status: approved; Applied at: none.",
|
||||
"DC-04": (
|
||||
" DB readback: Proposal: e987b07c-9c23-45ba-b95c-d6d04c6536c7; Status: pending_review; "
|
||||
"Source_ref: telegram-resolved-attachment:doc_996e2e86cd94."
|
||||
),
|
||||
"DC-05": " DB readback: kb_proposals: 26; canary proposal status: pending_review; canonical rows unchanged.",
|
||||
"DC-06": " DB readback: Proposal: f004bbb2-ac9a-481f-b7b8-74319373ba6a; Status: applied; Applied at: 2026-07-05.",
|
||||
}
|
||||
|
||||
|
||||
def load_packet() -> dict:
|
||||
return json.loads(receipt.DEFAULT_PACKET.read_text(encoding="utf-8"))
|
||||
|
|
@ -30,7 +41,7 @@ def good_capture(packet: dict) -> dict:
|
|||
{
|
||||
"prompt_id": item["prompt_id"],
|
||||
"sent_text": item["message"],
|
||||
"reply": replies[item["prompt_id"]],
|
||||
"reply": replies[item["prompt_id"]] + STRUCTURED_FIXTURE_READBACKS[item["prompt_id"]],
|
||||
"sent_at_utc": f"2026-07-10T03:0{index}:00Z",
|
||||
"reply_at_utc": f"2026-07-10T03:0{index}:30Z",
|
||||
"telegram_message_id": f"sent-{index}",
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ def test_manifest_hash_order_is_collation_independent() -> None:
|
|||
manifest = Path("ops/postgres_parity_manifest.sql").read_text()
|
||||
|
||||
assert 'collate "C"' in manifest
|
||||
assert "'server_address', host(inet_server_addr())" in manifest
|
||||
|
||||
|
||||
def test_toc_counts_database_objects(tmp_path: Path) -> None:
|
||||
|
|
|
|||
335
tests/test_gcp_generated_db_direct_claim_suite.py
Normal file
335
tests/test_gcp_generated_db_direct_claim_suite.py
Normal file
|
|
@ -0,0 +1,335 @@
|
|||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.run_gcp_generated_db_direct_claim_suite import (
|
||||
REVIEWED_CLOUDSQL_TOOL_SHA256,
|
||||
REVIEWED_MANIFEST_SQL_SHA256,
|
||||
build_cloudsql_wrapper,
|
||||
database_fingerprint,
|
||||
direct_prompts,
|
||||
normalize_database_manifest_rows,
|
||||
run_suite,
|
||||
validate_database_identity,
|
||||
validate_parity_receipt,
|
||||
)
|
||||
from scripts.run_leo_clone_bound_handler_checkpoint import (
|
||||
READ_ONLY_BRIDGE_COMMANDS,
|
||||
read_tool_proof,
|
||||
validate_read_only_bridge_argv,
|
||||
)
|
||||
|
||||
|
||||
def test_direct_prompt_catalog_is_the_exact_six_case_suite() -> None:
|
||||
prompts = direct_prompts()
|
||||
|
||||
assert [prompt["id"] for prompt in prompts] == [f"DC-{index:02d}" for index in range(1, 7)]
|
||||
assert "status" not in READ_ONLY_BRIDGE_COMMANDS
|
||||
validate_read_only_bridge_argv(["list-proposals", "--help"])
|
||||
validate_read_only_bridge_argv(["list-proposals", "--status", "rejected"])
|
||||
|
||||
|
||||
def test_wrapper_binds_private_read_only_database_and_records_calls(tmp_path: Path) -> None:
|
||||
wrapper = build_cloudsql_wrapper(
|
||||
cloudsql_tool=tmp_path / "cloudsql_memory_tool.py",
|
||||
tool_log=tmp_path / "calls.jsonl",
|
||||
target_db="teleo_clone_test",
|
||||
host="10.61.0.3",
|
||||
project="teleo-501523",
|
||||
password_secret="test-secret-name",
|
||||
run_nonce="nonce",
|
||||
)
|
||||
|
||||
assert "dbname=$TARGET_DATABASE" in wrapper
|
||||
assert '--db "$TARGET_DATABASE"' in wrapper
|
||||
assert '--canonical-db "$TARGET_DATABASE"' in wrapper
|
||||
assert "sslmode=require" in wrapper
|
||||
assert "begin transaction read only" in wrapper
|
||||
assert 'export PGOPTIONS="-c default_transaction_read_only=on"' in wrapper
|
||||
assert "default_transaction_read_only" in wrapper
|
||||
assert "host(inet_server_addr())" in wrapper
|
||||
assert '"phase": "start"' in wrapper
|
||||
assert '"phase": "end"' in wrapper
|
||||
assert "unset PGPASSWORD" in wrapper
|
||||
assert "systemctl restart" not in wrapper
|
||||
assert "send_message" not in wrapper
|
||||
|
||||
|
||||
def test_database_identity_requires_private_tls_and_read_only_target() -> None:
|
||||
validate_database_identity(
|
||||
{
|
||||
"current_database": "teleo_clone_test",
|
||||
"system_identifier": "1234",
|
||||
"server_address": "10.61.0.3",
|
||||
"ssl": True,
|
||||
"transaction_read_only": "on",
|
||||
"default_transaction_read_only": "on",
|
||||
},
|
||||
"teleo_clone_test",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="not RFC1918-private"):
|
||||
validate_database_identity(
|
||||
{
|
||||
"current_database": "teleo_clone_test",
|
||||
"system_identifier": "1234",
|
||||
"server_address": "34.1.2.3",
|
||||
"ssl": True,
|
||||
"transaction_read_only": "on",
|
||||
"default_transaction_read_only": "on",
|
||||
},
|
||||
"teleo_clone_test",
|
||||
)
|
||||
|
||||
|
||||
def test_manifest_normalization_excludes_nondeterministic_timing() -> None:
|
||||
output = "\n".join(
|
||||
[
|
||||
'BEGIN',
|
||||
'{"kind":"identity","database":"teleo_clone_test","captured_at":"2026-07-12T00:00:00Z"}',
|
||||
'{"kind":"schemas","items":["kb_stage","public"]}',
|
||||
'{"kind":"table","schema":"public","table":"claims","row_count":2,"rowset_md5":"abc"}',
|
||||
'{"kind":"performance","query":"count_claims","elapsed_ms":1.2,"observed_rows":2}',
|
||||
'ROLLBACK',
|
||||
]
|
||||
)
|
||||
|
||||
rows = normalize_database_manifest_rows(output)
|
||||
|
||||
assert all(row.get("kind") != "performance" for row in rows)
|
||||
assert next(row for row in rows if row["kind"] == "identity").get("captured_at") is None
|
||||
|
||||
|
||||
def test_database_fingerprint_uses_full_normalized_manifest(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
manifest_sql = tmp_path / "manifest.sql"
|
||||
manifest_sql.write_text("select 1;\n")
|
||||
args = type(
|
||||
"Args",
|
||||
(),
|
||||
{
|
||||
"host": "10.61.0.3",
|
||||
"target_db": "teleo_clone_test",
|
||||
"manifest_sql": manifest_sql,
|
||||
},
|
||||
)()
|
||||
output = "\n".join(
|
||||
[
|
||||
'{"kind":"identity","database":"teleo_clone_test","server_address":"10.61.0.3",'
|
||||
'"ssl":true,"transaction_read_only":"on","captured_at":"one"}',
|
||||
'{"kind":"schemas","items":["kb_stage","public"]}',
|
||||
'{"kind":"table","schema":"public","table":"claims","row_count":2,"rowset_md5":"abc"}',
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.database_password", lambda _args: "secret")
|
||||
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.run", lambda *_args, **_kwargs: output)
|
||||
|
||||
receipt = database_fingerprint(args)
|
||||
|
||||
assert receipt["table_count"] == 1
|
||||
assert receipt["total_rows"] == 2
|
||||
assert receipt["database"] == "teleo_clone_test"
|
||||
assert len(receipt["sha256"]) == 64
|
||||
|
||||
|
||||
def test_reviewed_helper_hashes_match_repo_files() -> None:
|
||||
import hashlib
|
||||
|
||||
assert (
|
||||
hashlib.sha256(Path("hermes-agent/leoclean-bin/cloudsql_memory_tool.py").read_bytes()).hexdigest()
|
||||
== REVIEWED_CLOUDSQL_TOOL_SHA256
|
||||
)
|
||||
assert hashlib.sha256(Path("ops/postgres_parity_manifest.sql").read_bytes()).hexdigest() == REVIEWED_MANIFEST_SQL_SHA256
|
||||
|
||||
|
||||
def parity_payload(target_db: str = "teleo_clone_test") -> dict[str, object]:
|
||||
return {
|
||||
"artifact": "canonical_postgres_parity_verification",
|
||||
"status": "pass",
|
||||
"scope": "local_restore",
|
||||
"problems": [],
|
||||
"details": {
|
||||
"target_database": target_db,
|
||||
"source_table_count": 2,
|
||||
"target_table_count": 2,
|
||||
"source_total_rows": 3,
|
||||
"target_total_rows": 3,
|
||||
"table_mismatches": [],
|
||||
"application_role_mismatches": {},
|
||||
"required_extension_mismatches": {},
|
||||
"performance_problems": [],
|
||||
"structural_hashes": {"columns": {"source": "same", "target": "same"}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_parity_receipt_is_bound_to_generated_target(tmp_path: Path) -> None:
|
||||
receipt_path = tmp_path / "parity.json"
|
||||
receipt_path.write_text(json.dumps(parity_payload()))
|
||||
|
||||
receipt = validate_parity_receipt(receipt_path, "teleo_clone_test")
|
||||
|
||||
assert receipt["target_database"] == "teleo_clone_test"
|
||||
assert receipt["target_table_count"] == 2
|
||||
|
||||
with pytest.raises(RuntimeError, match="target_database"):
|
||||
validate_parity_receipt(receipt_path, "teleo_clone_other")
|
||||
|
||||
|
||||
def test_generated_wrapper_executes_only_clone_bound_default_read_only_tool(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
gcloud = fake_bin / "gcloud"
|
||||
gcloud.write_text("#!/bin/sh\nprintf 'fake-password\\n'\n")
|
||||
gcloud.chmod(0o700)
|
||||
psql = fake_bin / "psql"
|
||||
psql.write_text(
|
||||
"#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' "
|
||||
"'{\"current_database\":\"teleo_clone_test\",\"system_identifier\":\"system-1\","
|
||||
"\"server_address\":\"10.61.0.3\",\"ssl\":true,\"transaction_read_only\":\"on\","
|
||||
"\"default_transaction_read_only\":\"on\"}'\n"
|
||||
)
|
||||
psql.chmod(0o700)
|
||||
tool = tmp_path / "tool.py"
|
||||
tool.write_text(
|
||||
"import json, os, sys\n"
|
||||
"print(json.dumps({'argv': sys.argv[1:], 'pgoptions': os.environ.get('PGOPTIONS'), "
|
||||
"'has_password': bool(os.environ.get('PGPASSWORD'))}))\n"
|
||||
)
|
||||
log = tmp_path / "calls.jsonl"
|
||||
wrapper = tmp_path / "teleo-kb"
|
||||
wrapper.write_text(
|
||||
build_cloudsql_wrapper(
|
||||
cloudsql_tool=tool,
|
||||
tool_log=log,
|
||||
target_db="teleo_clone_test",
|
||||
host="10.61.0.3",
|
||||
project="teleo-501523",
|
||||
password_secret="test-secret",
|
||||
run_nonce="test-nonce",
|
||||
system_exec_path=f"{fake_bin}:/usr/bin:/bin",
|
||||
)
|
||||
)
|
||||
wrapper.chmod(0o700)
|
||||
|
||||
completed = subprocess.run(
|
||||
[str(wrapper), "decision-matrix-status"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env={**os.environ, "HOME": str(tmp_path)},
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
tool_receipt = json.loads(completed.stdout)
|
||||
assert tool_receipt["pgoptions"] == "-c default_transaction_read_only=on"
|
||||
assert tool_receipt["has_password"] is True
|
||||
assert tool_receipt["argv"][:6] == [
|
||||
"--host",
|
||||
"10.61.0.3",
|
||||
"--db",
|
||||
"teleo_clone_test",
|
||||
"--canonical-db",
|
||||
"teleo_clone_test",
|
||||
]
|
||||
events = [json.loads(line) for line in log.read_text().splitlines()]
|
||||
assert [event["phase"] for event in events] == ["start", "end"]
|
||||
assert events[0]["database_identity"]["default_transaction_read_only"] == "on"
|
||||
assert "fake-password" not in log.read_text()
|
||||
proof = read_tool_proof(
|
||||
log,
|
||||
"teleo-prod-1",
|
||||
"teleo_clone_test",
|
||||
run_nonce="test-nonce",
|
||||
database_identity={"system_identifier": "system-1"},
|
||||
require_database_read_only=True,
|
||||
require_default_read_only=True,
|
||||
)
|
||||
assert proof["all_bound_to_supplied_target"] is True
|
||||
events[0]["database_identity"]["default_transaction_read_only"] = "off"
|
||||
unsafe_log = tmp_path / "unsafe-calls.jsonl"
|
||||
unsafe_log.write_text("".join(json.dumps(event) + "\n" for event in events))
|
||||
unsafe_proof = read_tool_proof(
|
||||
unsafe_log,
|
||||
"teleo-prod-1",
|
||||
"teleo_clone_test",
|
||||
run_nonce="test-nonce",
|
||||
database_identity={"system_identifier": "system-1"},
|
||||
require_database_read_only=True,
|
||||
require_default_read_only=True,
|
||||
)
|
||||
assert unsafe_proof["all_bound_to_supplied_target"] is False
|
||||
|
||||
|
||||
def test_run_suite_retains_failure_report_when_postflight_fails(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
output = tmp_path / "report.json"
|
||||
placeholder = tmp_path / "placeholder"
|
||||
placeholder.write_text("x")
|
||||
args = argparse.Namespace(
|
||||
target_db="teleo_clone_test",
|
||||
cloudsql_tool=placeholder,
|
||||
manifest_sql=placeholder,
|
||||
parity_receipt=placeholder,
|
||||
service="leo.service",
|
||||
live_profile=tmp_path,
|
||||
output=output,
|
||||
chat_id="1",
|
||||
user_id="2",
|
||||
user_name="test",
|
||||
run_id="test-run",
|
||||
turn_timeout=1,
|
||||
host="10.61.0.3",
|
||||
project="teleo-501523",
|
||||
password_secret="secret-name",
|
||||
)
|
||||
identity_calls = 0
|
||||
|
||||
def fake_identity(_args: argparse.Namespace) -> dict[str, object]:
|
||||
nonlocal identity_calls
|
||||
identity_calls += 1
|
||||
if identity_calls > 1:
|
||||
raise RuntimeError("postflight unavailable")
|
||||
return {
|
||||
"current_database": "teleo_clone_test",
|
||||
"system_identifier": "system-1",
|
||||
"server_address": "10.61.0.3",
|
||||
"ssl": True,
|
||||
"transaction_read_only": "on",
|
||||
"default_transaction_read_only": "on",
|
||||
}
|
||||
|
||||
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.validate_reviewed_file", lambda *args: "hash")
|
||||
monkeypatch.setattr(
|
||||
"scripts.run_gcp_generated_db_direct_claim_suite.validate_parity_receipt",
|
||||
lambda *args: {"target_database": "teleo_clone_test"},
|
||||
)
|
||||
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.service_state", lambda *args: {"MainPID": "1"})
|
||||
monkeypatch.setattr(
|
||||
"scripts.run_gcp_generated_db_direct_claim_suite.profile_hashes",
|
||||
lambda *args: {"wrapper": "hash"},
|
||||
)
|
||||
monkeypatch.setattr("scripts.run_gcp_generated_db_direct_claim_suite.database_identity", fake_identity)
|
||||
monkeypatch.setattr(
|
||||
"scripts.run_gcp_generated_db_direct_claim_suite.database_fingerprint",
|
||||
lambda *args: {"sha256": "fingerprint"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"scripts.run_gcp_generated_db_direct_claim_suite.canonical_status",
|
||||
lambda *args: {"high_signal_rows": {"claims": 1}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"scripts.run_gcp_generated_db_direct_claim_suite.bound.copy_profile",
|
||||
lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("runner setup failed")),
|
||||
)
|
||||
|
||||
report = asyncio.run(run_suite(args))
|
||||
retained = json.loads(output.read_text())
|
||||
|
||||
assert report["status"] == "fail"
|
||||
assert retained["status"] == "fail"
|
||||
assert {error["phase"] for error in retained["errors"]} >= {"execution", "database_identity_after"}
|
||||
assert "gateway_child_cleanup" in retained
|
||||
|
|
@ -119,7 +119,7 @@ def test_gcp_scope_accepts_matching_private_connectivity_receipt(tmp_path: Path)
|
|||
completed, payload = run_verifier(
|
||||
tmp_path,
|
||||
manifest_rows(database="teleo"),
|
||||
manifest_rows(database="teleo_gcp_copy", server_address="10.61.0.3", ssl=True),
|
||||
manifest_rows(database="teleo_gcp_copy", server_address="10.61.0.3/32", ssl=True),
|
||||
scope="gcp_staging",
|
||||
connectivity={
|
||||
"status": "pass",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,21 @@ sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
|||
|
||||
import working_leo_open_ended_benchmark as bench # noqa: E402
|
||||
|
||||
APPLIED_READBACK = (
|
||||
" DB readback: Proposal: `f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; "
|
||||
"Applied at: `2026-07-05`."
|
||||
)
|
||||
HELMER_READBACK = (
|
||||
" DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; Applied at: `none`."
|
||||
)
|
||||
PENDING_SOURCE_READBACK = (
|
||||
" DB readback: Proposal: `e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; "
|
||||
"Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`."
|
||||
)
|
||||
DEMO_READBACK = (
|
||||
" DB readback: kb_proposals: 26; canary proposal status: `pending_review`; canonical rows were not changed."
|
||||
)
|
||||
|
||||
|
||||
def test_prompts_are_open_ended_and_not_id_led():
|
||||
assert len(bench.OPEN_ENDED_PROMPTS) >= 5
|
||||
|
|
@ -168,6 +183,7 @@ def test_scores_direct_claim_followup_answer_as_pass():
|
|||
"row ids, applied_at, and postflight proof. Until readback confirms canonical rows changed, I cannot claim "
|
||||
"the database updated. Follow-up: I would ask which proposal or time window Cory means, then show the pending "
|
||||
"gap or the next guarded apply packet."
|
||||
+ APPLIED_READBACK
|
||||
)
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
||||
|
|
@ -180,11 +196,59 @@ def test_scores_direct_claim_natural_db_split_answer_as_pass():
|
|||
"applied_at empty. Everything else is pending_review or canceled. The substantive work is staged and "
|
||||
"approved but not applied. Next admin action: run the apply tool or draft the review packet. Want me to "
|
||||
"pull the full packet?"
|
||||
+ APPLIED_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
||||
|
||||
def test_gcp_readback_phrasings_count_as_semantic_proof() -> None:
|
||||
assert bench.matched_signal(
|
||||
"Approved proposal a64df080 has Applied at: none; `applied` status is not sufficient proof of "
|
||||
"canonical truth, which still requires review/apply.",
|
||||
"state_semantics",
|
||||
)
|
||||
assert bench.matched_signal(
|
||||
"Readback from the current tables: the decision-matrix schema is absent and proposal_votes do not exist.",
|
||||
"fresh_readback",
|
||||
)
|
||||
assert bench.matched_signal(
|
||||
"The pending proposal needs a reviewer/apply step and human/operator resolution before guarded apply.",
|
||||
"staging_or_review",
|
||||
)
|
||||
assert bench.matched_signal(
|
||||
"Explicit operator authorization is required before the operator-approved apply path.",
|
||||
"authorization",
|
||||
)
|
||||
assert bench.matched_signal(
|
||||
"For the current situation, I see one relevant proposal:\n"
|
||||
"`f004bbb2-ac9a-481f-b7b8-74319373ba6a`\nStatus: `applied`",
|
||||
"row_level_proof",
|
||||
)
|
||||
assert bench.matched_signal(
|
||||
"Do not infer proper matrix approval from status alone; there is no decision-matrix approval record.",
|
||||
"no_overclaim",
|
||||
)
|
||||
assert bench.matched_signal(APPLIED_READBACK, "structured_db_readback")
|
||||
assert bench.matched_signal(
|
||||
"Proposal f004bbb2-ac9a-481f-b7b8-74319373ba6a was applied at `2026-07-05 00:14:36 UTC`.",
|
||||
"structured_db_readback",
|
||||
)
|
||||
|
||||
|
||||
def test_direct_claim_keyword_bundle_without_structured_readback_fails() -> None:
|
||||
prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-01")
|
||||
reply = (
|
||||
"Proposed, approved, and applied are distinct. I would query canonical public.claims and kb_stage proposals "
|
||||
"for counts and readback, avoid overclaiming, then ask Cory which guarded apply packet to review next."
|
||||
)
|
||||
|
||||
score = bench.score_reply(prompt, reply)
|
||||
|
||||
assert score["pass"] is False
|
||||
assert score["signals"]["structured_db_readback"] is False
|
||||
|
||||
|
||||
def test_scores_direct_claim_natural_helmer_answer_as_pass():
|
||||
prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-02")
|
||||
reply = (
|
||||
|
|
@ -192,6 +256,7 @@ def test_scores_direct_claim_natural_helmer_answer_as_pass():
|
|||
"cea989fc is canceled. Proof-changing action: apply a64df080 through the admin review panel or apply tool. "
|
||||
"Until applied_at is set and public.claims/public.claim_edges rows appear, it is only in the proposal ledger. "
|
||||
"Want me to pull the full packet for review?"
|
||||
+ HELMER_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -204,6 +269,7 @@ def test_scores_direct_claim_natural_matrix_absent_answer_as_pass():
|
|||
"tables are absent in both kb_stage and public. a64df080 has manual m3ta approval in kb_stage.kb_proposals, "
|
||||
"not a matrix vote, and it is not applied to canonical public.*. Next action: use the admin apply tool or "
|
||||
"pull the packet; want me to show it?"
|
||||
+ HELMER_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -216,6 +282,7 @@ def test_scores_deployed_matrix_readback_answer_as_pass():
|
|||
"kb_stage.matrix_voters, proposal_votes, and proposal_decisions are absent, and the public.* equivalents "
|
||||
"are absent too. a64df080 is m3ta reviewer sign-off in kb_stage.kb_proposals, not approved by "
|
||||
"decision-matrix vote. Next Cory-style follow-up: scope the schema or treat m3ta sign-off as sufficient."
|
||||
+ HELMER_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -229,6 +296,7 @@ def test_scores_deployed_document_pointer_answer_as_pass():
|
|||
"and some will need source rows during apply. The proof-changing action is a reviewer/operator row-link audit "
|
||||
"and guarded apply contract for the pending backlog. Next Cory-style follow-up: draft the prioritized review "
|
||||
"packet."
|
||||
+ PENDING_SOURCE_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -242,6 +310,7 @@ def test_scores_deployed_demo_tier_answer_as_pass():
|
|||
"public.claim_evidence, or public.claim_edges and requires explicit operator authorization plus an apply tool "
|
||||
"and postflight readback. Tier 1 does not prove canonical public.* changed. Next Cory-style follow-up: choose "
|
||||
"a safe staging canary or authorize a Tier 2 apply."
|
||||
+ DEMO_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -256,6 +325,7 @@ def test_scores_deployed_demo_tier_does_not_show_claim_change_answer_as_pass():
|
|||
"public.claim_edges. What it does NOT show: the claim actually changing, because the proposal sits in "
|
||||
"pending_review and is not applied until a reviewer applies it. Next Cory-style follow-up: confirm whether "
|
||||
"you want the demo staged now, and whether you want a canary edge or a real pending proposal."
|
||||
+ DEMO_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -269,6 +339,7 @@ def test_scores_deployed_soul_identity_answer_as_pass():
|
|||
"identity changed requires new or updated rows in public.* plus a render/sync and postflight proof. A direct "
|
||||
"SOUL.md edit is not a canonical commit and not collective truth. Next Cory-style follow-up: inspect the "
|
||||
"SOUL.md diff and draft the corresponding KB proposal."
|
||||
+ APPLIED_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -283,6 +354,7 @@ def test_scores_deployed_soul_identity_does_not_touch_db_answer_as_pass():
|
|||
"requires a proposal staged to kb_stage.kb_proposals, reviewer approval, applied to public.* with postflight "
|
||||
"proof, and SOUL.md regenerated from the updated DB rows. Next Cory-style follow-up: stage the specific edit "
|
||||
"as a proposal and run the apply sequence with authorization."
|
||||
+ APPLIED_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -296,6 +368,7 @@ def test_scores_fresh_live_document_source_layer_answer_as_pass():
|
|||
"maps. The blocker is layered: reviewer attention, source rows, proposal type mismatch, and missing apply "
|
||||
"tooling. Next Cory-style follow-up: draft the apply-tooling spec or prep a review packet for the pending "
|
||||
"queue."
|
||||
+ PENDING_SOURCE_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -309,6 +382,7 @@ def test_scores_fresh_live_demo_apply_tooling_answer_as_pass():
|
|||
"or public.claim_edges is not safe to demo from chat; it requires explicit operator/admin authorization, "
|
||||
"apply tooling, and retained before/after postflight readback. Full loop is not yet, blocked on apply tooling. "
|
||||
"Next Cory-style follow-up: run the staging demo or draft teleo-kb apply-proposal."
|
||||
+ DEMO_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -323,6 +397,7 @@ def test_scores_fresh_live_demo_not_provable_from_chat_answer_as_pass():
|
|||
"operator/admin authorization plus before/after postflight readback. Canonical KB change is not provable "
|
||||
"from chat, and that apply path doesn't exist as a chat command. Next Cory-style follow-up: choose a safe "
|
||||
"staging canary or authorize the prepared apply packet."
|
||||
+ DEMO_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -336,6 +411,7 @@ def test_scores_fresh_live_soul_runtime_patch_answer_as_pass():
|
|||
"public.claim_edges. A SOUL.md edit without a proposal, review, apply, and postflight sequence is a runtime "
|
||||
"patch that cannot be audited and is at risk of being lost. Next Cory-style follow-up: stage the affected "
|
||||
"claim IDs as a proposal and apply after review."
|
||||
+ APPLIED_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -349,6 +425,7 @@ def test_scores_fresh_live_soul_did_not_change_identity_answer_as_pass():
|
|||
"public.personas, public.beliefs, or behavioral_rules. A direct SOUL.md edit stays local/runtime until a "
|
||||
"sync writes DB rows with before/after readback. Next Cory-style follow-up: audit the canonical identity "
|
||||
"rows, stage the missing proposal, or run the renderer/sync readback if an approved identity row exists."
|
||||
+ APPLIED_READBACK
|
||||
)
|
||||
|
||||
assert bench.score_reply(prompt, reply)["pass"] is True
|
||||
|
|
@ -428,7 +505,7 @@ def test_score_results_can_require_full_cory_style_catalog():
|
|||
assert score["pass"] is True
|
||||
|
||||
|
||||
def test_score_results_can_require_full_direct_claim_followup_catalog():
|
||||
def test_generic_keyword_bundle_cannot_pass_full_direct_claim_catalog():
|
||||
broad_reply = (
|
||||
"I will separate proposed, pending_review, approved, applied, and not applied. Approved is not the same as "
|
||||
"applied, and I will not claim the database changed until canonical readback confirms it. I will query "
|
||||
|
|
@ -460,7 +537,10 @@ def test_score_results_can_require_full_direct_claim_followup_catalog():
|
|||
include_cory_style=True,
|
||||
include_direct_claim_followups=True,
|
||||
)
|
||||
assert score["pass"] is True
|
||||
direct_scores = [item for item in score["scores"] if item["prompt_id"].startswith("DC-")]
|
||||
assert score["pass"] is False
|
||||
assert len(direct_scores) == len(bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS)
|
||||
assert all(item["signals"]["structured_db_readback"] is False for item in direct_scores)
|
||||
|
||||
|
||||
def test_score_result_subset_labels_retained_single_prompt_as_partial_pass():
|
||||
|
|
|
|||
Loading…
Reference in a new issue