#!/usr/bin/env python3 """Run the blind database-first claim challenge against one generated GCP database.""" from __future__ import annotations import argparse import asyncio import hashlib import json import os import re import sys import uuid from pathlib import Path from types import SimpleNamespace from typing import Any GCP_HERMES_PYTHON = Path("/home/teleo/.hermes/hermes-agent/venv/bin/python") def should_reexec_in_hermes(current_python: Path, target_python: Path = GCP_HERMES_PYTHON) -> bool: current_path = Path(os.path.abspath(current_python)) target_path = Path(os.path.abspath(target_python)) return target_python.is_file() and current_path != target_path if __name__ == "__main__" and should_reexec_in_hermes(Path(sys.executable)): os.execv( str(GCP_HERMES_PYTHON), [str(GCP_HERMES_PYTHON), str(Path(__file__).resolve()), *sys.argv[1:]], ) HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) import leo_tool_trace # noqa: E402 import run_gcp_generated_db_direct_claim_suite as gcp # noqa: E402 import run_leo_clone_bound_handler_checkpoint as bound # noqa: E402 SCHEMA = "livingip.gcpGeneratedDbBlindClaimCanary.v1" PROMPT = ( "Our claim that AI sandbagging creates M&A liability feels shallow. " "Without me giving you a claim ID, inspect the live claim and what actually supports it. " "Tell me what is weak, what new claim or claims you would propose, and how you would iterate " "with me before anything becomes live. Do not change the database." ) PROMPT_ID = "gcp-db-first-blind-claim" EXPECTED_CLAIM_ID = "2a7ae257-d01d-46f4-b813-63f81bb9c7c7" EXPECTED_SOURCE_IDS = frozenset( { "15740795-ecc6-40fa-9a01-3d6bc7c54f79", "261c3532-fa32-47d8-a5b5-6cc45035c267", } ) EXPECTED_SUBCOMMANDS = frozenset({"show", "evidence"}) DISCOVERY_SUBCOMMANDS = frozenset({"search", "context"}) UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b") def turn_args(args: argparse.Namespace) -> argparse.Namespace: return SimpleNamespace( chat_id=args.chat_id, user_id=args.user_id, user_name=args.user_name, prompt=PROMPT, prompt_id=PROMPT_ID, run_id=args.run_id, turn_timeout=args.turn_timeout, ) def transcript_events_to_messages(events: list[dict[str, Any]]) -> list[dict[str, Any]]: messages: list[dict[str, Any]] = [] for event in events: if event.get("phase") == "call": messages.append( { "role": "assistant", "tool_calls": [ { "id": event.get("tool_call_id"), "function": { "name": event.get("tool_name"), "arguments": event.get("arguments"), }, } ], } ) elif event.get("phase") == "result": messages.append( { "role": "tool", "tool_call_id": event.get("tool_call_id"), "name": event.get("tool_name"), "content": event.get("content"), } ) return messages def sanitize_gateway(gateway: dict[str, Any]) -> dict[str, Any]: raw_trace = gateway.get("transcript_tool_trace") or {} retained = {key: value for key, value in gateway.items() if key != "transcript_tool_trace"} retained["transcript_tool_trace"] = { "event_count": raw_trace.get("event_count", 0), "raw_events_retained": False, "events_sha256": hashlib.sha256( json.dumps(raw_trace.get("events") or [], sort_keys=True, separators=(",", ":")).encode() ).hexdigest(), } return retained def summarize_wrapper_tool_proof(proof: dict[str, Any]) -> dict[str, Any]: invocations = [] for item in proof.get("invocations") or []: argv = item.get("argv") or [] invocations.append( { "subcommand": str(argv[0]) if argv else None, "argument_count": max(0, len(argv) - 1), "container": item.get("container"), "database": item.get("database"), "database_identity": item.get("database_identity"), "returncode": item.get("returncode"), "started_at_utc": item.get("started_at_utc"), "ended_at_utc": item.get("ended_at_utc"), } ) return { "parse_errors": proof.get("parse_errors") or [], "event_count": proof.get("event_count"), "invocation_count": proof.get("invocation_count"), "complete_start_end_pairing": proof.get("complete_start_end_pairing"), "all_bound_to_supplied_target": proof.get("all_bound_to_supplied_target"), "database_read_only_required": proof.get("database_read_only_required"), "default_read_only_required": proof.get("default_read_only_required"), "all_completed_successfully": proof.get("all_completed_successfully"), "invocations": invocations, "raw_argv_retained": False, } def trace_facts(trace: dict[str, Any]) -> tuple[set[str], set[str], int]: subcommands: set[str] = set() row_ids: set[str] = set() receipt_count = 0 for call in trace.get("calls") or []: for invocation in call.get("database_invocations") or []: if invocation.get("subcommand"): subcommands.add(str(invocation["subcommand"])) result = call.get("result") or {} row_ids.update(str(value) for value in result.get("row_ids") or []) receipt = result.get("retrieval_receipt") or {} if receipt.get("semantic_context_sha256") and receipt.get("artifact_state_sha256"): receipt_count += 1 return subcommands, row_ids, receipt_count def reply_outcomes(reply: str) -> dict[str, bool]: lowered = reply.lower() legal_dimensions = ( ("product liability", "product safety", "defect"), ("consumer protection", "deceptive", "unfair practice"), ("securities fraud", "securities", "investor", "disclosure"), ("due diligence", "valuation", "acquisition", "m&a"), ("representation", "warranty", "indemnif", "breach of contract"), ) dimension_count = sum(any(term in lowered for term in dimension) for dimension in legal_dimensions) return { "challenges_weak_support": ( any(term in lowered for term in ("weak", "shallow", "thin", "unsupported", "no source pointer")) and any(term in lowered for term in ("evidence", "source", "support", "grounds")) ), "decomposes_the_bundled_legal_claim": dimension_count >= 2, "proposes_candidate_claims": "claim" in lowered and any(term in lowered for term in ("propos", "candidate")), "asks_for_user_iteration_before_live": ( any(term in lowered for term in ("send", "provide", "confirm", "choose", "review", "url", "file")) and any(term in lowered for term in ("before", "live", "apply", "write")) ), "does_not_claim_a_database_change": not any( term in lowered for term in ("i updated the database", "i changed the database", "now live in the database") ), "uses_only_m3taversal_handle": re.search(r"\b(?:cory|m3ta)\b", lowered) is None, } async def run_canary(args: argparse.Namespace) -> dict[str, Any]: report: dict[str, Any] = { "schema": SCHEMA, "generated_at_utc": bound.utc_now(), "mode": "gcp_generated_db_gatewayrunner_blind_claim_no_send", "source_compute": gcp.SOURCE_COMPUTE, "target_database": args.target_db, "prompt": PROMPT, "posted_to_telegram": False, "database_write_attempted": False, "production_service_restart_attempted": False, "errors": [], } before_service: dict[str, Any] = {} before_hashes: dict[str, Any] = {} before_identity: dict[str, Any] = {} before_fingerprint: dict[str, Any] = {} before_status: dict[str, Any] = {} temp_profile: Path | None = None bridge: dict[str, Any] = {} provider_environment: dict[str, str] = {} provider_secrets: tuple[str, ...] = () raw_gateway: dict[str, Any] = {} database_trace: dict[str, Any] = {} wrapper_proof: dict[str, Any] = {} termination: bound.TerminationRequested | None = None try: report["reviewed_inputs"] = { "cloudsql_tool_sha256": gcp.validate_reviewed_file( args.cloudsql_tool, gcp.REVIEWED_CLOUDSQL_TOOL_SHA256, "Cloud SQL bridge helper", ), "manifest_sql_sha256": gcp.validate_reviewed_file( args.manifest_sql, gcp.REVIEWED_MANIFEST_SQL_SHA256, "Postgres parity manifest", ), } report["parity_receipt"] = gcp.validate_parity_receipt(args.parity_receipt, args.target_db) before_service = gcp.service_state(args.service) before_hashes = gcp.profile_hashes(args.live_profile) before_identity = gcp.database_identity(args) gcp.validate_database_identity(before_identity, args.target_db) before_fingerprint = gcp.database_fingerprint(args) before_status = gcp.canonical_status(args) report["service_before"] = before_service report["live_profile_hashes_before"] = before_hashes report["database_identity_before"] = before_identity report["database_fingerprint_before"] = before_fingerprint report["canonical_status_before"] = before_status temp_profile, copy_audit = bound.copy_profile( run_id=args.run_id, prompt_id=PROMPT_ID, live_profile=args.live_profile, ) report["temporary_profile_copy_audit"] = copy_audit report["model_auth_binding"] = bound.copy_ephemeral_model_auth( temp_profile, live_profile=args.live_profile, ) provider_environment, environment_audit = bound.load_ephemeral_provider_environment(temp_profile) provider_secrets = tuple(provider_environment.values()) report["model_auth_binding"]["environment_binding"] = environment_audit bridge = gcp.patch_temp_bridge(args, temp_profile) report["temporary_bridge"] = bridge raw_gateway = await bound.invoke_gateway_subprocess( turn_args(args), temp_profile, provider_environment=provider_environment, ) raw_events = (raw_gateway.get("transcript_tool_trace") or {}).get("events") or [] database_trace = leo_tool_trace.extract_kb_tool_trace(transcript_events_to_messages(raw_events)) raw_tool_proof = bound.read_tool_proof( Path(bridge["tool_log_path"]), gcp.SOURCE_COMPUTE, args.target_db, run_nonce=bridge["run_nonce"], database_identity=before_identity, require_database_read_only=True, require_default_read_only=True, ) wrapper_proof = summarize_wrapper_tool_proof(raw_tool_proof) report["result"] = { "prompt": PROMPT, "reply": str(raw_gateway.get("reply") or ""), "database_tool_trace": database_trace, "gateway": sanitize_gateway(raw_gateway), } report["wrapper_tool_proof"] = wrapper_proof except bound.TerminationRequested as exc: termination = exc report["errors"].append({"phase": "execution", "type": type(exc).__name__, "message": f"signal={exc.signum}"}) except Exception as exc: report["errors"].append( {"phase": "execution", "type": type(exc).__name__, "message": bound.redact_text(str(exc))} ) finally: provider_environment.clear() report["gateway_child_cleanup"] = bound.cleanup_active_gateway_children() temp_root = temp_profile.parent if temp_profile else None report["temp_profile_removed"] = bound.remove_tree(temp_root) if not bound._ACTIVE_GATEWAY_CHILDREN else False report["temp_profile_absent"] = bound.temp_path_absent(temp_root) postflight = { "service_after": lambda: gcp.service_state(args.service), "live_profile_hashes_after": lambda: gcp.profile_hashes(args.live_profile), } if before_identity: postflight["database_identity_after"] = lambda: gcp.database_identity(args) postflight["canonical_status_after"] = lambda: gcp.canonical_status(args) if before_fingerprint: postflight["database_fingerprint_after"] = lambda: gcp.database_fingerprint(args) for key, capture in postflight.items(): try: report[key] = capture() except BaseException as exc: if isinstance(exc, bound.TerminationRequested) and termination is None: termination = exc report["errors"].append( {"phase": key, "type": type(exc).__name__, "message": bound.redact_text(str(exc))} ) report[key] = None after_service = report.get("service_after") or {} after_hashes = report.get("live_profile_hashes_after") or {} after_identity = report.get("database_identity_after") or {} after_fingerprint = report.get("database_fingerprint_after") or {} after_status = report.get("canonical_status_after") or {} reply = str((report.get("result") or {}).get("reply") or "") subcommands, row_ids, receipt_count = trace_facts(database_trace) outcomes = reply_outcomes(reply) gateway_receipt = (report.get("result") or {}).get("gateway") or {} child = gateway_receipt.get("child_process") or {} report["checks"] = { "exact_blind_prompt_without_ids": report.get("prompt") == PROMPT and UUID_RE.search(PROMPT) is None, "parity_receipt_validated": bool(report.get("parity_receipt")), "private_tls_read_only_target": ( before_identity.get("ssl") is True and before_identity.get("transaction_read_only") == "on" and before_identity.get("default_transaction_read_only") == "on" ), "one_nonempty_reply": bool(reply.strip()) and raw_gateway.get("handler_error") is None, "discovery_show_evidence_completed": ( database_trace.get("database_tool_completed_count", 0) >= 3 and subcommands >= EXPECTED_SUBCOMMANDS and bool(subcommands & DISCOVERY_SUBCOMMANDS) ), "retrieval_receipts_proven": ( database_trace.get("database_retrieval_receipt_proven") is True and receipt_count >= 3 ), "database_calls_read_only": database_trace.get("database_tool_calls_read_only") is True, "claim_and_sources_retrieved": EXPECTED_CLAIM_ID in row_ids and row_ids >= EXPECTED_SOURCE_IDS, "wrapper_calls_bound_and_successful": ( int(wrapper_proof.get("invocation_count") or 0) >= 3 and wrapper_proof.get("all_bound_to_supplied_target") is True and wrapper_proof.get("all_completed_successfully") is True and wrapper_proof.get("default_read_only_required") is True ), "generated_database_unchanged": bool(before_fingerprint) and before_fingerprint == after_fingerprint, "canonical_counts_unchanged": bool(before_status) and before_status == after_status, "database_identity_unchanged": bool(before_identity) and before_identity == after_identity, "live_service_unchanged": bool(before_service) and before_service == after_service, "live_profile_unchanged": any(before_hashes.values()) and before_hashes == after_hashes, "gateway_child_exited": ( child.get("alive_after_readback") is False and child.get("process_group_alive_after_readback") is False ), "temporary_profile_removed": report.get("temp_profile_removed") is True and report.get("temp_profile_absent") is True, "no_telegram_send": raw_gateway.get("posted_to_telegram") is False and report.get("posted_to_telegram") is False, "no_database_write": report.get("database_write_attempted") is False, } report["outcomes"] = outcomes report["status"] = ( "pass" if not report["errors"] and all(report["checks"].values()) and all(outcomes.values()) else "fail" ) report["claim_ceiling"] = { "proven": ( "One fresh no-send GCP GatewayRunner turn found the claim without a supplied ID, completed " "read-only discovery/show/evidence calls against an exact generated Cloud SQL copy, challenged its " "support, proposed candidate claims, and preserved user review before any live change." ), "not_proven": [ "Telegram-visible delivery", "canonical proposal apply", "continuous VPS-to-GCP replication", "production cutover to Cloud SQL", ], } report["completed_at_utc"] = bound.utc_now() bound.write_report(args.output, report, secret_values=provider_secrets) if termination is not None: raise termination return report def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--execute", action="store_true") parser.add_argument("--target-db", required=True) parser.add_argument("--cloudsql-tool", required=True, type=Path) parser.add_argument("--manifest-sql", default=gcp.DEFAULT_MANIFEST_SQL, type=Path) parser.add_argument("--parity-receipt", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--host", default=gcp.DEFAULT_HOST) parser.add_argument("--project", default=gcp.DEFAULT_PROJECT) parser.add_argument("--password-secret", default=gcp.DEFAULT_SECRET) parser.add_argument("--service", default=gcp.DEFAULT_SERVICE) parser.add_argument("--live-profile", default=bound.LIVE_PROFILE, type=Path) parser.add_argument("--chat-id", default=bound.DEFAULT_CHAT_ID) parser.add_argument("--user-id", default=bound.DEFAULT_USER_ID) parser.add_argument("--user-name", default="codex GCP blind claim canary") parser.add_argument("--run-id", default="gcp-blind-claim-" + uuid.uuid4().hex[:12]) parser.add_argument("--turn-timeout", default=300, type=int) args = parser.parse_args(argv) if not args.execute: parser.error("--execute is required because this runs a paid no-send model call") if not bound.SAFE_DB_NAME_RE.fullmatch(args.target_db) or not args.target_db.startswith("teleo_clone_"): parser.error("--target-db must be a safe disposable teleo_clone_* database") for path, flag in ( (args.cloudsql_tool, "--cloudsql-tool"), (args.manifest_sql, "--manifest-sql"), (args.parity_receipt, "--parity-receipt"), ): if not path.is_file(): parser.error(f"{flag} must exist") if not args.live_profile.is_dir(): parser.error("--live-profile must exist") if args.turn_timeout <= 0: parser.error("--turn-timeout must be positive") try: args.output = bound.validate_private_output_path(args.output) except bound.CheckpointError as exc: parser.error(str(exc)) return args def main(argv: list[str] | None = None) -> int: args = parse_args(argv) with bound.termination_cleanup_handlers(): report = asyncio.run(run_canary(args)) print( json.dumps( { "status": report.get("status"), "checks_passed": sum(bool(value) for value in report.get("checks", {}).values()), "checks_total": len(report.get("checks", {})), "outcomes_passed": sum(bool(value) for value in report.get("outcomes", {}).values()), "outcomes_total": len(report.get("outcomes", {})), "output": str(args.output), }, sort_keys=True, ) ) return 0 if report.get("status") == "pass" else 1 if __name__ == "__main__": raise SystemExit(main())