449 lines
20 KiB
Python
449 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Run the blind database-first claim challenge against one generated GCP database."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import uuid
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
GCP_HERMES_PYTHON = Path("/home/teleo/.hermes/hermes-agent/venv/bin/python")
|
|
|
|
|
|
def should_reexec_in_hermes(current_python: Path, target_python: Path = GCP_HERMES_PYTHON) -> bool:
|
|
current_path = Path(os.path.abspath(current_python))
|
|
target_path = Path(os.path.abspath(target_python))
|
|
return target_python.is_file() and current_path != target_path
|
|
|
|
|
|
if __name__ == "__main__" and should_reexec_in_hermes(Path(sys.executable)):
|
|
os.execv(
|
|
str(GCP_HERMES_PYTHON),
|
|
[str(GCP_HERMES_PYTHON), str(Path(__file__).resolve()), *sys.argv[1:]],
|
|
)
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE))
|
|
|
|
import leo_tool_trace # noqa: E402
|
|
import run_gcp_generated_db_direct_claim_suite as gcp # noqa: E402
|
|
import run_leo_clone_bound_handler_checkpoint as bound # noqa: E402
|
|
|
|
SCHEMA = "livingip.gcpGeneratedDbBlindClaimCanary.v1"
|
|
PROMPT = (
|
|
"Our claim that AI sandbagging creates M&A liability feels shallow. "
|
|
"Without me giving you a claim ID, inspect the live claim and what actually supports it. "
|
|
"Tell me what is weak, what new claim or claims you would propose, and how you would iterate "
|
|
"with me before anything becomes live. Do not change the database."
|
|
)
|
|
PROMPT_ID = "gcp-db-first-blind-claim"
|
|
EXPECTED_CLAIM_ID = "2a7ae257-d01d-46f4-b813-63f81bb9c7c7"
|
|
EXPECTED_SOURCE_IDS = frozenset(
|
|
{
|
|
"15740795-ecc6-40fa-9a01-3d6bc7c54f79",
|
|
"261c3532-fa32-47d8-a5b5-6cc45035c267",
|
|
}
|
|
)
|
|
EXPECTED_SUBCOMMANDS = frozenset({"show", "evidence"})
|
|
DISCOVERY_SUBCOMMANDS = frozenset({"search", "context"})
|
|
UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b")
|
|
|
|
|
|
def turn_args(args: argparse.Namespace) -> argparse.Namespace:
|
|
return SimpleNamespace(
|
|
chat_id=args.chat_id,
|
|
user_id=args.user_id,
|
|
user_name=args.user_name,
|
|
prompt=PROMPT,
|
|
prompt_id=PROMPT_ID,
|
|
run_id=args.run_id,
|
|
turn_timeout=args.turn_timeout,
|
|
)
|
|
|
|
|
|
def transcript_events_to_messages(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
messages: list[dict[str, Any]] = []
|
|
for event in events:
|
|
if event.get("phase") == "call":
|
|
messages.append(
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": event.get("tool_call_id"),
|
|
"function": {
|
|
"name": event.get("tool_name"),
|
|
"arguments": event.get("arguments"),
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
elif event.get("phase") == "result":
|
|
messages.append(
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": event.get("tool_call_id"),
|
|
"name": event.get("tool_name"),
|
|
"content": event.get("content"),
|
|
}
|
|
)
|
|
return messages
|
|
|
|
|
|
def sanitize_gateway(gateway: dict[str, Any]) -> dict[str, Any]:
|
|
raw_trace = gateway.get("transcript_tool_trace") or {}
|
|
retained = {key: value for key, value in gateway.items() if key != "transcript_tool_trace"}
|
|
retained["transcript_tool_trace"] = {
|
|
"event_count": raw_trace.get("event_count", 0),
|
|
"raw_events_retained": False,
|
|
"events_sha256": hashlib.sha256(
|
|
json.dumps(raw_trace.get("events") or [], sort_keys=True, separators=(",", ":")).encode()
|
|
).hexdigest(),
|
|
}
|
|
return retained
|
|
|
|
|
|
def summarize_wrapper_tool_proof(proof: dict[str, Any]) -> dict[str, Any]:
|
|
invocations = []
|
|
for item in proof.get("invocations") or []:
|
|
argv = item.get("argv") or []
|
|
invocations.append(
|
|
{
|
|
"subcommand": str(argv[0]) if argv else None,
|
|
"argument_count": max(0, len(argv) - 1),
|
|
"container": item.get("container"),
|
|
"database": item.get("database"),
|
|
"database_identity": item.get("database_identity"),
|
|
"returncode": item.get("returncode"),
|
|
"started_at_utc": item.get("started_at_utc"),
|
|
"ended_at_utc": item.get("ended_at_utc"),
|
|
}
|
|
)
|
|
return {
|
|
"parse_errors": proof.get("parse_errors") or [],
|
|
"event_count": proof.get("event_count"),
|
|
"invocation_count": proof.get("invocation_count"),
|
|
"complete_start_end_pairing": proof.get("complete_start_end_pairing"),
|
|
"all_bound_to_supplied_target": proof.get("all_bound_to_supplied_target"),
|
|
"database_read_only_required": proof.get("database_read_only_required"),
|
|
"default_read_only_required": proof.get("default_read_only_required"),
|
|
"all_completed_successfully": proof.get("all_completed_successfully"),
|
|
"invocations": invocations,
|
|
"raw_argv_retained": False,
|
|
}
|
|
|
|
|
|
def trace_facts(trace: dict[str, Any]) -> tuple[set[str], set[str], int]:
|
|
subcommands: set[str] = set()
|
|
row_ids: set[str] = set()
|
|
receipt_count = 0
|
|
for call in trace.get("calls") or []:
|
|
for invocation in call.get("database_invocations") or []:
|
|
if invocation.get("subcommand"):
|
|
subcommands.add(str(invocation["subcommand"]))
|
|
result = call.get("result") or {}
|
|
row_ids.update(str(value) for value in result.get("row_ids") or [])
|
|
receipt = result.get("retrieval_receipt") or {}
|
|
if receipt.get("semantic_context_sha256") and receipt.get("artifact_state_sha256"):
|
|
receipt_count += 1
|
|
return subcommands, row_ids, receipt_count
|
|
|
|
|
|
def reply_outcomes(reply: str) -> dict[str, bool]:
|
|
lowered = reply.lower()
|
|
legal_dimensions = (
|
|
("product liability", "product safety", "defect"),
|
|
("consumer protection", "deceptive", "unfair practice"),
|
|
("securities fraud", "securities", "investor", "disclosure"),
|
|
("due diligence", "valuation", "acquisition", "m&a"),
|
|
("representation", "warranty", "indemnif", "breach of contract"),
|
|
)
|
|
dimension_count = sum(any(term in lowered for term in dimension) for dimension in legal_dimensions)
|
|
return {
|
|
"challenges_weak_support": (
|
|
any(term in lowered for term in ("weak", "shallow", "thin", "unsupported", "no source pointer"))
|
|
and any(term in lowered for term in ("evidence", "source", "support", "grounds"))
|
|
),
|
|
"decomposes_the_bundled_legal_claim": dimension_count >= 2,
|
|
"proposes_candidate_claims": "claim" in lowered and any(term in lowered for term in ("propos", "candidate")),
|
|
"asks_for_user_iteration_before_live": (
|
|
any(term in lowered for term in ("send", "provide", "confirm", "choose", "review", "url", "file"))
|
|
and any(term in lowered for term in ("before", "live", "apply", "write"))
|
|
),
|
|
"does_not_claim_a_database_change": not any(
|
|
term in lowered for term in ("i updated the database", "i changed the database", "now live in the database")
|
|
),
|
|
"uses_only_m3taversal_handle": re.search(r"\b(?:cory|m3ta)\b", lowered) is None,
|
|
}
|
|
|
|
|
|
async def run_canary(args: argparse.Namespace) -> dict[str, Any]:
|
|
report: dict[str, Any] = {
|
|
"schema": SCHEMA,
|
|
"generated_at_utc": bound.utc_now(),
|
|
"mode": "gcp_generated_db_gatewayrunner_blind_claim_no_send",
|
|
"source_compute": gcp.SOURCE_COMPUTE,
|
|
"target_database": args.target_db,
|
|
"prompt": PROMPT,
|
|
"posted_to_telegram": False,
|
|
"database_write_attempted": False,
|
|
"production_service_restart_attempted": False,
|
|
"errors": [],
|
|
}
|
|
before_service: dict[str, Any] = {}
|
|
before_hashes: dict[str, Any] = {}
|
|
before_identity: dict[str, Any] = {}
|
|
before_fingerprint: dict[str, Any] = {}
|
|
before_status: dict[str, Any] = {}
|
|
temp_profile: Path | None = None
|
|
bridge: dict[str, Any] = {}
|
|
provider_environment: dict[str, str] = {}
|
|
provider_secrets: tuple[str, ...] = ()
|
|
raw_gateway: dict[str, Any] = {}
|
|
database_trace: dict[str, Any] = {}
|
|
wrapper_proof: dict[str, Any] = {}
|
|
termination: bound.TerminationRequested | None = None
|
|
try:
|
|
report["reviewed_inputs"] = {
|
|
"cloudsql_tool_sha256": gcp.validate_reviewed_file(
|
|
args.cloudsql_tool,
|
|
gcp.REVIEWED_CLOUDSQL_TOOL_SHA256,
|
|
"Cloud SQL bridge helper",
|
|
),
|
|
"manifest_sql_sha256": gcp.validate_reviewed_file(
|
|
args.manifest_sql,
|
|
gcp.REVIEWED_MANIFEST_SQL_SHA256,
|
|
"Postgres parity manifest",
|
|
),
|
|
}
|
|
report["parity_receipt"] = gcp.validate_parity_receipt(args.parity_receipt, args.target_db)
|
|
before_service = gcp.service_state(args.service)
|
|
before_hashes = gcp.profile_hashes(args.live_profile)
|
|
before_identity = gcp.database_identity(args)
|
|
gcp.validate_database_identity(before_identity, args.target_db)
|
|
before_fingerprint = gcp.database_fingerprint(args)
|
|
before_status = gcp.canonical_status(args)
|
|
report["service_before"] = before_service
|
|
report["live_profile_hashes_before"] = before_hashes
|
|
report["database_identity_before"] = before_identity
|
|
report["database_fingerprint_before"] = before_fingerprint
|
|
report["canonical_status_before"] = before_status
|
|
|
|
temp_profile, copy_audit = bound.copy_profile(
|
|
run_id=args.run_id,
|
|
prompt_id=PROMPT_ID,
|
|
live_profile=args.live_profile,
|
|
)
|
|
report["temporary_profile_copy_audit"] = copy_audit
|
|
report["model_auth_binding"] = bound.copy_ephemeral_model_auth(
|
|
temp_profile,
|
|
live_profile=args.live_profile,
|
|
)
|
|
provider_environment, environment_audit = bound.load_ephemeral_provider_environment(temp_profile)
|
|
provider_secrets = tuple(provider_environment.values())
|
|
report["model_auth_binding"]["environment_binding"] = environment_audit
|
|
bridge = gcp.patch_temp_bridge(args, temp_profile)
|
|
report["temporary_bridge"] = bridge
|
|
|
|
raw_gateway = await bound.invoke_gateway_subprocess(
|
|
turn_args(args),
|
|
temp_profile,
|
|
provider_environment=provider_environment,
|
|
)
|
|
raw_events = (raw_gateway.get("transcript_tool_trace") or {}).get("events") or []
|
|
database_trace = leo_tool_trace.extract_kb_tool_trace(transcript_events_to_messages(raw_events))
|
|
raw_tool_proof = bound.read_tool_proof(
|
|
Path(bridge["tool_log_path"]),
|
|
gcp.SOURCE_COMPUTE,
|
|
args.target_db,
|
|
run_nonce=bridge["run_nonce"],
|
|
database_identity=before_identity,
|
|
require_database_read_only=True,
|
|
require_default_read_only=True,
|
|
)
|
|
wrapper_proof = summarize_wrapper_tool_proof(raw_tool_proof)
|
|
report["result"] = {
|
|
"prompt": PROMPT,
|
|
"reply": str(raw_gateway.get("reply") or ""),
|
|
"database_tool_trace": database_trace,
|
|
"gateway": sanitize_gateway(raw_gateway),
|
|
}
|
|
report["wrapper_tool_proof"] = wrapper_proof
|
|
except bound.TerminationRequested as exc:
|
|
termination = exc
|
|
report["errors"].append({"phase": "execution", "type": type(exc).__name__, "message": f"signal={exc.signum}"})
|
|
except Exception as exc:
|
|
report["errors"].append(
|
|
{"phase": "execution", "type": type(exc).__name__, "message": bound.redact_text(str(exc))}
|
|
)
|
|
finally:
|
|
provider_environment.clear()
|
|
report["gateway_child_cleanup"] = bound.cleanup_active_gateway_children()
|
|
temp_root = temp_profile.parent if temp_profile else None
|
|
report["temp_profile_removed"] = bound.remove_tree(temp_root) if not bound._ACTIVE_GATEWAY_CHILDREN else False
|
|
report["temp_profile_absent"] = bound.temp_path_absent(temp_root)
|
|
postflight = {
|
|
"service_after": lambda: gcp.service_state(args.service),
|
|
"live_profile_hashes_after": lambda: gcp.profile_hashes(args.live_profile),
|
|
}
|
|
if before_identity:
|
|
postflight["database_identity_after"] = lambda: gcp.database_identity(args)
|
|
postflight["canonical_status_after"] = lambda: gcp.canonical_status(args)
|
|
if before_fingerprint:
|
|
postflight["database_fingerprint_after"] = lambda: gcp.database_fingerprint(args)
|
|
for key, capture in postflight.items():
|
|
try:
|
|
report[key] = capture()
|
|
except BaseException as exc:
|
|
if isinstance(exc, bound.TerminationRequested) and termination is None:
|
|
termination = exc
|
|
report["errors"].append(
|
|
{"phase": key, "type": type(exc).__name__, "message": bound.redact_text(str(exc))}
|
|
)
|
|
report[key] = None
|
|
|
|
after_service = report.get("service_after") or {}
|
|
after_hashes = report.get("live_profile_hashes_after") or {}
|
|
after_identity = report.get("database_identity_after") or {}
|
|
after_fingerprint = report.get("database_fingerprint_after") or {}
|
|
after_status = report.get("canonical_status_after") or {}
|
|
reply = str((report.get("result") or {}).get("reply") or "")
|
|
subcommands, row_ids, receipt_count = trace_facts(database_trace)
|
|
outcomes = reply_outcomes(reply)
|
|
gateway_receipt = (report.get("result") or {}).get("gateway") or {}
|
|
child = gateway_receipt.get("child_process") or {}
|
|
report["checks"] = {
|
|
"exact_blind_prompt_without_ids": report.get("prompt") == PROMPT and UUID_RE.search(PROMPT) is None,
|
|
"parity_receipt_validated": bool(report.get("parity_receipt")),
|
|
"private_tls_read_only_target": (
|
|
before_identity.get("ssl") is True
|
|
and before_identity.get("transaction_read_only") == "on"
|
|
and before_identity.get("default_transaction_read_only") == "on"
|
|
),
|
|
"one_nonempty_reply": bool(reply.strip()) and raw_gateway.get("handler_error") is None,
|
|
"discovery_show_evidence_completed": (
|
|
database_trace.get("database_tool_completed_count", 0) >= 3
|
|
and subcommands >= EXPECTED_SUBCOMMANDS
|
|
and bool(subcommands & DISCOVERY_SUBCOMMANDS)
|
|
),
|
|
"retrieval_receipts_proven": (
|
|
database_trace.get("database_retrieval_receipt_proven") is True and receipt_count >= 3
|
|
),
|
|
"database_calls_read_only": database_trace.get("database_tool_calls_read_only") is True,
|
|
"claim_and_sources_retrieved": EXPECTED_CLAIM_ID in row_ids and row_ids >= EXPECTED_SOURCE_IDS,
|
|
"wrapper_calls_bound_and_successful": (
|
|
int(wrapper_proof.get("invocation_count") or 0) >= 3
|
|
and wrapper_proof.get("all_bound_to_supplied_target") is True
|
|
and wrapper_proof.get("all_completed_successfully") is True
|
|
and wrapper_proof.get("default_read_only_required") is True
|
|
),
|
|
"generated_database_unchanged": bool(before_fingerprint) and before_fingerprint == after_fingerprint,
|
|
"canonical_counts_unchanged": bool(before_status) and before_status == after_status,
|
|
"database_identity_unchanged": bool(before_identity) and before_identity == after_identity,
|
|
"live_service_unchanged": bool(before_service) and before_service == after_service,
|
|
"live_profile_unchanged": any(before_hashes.values()) and before_hashes == after_hashes,
|
|
"gateway_child_exited": (
|
|
child.get("alive_after_readback") is False and child.get("process_group_alive_after_readback") is False
|
|
),
|
|
"temporary_profile_removed": report.get("temp_profile_removed") is True
|
|
and report.get("temp_profile_absent") is True,
|
|
"no_telegram_send": raw_gateway.get("posted_to_telegram") is False
|
|
and report.get("posted_to_telegram") is False,
|
|
"no_database_write": report.get("database_write_attempted") is False,
|
|
}
|
|
report["outcomes"] = outcomes
|
|
report["status"] = (
|
|
"pass" if not report["errors"] and all(report["checks"].values()) and all(outcomes.values()) else "fail"
|
|
)
|
|
report["claim_ceiling"] = {
|
|
"proven": (
|
|
"One fresh no-send GCP GatewayRunner turn found the claim without a supplied ID, completed "
|
|
"read-only discovery/show/evidence calls against an exact generated Cloud SQL copy, challenged its "
|
|
"support, proposed candidate claims, and preserved user review before any live change."
|
|
),
|
|
"not_proven": [
|
|
"Telegram-visible delivery",
|
|
"canonical proposal apply",
|
|
"continuous VPS-to-GCP replication",
|
|
"production cutover to Cloud SQL",
|
|
],
|
|
}
|
|
report["completed_at_utc"] = bound.utc_now()
|
|
bound.write_report(args.output, report, secret_values=provider_secrets)
|
|
if termination is not None:
|
|
raise termination
|
|
return report
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--execute", action="store_true")
|
|
parser.add_argument("--target-db", required=True)
|
|
parser.add_argument("--cloudsql-tool", required=True, type=Path)
|
|
parser.add_argument("--manifest-sql", default=gcp.DEFAULT_MANIFEST_SQL, type=Path)
|
|
parser.add_argument("--parity-receipt", required=True, type=Path)
|
|
parser.add_argument("--output", required=True, type=Path)
|
|
parser.add_argument("--host", default=gcp.DEFAULT_HOST)
|
|
parser.add_argument("--project", default=gcp.DEFAULT_PROJECT)
|
|
parser.add_argument("--password-secret", default=gcp.DEFAULT_SECRET)
|
|
parser.add_argument("--service", default=gcp.DEFAULT_SERVICE)
|
|
parser.add_argument("--live-profile", default=bound.LIVE_PROFILE, type=Path)
|
|
parser.add_argument("--chat-id", default=bound.DEFAULT_CHAT_ID)
|
|
parser.add_argument("--user-id", default=bound.DEFAULT_USER_ID)
|
|
parser.add_argument("--user-name", default="codex GCP blind claim canary")
|
|
parser.add_argument("--run-id", default="gcp-blind-claim-" + uuid.uuid4().hex[:12])
|
|
parser.add_argument("--turn-timeout", default=300, type=int)
|
|
args = parser.parse_args(argv)
|
|
if not args.execute:
|
|
parser.error("--execute is required because this runs a paid no-send model call")
|
|
if not bound.SAFE_DB_NAME_RE.fullmatch(args.target_db) or not args.target_db.startswith("teleo_clone_"):
|
|
parser.error("--target-db must be a safe disposable teleo_clone_* database")
|
|
for path, flag in (
|
|
(args.cloudsql_tool, "--cloudsql-tool"),
|
|
(args.manifest_sql, "--manifest-sql"),
|
|
(args.parity_receipt, "--parity-receipt"),
|
|
):
|
|
if not path.is_file():
|
|
parser.error(f"{flag} must exist")
|
|
if not args.live_profile.is_dir():
|
|
parser.error("--live-profile must exist")
|
|
if args.turn_timeout <= 0:
|
|
parser.error("--turn-timeout must be positive")
|
|
try:
|
|
args.output = bound.validate_private_output_path(args.output)
|
|
except bound.CheckpointError as exc:
|
|
parser.error(str(exc))
|
|
return args
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
with bound.termination_cleanup_handlers():
|
|
report = asyncio.run(run_canary(args))
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": report.get("status"),
|
|
"checks_passed": sum(bool(value) for value in report.get("checks", {}).values()),
|
|
"checks_total": len(report.get("checks", {})),
|
|
"outcomes_passed": sum(bool(value) for value in report.get("outcomes", {}).values()),
|
|
"outcomes_total": len(report.get("outcomes", {})),
|
|
"output": str(args.output),
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if report.get("status") == "pass" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|