#!/usr/bin/env python3 """Verify and condense one live VPS database-first out-of-sample canary.""" from __future__ import annotations import argparse import hashlib import json import re from pathlib import Path from typing import Any SCHEMA = "livingip.leoDbFirstOosCanaryReceipt.v1" EXPECTED_PROMPT = ( "Our claim that AI sandbagging creates M&A liability feels shallow. " "Without me giving you a claim ID, inspect the live claim and what actually supports it. " "Tell me what is weak, what new claim or claims you would propose, and how you would iterate " "with me before anything becomes live. Do not change the database." ) EXPECTED_CLAIM_ID = "2a7ae257-d01d-46f4-b813-63f81bb9c7c7" EXPECTED_SOURCE_IDS = frozenset( { "15740795-ecc6-40fa-9a01-3d6bc7c54f79", "261c3532-fa32-47d8-a5b5-6cc45035c267", } ) EXPECTED_SUBCOMMANDS = frozenset({"search", "show", "evidence"}) SHA_RE = re.compile(r"^[0-9a-f]{40}$") UUID_RE = re.compile( r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b" ) WORD_RE = re.compile(r"\b\w+(?:[-']\w+)*\b") def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _contains_all(value: str, terms: tuple[str, ...]) -> bool: lowered = value.lower() return all(term.lower() in lowered for term in terms) def _tool_facts(trace: dict[str, Any]) -> tuple[set[str], set[str], bool]: subcommands: set[str] = set() row_ids: set[str] = set() completed_without_error = True for call in trace.get("calls") or []: for invocation in call.get("database_invocations") or []: subcommand = str(invocation.get("subcommand") or "") if subcommand: subcommands.add(subcommand) result = call.get("result") or {} row_ids.update(str(value) for value in result.get("row_ids") or []) completed_without_error = completed_without_error and bool( result.get("nonempty") is True and result.get("error_detected") is False ) return subcommands, row_ids, completed_without_error def verify_report( report: dict[str, Any], *, deploy_head: str, deploy_stamp: str, source_report_sha256: str, ) -> dict[str, Any]: results = report.get("results") or [] result = results[0] if len(results) == 1 and isinstance(results[0], dict) else {} prompt = str(result.get("prompt") or "") reply = str(result.get("reply") or "") trace = result.get("database_tool_trace") or {} subcommands, row_ids, completed_without_error = _tool_facts(trace) context_records = result.get("database_context_trace") or [] post_record = next( (record for record in reversed(context_records) if record.get("event") == "post_llm_call"), {}, ) service = report.get("service_before_after") or {} service_before = service.get("before") or {} service_after = service.get("after") or {} behavior_before = (report.get("live_behavior_manifest_before") or {}).get("behavior_sha256") behavior_after = (report.get("live_behavior_manifest_after") or {}).get("behavior_sha256") word_count = len(WORD_RE.findall(reply)) checks = { "exact_blind_prompt": prompt == EXPECTED_PROMPT and not UUID_RE.search(prompt), "one_nonempty_reply": len(results) == 1 and bool(reply.strip()), "handler_completed": report.get("pass_runtime") is True, "not_posted_to_telegram": report.get("posted_to_telegram") is False, "harness_did_not_mutate_kb": report.get("mutates_kb_by_harness") is False, "database_tool_call_proven": trace.get("database_tool_call_proven") is True, "all_three_database_tools_completed": ( trace.get("database_tool_completed_count") == 3 and subcommands >= EXPECTED_SUBCOMMANDS and completed_without_error ), "database_tools_read_only": ( trace.get("database_tool_calls_read_only") is True and trace.get("access_modes") == ["read_only"] ), "retrieval_receipt_proven": trace.get("database_retrieval_receipt_proven") is True, "claim_and_sources_returned": ( EXPECTED_CLAIM_ID in row_ids and row_ids >= EXPECTED_SOURCE_IDS ), "claim_support_challenged": _contains_all( reply, ("no_source_pointer", "grounds", "verified artifact", "illustrates"), ), "shallow_claim_decomposed": ( _contains_all(reply, ("product liability", "consumer protection", "securities fraud")) and any(term in reply.lower() for term in ("split", "decompos")) ), "candidate_claims_proposed": _contains_all( reply, ("empirical claims", "normative claim", "proposal packet"), ), "user_iteration_requested_before_live": ( _contains_all(reply, ("url or file", "prepare-source")) and any(term in reply.lower() for term in ("no writes yet", "before anything becomes live")) ), "delivered_reply_within_budget": ( word_count <= 220 and report.get("database_response_validation_all_ok") is True and post_record.get("delivered_issues") == [] and post_record.get("validated") is True ), "canonical_counts_unchanged": ( report.get("db_counts_changed") is False and report.get("db_counts_before") == report.get("db_counts_after") ), "live_behavior_profile_unchanged": ( report.get("changed_live_profile") is False and report.get("live_behavior_manifest_unchanged") is True and bool(behavior_before) and behavior_before == behavior_after ), "gateway_process_unchanged": ( service.get("unchanged_from_preexisting_live_readback") is True and service_before == service_after and service_after.get("ActiveState") == "active" and service_after.get("SubState") == "running" and service_after.get("NRestarts") == "0" ), "temporary_profile_removed": report.get("temp_profile_removed") is True, "deploy_head_matches_stamp": ( bool(SHA_RE.fullmatch(deploy_head)) and deploy_head == deploy_stamp ), } outcomes = { "found_the_relevant_claim_without_a_supplied_id": ( checks["exact_blind_prompt"] and checks["claim_and_sources_returned"] ), "inspected_the_claim_and_its_actual_support": ( checks["all_three_database_tools_completed"] and checks["claim_support_challenged"] ), "recognized_that_the_claim_was_shallow": checks["shallow_claim_decomposed"], "proposed_better_candidate_claims": checks["candidate_claims_proposed"], "kept_the_user_in_the_loop_before_any_live_change": checks["user_iteration_requested_before_live"], "did_not_turn_the_test_conversation_into_hidden_training": ( checks["live_behavior_profile_unchanged"] and checks["temporary_profile_removed"] ), "did_not_change_canonical_knowledge": checks["canonical_counts_unchanged"], } passed = all(checks.values()) and all(outcomes.values()) return { "schema": SCHEMA, "status": "pass" if passed else "fail", "generated_at_utc": report.get("generated_at_utc"), "proof_tier": "live_vps_gatewayrunner_temp_profile_no_telegram_post_no_apply", "deployment": { "deploy_head": deploy_head, "deploy_stamp": deploy_stamp, "head_matches_stamp": deploy_head == deploy_stamp, }, "source_report_sha256": source_report_sha256, "prompt": prompt, "reply": reply, "reply_sha256": hashlib.sha256(reply.encode("utf-8")).hexdigest(), "reply_contract_word_count": word_count, "checks": checks, "outcomes": outcomes, "database_tool_proof": { "subcommands": sorted(subcommands), "claim_id": EXPECTED_CLAIM_ID, "source_ids": sorted(EXPECTED_SOURCE_IDS), "trace": trace, }, "state_readback": { "database_counts_before": report.get("db_counts_before"), "database_counts_after": report.get("db_counts_after"), "behavior_sha256_before": behavior_before, "behavior_sha256_after": behavior_after, "service_before": service_before, "service_after": service_after, "temporary_profile_removed": report.get("temp_profile_removed"), }, "claim_ceiling": { "proven": ( "A fresh-session live VPS handler turn found an unfamiliar claim without a supplied ID, completed " "read-only search/show/evidence calls against the canonical KB, challenged weak support, proposed " "candidate replacements, and preserved the review-before-live boundary." ), "not_proven": [ "Telegram-visible user-message delivery", "canonical proposal apply", "database-to-identity or SOUL recomposition", "GCP runtime or database parity with this VPS state", ], }, } def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--report", type=Path, required=True) parser.add_argument("--deploy-head", required=True) parser.add_argument("--deploy-stamp", required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() report = json.loads(args.report.read_text(encoding="utf-8")) receipt = verify_report( report, deploy_head=args.deploy_head, deploy_stamp=args.deploy_stamp, source_report_sha256=_sha256_file(args.report), ) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(json.dumps({"output": str(args.output), "status": receipt["status"]}, sort_keys=True)) return 0 if receipt["status"] == "pass" else 1 if __name__ == "__main__": raise SystemExit(main())