240 lines
10 KiB
Python
240 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify and condense one live VPS database-first out-of-sample canary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
SCHEMA = "livingip.leoDbFirstOosCanaryReceipt.v1"
|
|
EXPECTED_PROMPT = (
|
|
"Our claim that AI sandbagging creates M&A liability feels shallow. "
|
|
"Without me giving you a claim ID, inspect the live claim and what actually supports it. "
|
|
"Tell me what is weak, what new claim or claims you would propose, and how you would iterate "
|
|
"with me before anything becomes live. Do not change the database."
|
|
)
|
|
EXPECTED_CLAIM_ID = "2a7ae257-d01d-46f4-b813-63f81bb9c7c7"
|
|
EXPECTED_SOURCE_IDS = frozenset(
|
|
{
|
|
"15740795-ecc6-40fa-9a01-3d6bc7c54f79",
|
|
"261c3532-fa32-47d8-a5b5-6cc45035c267",
|
|
}
|
|
)
|
|
EXPECTED_SUBCOMMANDS = frozenset({"search", "show", "evidence"})
|
|
SHA_RE = re.compile(r"^[0-9a-f]{40}$")
|
|
UUID_RE = re.compile(
|
|
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b"
|
|
)
|
|
WORD_RE = re.compile(r"\b\w+(?:[-']\w+)*\b")
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _contains_all(value: str, terms: tuple[str, ...]) -> bool:
|
|
lowered = value.lower()
|
|
return all(term.lower() in lowered for term in terms)
|
|
|
|
|
|
def _tool_facts(trace: dict[str, Any]) -> tuple[set[str], set[str], bool]:
|
|
subcommands: set[str] = set()
|
|
row_ids: set[str] = set()
|
|
completed_without_error = True
|
|
for call in trace.get("calls") or []:
|
|
for invocation in call.get("database_invocations") or []:
|
|
subcommand = str(invocation.get("subcommand") or "")
|
|
if subcommand:
|
|
subcommands.add(subcommand)
|
|
result = call.get("result") or {}
|
|
row_ids.update(str(value) for value in result.get("row_ids") or [])
|
|
completed_without_error = completed_without_error and bool(
|
|
result.get("nonempty") is True and result.get("error_detected") is False
|
|
)
|
|
return subcommands, row_ids, completed_without_error
|
|
|
|
|
|
def verify_report(
|
|
report: dict[str, Any],
|
|
*,
|
|
deploy_head: str,
|
|
deploy_stamp: str,
|
|
source_report_sha256: str,
|
|
) -> dict[str, Any]:
|
|
results = report.get("results") or []
|
|
result = results[0] if len(results) == 1 and isinstance(results[0], dict) else {}
|
|
prompt = str(result.get("prompt") or "")
|
|
reply = str(result.get("reply") or "")
|
|
trace = result.get("database_tool_trace") or {}
|
|
subcommands, row_ids, completed_without_error = _tool_facts(trace)
|
|
context_records = result.get("database_context_trace") or []
|
|
post_record = next(
|
|
(record for record in reversed(context_records) if record.get("event") == "post_llm_call"),
|
|
{},
|
|
)
|
|
service = report.get("service_before_after") or {}
|
|
service_before = service.get("before") or {}
|
|
service_after = service.get("after") or {}
|
|
behavior_before = (report.get("live_behavior_manifest_before") or {}).get("behavior_sha256")
|
|
behavior_after = (report.get("live_behavior_manifest_after") or {}).get("behavior_sha256")
|
|
word_count = len(WORD_RE.findall(reply))
|
|
|
|
checks = {
|
|
"exact_blind_prompt": prompt == EXPECTED_PROMPT and not UUID_RE.search(prompt),
|
|
"one_nonempty_reply": len(results) == 1 and bool(reply.strip()),
|
|
"handler_completed": report.get("pass_runtime") is True,
|
|
"not_posted_to_telegram": report.get("posted_to_telegram") is False,
|
|
"harness_did_not_mutate_kb": report.get("mutates_kb_by_harness") is False,
|
|
"database_tool_call_proven": trace.get("database_tool_call_proven") is True,
|
|
"all_three_database_tools_completed": (
|
|
trace.get("database_tool_completed_count") == 3
|
|
and subcommands >= EXPECTED_SUBCOMMANDS
|
|
and completed_without_error
|
|
),
|
|
"database_tools_read_only": (
|
|
trace.get("database_tool_calls_read_only") is True
|
|
and trace.get("access_modes") == ["read_only"]
|
|
),
|
|
"retrieval_receipt_proven": trace.get("database_retrieval_receipt_proven") is True,
|
|
"claim_and_sources_returned": (
|
|
EXPECTED_CLAIM_ID in row_ids and row_ids >= EXPECTED_SOURCE_IDS
|
|
),
|
|
"claim_support_challenged": _contains_all(
|
|
reply,
|
|
("no_source_pointer", "grounds", "verified artifact", "illustrates"),
|
|
),
|
|
"shallow_claim_decomposed": (
|
|
_contains_all(reply, ("product liability", "consumer protection", "securities fraud"))
|
|
and any(term in reply.lower() for term in ("split", "decompos"))
|
|
),
|
|
"candidate_claims_proposed": _contains_all(
|
|
reply,
|
|
("empirical claims", "normative claim", "proposal packet"),
|
|
),
|
|
"user_iteration_requested_before_live": (
|
|
_contains_all(reply, ("url or file", "prepare-source"))
|
|
and any(term in reply.lower() for term in ("no writes yet", "before anything becomes live"))
|
|
),
|
|
"delivered_reply_within_budget": (
|
|
word_count <= 220
|
|
and report.get("database_response_validation_all_ok") is True
|
|
and post_record.get("delivered_issues") == []
|
|
and post_record.get("validated") is True
|
|
),
|
|
"canonical_counts_unchanged": (
|
|
report.get("db_counts_changed") is False
|
|
and report.get("db_counts_before") == report.get("db_counts_after")
|
|
),
|
|
"live_behavior_profile_unchanged": (
|
|
report.get("changed_live_profile") is False
|
|
and report.get("live_behavior_manifest_unchanged") is True
|
|
and bool(behavior_before)
|
|
and behavior_before == behavior_after
|
|
),
|
|
"gateway_process_unchanged": (
|
|
service.get("unchanged_from_preexisting_live_readback") is True
|
|
and service_before == service_after
|
|
and service_after.get("ActiveState") == "active"
|
|
and service_after.get("SubState") == "running"
|
|
and service_after.get("NRestarts") == "0"
|
|
),
|
|
"temporary_profile_removed": report.get("temp_profile_removed") is True,
|
|
"deploy_head_matches_stamp": (
|
|
bool(SHA_RE.fullmatch(deploy_head))
|
|
and deploy_head == deploy_stamp
|
|
),
|
|
}
|
|
outcomes = {
|
|
"found_the_relevant_claim_without_a_supplied_id": (
|
|
checks["exact_blind_prompt"] and checks["claim_and_sources_returned"]
|
|
),
|
|
"inspected_the_claim_and_its_actual_support": (
|
|
checks["all_three_database_tools_completed"] and checks["claim_support_challenged"]
|
|
),
|
|
"recognized_that_the_claim_was_shallow": checks["shallow_claim_decomposed"],
|
|
"proposed_better_candidate_claims": checks["candidate_claims_proposed"],
|
|
"kept_the_user_in_the_loop_before_any_live_change": checks["user_iteration_requested_before_live"],
|
|
"did_not_turn_the_test_conversation_into_hidden_training": (
|
|
checks["live_behavior_profile_unchanged"] and checks["temporary_profile_removed"]
|
|
),
|
|
"did_not_change_canonical_knowledge": checks["canonical_counts_unchanged"],
|
|
}
|
|
passed = all(checks.values()) and all(outcomes.values())
|
|
return {
|
|
"schema": SCHEMA,
|
|
"status": "pass" if passed else "fail",
|
|
"generated_at_utc": report.get("generated_at_utc"),
|
|
"proof_tier": "live_vps_gatewayrunner_temp_profile_no_telegram_post_no_apply",
|
|
"deployment": {
|
|
"deploy_head": deploy_head,
|
|
"deploy_stamp": deploy_stamp,
|
|
"head_matches_stamp": deploy_head == deploy_stamp,
|
|
},
|
|
"source_report_sha256": source_report_sha256,
|
|
"prompt": prompt,
|
|
"reply": reply,
|
|
"reply_sha256": hashlib.sha256(reply.encode("utf-8")).hexdigest(),
|
|
"reply_contract_word_count": word_count,
|
|
"checks": checks,
|
|
"outcomes": outcomes,
|
|
"database_tool_proof": {
|
|
"subcommands": sorted(subcommands),
|
|
"claim_id": EXPECTED_CLAIM_ID,
|
|
"source_ids": sorted(EXPECTED_SOURCE_IDS),
|
|
"trace": trace,
|
|
},
|
|
"state_readback": {
|
|
"database_counts_before": report.get("db_counts_before"),
|
|
"database_counts_after": report.get("db_counts_after"),
|
|
"behavior_sha256_before": behavior_before,
|
|
"behavior_sha256_after": behavior_after,
|
|
"service_before": service_before,
|
|
"service_after": service_after,
|
|
"temporary_profile_removed": report.get("temp_profile_removed"),
|
|
},
|
|
"claim_ceiling": {
|
|
"proven": (
|
|
"A fresh-session live VPS handler turn found an unfamiliar claim without a supplied ID, completed "
|
|
"read-only search/show/evidence calls against the canonical KB, challenged weak support, proposed "
|
|
"candidate replacements, and preserved the review-before-live boundary."
|
|
),
|
|
"not_proven": [
|
|
"Telegram-visible user-message delivery",
|
|
"canonical proposal apply",
|
|
"database-to-identity or SOUL recomposition",
|
|
"GCP runtime or database parity with this VPS state",
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--report", type=Path, required=True)
|
|
parser.add_argument("--deploy-head", required=True)
|
|
parser.add_argument("--deploy-stamp", required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
report = json.loads(args.report.read_text(encoding="utf-8"))
|
|
receipt = verify_report(
|
|
report,
|
|
deploy_head=args.deploy_head,
|
|
deploy_stamp=args.deploy_stamp,
|
|
source_report_sha256=_sha256_file(args.report),
|
|
)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(json.dumps({"output": str(args.output), "status": receipt["status"]}, sort_keys=True))
|
|
return 0 if receipt["status"] == "pass" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|