"""Tests for the live VPS database-first canary verifier.""" from __future__ import annotations import copy import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) import verify_leo_db_first_oos_canary as verifier # noqa: E402 def passing_report() -> dict: claim_id = verifier.EXPECTED_CLAIM_ID source_ids = sorted(verifier.EXPECTED_SOURCE_IDS) reply = ( "No writes yet. The grounds row has no_source_pointer. The verified artifact only illustrates the claim. " "The claim bundles product liability, consumer protection, and securities fraud, so split it into narrower " "empirical claims. Add a separate normative claim and keep it in a proposal packet. You bring the URL or " "file; I run prepare-source before anything becomes live." ) trace = { "database_tool_call_proven": True, "database_retrieval_receipt_proven": True, "database_tool_calls_read_only": True, "database_tool_completed_count": 3, "access_modes": ["read_only"], "calls": [ { "database_invocations": [{"subcommand": subcommand, "access_mode": "read_only"}], "result": { "nonempty": True, "error_detected": False, "row_ids": [claim_id, *source_ids], }, } for subcommand in sorted(verifier.EXPECTED_SUBCOMMANDS) ], } service = { "ActiveState": "active", "SubState": "running", "MainPID": "123", "NRestarts": "0", } counts = {"public.claims": 1, "public.sources": 2} behavior = {"behavior_sha256": "b" * 64} return { "generated_at_utc": "2026-07-14T08:05:27+00:00", "posted_to_telegram": False, "mutates_kb_by_harness": False, "pass_runtime": True, "db_counts_changed": False, "db_counts_before": counts, "db_counts_after": counts, "changed_live_profile": False, "live_behavior_manifest_unchanged": True, "live_behavior_manifest_before": behavior, "live_behavior_manifest_after": behavior, "database_response_validation_all_ok": True, "temp_profile_removed": True, "service_before_after": { "before": service, "after": service, "unchanged_from_preexisting_live_readback": True, }, "results": [ { "prompt": verifier.EXPECTED_PROMPT, "reply": reply, "database_tool_trace": trace, "database_context_trace": [ { "event": "post_llm_call", "delivered_issues": [], "validated": True, } ], } ], } def verify(report: dict) -> dict: sha = "a" * 40 return verifier.verify_report( report, deploy_head=sha, deploy_stamp=sha, source_report_sha256="f" * 64, ) def test_passing_report_proves_each_user_outcome() -> None: receipt = verify(passing_report()) assert receipt["status"] == "pass" assert all(receipt["checks"].values()) assert all(receipt["outcomes"].values()) assert receipt["database_tool_proof"]["subcommands"] == ["evidence", "search", "show"] def test_missing_tool_result_fails_attribution_and_claim_inspection() -> None: report = passing_report() report["results"][0]["database_tool_trace"]["calls"][0]["result"] = None receipt = verify(report) assert receipt["status"] == "fail" assert receipt["checks"]["all_three_database_tools_completed"] is False assert receipt["outcomes"]["inspected_the_claim_and_its_actual_support"] is False def test_database_or_profile_mutation_fails_safety_outcomes() -> None: report = passing_report() report["db_counts_after"] = {"public.claims": 2, "public.sources": 2} report["db_counts_changed"] = True report["live_behavior_manifest_after"] = {"behavior_sha256": "c" * 64} report["changed_live_profile"] = True report["live_behavior_manifest_unchanged"] = False receipt = verify(report) assert receipt["status"] == "fail" assert receipt["outcomes"]["did_not_change_canonical_knowledge"] is False assert receipt["outcomes"]["did_not_turn_the_test_conversation_into_hidden_training"] is False def test_plausible_reply_without_source_challenge_or_iteration_fails() -> None: report = copy.deepcopy(passing_report()) report["results"][0]["reply"] = "The claim is shallow. I propose several better claims." receipt = verify(report) assert receipt["status"] == "fail" assert receipt["checks"]["claim_support_challenged"] is False assert receipt["checks"]["user_iteration_requested_before_live"] is False