139 lines
4.8 KiB
Python
139 lines
4.8 KiB
Python
"""Tests for the live VPS database-first canary verifier."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
import verify_leo_db_first_oos_canary as verifier # noqa: E402
|
|
|
|
|
|
def passing_report() -> dict:
|
|
claim_id = verifier.EXPECTED_CLAIM_ID
|
|
source_ids = sorted(verifier.EXPECTED_SOURCE_IDS)
|
|
reply = (
|
|
"No writes yet. The grounds row has no_source_pointer. The verified artifact only illustrates the claim. "
|
|
"The claim bundles product liability, consumer protection, and securities fraud, so split it into narrower "
|
|
"empirical claims. Add a separate normative claim and keep it in a proposal packet. You bring the URL or "
|
|
"file; I run prepare-source before anything becomes live."
|
|
)
|
|
trace = {
|
|
"database_tool_call_proven": True,
|
|
"database_retrieval_receipt_proven": True,
|
|
"database_tool_calls_read_only": True,
|
|
"database_tool_completed_count": 3,
|
|
"access_modes": ["read_only"],
|
|
"calls": [
|
|
{
|
|
"database_invocations": [{"subcommand": subcommand, "access_mode": "read_only"}],
|
|
"result": {
|
|
"nonempty": True,
|
|
"error_detected": False,
|
|
"row_ids": [claim_id, *source_ids],
|
|
},
|
|
}
|
|
for subcommand in sorted(verifier.EXPECTED_SUBCOMMANDS)
|
|
],
|
|
}
|
|
service = {
|
|
"ActiveState": "active",
|
|
"SubState": "running",
|
|
"MainPID": "123",
|
|
"NRestarts": "0",
|
|
}
|
|
counts = {"public.claims": 1, "public.sources": 2}
|
|
behavior = {"behavior_sha256": "b" * 64}
|
|
return {
|
|
"generated_at_utc": "2026-07-14T08:05:27+00:00",
|
|
"posted_to_telegram": False,
|
|
"mutates_kb_by_harness": False,
|
|
"pass_runtime": True,
|
|
"db_counts_changed": False,
|
|
"db_counts_before": counts,
|
|
"db_counts_after": counts,
|
|
"changed_live_profile": False,
|
|
"live_behavior_manifest_unchanged": True,
|
|
"live_behavior_manifest_before": behavior,
|
|
"live_behavior_manifest_after": behavior,
|
|
"database_response_validation_all_ok": True,
|
|
"temp_profile_removed": True,
|
|
"service_before_after": {
|
|
"before": service,
|
|
"after": service,
|
|
"unchanged_from_preexisting_live_readback": True,
|
|
},
|
|
"results": [
|
|
{
|
|
"prompt": verifier.EXPECTED_PROMPT,
|
|
"reply": reply,
|
|
"database_tool_trace": trace,
|
|
"database_context_trace": [
|
|
{
|
|
"event": "post_llm_call",
|
|
"delivered_issues": [],
|
|
"validated": True,
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
def verify(report: dict) -> dict:
|
|
sha = "a" * 40
|
|
return verifier.verify_report(
|
|
report,
|
|
deploy_head=sha,
|
|
deploy_stamp=sha,
|
|
source_report_sha256="f" * 64,
|
|
)
|
|
|
|
|
|
def test_passing_report_proves_each_user_outcome() -> None:
|
|
receipt = verify(passing_report())
|
|
|
|
assert receipt["status"] == "pass"
|
|
assert all(receipt["checks"].values())
|
|
assert all(receipt["outcomes"].values())
|
|
assert receipt["database_tool_proof"]["subcommands"] == ["evidence", "search", "show"]
|
|
|
|
|
|
def test_missing_tool_result_fails_attribution_and_claim_inspection() -> None:
|
|
report = passing_report()
|
|
report["results"][0]["database_tool_trace"]["calls"][0]["result"] = None
|
|
|
|
receipt = verify(report)
|
|
|
|
assert receipt["status"] == "fail"
|
|
assert receipt["checks"]["all_three_database_tools_completed"] is False
|
|
assert receipt["outcomes"]["inspected_the_claim_and_its_actual_support"] is False
|
|
|
|
|
|
def test_database_or_profile_mutation_fails_safety_outcomes() -> None:
|
|
report = passing_report()
|
|
report["db_counts_after"] = {"public.claims": 2, "public.sources": 2}
|
|
report["db_counts_changed"] = True
|
|
report["live_behavior_manifest_after"] = {"behavior_sha256": "c" * 64}
|
|
report["changed_live_profile"] = True
|
|
report["live_behavior_manifest_unchanged"] = False
|
|
|
|
receipt = verify(report)
|
|
|
|
assert receipt["status"] == "fail"
|
|
assert receipt["outcomes"]["did_not_change_canonical_knowledge"] is False
|
|
assert receipt["outcomes"]["did_not_turn_the_test_conversation_into_hidden_training"] is False
|
|
|
|
|
|
def test_plausible_reply_without_source_challenge_or_iteration_fails() -> None:
|
|
report = copy.deepcopy(passing_report())
|
|
report["results"][0]["reply"] = "The claim is shallow. I propose several better claims."
|
|
|
|
receipt = verify(report)
|
|
|
|
assert receipt["status"] == "fail"
|
|
assert receipt["checks"]["claim_support_challenged"] is False
|
|
assert receipt["checks"]["user_iteration_requested_before_live"] is False
|