135 lines
5.2 KiB
Python
135 lines
5.2 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
|
|
from scripts import verify_leo_unseen_reasoning_chain as verifier
|
|
|
|
CLAIM_ID = "fd159490-280d-4ede-84ef-faa169cff766"
|
|
|
|
|
|
def execution_manifest(execution_sha256: str, previous: str | None, *, first: bool) -> dict:
|
|
return {
|
|
"execution_sha256": execution_sha256,
|
|
"attribution": {"status": "complete"},
|
|
"session_boundary": {
|
|
"conversation": {
|
|
"previous_execution_sha256": previous,
|
|
"prior_turn_state_bound": True,
|
|
"history_prefix_preserved": True,
|
|
"starts_from_empty_conversation": first,
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def passing_report() -> dict:
|
|
replies = [
|
|
(
|
|
"The claim fd159490 has confidence 0.9. Its grounds row is no_source_pointer and the only other support "
|
|
"is an internal document with no external primary source. First narrower test: test one tradition."
|
|
),
|
|
(
|
|
"Assumptions versus observed support: the categorical leap is assumed. Falsifier: one contrary case. "
|
|
"PROPOSAL A is empirical. PROPOSAL B is structural. Both are staged only; nothing applied. fd159490."
|
|
),
|
|
(
|
|
"Revised Proposal A separates the mechanism from the outcome. Ostrom, Governing the Commons, is the "
|
|
"primary per-case source. Stage pending_review, then human review; approved is not apply, and applied_at "
|
|
"must be non-null. Nothing staged or changed here."
|
|
),
|
|
]
|
|
fingerprints = {
|
|
"fingerprint_sha256": "f" * 64,
|
|
"table_count": 39,
|
|
"total_rows": 52167,
|
|
}
|
|
first_sha = "a" * 64
|
|
second_sha = "b" * 64
|
|
third_sha = "c" * 64
|
|
return {
|
|
"remote_run_id": "run-1",
|
|
"db_counts_changed": False,
|
|
"db_fingerprint_unchanged": True,
|
|
"db_fingerprint_before": fingerprints,
|
|
"db_fingerprint_after": fingerprints,
|
|
"safety_gate": {"status": "pass"},
|
|
"execution_manifest_summary": {
|
|
"all_turns_attribution_complete": True,
|
|
"harness_source": {"git_head": "d" * 40},
|
|
},
|
|
"results": [
|
|
{
|
|
"prompt_id": verifier.PROMPTS[0]["id"],
|
|
"prompt": verifier.PROMPTS[0]["message"],
|
|
"reply": replies[0],
|
|
"database_tool_trace": {
|
|
"calls": [
|
|
{
|
|
"database_invocations": [{"subcommand": "context"}],
|
|
"result": {"row_ids": [CLAIM_ID]},
|
|
}
|
|
]
|
|
},
|
|
"execution_manifest": execution_manifest(first_sha, None, first=True),
|
|
},
|
|
{
|
|
"prompt_id": verifier.PROMPTS[1]["id"],
|
|
"prompt": verifier.PROMPTS[1]["message"],
|
|
"reply": replies[1],
|
|
"database_tool_trace": {
|
|
"calls": [
|
|
{
|
|
"database_invocations": [
|
|
{"subcommand": "show"},
|
|
{"subcommand": "evidence"},
|
|
],
|
|
"result": {"row_ids": [CLAIM_ID]},
|
|
}
|
|
]
|
|
},
|
|
"execution_manifest": execution_manifest(second_sha, first_sha, first=False),
|
|
},
|
|
{
|
|
"prompt_id": verifier.PROMPTS[2]["id"],
|
|
"prompt": verifier.PROMPTS[2]["message"],
|
|
"reply": replies[2],
|
|
"database_tool_trace": {"calls": []},
|
|
"execution_manifest": execution_manifest(third_sha, second_sha, first=False),
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def test_unseen_reasoning_chain_passes_only_with_db_challenge_revision_and_apply_boundary() -> None:
|
|
receipt = verifier.verify_report(passing_report(), source_report_sha256="e" * 64)
|
|
|
|
assert receipt["status"] == "pass"
|
|
assert all(receipt["checks"].values())
|
|
assert all(receipt["outcomes"].values())
|
|
assert receipt["selected_claim_ids"] == [CLAIM_ID]
|
|
assert receipt["execution_sha256_chain"] == ["a" * 64, "b" * 64, "c" * 64]
|
|
|
|
|
|
def test_unseen_reasoning_chain_rejects_a_challenge_that_does_not_reinspect_evidence() -> None:
|
|
report = copy.deepcopy(passing_report())
|
|
report["results"][1]["database_tool_trace"]["calls"][0]["database_invocations"] = [
|
|
{"subcommand": "show"}
|
|
]
|
|
|
|
receipt = verifier.verify_report(report, source_report_sha256="e" * 64)
|
|
|
|
assert receipt["status"] == "fail"
|
|
assert receipt["checks"]["same_claim_body_and_evidence_reinspected_after_challenge"] is False
|
|
assert receipt["outcomes"]["survives_a_claim_body_and_evidence_challenge"] is False
|
|
|
|
|
|
def test_unseen_reasoning_chain_rejects_broken_conversation_parent_hash() -> None:
|
|
report = copy.deepcopy(passing_report())
|
|
report["results"][2]["execution_manifest"]["session_boundary"]["conversation"][
|
|
"previous_execution_sha256"
|
|
] = "0" * 64
|
|
|
|
receipt = verifier.verify_report(report, source_report_sha256="e" * 64)
|
|
|
|
assert receipt["status"] == "fail"
|
|
assert receipt["checks"]["all_turn_execution_manifests_complete"] is False
|