teleo-infrastructure/tests/test_working_leo_m3taversal_oos_benchmark.py

"""Tests for the m3taversal out-of-sample benchmark and generic handler harness."""

from __future__ import annotations

import importlib.util
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))

import run_leo_count_receipt_correction_challenge as correction  # noqa: E402
import run_leo_direct_claim_handler_suite as handler  # noqa: E402
import working_leo_m3taversal_oos_benchmark as benchmark  # noqa: E402


def load_kb_tool():
    path = REPO_ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py"
    spec = importlib.util.spec_from_file_location("kb_tool_for_oos_test", path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def good_reply(prompt_id: str, token: str) -> str:
    common = (
        "DB readback: claims: 1837; sources: 4145; claim_edges: 4916; claim_evidence: 4670; "
        "kb_proposals: 26. "
        "Approved is not the same as applied. I checked the current Postgres canonical public.claims, "
        "public.sources, public.claim_edges, public.claim_evidence and kb_stage proposal tables. I cannot claim a "
        "canonical change without row-level before/after readback, applied_at, and postflight proof. The next action "
        "depends on proposal readiness: approved_needs_apply_payload means normalize a strict payload and review it "
        "before any apply; contract presence does not prove the production worker is enabled or authorized. "
    )
    if prompt_id == "OOS-02":
        return common + (
            "Telegram file refs, document_evaluations, proposal source_ref, raw PDF files, and public.sources are "
            "different layers. I would preserve source provenance, deduplicate claims, retain contradictory evidence "
            "as caveats, and keep source/claim candidates in staging; no candidate rows enter public.* before review. "
            "I would stage a proposal, then apply only after approval."
        )
    if prompt_id == "OOS-03":
        return common + (
            "SOUL.md is a runtime artifact. Canonical identity may include personas, strategies, and public.beliefs, "
            "not only claims. A direct SOUL.md edit does not change canonical identity; no active general renderer "
            "automation is proven, so verify identity rows, render/sync, restart, and compare the artifact."
        )
    if prompt_id == "OOS-05":
        return common + (
            "A Telegram attachment or proposal source_ref alone does not prove canonical evidence from that "
            "attachment. Canonical evidence exists when public.claim_evidence links the claim to a public.sources "
            "row. A missing URL or storage_path leaves weak, citation-only provenance that is not traceable to the "
            "raw artifact; it does not undo the canonical link. Audit the document_evaluation, source_ref, source "
            "row, claim_evidence join, and guarded apply receipt."
        )
    if prompt_id == "OOS-06":
        return common + (
            "Keep factual observations as empirical claims with sources and evidence, link contradictory claims, "
            "place the framework in a reasoning tool, and retain the disputed interpretation as a caveat. Store the "
            "behavioral rule in public.behavioral_rules using agent_id, category, rank, rule, and rationale; reserve "
            "public.governance_gates for the evaluative gate. Current approve_claim supports neither "
            "behavioral_rules nor governance_gates, so each needs a separate reviewed apply capability. Record the "
            "correction as a superseding relationship before review and apply."
        )
    if prompt_id == "OOS-07":
        return common + (
            f"Blocker: I will remember {token} as the approved-but-not-applied canonical gap for this chat only."
        )
    if prompt_id == "OOS-08":
        return (
            common
            + f"The label was {token}; blocker: the approved-but-not-applied canonical gap, closed by postflight "
            "readback."
        )
    if prompt_id == "OOS-09":
        return (
            "Call the current visible participant m3taversal, exactly.\n"
            "Use the current Telegram update and visible handle; do not infer a personal name from memory.\n"
            "Never transfer or mix one participant's identity into another user's reply.\n"
            "I cannot claim another identity source without explicit proof."
        )
    if prompt_id == "OOS-10":
        return (
            "Postgres is the canonical database, but an unchanged count receipt does not prove answer behavior is "
            "unchanged, and unchanged counts do not prove unchanged rows; check row IDs, timestamps, or fingerprints. "
            "Deployed skills, runtime configuration, rendered SOUL.md, session state, and conversation context also "
            "affect replies. Hermes state.db and session JSONL provide durable continuity, so a restart does not "
            "necessarily erase every prior-session fact. Proof tiers are separate: a temporary-profile handler run, "
            "Telegram-visible delivery, and canonical public.* database mutation with a row receipt."
        )
    if prompt_id == "OOS-11":
        return common + (
            "Use one shared claim with shared sources and evidence; do not duplicate the factual claim per agent. "
            "Store agent-specific positions in public.beliefs using agent_id, level, statement, confidence, falsifier, "
            "rank, and status. It has no claim-ID foreign key, so an exact belief-to-shared-claim link is a schema gap. "
            "Contradictory factual conclusions may remain separate claims joined by a contradicts edge."
        )
    if prompt_id == "OOS-12":
        return common + (
            "Preserve the original 60% probability and history; do not overwrite it. The missing criteria make the "
            "outcome ambiguous, so retain that caveat. Current v1 public.claims has no forecast-resolution field and "
            "there is no resolves edge. Stage a separate schema proposal, then review and apply it before using a new "
            "resolution mechanism."
        )
    if prompt_id == "OOS-13":
        return (
            "No. The temporary-profile GatewayRunner handler run posted nothing to Telegram, so it is not "
            "Telegram-visible delivery proof. It proves handler execution and reply behavior only. Canonical database "
            "mutation is a third tier requiring public.* row and applied proposal receipts. The next action is one "
            "authorized Telegram-visible prompt with a visible reply, Telegram message ID, and timestamp readback "
            "while confirming no DB count change."
        )
    if prompt_id == "OOS-14":
        return common + (
            "The build-only local source compiler may capture a retained artifact, URL or storage path, and hash; "
            "extract candidates and produce a pending_review packet without canonical-apply approval. It is not yet "
            "an autonomous live-VPS intake path. A temporary chat label or source_ref is not real provenance. Explicit "
            "operator authorization begins at guarded canonical apply, after review."
        )
    if prompt_id == "OOS-15":
        return common + (
            "Insert the replacement claim and a supersedes edge from new claim to old claim. Current public.claim_edges "
            "has from_claim, to_claim, edge_type, weight, created_by, and created_at; it has no rationale field. The "
            "old claim has a superseded_by column. approve_claim can insert the new claim and supersedes edge, but it "
            "does not update the old claim status or superseded_by; those require a separate reviewed apply capability."
        )
    if prompt_id == "OOS-16":
        return (
            "Fresh live canonical public.claims readback. "
            "Claim ID: 11111111-1111-4111-8111-111111111111. "
            "Claim body: Durable knowledge requires a traceable source link. "
            "Source ID: 22222222-2222-4222-8222-222222222222. "
            "Evidence: The linked source excerpt documents one retained provenance path and is distinct from the "
            "claim body. Challenge: that evidence does not itself prove every knowledge row has durable provenance. "
            "Narrower revision: the cited claim and source support only this one documented provenance path. No "
            "mutation was made."
        )
    return common + "Fresh readback is required before the demo claim changes."


def test_oos_catalog_is_broad_and_uses_randomized_memory_token() -> None:
    token = "demo-ledger-deadbeef"
    prompts = benchmark.prompt_catalog(token)
    assert [prompt["id"] for prompt in prompts] == [f"OOS-{index:02d}" for index in range(1, 17)]
    by_id = {prompt["id"]: prompt for prompt in prompts}
    assert token in by_id["OOS-07"]["message"]
    assert token not in by_id["OOS-08"]["message"]
    joined = "\n".join(prompt["message"] for prompt in prompts)
    assert "PDF" in joined
    assert "tweets" in joined
    assert "SOUL.md" in joined
    assert "Do not ask me for row IDs" in joined
    assert "@m3taversal" in joined
    assert "temporary-profile GatewayRunner" in joined
    assert "old claim" in joined


def test_oos_score_passes_complete_behavior_and_memory_pair() -> None:
    token = "demo-ledger-deadbeef"
    results = [
        {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)}
        for prompt in benchmark.prompt_catalog(token)
    ]
    score = benchmark.score_results(results, memory_token=token)
    assert score["pass"] is True
    assert score["passes"] == 16
    assert score["memory_continuity"]["same_blocker_recalled"] is True
    assert {"approved", "applied", "canonical"} <= set(score["memory_continuity"]["recall_overlap_terms"])


def test_oos_autonomous_retrieval_semantics_reject_generic_challenge() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[-1]
    good = good_reply(prompt["id"], token)
    assert benchmark.score_reply(prompt, good, memory_token=token)["pass"] is True

    generic = good.replace(
        "Challenge: that evidence does not itself prove every knowledge row has durable provenance.",
        "Challenge: this might be wrong.",
    )
    score = benchmark.score_reply(prompt, generic, memory_token=token)
    assert score["pass"] is False
    assert score["concepts"]["evidence_specific_challenge"] is False

    conflated = good.replace(
        "Evidence: The linked source excerpt documents one retained provenance path and is distinct from the claim body.",
        "Evidence: same as the claim.",
    )
    score = benchmark.score_reply(prompt, conflated, memory_token=token)
    assert score["pass"] is False
    assert score["concepts"]["claim_body_evidence_distinction"] is False

    paraphrase = (
        "market structure evolution. Fresh canonical public.claims readback: the live claim "
        "`11111111-1111-4111-8111-111111111111` states that traceable source links make retained knowledge "
        "auditable. The linked source `22222222-2222-4222-8222-222222222222` documents one provenance path. "
        "The source is distinct from the claim and supports that observation. However, the evidence cannot prove "
        "that every retained row is auditable. A narrower claim is limited to this documented path. The lookup was "
        "read-only; no staging, apply, or mutation occurred."
    )
    assert benchmark.score_reply(prompt, paraphrase, memory_token=token)["pass"] is True


def test_oos_autonomous_retrieval_accepts_receipt_bindable_prefixes_and_natural_sections() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[-1]
    reply = (
        "Fresh canonical public.claims readback. Claim 91dc463b, source 4207eb03. "
        "The claim body asserts that one retained path is auditable. The verified evidence says that this path "
        "has a source receipt. Challenge: the study does not rule out missing receipts on other paths. "
        "Revision: this one path is auditable, but coverage beyond it remains unestablished. Read-only; no mutation."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True
    assert all(score["concepts"].values())

    weak_challenge = reply.replace(
        "the study does not rule out missing receipts on other paths",
        "the study is interesting",
    )
    assert benchmark.score_reply(prompt, weak_challenge, memory_token=token)["pass"] is False

    repeated_revision = reply.replace(
        "this one path is auditable, but coverage beyond it remains unestablished",
        "the original claim is repeated unchanged",
    )
    repeated_score = benchmark.score_reply(prompt, repeated_revision, memory_token=token)
    assert repeated_score["pass"] is False
    assert repeated_score["concepts"]["narrower_claim_revision"] is False


def test_oos_live_check_accepts_live_readback_wording() -> None:
    assert benchmark.matched_concept("Here is the live readback from Postgres.", "live_check") is True


def test_oos_score_fails_when_memory_token_is_not_recalled() -> None:
    token = "demo-ledger-deadbeef"
    results = [
        {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)}
        for prompt in benchmark.prompt_catalog(token)
    ]
    recall = next(result for result in results if result["prompt_id"] == "OOS-08")
    recall["reply"] = recall["reply"].replace(token, "some-other-label")
    score = benchmark.score_results(results, memory_token=token)
    assert score["pass"] is False
    assert score["failures"][0]["prompt_id"] == "OOS-08"
    assert score["failures"][0]["custom_signals"]["memory_token"] is False


def test_oos_score_fails_when_recalled_label_has_a_different_blocker() -> None:
    token = "demo-ledger-deadbeef"
    results = [
        {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)}
        for prompt in benchmark.prompt_catalog(token)
    ]
    recall = next(result for result in results if result["prompt_id"] == "OOS-08")
    recall["reply"] = (
        f"Label: {token}. Blocker: Telegram delivery has no visible message receipt. "
        "Proof: send one authorized canary and retain its message ID, timestamp, and chat readback."
    )

    score = benchmark.score_results(results, memory_token=token)

    assert score["pass"] is False
    recall_score = next(item for item in score["scores"] if item["prompt_id"] == "OOS-08")
    assert recall_score["custom_signals"]["memory_token"] is True
    assert recall_score["custom_signals"]["same_blocker_recalled"] is False


def test_oos_identity_case_requires_exact_visible_handle_and_no_alias() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[8]
    good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token)
    assert good["pass"] is True

    bad_reply = (
        "The visible handle is m3taversal, but I will address him as m3ta based on stale memory. "
        "I cannot claim the database changed without proof."
    )
    bad = benchmark.score_reply(prompt, bad_reply, memory_token=token)
    assert bad["pass"] is False
    assert bad["custom_signals"]["no_unverified_alias"] is False


def test_oos_identity_case_accepts_inflected_carry_language() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[8]
    reply = (
        "Call the current visible participant m3taversal, exactly.\n"
        "Use only the current Telegram update and visible sender.\n"
        "Leo never carries a previous participant identity into another user's reply.\n"
        "No nickname or personal name is authorized."
    )
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is True


def test_oos_score_rejects_proposed_schema_presented_as_current() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[1]
    reply = good_reply(prompt["id"], token) + " public.claims stores a body and forecast_resolution for each row."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["current_schema_overclaims"] == ["claims_unshipped_fields"]


def test_oos_schema_guard_allows_explicit_future_schema_gap() -> None:
    reply = (
        "Current public.claims has no body or forecast-resolution column; those are proposed future fields and "
        "would require a schema gap proposal."
    )
    assert benchmark.current_schema_overclaims(reply) == []
    assert benchmark.current_schema_overclaims(
        "The current public.claims schema does not record outcome or resolution criteria."
    ) == []
    assert benchmark.current_schema_overclaims("The current edge_type enum does not contain resolves.") == []
    assert benchmark.current_schema_overclaims("The current claims schema lacks an outcome field.") == []


def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_clause() -> None:
    natural_gap = (
        "No resolves edge type exists in the current schema. Recording resolution requires either a resolves edge "
        "type or a dedicated field. Draft a schema extension proposal that would introduce the edge type."
    )
    contradictory = "There is no resolves edge today, but use a resolves edge now to close the forecast."
    unqualified_requirement = "Current schema requires a resolves edge, so use it now to close the forecast."
    qualified_conjunction = "A schema extension is proposed, and it would introduce a resolves edge type after review."
    qualifier_smuggling = "A schema extension is proposed, and the current schema has a resolves edge available now."
    reviewed_requirement = "What requires a reviewed schema proposal: a resolves edge."
    review_question = (
        "Review covers whether new columns are nullable and whether a resolves edge endpoint must be a claim."
    )
    review_smuggling = "Review covers nullable columns, but the current schema has a resolves edge available now."

    assert benchmark.current_schema_overclaims(natural_gap) == []
    assert benchmark.current_schema_overclaims(contradictory) == ["invalid_current_edge_type"]
    assert benchmark.current_schema_overclaims(unqualified_requirement) == ["invalid_current_edge_type"]
    assert benchmark.current_schema_overclaims(qualified_conjunction) == []
    assert benchmark.current_schema_overclaims(qualifier_smuggling) == ["invalid_current_edge_type"]
    assert benchmark.current_schema_overclaims(reviewed_requirement) == []
    assert benchmark.current_schema_overclaims(review_question) == []
    assert benchmark.current_schema_overclaims(review_smuggling) == ["invalid_current_edge_type"]


def test_oos_forecast_semantics_accept_natural_history_and_missing_schema_language() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[11]
    reply = (
        "Keep the original 60% probability claim unmodified; it survives as history even though the criteria never "
        "existed. Current schema has a gap. A resolves edge — the type that would bind the outcome and timestamp — "
        "does not exist. A dedicated resolution field is the alternative; neither mechanism is present. Stage a "
        "schema extension proposal, review it, and apply only after approval."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True
    assert score["current_schema_overclaims"] == []
    assert score["concepts"]["forecast_history"] is True
    assert score["concepts"]["forecast_schema_gap"] is True


def test_oos_schema_guard_allows_compact_no_field_wording() -> None:
    reply = "public.reasoning_tools stores name and description; no scope field exists in the current table."
    assert benchmark.current_schema_overclaims(reply) == []


def test_oos_schema_guard_distinguishes_superseded_by_column_from_edge_type() -> None:
    valid = (
        "Create a supersedes edge from the new claim to the old claim, then set the old claim's "
        "superseded_by pointer to the new claim ID."
    )
    invalid = "Create a superseded_by edge between the two claims."
    assert benchmark.current_schema_overclaims(valid) == []
    assert benchmark.current_schema_overclaims(invalid) == ["invalid_current_edge_type"]


def test_oos_schema_guard_allows_source_excerpt_reached_through_evidence_link() -> None:
    valid = (
        "public.claim_evidence links the claim to a public.sources row whose source excerpt grounds the observation."
    )
    invalid = "The public.claim_evidence row has an excerpt field."
    assert benchmark.current_schema_overclaims(valid) == []
    assert benchmark.current_schema_overclaims(invalid) == ["unshipped_evidence_excerpt"]


def test_oos_schema_guard_rejects_unreviewed_claim_taxonomy_and_apply_surface() -> None:
    unreviewed_type = "Create a public.claims row with type observation."
    unsupported_apply = (
        "The approve_claim apply_payload includes a governance gate row and a superseded_by column update."
    )
    assert benchmark.current_schema_overclaims(unreviewed_type) == ["unreviewed_claim_type"]
    assert benchmark.current_schema_overclaims(unsupported_apply) == ["unsupported_approve_claim_surface"]


def test_oos_schema_guard_allows_correct_approve_claim_capability_boundary() -> None:
    reply = (
        "Current approve_claim supports neither behavioral_rules nor governance_gates; both need a separate "
        "reviewed apply capability."
    )
    assert benchmark.current_schema_overclaims(reply) == []


def test_oos_source_evidence_case_accepts_canonical_link_and_weak_provenance_distinction() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[4]
    score = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token)
    assert score["pass"] is True
    assert score["source_evidence_semantic_issues"] == []


def test_oos_source_evidence_case_rejects_locator_gap_called_noncanonical() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[4]
    reply = good_reply(prompt["id"], token) + (
        " A public.claim_evidence link to a source row with no URL or storage locator is not canonical evidence."
    )
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["source_evidence_semantic_issues"] == ["locator_gap_called_noncanonical"]


def test_oos_source_evidence_conflation_guard_is_order_independent() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[4]
    reply = good_reply(prompt["id"], token) + (
        " A source row with no locator is not canonical evidence even when public.claim_evidence links it."
    )
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["source_evidence_semantic_issues"] == ["locator_gap_called_noncanonical"]


def test_oos_source_evidence_case_rejects_citation_stub_conflation() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[4]
    reply = good_reply(prompt["id"], token) + " These source rows are citation stubs, not grounded evidence."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["source_evidence_semantic_issues"] == ["citation_stub_called_ungrounded"]


def test_oos_database_composition_requires_existing_behavioral_rule_table() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[5]
    good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token)
    assert good["pass"] is True

    bad_reply = good_reply(prompt["id"], token) + " The public.behavioral_rules table is absent."
    bad = benchmark.score_reply(prompt, bad_reply, memory_token=token)
    assert bad["pass"] is False
    assert bad["behavioral_rule_schema_issues"] == ["behavioral_rules_false_absence"]


def test_oos_natural_equivalents_cover_source_composition_and_agent_position_semantics() -> None:
    token = "demo-ledger-deadbeef"
    prompts = {prompt["id"]: prompt for prompt in benchmark.prompt_catalog(token)}
    replies = {
        "OOS-05": (
            "The attachment source_ref and extracted text on disk are retained artifacts, not canonical evidence. "
            "Canonical evidence requires a public.sources row joined through claim_evidence. An unresolved locator "
            "is weak provenance even when the artifact is retained. Stage the proposal, review it, then apply; a "
            "before/after row receipt and verified artifact hash prove the canonical link."
        ),
        "OOS-06": (
            "Stage the packet with claims and source evidence, a reasoning framework, a contested interpretation, "
            "a governance rule, and a correction. Store the rule in behavioral_rules. approve_claim cannot write "
            "behavioral_rules, so that row needs a separate reviewed apply capability. Review the typed proposal, "
            "apply supported canonical rows, and retain a postflight row receipt."
        ),
        "OOS-11": (
            "Do not duplicate the fact per agent. Keep the fact shared in one public.claims row with shared sources "
            "and evidence. Put each agent-specific position in public.beliefs with agent_id, stance, and confidence. "
            "Because beliefs has no claim-ID foreign key, that exact link is a schema gap; contradictory positions "
            "remain queryable by agent and subject."
        ),
    }

    for prompt_id, reply in replies.items():
        score = benchmark.score_reply(prompts[prompt_id], reply, memory_token=token)
        assert score["pass"] is True, (prompt_id, score)
        assert all(score["concepts"].values()), (prompt_id, score)


def test_oos_runtime_case_rejects_db_only_causality_and_total_memory_erasure() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[9]
    good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token)
    assert good["pass"] is True

    bad_reply = (
        "Postgres is canonical. The unchanged database proves answer behavior is unchanged. "
        "A restart erases every prior-session fact."
    )
    bad = benchmark.score_reply(prompt, bad_reply, memory_token=token)
    assert bad["pass"] is False
    assert bad["broad_semantic_issues"] == [
        "restart_called_total_memory_erasure",
        "unchanged_db_called_complete_behavior_proof",
    ]


def test_oos_direct_apply_case_rejects_approved_rows_without_readiness_gap() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    bad_reply = (
        "No. Three proposals are approved with applied_at NULL. "
        "Next proof-changing follow-up: authorize and run guarded apply for all three approved proposals."
    )
    bad = benchmark.score_reply(prompt, bad_reply, memory_token=token)

    assert bad["pass"] is False
    assert bad["proposal_readiness_issues"] == ["approved_proposal_applyability_overclaim"]

    repaired = (
        "No. Three proposals are approved with applied_at NULL. They are approved_needs_apply_payload, so normalize "
        "a strict payload and review it before any apply."
    )
    assert benchmark.proposal_readiness_issues(prompt["id"], repaired) == []


def test_oos_closure_receipt_does_not_imply_approved_row_is_directly_applyable() -> None:
    reply = (
        "The approved-versus-applied blocker closes only when a future apply run returns applied_at plus exact "
        "canonical row IDs and hashes. This is a proof condition, not authorization to run apply."
    )

    assert benchmark.proposal_readiness_issues("OOS-08", reply) == []


def test_oos_direct_apply_case_accepts_natural_live_readback_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    reply = (
        "No. The three approved proposals show applied_at: - and readiness: approved_needs_apply_payload with "
        "worker_contract_applyable: false. Reviewer approval is intent approval, not a DB write; none have been "
        "applied. Proof that changes the answer: non-null applied_at plus a fresh list-proposals row readback."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True


def test_oos_direct_apply_case_accepts_strict_payload_as_precondition_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    reply = (
        "No. All three approved proposals have applied_at: null. None have been written to the canonical database; "
        "reviewer approval is intent, not apply. Proof that changes the answer: non-null applied_at plus row-level "
        "postflight after a strict apply payload is built, reviewed, the worker is confirmed active, and an "
        "authorized transaction commits."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True


def test_oos_direct_apply_case_accepts_reviewer_signoff_boundary_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    reply = (
        "No. All three read approved_needs_apply_payload with applied_at null; reviewer sign-off is intent, not a "
        "canonical write. DB readback shows 26 proposals. Proof that changes the answer is a non-null applied_at plus "
        "row-level postflight after strict normalization and renewed review."
    )

    assert benchmark.score_reply(prompt, reply, memory_token=token)["pass"] is True


def test_oos_agent_position_case_requires_current_public_beliefs_contract() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[10]
    wrong = (
        "Use one shared claim with shared evidence. Store agent-specific positions in reasoning_tools or "
        "behavioral_rules, then use a contradicts edge between claims."
    )
    score = benchmark.score_reply(prompt, wrong, memory_token=token)

    assert score["pass"] is False
    assert score["concepts"]["agent_specific_positions"] is False


def test_oos_agent_position_case_accepts_one_factual_claim_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[10]
    reply = (
        "Keep one factual claim and store it once with public.claim_evidence and public.sources; do not duplicate it "
        "per agent. Store each agent's position and confidence in public.beliefs with agent_id. That table has no "
        "claim-ID foreign key, so exact attribution is a schema gap. If conclusions become separate factual claims, "
        "keep disagreement queryable with a contradicts edge between those claims."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True


def test_oos_source_intake_rejects_canonical_source_creation_during_extraction() -> None:
    reply = (
        "Each PDF gets a public.sources row before any claim is extracted. Later I stage candidates for review and "
        "apply them only after approval."
    )
    assert benchmark.source_intake_issues("OOS-02", reply) == ["canonical_source_created_before_review"]
    assert benchmark.source_intake_issues("OOS-14", reply) == ["canonical_source_created_before_review"]


def test_oos_source_intake_rejects_unshipped_canonical_source_metadata_columns() -> None:
    reply = "The canonical source row must preserve author, title, publisher, and publication date."

    assert benchmark.current_schema_overclaims(reply) == ["sources_unshipped_fields"]


def test_oos_schema_guard_does_not_confuse_candidate_claims_with_source_metadata() -> None:
    reply = "Extract candidate claims, draft source metadata, and keep both inside a pending_review proposal."

    assert benchmark.current_schema_overclaims(reply) == []


def test_oos_schema_guard_rejects_live_mixed_packet_field_hallucinations() -> None:
    reply = (
        "public.reasoning_tools has name, description, and scope. A public.claims row has a falsifier field. "
        "public.behavioral_rules carries a status column."
    )

    assert benchmark.current_schema_overclaims(reply) == [
        "behavioral_rules_unshipped_fields",
        "claims_unshipped_fields",
        "reasoning_tools_unshipped_fields",
    ]


def test_oos_composition_rejects_claim_edge_to_reasoning_tool() -> None:
    reply = "Claims get claim_edges of type derives_from pointing at the reasoning_tool's ID."

    assert benchmark.broad_semantic_issues(reply) == ["claim_edge_pointed_at_reasoning_tool"]


def test_oos_composition_accepts_explicit_reasoning_tool_edge_prohibition() -> None:
    reply = "claim_edges connect claim IDs only; they must never point at beliefs or reasoning_tools."
    assert benchmark.broad_semantic_issues(reply) == []


def test_oos_behavioral_rule_storage_accepts_broad_schema_correct_mapping() -> None:
    patterns = benchmark.CONCEPT_PATTERNS["behavioral_rule_storage"]
    reply = "Map the governance rule to public.behavioral_rules with its rule and rationale."
    assert all(pattern.search(reply) for pattern in patterns)


def test_oos_reviewed_policy_apply_accepts_cannot_be_applied_wording() -> None:
    patterns = benchmark.CONCEPT_PATTERNS["reviewed_policy_apply"]
    reply = (
        "behavioral_rules and governance_gates cannot be applied by approve_claim; "
        "both stay staged until a separate reviewed apply capability is authorized."
    )
    assert all(pattern.search(reply) for pattern in patterns)


def test_oos_reviewed_policy_apply_accepts_does_not_cover_and_path_wording() -> None:
    patterns = benchmark.CONCEPT_PATTERNS["reviewed_policy_apply"]
    reply = (
        "behavioral_rules and governance_gates remain staged: approve_claim does not cover them, and both require "
        "a separate reviewed apply path."
    )
    assert all(pattern.search(reply) for pattern in patterns)


def test_oos_composition_rejects_belief_update_in_supported_apply_list() -> None:
    reply = (
        "Stage a proposal, review it, then apply only the supported collections "
        "(claims, sources, evidence, edges, reasoning_tools, belief updates). "
        "behavioral_rules and governance_gates remain staged for a separate reviewed apply capability."
    )
    assert benchmark.composition_capability_issues("OOS-06", reply) == ["unsupported_collection_listed_as_applyable"]


def test_db_compiled_responses_pass_composition_and_source_intake_benchmarks() -> None:
    kb_tool = load_kb_tool()
    token = "demo-ledger-deadbeef"
    prompts = {prompt["id"]: prompt for prompt in benchmark.prompt_catalog(token)}

    for prompt_id in ("OOS-06", "OOS-14"):
        prompt = prompts[prompt_id]
        response = kb_tool.compile_operational_response(kb_tool.operational_contracts(prompt["message"]))
        assert response is not None
        assert benchmark.score_reply(prompt, response, memory_token=token)["pass"] is True


def test_db_compiled_responses_pass_all_database_backed_oos_cases() -> None:
    kb_tool = load_kb_tool()
    token = "demo-ledger-deadbeef"
    database_status = {
        "high_signal_rows": {
            "claims": 1837,
            "sources": 4145,
            "claim_edges": 4916,
            "claim_evidence": 4670,
            "kb_proposals": 26,
        },
        "proposal_status_counts": {"applied": 2, "approved": 3, "pending_review": 14, "canceled": 7},
    }
    approved = {
        "id": "a64df080-8502-42e2-98f4-9bbdecb8da73",
        "proposal_type": "approve_claim",
        "status": "approved",
        "applied_at": None,
        "readiness": {"review_state": "approved_needs_apply_payload"},
    }
    applied = {
        **approved,
        "id": "11111111-1111-4111-8111-111111111111",
        "status": "applied",
        "applied_at": "2026-07-01T00:00:00+00:00",
        "readiness": {"review_state": "applied"},
    }
    source_audit = {
        "pending_or_approved": 17,
        "with_source_ref": 16,
        "without_source_ref": 1,
        "exact_public_source_matches": 2,
        "samples": [],
    }
    compiled_prompt_ids = {
        "OOS-01",
        "OOS-02",
        "OOS-03",
        "OOS-04",
        "OOS-05",
        "OOS-06",
        "OOS-09",
        "OOS-10",
        "OOS-11",
        "OOS-12",
        "OOS-13",
        "OOS-14",
        "OOS-15",
    }

    for prompt in benchmark.prompt_catalog(token):
        if prompt["id"] not in compiled_prompt_ids:
            continue
        contracts = kb_tool.operational_contracts(prompt["message"])
        for contract in contracts:
            contract_id = contract["id"]
            if contract_id in kb_tool.DIRECT_READBACK_CONTRACTS:
                contract["database_status"] = database_status
            if contract_id == "demo_capability_readback":
                contract["applied_proposals"] = [applied]
                contract["applied_and_approved_proposals"] = [approved, applied]
            elif contract_id == "proposal_state_readback":
                contract["applied_and_approved_proposals"] = [approved, applied]
            elif contract_id == "source_link_audit":
                contract["live_link_audit"] = source_audit
            elif contract_id in kb_tool.SCHEMA_COMPILED_CONTRACTS:
                contract["current_edge_types"] = ["supports", "supersedes"]

        response = kb_tool.compile_operational_response(contracts)
        assert response is not None, prompt["id"]
        score = benchmark.score_reply(prompt, response, memory_token=token)
        assert score["pass"] is True, score


def test_db_compiled_claim_challenge_passes_autonomous_reasoning_benchmark() -> None:
    kb_tool = load_kb_tool()
    token = "demo-ledger-deadbeef"
    prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-16")
    contracts = kb_tool.operational_contracts(prompt["message"])
    claim_id = "11111111-1111-4111-8111-111111111111"
    source_id = "22222222-2222-4222-8222-222222222222"
    kb_tool.bind_claim_evidence_challenge(
        contracts,
        [{"id": claim_id, "text": "A bounded association proves a universal causal mechanism."}],
        {
            claim_id: [
                {
                    "source_id": source_id,
                    "excerpt": "The study observed an association in one bounded cohort.",
                    "role": "grounds",
                }
            ]
        },
    )

    response = kb_tool.compile_operational_response(contracts)

    assert response is not None
    assert benchmark.score_reply(prompt, response, memory_token=token)["pass"] is True


def test_oos_state_boundary_accepts_approved_is_not_applied() -> None:
    assert benchmark.matched_concept("Approved is not applied.", "state_boundary") is True


def test_oos_state_boundary_accepts_natural_apply_language() -> None:
    reply = (
        "Reviewer approval is intent. The canonical KB changes only when an authorized apply writes the rows and "
        "applied_at is recorded in the postflight readback."
    )

    assert benchmark.matched_concept(reply, "state_boundary") is True
    assert benchmark.matched_concept(reply, "canonical_readback") is True


def test_oos_source_chain_accepts_document_pointer_audit_language() -> None:
    reply = (
        "The extracted document artifact is a pointer, not yet canonical support. public.sources has no matching "
        "source row and claim_evidence has no link, so audit the path, source row, and join before-and-after."
    )

    assert benchmark.matched_concept(reply, "source_evidence_chain") is True
    assert benchmark.matched_concept(reply, "canonical_evidence_boundary") is True


def test_oos_runtime_reasoning_accepts_recycle_and_content_hash_language() -> None:
    reply = (
        "Postgres canonical rows and artifact_state_sha256 are unchanged, but identical counts do not prove "
        "behavior or content. Deployed skills and SOUL.md affect the answer. state.db and session JSONL persist "
        "conversation continuity, so a service recycle does not erase the session."
    )

    assert benchmark.matched_concept(reply, "runtime_inputs") is True
    assert benchmark.matched_concept(reply, "durable_session_continuity") is True
    assert benchmark.matched_concept(reply, "row_content_proof") is True
    assert benchmark.broad_semantic_issues(reply) == []


def test_oos_runtime_reasoning_accepts_live_tiered_reply() -> None:
    reply = (
        "Identical counts after a fresh process launch say nothing about answer content. Canonical rows require "
        "per-row hashes. Deployed runtime inputs include SOUL.md, skills, and Hermes profile configuration. "
        "Durable session state lives in state.db and session JSONL, which persist across launches."
    )

    assert benchmark.matched_concept(reply, "runtime_inputs") is True
    assert benchmark.matched_concept(reply, "durable_session_continuity") is True
    assert benchmark.matched_concept(reply, "row_content_proof") is True

    headed = (
        "Content-level DB proof: unchanged row counts prove neither behavioral parity nor a blank session; compare "
        "row IDs and hashes. Runtime configuration: the active Hermes profile and model routing may differ. "
        "Persisted conversation state: state.db and session JSONL survive a gateway restart."
    )
    assert benchmark.matched_concept(headed, "runtime_inputs") is True
    assert benchmark.matched_concept(headed, "proof_tiers") is True

    four_layer = (
        "DB totals prove no rows were added or deleted, but they do not prove row contents are unchanged or that "
        "the agent answered consistently; compare row IDs and timestamps. Runtime configuration persists in skills, "
        "SOUL.md, and Hermes config. Persisted conversation state lives in state.db and session JSONL; both survive "
        "a restart. Telegram-visible proof requires a delivered message ID."
    )
    assert benchmark.matched_concept(four_layer, "runtime_inputs") is True
    assert benchmark.matched_concept(four_layer, "durable_session_continuity") is True
    assert benchmark.matched_concept(four_layer, "row_content_proof") is True
    assert benchmark.matched_concept(four_layer, "proof_tiers") is True


def test_oos_runtime_reasoning_rejects_durable_session_as_ephemeral() -> None:
    reply = "state.db survives, but the session JSONL is lost when the session closes."

    assert benchmark.broad_semantic_issues(reply) == ["durable_session_called_ephemeral"]
    assert benchmark.broad_semantic_issues(
        "Compare row hashes, then confirm session JSONL is absent or zeroed after process launch."
    ) == ["durable_session_called_ephemeral"]
    assert benchmark.broad_semantic_issues(
        "state.db and session JSONL. These carry prior-turn context. A service recycle can clear them."
    ) == ["durable_session_called_ephemeral"]


def test_oos_agent_positions_accept_attribution_without_literal_column_name() -> None:
    reply = (
        "public.beliefs stores each stance attributed to that agent with confidence. There is no claim-ID foreign "
        "key, so a direct link to the shared claim remains a schema gap."
    )

    assert benchmark.matched_concept(reply, "agent_specific_positions") is True

    natural = (
        "The fact and its source evidence live in one shared public.claims row, and both analysts do not fork the "
        "claim. Each agent files one public.beliefs row for its position. There is no claim-ID foreign key, so that "
        "link is a schema gap."
    )
    assert benchmark.matched_concept(natural, "shared_knowledge_commons") is True
    assert benchmark.matched_concept(natural, "agent_specific_positions") is True

    observed = (
        "Facts are shared, not per-agent. One claim row, one source, and one evidence link remain authoritative. "
        "Duplicating a claim per agent creates divergence."
    )
    assert benchmark.matched_concept(observed, "shared_knowledge_commons") is True

    live_reply = (
        "The shared source material and the observation both sides accept go into one claim row plus shared sources "
        "and claim_evidence. Each agent's divergent reading goes into one public.beliefs row per agent."
    )
    assert benchmark.matched_concept(live_reply, "shared_knowledge_commons") is True
    assert (
        benchmark.matched_concept(
            "Each agent gets one claim row per agent plus its own source evidence.", "shared_knowledge_commons"
        )
        is False
    )

    live_structural = (
        "One shared structural claim row plus its shared sources and evidence serves both agents. Duplicating the "
        "factual claim creates divergent trails. Each agent's interpretation goes into a separate public.beliefs "
        "row with agent_id. There is no claim-ID foreign key, which is a schema gap."
    )
    assert benchmark.matched_concept(live_structural, "shared_knowledge_commons") is True
    assert benchmark.matched_concept(live_structural, "agent_specific_positions") is True

    live_canonical = (
        "Shared source material lands as one canonical claim with shared sources and evidence rows. The two "
        "readings each become one public.beliefs row, one per agent. Beliefs has no claim-ID foreign key, so the "
        "link remains a schema gap."
    )
    assert benchmark.matched_concept(live_canonical, "shared_knowledge_commons") is True
    assert benchmark.matched_concept(live_canonical, "agent_specific_positions") is True

    share_the_fact = (
        "Share the fact. Duplicate nothing. One claim row holds the observation, and both agents use the same "
        "source evidence. Each agent's position goes into a public.beliefs row with agent_id and confidence. "
        "Beliefs has no claim-ID foreign key, so the disagreement is queryable by convention; claim_edges cannot "
        "link a belief to a claim."
    )
    assert benchmark.matched_concept(share_the_fact, "shared_knowledge_commons") is True
    assert benchmark.matched_concept(share_the_fact, "agent_specific_positions") is True

    agreed_proposition = (
        "One claim row holds the agreed proposition; both agents' sources and evidence attach to it. Each agent's "
        "divergent interpretation goes into one public.beliefs row. Beliefs has no claim_id foreign key, so an "
        "explicit belief-to-claim link remains a schema gap."
    )
    assert benchmark.matched_concept(agreed_proposition, "shared_knowledge_commons") is True
    assert benchmark.matched_concept(agreed_proposition, "agent_specific_positions") is True

    duplicated_claims = (
        "Store one separate public.claims row per agent. One claim row per agent holds the agreed proposition; "
        "both agents' sources and evidence attach to their separate rows. Each agent's divergent interpretation "
        "goes into one public.beliefs row per agent. Beliefs has no claim_id foreign key, so the link is a schema "
        "gap. The positions disagree."
    )
    assert benchmark.asserts_per_agent_claim_duplication(duplicated_claims) is True
    assert benchmark.matched_concept(duplicated_claims, "shared_knowledge_commons") is False
    prompt = next(item for item in benchmark.prompt_catalog("demo-ledger-deadbeef") if item["id"] == "OOS-11")
    assert benchmark.score_reply(prompt, duplicated_claims, memory_token="demo-ledger-deadbeef")["pass"] is False
    assert benchmark.asserts_per_agent_claim_duplication(
        "Do not create one public.claims row per agent; keep one shared claim row."
    ) is False
    assert benchmark.asserts_per_agent_claim_duplication(
        "Each agent gets its own public.claims row for the shared fact."
    ) is True
    assert benchmark.asserts_per_agent_claim_duplication(
        "Each agent should not get its own public.claims row; keep the factual row shared."
    ) is False
    assert benchmark.asserts_per_agent_claim_duplication(
        "Use one claim row per agent instead of one shared claim row."
    ) is True
    assert benchmark.asserts_per_agent_claim_duplication(
        "Use one shared claim row instead of one claim row per agent."
    ) is False

    unrelated_denial = (
        "Do not invent belief-to-claim edges while storing one public.claims row per agent: one claim row holds "
        "the agreed proposition for each agent, and both agents' sources and evidence attach to those rows. Retain "
        "that source evidence. Each agent's divergent position goes into one public.beliefs row per agent. Beliefs "
        "has no claim_id foreign key, so the schema has a gap. The positions contradict."
    )
    assert benchmark.asserts_per_agent_claim_duplication(unrelated_denial) is True
    assert benchmark.score_reply(prompt, unrelated_denial, memory_token="demo-ledger-deadbeef")["pass"] is False

    faithful_paraphrase = (
        "Keep the proposition canonical once and retain both analysts' source evidence on it. Record each analyst's "
        "interpretation in public.beliefs with agent attribution. Because beliefs has no claim_id foreign key, the "
        "association remains a schema gap; contradictory interpretations remain queryable by agent."
    )
    assert benchmark.matched_concept(faithful_paraphrase, "shared_knowledge_commons") is True
    assert benchmark.score_reply(prompt, faithful_paraphrase, memory_token="demo-ledger-deadbeef")["pass"] is True

    named_duplicates = (
        "Keep one public.claims row for Alice and one for Bob, each holding the agreed proposition, with both "
        "agents' source evidence attached to both copies. Retain that evidence. Each agent's divergent position "
        "goes into one public.beliefs row with agent attribution. Beliefs has no claim_id foreign key, so the schema "
        "has a gap. The positions contradict."
    )
    assert benchmark.asserts_per_agent_claim_duplication(named_duplicates) is True
    assert benchmark.score_reply(prompt, named_duplicates, memory_token="demo-ledger-deadbeef")["pass"] is False

    single_assertion = (
        "Deduplicate the proposition into a single canonical assertion and retain both analysts' evidence on that "
        "one record. Put each conflicting interpretation in public.beliefs with agent attribution. Beliefs lacks a "
        "claim_id foreign key, so the association remains a schema gap and disagreement stays queryable."
    )
    assert benchmark.score_reply(prompt, single_assertion, memory_token="demo-ledger-deadbeef")["pass"] is True

    namespaced_copies = (
        "Keep the proposition canonical once under each agent; retain both analysts' source evidence and attach it "
        "to their respective copies. Put each conflicting interpretation in public.beliefs with agent attribution. "
        "Beliefs lacks a claim_id foreign key, so the association remains a schema gap and disagreement stays "
        "queryable."
    )
    assert benchmark.asserts_per_agent_claim_duplication(namespaced_copies) is True
    assert benchmark.score_reply(prompt, namespaced_copies, memory_token="demo-ledger-deadbeef")["pass"] is False


def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None:
    token = "demo-ledger-deadbeef"
    set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07")
    recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08")
    set_reply = (
        f"Label: {token}\nBlocker: beliefs has no claim_id foreign key, so agent positions are not queryable "
        "against the shared claim."
    )
    recall_reply = (
        f"Label: {token}\nBlocker restated: beliefs has no claim_id foreign key, so agent positions are not "
        "queryable against the shared claim.\nProof: postflight readback of the new foreign key and linked row."
    )

    score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": set_reply},
            {"prompt_id": recall_prompt["id"], "reply": recall_reply},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )

    assert score["pass"] is True
    assert score["memory_continuity"]["same_blocker_recalled"] is True
    assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False

    different_schema_gap = (
        f"Label: {token}\nBlocker restated: public.sources has no author field, so source attribution remains a "
        "schema gap.\nProof: postflight readback of a reviewed source-schema extension."
    )
    changed_score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": set_reply},
            {"prompt_id": recall_prompt["id"], "reply": different_schema_gap},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )
    assert changed_score["pass"] is False
    assert changed_score["memory_continuity"]["same_blocker_recalled"] is False

    different_foreign_key = (
        f"Label: {token}\nBlocker restated: public.sources lacks a foreign key to provenance metadata, so the "
        "source relation remains a schema gap.\nProof: postflight readback of the source relation."
    )
    foreign_key_score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": set_reply},
            {"prompt_id": recall_prompt["id"], "reply": different_foreign_key},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )
    assert foreign_key_score["pass"] is False
    assert foreign_key_score["memory_continuity"]["same_blocker_recalled"] is False

    different_mechanism = (
        f"Label: {token}\nBlocker restated: beliefs.claim_id lacks an index, so position lookups remain a schema "
        "gap.\nProof: postflight readback of the new index and query plan."
    )
    index_score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": set_reply},
            {"prompt_id": recall_prompt["id"], "reply": different_mechanism},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )
    assert index_score["pass"] is False
    assert index_score["memory_continuity"]["same_blocker_recalled"] is False


def test_oos_memory_recall_accepts_apply_receipt_correction_without_losing_the_token() -> None:
    token = "demo-ledger-deadbeef"
    prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08")
    reply = (
        "Proof-boundary correction: aggregate counts need not change. Approved content is not directly applyable "
        "when the payload or write surface is unsupported; belief updates and behavioral_rules remain staged for a "
        "separate reviewed apply capability. "
        f"Label: {token}. Blocker: approved proposals still have applied_at null and no canonical row readback. "
        "After an authorized apply, verify proposal-level applied_at plus before/after row IDs and content hashes."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True
    assert score["invalid_count_invariant_detected"] is False
    assert score["proposal_readiness_issues"] == []


def test_oos_memory_accepts_natural_mnemonic_without_fixed_blocker_label() -> None:
    token = "blind-ledger-deadbeef1234"
    set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07")
    recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08")
    set_reply = (
        f"Mnemonic tag {token}. Three approved proposals; zero applied_at values; zero canonical rows written. "
        "Approval is a reviewer signature, applied_at is the guarded-apply receipt, and canonical readback is a "
        "public.* row ID plus hash."
    )
    recall_reply = (
        f"Mnemonic retrieved: {token}. Three proposals carry reviewer approval; applied_at is null on all three; "
        "no public.* row ID or hash postflight exists. Resolution requires proposal-level applied_at plus the target "
        "row ID and matching content hash."
    )

    score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": set_reply},
            {"prompt_id": recall_prompt["id"], "reply": recall_reply},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )

    assert score["pass"] is True
    assert score["memory_continuity"]["same_blocker_recalled"] is True

    changed_recall = (
        f"Mnemonic retrieved: {token}. Telegram delivery remains blocked: applied_at and canonical readback exist, "
        "but the visible message receipt is absent. Resolution requires message ID, timestamp, chat readback, and "
        "a postflight row ID plus hash."
    )
    changed_score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": set_reply},
            {"prompt_id": recall_prompt["id"], "reply": changed_recall},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )
    assert changed_score["pass"] is False
    assert changed_score["memory_continuity"]["same_blocker_recalled"] is False
    assert changed_score["memory_continuity"]["source_blocker_signature"] == [
        "applied_missing",
        "canonical_missing",
        "reviewer_approval",
    ]
    assert changed_score["memory_continuity"]["recall_blocker_signature"] == ["telegram_missing"]

    vague = f"Mnemonic tag {token}. Database schema evidence remains unclear."
    assert benchmark.matched_concept(vague, "blocker_definition") is False

    no_blocker = f"Mnemonic tag {token}. No database blocker exists; all canonical rows are current and complete."
    assert benchmark.matched_concept(no_blocker, "blocker_definition") is False
    assert benchmark.score_reply(set_prompt, no_blocker, memory_token=token)["pass"] is False
    modified_no_blocker = (
        f"Mnemonic tag {token}. No meaningful database blocker exists; all canonical rows are current and complete."
    )
    assert benchmark.matched_concept(modified_no_blocker, "blocker_definition") is False
    assert benchmark.score_reply(set_prompt, modified_no_blocker, memory_token=token)["pass"] is False

    faithful_set = (
        f"Mnemonic: {token} \u2014 three approved proposals still have applied_at NULL and no canonical rows; closure "
        "requires row IDs and matching hashes."
    )
    faithful_recall = (
        f"Mnemonic retrieved: {token}. Reviewer sign-off exists, but applied_at remains unset and no rows have landed "
        "in public.claims. Close it with a postflight readback of the proposal timestamp, target claim row ID, and "
        "content hash."
    )
    faithful_score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": faithful_set},
            {"prompt_id": recall_prompt["id"], "reply": faithful_recall},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )
    assert faithful_score["pass"] is True

    opposite_recall = (
        f"Mnemonic retrieved: {token}. Reviewer approval is present and applied_at is non-null; the canonical row "
        "exists. Telegram delivery is missing. Closure proof: postflight readback of the message ID, timestamp, and "
        "public row."
    )
    opposite_score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": set_reply},
            {"prompt_id": recall_prompt["id"], "reply": opposite_recall},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )
    assert opposite_score["pass"] is False
    assert opposite_score["memory_continuity"]["same_blocker_recalled"] is False

    reversed_flags = (
        f"Mnemonic retrieved: {token}. Reviewer approval is present. The applied_at missing flag is false; the "
        "canonical gap is closed. Telegram delivery remains blocked. Closure proof: postflight readback of message "
        "ID, timestamp, and public row."
    )
    reversed_score = benchmark.score_results(
        [
            {"prompt_id": set_prompt["id"], "reply": set_reply},
            {"prompt_id": recall_prompt["id"], "reply": reversed_flags},
        ],
        memory_token=token,
        catalog=[set_prompt, recall_prompt],
    )
    assert reversed_score["pass"] is False
    assert reversed_score["memory_continuity"]["same_blocker_recalled"] is False


def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
    reply = (
        "Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved "
        "field, and there is no resolves edge. Stage a proposal, obtain reviewer approval, then apply it and "
        "record applied_at."
    )

    assert benchmark.matched_concept(reply, "forecast_history") is True
    assert benchmark.matched_concept(reply, "forecast_schema_gap") is True
    assert benchmark.matched_concept(reply, "staged_review_apply") is True
    assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(reply)
    assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(
        "The schema is missing a resolves edge."
    )

    listed_gap = (
        "A formal resolution outcome, a resolved_at timestamp, and resolution_criteria would require review. "
        "None of those fields exist in the current schema. There is no resolves edge type."
    )
    assert benchmark.matched_concept(listed_gap, "forecast_schema_gap") is True

    live_reply = (
        "The original 60% probability stays in its claim. The absence of success criteria is a second claim. "
        "Overwriting the original confidence would destroy history. Resolved status and resolution criteria are "
        "not current columns, and no resolves edge type exists. Stage a reviewed proposal before any apply."
    )
    assert benchmark.matched_concept(live_reply, "forecast_history") is True
    assert benchmark.matched_concept(live_reply, "forecast_schema_gap") is True

    live_untouched = (
        "The original 60% row, including its confidence, stays untouched because resolution criteria were never "
        "defined. A new outcome claim may be staged, while a resolves edge and resolution fields require a "
        "reviewed schema proposal before apply."
    )
    assert benchmark.matched_concept(live_untouched, "forecast_history") is True

    current_claims_schema = (
        "The original 0.60 forecast stays untouched. No field on the current claims schema records resolution "
        "criteria, outcome, or resolved_at, and the current edge_type enum has no resolves type. Stage a reviewed "
        "schema proposal before an explicitly authorized apply."
    )
    assert benchmark.matched_concept(current_claims_schema, "forecast_schema_gap") is True
    explicit_denial = (
        "The schema has no resolution fields. Do not invent a resolves edge or forecast-resolution fields; neither "
        "exists in the current schema."
    )
    assert benchmark.matched_concept(explicit_denial, "forecast_schema_gap") is True

    invented_current_schema = (
        "The original 0.60 forecast stays untouched because resolution criteria were never defined. No field "
        "review is required because the current public.claims schema already records resolution criteria, outcome, "
        "and resolved_at. The current edge_type enum already contains resolves, so no resolves addition is needed. "
        "Stage a reviewed proposal before explicitly authorized apply."
    )
    assert benchmark.matched_concept(invented_current_schema, "forecast_schema_gap") is False
    assert benchmark.current_schema_overclaims(invented_current_schema) == [
        "claims_unshipped_fields",
        "invalid_current_edge_type",
    ]
    prompt = next(item for item in benchmark.prompt_catalog("demo-ledger-deadbeef") if item["id"] == "OOS-12")
    assert benchmark.score_reply(prompt, invented_current_schema, memory_token="demo-ledger-deadbeef")["pass"] is False

    exposed_fields = (
        "Preserve the original 0.60 forecast untouched because its criteria were never defined. Current "
        "public.claims exposes resolution_criteria, outcome, and resolved_at, but there is no forecast-resolution "
        "field; the enum has no resolves type. Retain the missing-criteria caveat. Stage a reviewed proposal before "
        "any apply. This is a schema gap."
    )
    exposed_score = benchmark.score_reply(prompt, exposed_fields, memory_token="demo-ledger-deadbeef")
    assert exposed_score["pass"] is False
    assert exposed_score["current_schema_overclaims"] == ["claims_unshipped_fields"]

    inverted_resolves = (
        "Preserve the original 0.60 forecast untouched because its criteria were never defined. The current schema "
        "has no forecast-resolution field. A resolves edge is not absent; it is already supported by edge_type. "
        "Retain the missing-criteria caveat and stage a reviewed proposal before any apply. This is a schema gap."
    )
    inverted_score = benchmark.score_reply(prompt, inverted_resolves, memory_token="demo-ledger-deadbeef")
    assert inverted_score["pass"] is False
    assert inverted_score["current_schema_overclaims"] == ["invalid_current_edge_type"]

    faithful_gap = (
        "Keep the original 60% forecast unchanged because its criteria were never defined. The current public.claims "
        "schema lacks resolution_criteria and resolved_at, and edge_type lacks resolves. Retain the ambiguity. Stage "
        "a reviewed schema proposal before apply; this remains a schema gap."
    )
    faithful_gap_score = benchmark.score_reply(prompt, faithful_gap, memory_token="demo-ledger-deadbeef")
    assert faithful_gap_score["pass"] is True
    assert faithful_gap_score["current_schema_overclaims"] == []

    generic_current_schema = (
        "Keep the original 60% forecast unchanged because its criteria were never defined. The current schema has "
        "resolution criteria, outcome, and resolved_at already. No resolved_at migration is needed, and the current "
        "edge_type enum has no resolves type. Retain the ambiguity and stage a reviewed proposal before apply. This "
        "remains a schema gap."
    )
    generic_score = benchmark.score_reply(prompt, generic_current_schema, memory_token="demo-ledger-deadbeef")
    assert generic_score["pass"] is False
    assert generic_score["current_schema_overclaims"] == ["claims_unshipped_fields"]

    models_fields = (
        "Preserve the original 0.60 forecast untouched because its criteria were never defined. Current public.claims "
        "natively models resolution_criteria, outcome, and resolved_at, but there is no forecast-resolution field; "
        "the enum has no resolves type. Retain the missing-criteria caveat. Stage a reviewed proposal before any "
        "apply. This is a schema gap."
    )
    models_score = benchmark.score_reply(prompt, models_fields, memory_token="demo-ledger-deadbeef")
    assert models_score["pass"] is False
    assert models_score["current_schema_overclaims"] == ["claims_unshipped_fields"]

    enumerates_resolves = (
        "Preserve the original 0.60 forecast untouched because its criteria were never defined. The current schema "
        "has no forecast-resolution field. The current edge_type enumerates resolves, which is not missing. Retain "
        "the missing-criteria caveat and stage a reviewed proposal before any apply. This is a schema gap."
    )
    enumerates_score = benchmark.score_reply(prompt, enumerates_resolves, memory_token="demo-ledger-deadbeef")
    assert enumerates_score["pass"] is False
    assert enumerates_score["current_schema_overclaims"] == ["invalid_current_edge_type"]

    cannot_express = (
        "Keep the original 60% forecast unchanged because its criteria were never defined. The current public.claims "
        "schema doesn't model resolution_criteria or resolved_at, and edge_type cannot express resolves. Retain the "
        "ambiguity. Stage a reviewed schema proposal before apply; this remains a schema gap."
    )
    cannot_express_score = benchmark.score_reply(prompt, cannot_express, memory_token="demo-ledger-deadbeef")
    assert cannot_express_score["pass"] is True
    assert cannot_express_score["current_schema_overclaims"] == []

    synonym_overclaim = (
        "Keep the original 60% forecast unchanged because its criteria were never defined. The current schema tracks "
        "resolution criteria, outcome, and resolved_at. The current edge_type enum recognizes resolves. Retain the "
        "ambiguity and stage a reviewed proposal before apply."
    )
    synonym_score = benchmark.score_reply(prompt, synonym_overclaim, memory_token="demo-ledger-deadbeef")
    assert synonym_score["pass"] is False
    assert synonym_score["current_schema_overclaims"] == [
        "claims_unshipped_fields",
        "invalid_current_edge_type",
    ]


def test_oos_schema_overclaim_accepts_anaphoric_semicolon_denial() -> None:
    safe = "A resolves edge type; it is not in the current edge type list and must not be invented."
    unsafe = "A resolves edge type; it is current and supported."

    assert benchmark.current_schema_overclaims(safe) == []
    assert benchmark.current_schema_overclaims(unsafe) == ["invalid_current_edge_type"]


def test_oos_schema_overclaim_distinguishes_claim_body_readback_from_a_body_column() -> None:
    readback = (
        "A fresh query on the claim ID returns the expected row with matching body, confidence, and source links."
    )
    plural_readback = "The retrieved claims have matching body text and source links."
    qualified_plural_readback = "Fresh public.claims rows have matching body text and source links."
    field_claim = "The public.claims table has a body field."

    assert benchmark.current_schema_overclaims(readback) == []
    assert benchmark.current_schema_overclaims(plural_readback) == []
    assert benchmark.current_schema_overclaims(qualified_plural_readback) == []
    assert benchmark.current_schema_overclaims(field_claim) == ["claims_unshipped_fields"]


def test_oos_count_invariant_rejects_a_later_contradiction_despite_a_disclaimer() -> None:
    wrong = "The aggregate count readback must show totals that differ by the exact payload delta."
    contradictory = (
        "Aggregate counts need not change for unrelated updates. For this apply, database totals must change."
    )
    same_clause = "Although aggregate counts need not change in general, database totals must change for this apply."
    safe = "Aggregate counts need not change because in-place updates preserve counts."

    assert benchmark.asserts_invalid_count_invariant(wrong) is True
    assert benchmark.asserts_invalid_count_invariant(contradictory) is True
    assert benchmark.asserts_invalid_count_invariant(same_clause) is True
    assert benchmark.asserts_invalid_count_invariant(safe) is False


def test_oos_applyability_rejects_unsupported_surface_despite_a_disclaimer() -> None:
    reply = (
        "Beliefs and behavioral_rules are written by the authorized apply. "
        "approve_claim supports claims only. The current apply also writes beliefs."
    )
    safe = "The current apply does not write beliefs; belief updates remain staged for a separate reviewed apply."
    same_clause = "Although the current apply does not write beliefs, it writes behavioral_rules."

    assert benchmark.proposal_readiness_issues("OOS-08", reply) == ["unsupported_apply_surface_overclaim"]
    assert benchmark.proposal_readiness_issues("OOS-08", same_clause) == ["unsupported_apply_surface_overclaim"]
    assert benchmark.proposal_readiness_issues("OOS-08", safe) == []


def test_oos_schema_overclaim_rejects_reverse_unreviewed_claim_types() -> None:
    observation = "Create a new observation claim in public.claims."
    hypothesis = "Create a new hypothesis claim in public.claims."

    assert benchmark.current_schema_overclaims(observation) == ["unreviewed_claim_type"]
    assert benchmark.current_schema_overclaims(hypothesis) == ["unreviewed_claim_type"]


def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None:
    reply = (
        "Challenge: the sources and claim bodies do not establish causality; the conclusion is not paid for by "
        "the evidence. A narrower candidate claim says only that the association is observed, while the mechanism "
        "remains undemonstrated."
    )

    assert benchmark.matched_concept(reply, "evidence_specific_challenge") is True
    assert benchmark.matched_concept(reply, "narrower_claim_revision") is True


def test_oos_autonomous_retrieval_accepts_structured_live_quote_blocks() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[-1]
    reply = (
        "market structure evolution. Claim ec8285ff-5f7a-4d67-baad-6d0fb0145951, type structural: "
        '\n\n"The claim body asserts a legal preemption paradox."\n\n'
        "Linked evidence source 2bbdc047-8e43-46ed-adc4-ebbb3e2ce345, artifact status no_source_pointer: "
        '\n\n"A dissent identifies the textual tension."\n\n'
        "Inference the evidence does not establish: a dissent is not binding law and does not establish that "
        "operators changed behavior. Narrower revision: the dissent identifies a textual tension that remains "
        "unresolved. Fresh canonical readback; read-only and no mutation."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True
    assert all(score["concepts"].values())

    conflated = reply.replace(
        "A dissent identifies the textual tension.",
        "The claim body asserts a legal preemption paradox.",
    )
    assert benchmark.matched_concept(conflated, "claim_body_evidence_distinction") is False


def test_oos_semantic_scorer_accepts_natural_receipt_and_policy_paraphrases() -> None:
    assert benchmark.matched_concept(
        "Identical counts prove nothing about content; compare artifact_state_sha256.",
        "row_content_proof",
    )
    assert benchmark.matched_concept(
        "approve_claim does not support behavioral_rules; the separate apply capability must be reviewed and authorized.",
        "reviewed_policy_apply",
    )
    assert benchmark.matched_concept(
        "behavioral_rules are outside approve_claim's apply capability and need a separate authorization.",
        "reviewed_policy_apply",
    )
    assert benchmark.matched_concept(
        "Leave the original claim untouched with confidence 0.6 because no resolution criteria existed.",
        "forecast_history",
    )
    assert benchmark.matched_concept(
        "Challenged inference: the claim body and evidence do not establish causality.",
        "evidence_specific_challenge",
    )
    assert benchmark.matched_concept(
        "Revision: adoption is documented, while lock-in remains undemonstrated.",
        "narrower_claim_revision",
    )
    assert benchmark.matched_concept(
        "public.sources contains no matching row and claim_evidence contains no link, so canonical evidence is absent.",
        "canonical_evidence_boundary",
    )
    assert benchmark.matched_concept(
        "Agent divergence lives in public.beliefs, one row per agent position; no claim-ID foreign key exists.",
        "agent_specific_positions",
    )
    assert benchmark.matched_concept(
        "The original forecast remains a historical record at confidence 0.60; without a resolution rule the "
        "outcome is ambiguous.",
        "forecast_history",
    )
    assert benchmark.matched_concept(
        "Narrower revision: the snapshots document recurrence but do not establish an increasing rate.",
        "narrower_claim_revision",
    )


def test_oos_applyability_gap_accepts_human_readable_payload_wording() -> None:
    reply = "The approved proposal has no apply payload, so normalization and renewed review must precede apply."

    assert benchmark.proposal_readiness_issues("OOS-07", reply) == []


def test_oos_schema_gap_accepts_explicitly_absent_edge_type() -> None:
    reply = "Current claim_edges has edge_type; there is no `resolves` edge type."
    assert benchmark.current_schema_overclaims(reply) == []


def test_oos_source_intake_accepts_prepared_input_staging_boundary() -> None:
    patterns = benchmark.CONCEPT_PATTERNS["bounded_intake_tier"]
    reply = (
        "VPS source-to-proposal staging is shipped for prepared inputs; autonomous chat attachment extraction "
        "is not shipped."
    )
    assert all(pattern.search(reply) for pattern in patterns)


def test_oos_source_intake_accepts_without_authorization_and_created_by_column() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[13]
    reply = (
        "Immediately capture and hash the artifact without authorization, retain its URL or storage path, and stage "
        "candidates in a pending_review proposal. The real source identity is the URL, storage path, and content hash; "
        "a chat label or source_ref is not a source. created_by is metadata, not an instruction to create a chat label. "
        "Approval begins at canonical apply where reviewed rows would write to public.*. VPS source-to-proposal "
        "staging is shipped for prepared inputs; autonomous chat attachment extraction is not shipped."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["concepts"]["staging_without_apply_authorization"] is True
    assert score["broad_semantic_issues"] == []
    assert score["pass"] is True


def test_oos_runtime_case_accepts_explicit_persistence_tiers() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[9]
    reply = (
        "No to both. Unchanged totals do not prove behavior is unchanged. Individual rows could have been updated. "
        "Proof tiers: Tier 1 is canonical DB counts; Tier 2 is runtime skills, config, SOUL.md, state.db and session "
        "JSONL, which can preserve continuity across restart; Tier 3 is transient context."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True


def test_oos_runtime_case_accepts_not_erased_by_restart_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[9]
    reply = (
        "Neither. Unchanged database totals say nothing about answer behavior, and individual rows may be updated. "
        "Proof tiers: Tier 1 is canonical DB rows and hashes; Tier 2 is runtime skills, config and SOUL.md; Tier 3 is "
        "state.db and session JSONL, which preserve continuity. Facts are not erased by restart."
    )

    assert benchmark.score_reply(prompt, reply, memory_token=token)["pass"] is True


def test_oos_composition_rejects_belief_edges_and_per_step_applied_at() -> None:
    reply = "The old belief row gets a supersedes edge. Each step gets a non-null applied_at before the next runs."

    assert benchmark.broad_semantic_issues(reply) == [
        "applied_at_assigned_to_each_apply_step",
        "belief_row_given_claim_edge",
    ]


def test_oos_handler_case_rejects_telegram_live_overclaim() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[12]
    reply = good_reply(prompt["id"], token) + (
        " The Telegram path is live and proven even though the run posted nothing to Telegram."
    )
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["broad_semantic_issues"] == ["handler_proof_called_telegram_live"]


def test_oos_forecast_case_rejects_historical_probability_overwrite() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[11]
    reply = good_reply(prompt["id"], token) + " Overwrite the original probability with the actual resolved outcome."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["broad_semantic_issues"] == ["forecast_history_rewrite"]


def test_oos_source_intake_case_rejects_temporary_label_as_source() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[13]
    reply = good_reply(prompt["id"], token) + " Create the temporary chat label as a canonical source_ref."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["broad_semantic_issues"] == ["temporary_label_promoted_to_source"]


def test_oos_score_rejects_architecture_lecture_length() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[0]
    reply = good_reply(prompt["id"], token) + " detail" * 301
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["response_too_long"] is True
    assert score["max_response_words"] == benchmark.DEFAULT_MAX_RESPONSE_WORDS


def test_oos_score_rejects_blanket_all_five_counts_must_move_claim() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    reply = good_reply(prompt["id"], token) + " All five counts must move after every apply."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["invalid_count_invariant_detected"] is True


def test_oos_score_rejects_blanket_all_five_counts_higher_claim() -> None:
    reply = "The proof is all five canonical counts higher than the baseline."
    assert benchmark.asserts_invalid_count_invariant(reply) is True


def test_oos_score_does_not_penalize_an_explicit_retraction() -> None:
    reply = "The claim that all five counts must move is wrong; packet-specific row readback is required."
    assert benchmark.asserts_invalid_count_invariant(reply) is False


def test_count_receipt_correction_requires_packet_specific_row_receipts() -> None:
    reply = """
    No. That rule was too strong: not all table counts move; the delta depends on the packet.
    public.claims and public.sources stay unchanged when reused. For attach_evidence with an existing claim and
    existing source, public.claim_evidence gets a new row. For an edge-only packet, public.claim_edges gets a new
    row. kb_stage.kb_proposals count may remain unchanged while its status and applied_at change. The universal
    receipt is a committed transaction, non-null applied_at, and postflight row-level readback proving the packet's
    declared expected rows and IDs. public.claims, public.sources, public.claim_edges, and public.claim_evidence are
    checked table by table against that packet.
    """
    score = correction.score_reply(reply)
    assert score["pass"] is True


def test_count_receipt_correction_rejects_repeated_bad_invariant() -> None:
    reply = "All five counts must move after every apply."
    score = correction.score_reply(reply)
    assert score["pass"] is False
    assert score["invalid_count_invariant_detected"] is True


def test_generic_handler_template_accepts_an_oos_prompt_set() -> None:
    script = handler.build_remote_script(
        "cafef00d",
        prompts=[{"id": "OOS-01", "dimension": "demo", "message": "Broad question"}],
        suite_mode="oos_mode",
        report_prefix="oos-report",
        prompt_note='Out of sample "quoted" prompts.',
    )
    assert '"mode": "oos_mode"' in script
    assert 'RUN_ID = "cafef00d"' in script
    assert 'REPORT_PREFIX = "oos-report"' in script
    assert 'Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")' in script
    assert 'Out of sample \\"quoted\\" prompts.' in script
    assert '"id": "OOS-01"' in script
    assert "__PROMPTS_JSON__" not in script


def test_generic_handler_template_rejects_unsafe_report_prefix() -> None:
    try:
        handler.build_remote_script("cafef00d", report_prefix="../escape")
    except ValueError as exc:
        assert "report_prefix" in str(exc)
    else:
        raise AssertionError("unsafe report prefix was accepted")