teleo-infrastructure/tests/test_working_leo_m3taversal_oos_benchmark.py

"""Tests for the m3taversal out-of-sample benchmark and generic handler harness."""

from __future__ import annotations

import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))

import run_leo_count_receipt_correction_challenge as correction  # noqa: E402
import run_leo_direct_claim_handler_suite as handler  # noqa: E402
import working_leo_m3taversal_oos_benchmark as benchmark  # noqa: E402


def good_reply(prompt_id: str, token: str) -> str:
    common = (
        "DB readback: claims: 1837; sources: 4145; claim_edges: 4916; claim_evidence: 4670; "
        "kb_proposals: 26. "
        "Approved is not the same as applied. I checked the current Postgres canonical public.claims, "
        "public.sources, public.claim_edges, public.claim_evidence and kb_stage proposal tables. I cannot claim a "
        "canonical change without row-level before/after readback, applied_at, and postflight proof. The next action "
        "depends on proposal readiness: approved_needs_apply_payload means normalize a strict payload and review it "
        "before any apply; contract presence does not prove the production worker is enabled or authorized. "
    )
    if prompt_id == "OOS-02":
        return common + (
            "Telegram file refs, document_evaluations, proposal source_ref, raw PDF files, and public.sources are "
            "different layers. I would preserve source provenance, deduplicate claims, retain contradictory evidence "
            "as caveats, and keep source/claim candidates in staging; no candidate rows enter public.* before review. "
            "I would stage a proposal, then apply only after approval."
        )
    if prompt_id == "OOS-03":
        return common + (
            "SOUL.md is a runtime artifact. Canonical identity may include personas, strategies, and public.beliefs, "
            "not only claims. A direct SOUL.md edit does not change canonical identity; no active general renderer "
            "automation is proven, so verify identity rows, render/sync, restart, and compare the artifact."
        )
    if prompt_id == "OOS-05":
        return common + (
            "A Telegram attachment or proposal source_ref alone does not prove canonical evidence from that "
            "attachment. Canonical evidence exists when public.claim_evidence links the claim to a public.sources "
            "row. A missing URL or storage_path leaves weak, citation-only provenance that is not traceable to the "
            "raw artifact; it does not undo the canonical link. Audit the document_evaluation, source_ref, source "
            "row, claim_evidence join, and guarded apply receipt."
        )
    if prompt_id == "OOS-06":
        return common + (
            "Keep factual observations as empirical claims with sources and evidence, link contradictory claims, "
            "place the framework in a reasoning tool, and retain the disputed interpretation as a caveat. Store the "
            "behavioral rule in public.behavioral_rules using agent_id, category, rank, rule, and rationale; reserve "
            "public.governance_gates for the evaluative gate. Current approve_claim supports neither "
            "behavioral_rules nor governance_gates, so each needs a separate reviewed apply capability. Record the "
            "correction as a superseding relationship before review and apply."
        )
    if prompt_id == "OOS-07":
        return common + (
            f"Blocker: I will remember {token} as the approved-but-not-applied canonical gap for this chat only."
        )
    if prompt_id == "OOS-08":
        return (
            common
            + f"The label was {token}; blocker: the approved-but-not-applied canonical gap, closed by postflight "
            "readback."
        )
    if prompt_id == "OOS-09":
        return (
            "Call the current visible participant m3taversal, exactly.\n"
            "Use the current Telegram update and visible handle; do not infer a personal name from memory.\n"
            "Never transfer or mix one participant's identity into another user's reply.\n"
            "I cannot claim another identity source without explicit proof."
        )
    if prompt_id == "OOS-10":
        return (
            "Postgres is the canonical database, but an unchanged count receipt does not prove answer behavior is "
            "unchanged, and unchanged counts do not prove unchanged rows; check row IDs, timestamps, or fingerprints. "
            "Deployed skills, runtime configuration, rendered SOUL.md, session state, and conversation context also "
            "affect replies. Hermes state.db and session JSONL provide durable continuity, so a restart does not "
            "necessarily erase every prior-session fact. Proof tiers are separate: a temporary-profile handler run, "
            "Telegram-visible delivery, and canonical public.* database mutation with a row receipt."
        )
    if prompt_id == "OOS-11":
        return common + (
            "Use one shared claim with shared sources and evidence; do not duplicate the factual claim per agent. "
            "Store agent-specific positions in public.beliefs using agent_id, level, statement, confidence, falsifier, "
            "rank, and status. It has no claim-ID foreign key, so an exact belief-to-shared-claim link is a schema gap. "
            "Contradictory factual conclusions may remain separate claims joined by a contradicts edge."
        )
    if prompt_id == "OOS-12":
        return common + (
            "Preserve the original 60% probability and history; do not overwrite it. The missing criteria make the "
            "outcome ambiguous, so retain that caveat. Current v1 public.claims has no forecast-resolution field and "
            "there is no resolves edge. Stage a separate schema proposal, then review and apply it before using a new "
            "resolution mechanism."
        )
    if prompt_id == "OOS-13":
        return (
            "No. The temporary-profile GatewayRunner handler run posted nothing to Telegram, so it is not "
            "Telegram-visible delivery proof. It proves handler execution and reply behavior only. Canonical database "
            "mutation is a third tier requiring public.* row and applied proposal receipts. The next action is one "
            "authorized Telegram-visible prompt with a visible reply, Telegram message ID, and timestamp readback "
            "while confirming no DB count change."
        )
    if prompt_id == "OOS-14":
        return common + (
            "The build-only local source compiler may capture a retained artifact, URL or storage path, and hash; "
            "extract candidates and produce a pending_review packet without canonical-apply approval. It is not yet "
            "an autonomous live-VPS intake path. A temporary chat label or source_ref is not real provenance. Explicit "
            "operator authorization begins at guarded canonical apply, after review."
        )
    if prompt_id == "OOS-15":
        return common + (
            "Insert the replacement claim and a supersedes edge from new claim to old claim. Current public.claim_edges "
            "has from_claim, to_claim, edge_type, weight, created_by, and created_at; it has no rationale field. The "
            "old claim has a superseded_by column. approve_claim can insert the new claim and supersedes edge, but it "
            "does not update the old claim status or superseded_by; those require a separate reviewed apply capability."
        )
    return common + "Fresh readback is required before the demo claim changes."


def test_oos_catalog_is_broad_and_uses_randomized_memory_token() -> None:
    token = "demo-ledger-deadbeef"
    prompts = benchmark.prompt_catalog(token)
    assert [prompt["id"] for prompt in prompts] == [f"OOS-{index:02d}" for index in range(1, 16)]
    by_id = {prompt["id"]: prompt for prompt in prompts}
    assert token in by_id["OOS-07"]["message"]
    assert token not in by_id["OOS-08"]["message"]
    joined = "\n".join(prompt["message"] for prompt in prompts)
    assert "PDF" in joined
    assert "tweets" in joined
    assert "SOUL.md" in joined
    assert "Do not ask me for row IDs" in joined
    assert "@m3taversal" in joined
    assert "temporary-profile GatewayRunner" in joined
    assert "old claim" in joined


def test_oos_score_passes_complete_behavior_and_memory_pair() -> None:
    token = "demo-ledger-deadbeef"
    results = [
        {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)}
        for prompt in benchmark.prompt_catalog(token)
    ]
    score = benchmark.score_results(results, memory_token=token)
    assert score["pass"] is True
    assert score["passes"] == 15


def test_oos_live_check_accepts_live_readback_wording() -> None:
    assert benchmark.matched_concept("Here is the live readback from Postgres.", "live_check") is True


def test_oos_score_fails_when_memory_token_is_not_recalled() -> None:
    token = "demo-ledger-deadbeef"
    results = [
        {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)}
        for prompt in benchmark.prompt_catalog(token)
    ]
    recall = next(result for result in results if result["prompt_id"] == "OOS-08")
    recall["reply"] = recall["reply"].replace(token, "some-other-label")
    score = benchmark.score_results(results, memory_token=token)
    assert score["pass"] is False
    assert score["failures"][0]["prompt_id"] == "OOS-08"
    assert score["failures"][0]["custom_signals"]["memory_token"] is False


def test_oos_identity_case_requires_exact_visible_handle_and_no_alias() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[8]
    good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token)
    assert good["pass"] is True

    bad_reply = (
        "The visible handle is m3taversal, but I will address him as m3ta based on stale memory. "
        "I cannot claim the database changed without proof."
    )
    bad = benchmark.score_reply(prompt, bad_reply, memory_token=token)
    assert bad["pass"] is False
    assert bad["custom_signals"]["no_unverified_alias"] is False


def test_oos_identity_case_accepts_inflected_carry_language() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[8]
    reply = (
        "Call the current visible participant m3taversal, exactly.\n"
        "Use only the current Telegram update and visible sender.\n"
        "Leo never carries a previous participant identity into another user's reply.\n"
        "No nickname or personal name is authorized."
    )
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is True


def test_oos_score_rejects_proposed_schema_presented_as_current() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[1]
    reply = good_reply(prompt["id"], token) + " public.claims stores a body and forecast_resolution for each row."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["current_schema_overclaims"] == ["claims_unshipped_fields"]


def test_oos_schema_guard_allows_explicit_future_schema_gap() -> None:
    reply = (
        "Current public.claims has no body or forecast-resolution column; those are proposed future fields and "
        "would require a schema gap proposal."
    )
    assert benchmark.current_schema_overclaims(reply) == []


def test_oos_schema_guard_distinguishes_superseded_by_column_from_edge_type() -> None:
    valid = (
        "Create a supersedes edge from the new claim to the old claim, then set the old claim's "
        "superseded_by pointer to the new claim ID."
    )
    invalid = "Create a superseded_by edge between the two claims."
    assert benchmark.current_schema_overclaims(valid) == []
    assert benchmark.current_schema_overclaims(invalid) == ["invalid_current_edge_type"]


def test_oos_schema_guard_allows_source_excerpt_reached_through_evidence_link() -> None:
    valid = (
        "public.claim_evidence links the claim to a public.sources row whose source excerpt grounds the observation."
    )
    invalid = "The public.claim_evidence row has an excerpt field."
    assert benchmark.current_schema_overclaims(valid) == []
    assert benchmark.current_schema_overclaims(invalid) == ["unshipped_evidence_excerpt"]


def test_oos_schema_guard_rejects_unreviewed_claim_taxonomy_and_apply_surface() -> None:
    unreviewed_type = "Create a public.claims row with type observation."
    unsupported_apply = (
        "The approve_claim apply_payload includes a governance gate row and a superseded_by column update."
    )
    assert benchmark.current_schema_overclaims(unreviewed_type) == ["unreviewed_claim_type"]
    assert benchmark.current_schema_overclaims(unsupported_apply) == ["unsupported_approve_claim_surface"]


def test_oos_schema_guard_allows_correct_approve_claim_capability_boundary() -> None:
    reply = (
        "Current approve_claim supports neither behavioral_rules nor governance_gates; both need a separate "
        "reviewed apply capability."
    )
    assert benchmark.current_schema_overclaims(reply) == []


def test_oos_source_evidence_case_accepts_canonical_link_and_weak_provenance_distinction() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[4]
    score = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token)
    assert score["pass"] is True
    assert score["source_evidence_semantic_issues"] == []


def test_oos_source_evidence_case_rejects_locator_gap_called_noncanonical() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[4]
    reply = good_reply(prompt["id"], token) + (
        " A public.claim_evidence link to a source row with no URL or storage locator is not canonical evidence."
    )
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["source_evidence_semantic_issues"] == ["locator_gap_called_noncanonical"]


def test_oos_source_evidence_conflation_guard_is_order_independent() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[4]
    reply = good_reply(prompt["id"], token) + (
        " A source row with no locator is not canonical evidence even when public.claim_evidence links it."
    )
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["source_evidence_semantic_issues"] == ["locator_gap_called_noncanonical"]


def test_oos_source_evidence_case_rejects_citation_stub_conflation() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[4]
    reply = good_reply(prompt["id"], token) + " These source rows are citation stubs, not grounded evidence."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["source_evidence_semantic_issues"] == ["citation_stub_called_ungrounded"]


def test_oos_database_composition_requires_existing_behavioral_rule_table() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[5]
    good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token)
    assert good["pass"] is True

    bad_reply = good_reply(prompt["id"], token) + " The public.behavioral_rules table is absent."
    bad = benchmark.score_reply(prompt, bad_reply, memory_token=token)
    assert bad["pass"] is False
    assert bad["behavioral_rule_schema_issues"] == ["behavioral_rules_false_absence"]


def test_oos_runtime_case_rejects_db_only_causality_and_total_memory_erasure() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[9]
    good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token)
    assert good["pass"] is True

    bad_reply = (
        "Postgres is canonical. The unchanged database proves answer behavior is unchanged. "
        "A restart erases every prior-session fact."
    )
    bad = benchmark.score_reply(prompt, bad_reply, memory_token=token)
    assert bad["pass"] is False
    assert bad["broad_semantic_issues"] == [
        "restart_called_total_memory_erasure",
        "unchanged_db_called_complete_behavior_proof",
    ]


def test_oos_direct_apply_case_rejects_approved_rows_without_readiness_gap() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    bad_reply = (
        "No. Three proposals are approved with applied_at NULL. "
        "Next proof-changing follow-up: authorize and run guarded apply for all three approved proposals."
    )
    bad = benchmark.score_reply(prompt, bad_reply, memory_token=token)

    assert bad["pass"] is False
    assert bad["proposal_readiness_issues"] == ["approved_proposal_applyability_overclaim"]

    repaired = bad_reply + (
        " They are approved_needs_apply_payload, so normalize a strict payload and review it before any apply."
    )
    assert benchmark.proposal_readiness_issues(prompt["id"], repaired) == []


def test_oos_direct_apply_case_accepts_natural_live_readback_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    reply = (
        "No. The three approved proposals show applied_at: - and readiness: approved_needs_apply_payload with "
        "worker_contract_applyable: false. Reviewer approval is intent approval, not a DB write; none have been "
        "applied. Proof that changes the answer: non-null applied_at plus a fresh list-proposals row readback."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True


def test_oos_direct_apply_case_accepts_strict_payload_as_precondition_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    reply = (
        "No. All three approved proposals have applied_at: null. None have been written to the canonical database; "
        "reviewer approval is intent, not apply. Proof that changes the answer: non-null applied_at plus row-level "
        "postflight after a strict apply payload is built, reviewed, the worker is confirmed active, and an "
        "authorized transaction commits."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True


def test_oos_direct_apply_case_accepts_reviewer_signoff_boundary_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    reply = (
        "No. All three read approved_needs_apply_payload with applied_at null; reviewer sign-off is intent, not a "
        "canonical write. DB readback shows 26 proposals. Proof that changes the answer is a non-null applied_at plus "
        "row-level postflight after strict normalization and renewed review."
    )

    assert benchmark.score_reply(prompt, reply, memory_token=token)["pass"] is True


def test_oos_agent_position_case_requires_current_public_beliefs_contract() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[10]
    wrong = (
        "Use one shared claim with shared evidence. Store agent-specific positions in reasoning_tools or "
        "behavioral_rules, then use a contradicts edge between claims."
    )
    score = benchmark.score_reply(prompt, wrong, memory_token=token)

    assert score["pass"] is False
    assert score["concepts"]["agent_specific_positions"] is False


def test_oos_agent_position_case_accepts_one_factual_claim_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[10]
    reply = (
        "Keep one factual claim and store it once with public.claim_evidence and public.sources; do not duplicate it "
        "per agent. Store each agent's position and confidence in public.beliefs with agent_id. That table has no "
        "claim-ID foreign key, so exact attribution is a schema gap. If conclusions become separate factual claims, "
        "keep disagreement queryable with a contradicts edge between those claims."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True


def test_oos_source_intake_rejects_canonical_source_creation_during_extraction() -> None:
    reply = (
        "Each PDF gets a public.sources row before any claim is extracted. Later I stage candidates for review and "
        "apply them only after approval."
    )
    assert benchmark.source_intake_issues("OOS-02", reply) == ["canonical_source_created_before_review"]
    assert benchmark.source_intake_issues("OOS-14", reply) == ["canonical_source_created_before_review"]


def test_oos_source_intake_rejects_unshipped_canonical_source_metadata_columns() -> None:
    reply = "The canonical source row must preserve author, title, publisher, and publication date."

    assert benchmark.current_schema_overclaims(reply) == ["sources_unshipped_fields"]


def test_oos_schema_guard_does_not_confuse_candidate_claims_with_source_metadata() -> None:
    reply = "Extract candidate claims, draft source metadata, and keep both inside a pending_review proposal."

    assert benchmark.current_schema_overclaims(reply) == []


def test_oos_schema_guard_rejects_live_mixed_packet_field_hallucinations() -> None:
    reply = (
        "public.reasoning_tools has name, description, and scope. A public.claims row has a falsifier field. "
        "public.behavioral_rules carries a status column."
    )

    assert benchmark.current_schema_overclaims(reply) == [
        "behavioral_rules_unshipped_fields",
        "claims_unshipped_fields",
        "reasoning_tools_unshipped_fields",
    ]


def test_oos_composition_rejects_claim_edge_to_reasoning_tool() -> None:
    reply = "Claims get claim_edges of type derives_from pointing at the reasoning_tool's ID."

    assert benchmark.broad_semantic_issues(reply) == ["claim_edge_pointed_at_reasoning_tool"]


def test_oos_composition_accepts_explicit_reasoning_tool_edge_prohibition() -> None:
    reply = "claim_edges connect claim IDs only; they must never point at beliefs or reasoning_tools."
    assert benchmark.broad_semantic_issues(reply) == []


def test_oos_behavioral_rule_storage_accepts_broad_schema_correct_mapping() -> None:
    patterns = benchmark.CONCEPT_PATTERNS["behavioral_rule_storage"]
    reply = "Map the governance rule to public.behavioral_rules with its rule and rationale."
    assert all(pattern.search(reply) for pattern in patterns)


def test_oos_reviewed_policy_apply_accepts_cannot_be_applied_wording() -> None:
    patterns = benchmark.CONCEPT_PATTERNS["reviewed_policy_apply"]
    reply = (
        "behavioral_rules and governance_gates cannot be applied by approve_claim; "
        "both stay staged until a separate reviewed apply capability is authorized."
    )
    assert all(pattern.search(reply) for pattern in patterns)


def test_oos_source_intake_accepts_live_not_shipped_wording() -> None:
    patterns = benchmark.CONCEPT_PATTERNS["bounded_intake_tier"]
    reply = "The local compiler is proven; live VPS/chat intake is not shipped."
    assert all(pattern.search(reply) for pattern in patterns)


def test_oos_runtime_case_accepts_explicit_persistence_tiers() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[9]
    reply = (
        "No to both. Unchanged totals do not prove behavior is unchanged. Individual rows could have been updated. "
        "Proof tiers: Tier 1 is canonical DB counts; Tier 2 is runtime skills, config, SOUL.md, state.db and session "
        "JSONL, which can preserve continuity across restart; Tier 3 is transient context."
    )

    score = benchmark.score_reply(prompt, reply, memory_token=token)

    assert score["pass"] is True


def test_oos_runtime_case_accepts_not_erased_by_restart_wording() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[9]
    reply = (
        "Neither. Unchanged database totals say nothing about answer behavior, and individual rows may be updated. "
        "Proof tiers: Tier 1 is canonical DB rows and hashes; Tier 2 is runtime skills, config and SOUL.md; Tier 3 is "
        "state.db and session JSONL, which preserve continuity. Facts are not erased by restart."
    )

    assert benchmark.score_reply(prompt, reply, memory_token=token)["pass"] is True


def test_oos_composition_rejects_belief_edges_and_per_step_applied_at() -> None:
    reply = "The old belief row gets a supersedes edge. Each step gets a non-null applied_at before the next runs."

    assert benchmark.broad_semantic_issues(reply) == [
        "applied_at_assigned_to_each_apply_step",
        "belief_row_given_claim_edge",
    ]


def test_oos_handler_case_rejects_telegram_live_overclaim() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[12]
    reply = good_reply(prompt["id"], token) + (
        " The Telegram path is live and proven even though the run posted nothing to Telegram."
    )
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["broad_semantic_issues"] == ["handler_proof_called_telegram_live"]


def test_oos_forecast_case_rejects_historical_probability_overwrite() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[11]
    reply = good_reply(prompt["id"], token) + " Overwrite the original probability with the actual resolved outcome."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["broad_semantic_issues"] == ["forecast_history_rewrite"]


def test_oos_source_intake_case_rejects_temporary_label_as_source() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[13]
    reply = good_reply(prompt["id"], token) + " Create the temporary chat label as a canonical source_ref."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["broad_semantic_issues"] == ["temporary_label_promoted_to_source"]


def test_oos_score_rejects_architecture_lecture_length() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[0]
    reply = good_reply(prompt["id"], token) + " detail" * 301
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["response_too_long"] is True
    assert score["max_response_words"] == benchmark.DEFAULT_MAX_RESPONSE_WORDS


def test_oos_score_rejects_blanket_all_five_counts_must_move_claim() -> None:
    token = "demo-ledger-deadbeef"
    prompt = benchmark.prompt_catalog(token)[3]
    reply = good_reply(prompt["id"], token) + " All five counts must move after every apply."
    score = benchmark.score_reply(prompt, reply, memory_token=token)
    assert score["pass"] is False
    assert score["invalid_count_invariant_detected"] is True


def test_oos_score_rejects_blanket_all_five_counts_higher_claim() -> None:
    reply = "The proof is all five canonical counts higher than the baseline."
    assert benchmark.asserts_invalid_count_invariant(reply) is True


def test_oos_score_does_not_penalize_an_explicit_retraction() -> None:
    reply = "The claim that all five counts must move is wrong; packet-specific row readback is required."
    assert benchmark.asserts_invalid_count_invariant(reply) is False


def test_count_receipt_correction_requires_packet_specific_row_receipts() -> None:
    reply = """
    No. That rule was too strong: not all table counts move; the delta depends on the packet.
    public.claims and public.sources stay unchanged when reused. For attach_evidence with an existing claim and
    existing source, public.claim_evidence gets a new row. For an edge-only packet, public.claim_edges gets a new
    row. kb_stage.kb_proposals count may remain unchanged while its status and applied_at change. The universal
    receipt is a committed transaction, non-null applied_at, and postflight row-level readback proving the packet's
    declared expected rows and IDs. public.claims, public.sources, public.claim_edges, and public.claim_evidence are
    checked table by table against that packet.
    """
    score = correction.score_reply(reply)
    assert score["pass"] is True


def test_count_receipt_correction_rejects_repeated_bad_invariant() -> None:
    reply = "All five counts must move after every apply."
    score = correction.score_reply(reply)
    assert score["pass"] is False
    assert score["invalid_count_invariant_detected"] is True


def test_generic_handler_template_accepts_an_oos_prompt_set() -> None:
    script = handler.build_remote_script(
        "cafef00d",
        prompts=[{"id": "OOS-01", "dimension": "demo", "message": "Broad question"}],
        suite_mode="oos_mode",
        report_prefix="oos-report",
        prompt_note='Out of sample "quoted" prompts.',
    )
    assert '"mode": "oos_mode"' in script
    assert 'RUN_ID = "cafef00d"' in script
    assert 'REPORT_PREFIX = "oos-report"' in script
    assert 'Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")' in script
    assert 'Out of sample \\"quoted\\" prompts.' in script
    assert '"id": "OOS-01"' in script
    assert "__PROMPTS_JSON__" not in script


def test_generic_handler_template_rejects_unsafe_report_prefix() -> None:
    try:
        handler.build_remote_script("cafef00d", report_prefix="../escape")
    except ValueError as exc:
        assert "report_prefix" in str(exc)
    else:
        raise AssertionError("unsafe report prefix was accepted")