"""Tests for the m3taversal out-of-sample benchmark and generic handler harness.""" from __future__ import annotations import importlib.util import sys from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "scripts")) import run_leo_count_receipt_correction_challenge as correction # noqa: E402 import run_leo_direct_claim_handler_suite as handler # noqa: E402 import working_leo_m3taversal_oos_benchmark as benchmark # noqa: E402 def load_kb_tool(): path = REPO_ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py" spec = importlib.util.spec_from_file_location("kb_tool_for_oos_test", path) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def good_reply(prompt_id: str, token: str) -> str: common = ( "DB readback: claims: 1837; sources: 4145; claim_edges: 4916; claim_evidence: 4670; " "kb_proposals: 26. " "Approved is not the same as applied. I checked the current Postgres canonical public.claims, " "public.sources, public.claim_edges, public.claim_evidence and kb_stage proposal tables. I cannot claim a " "canonical change without row-level before/after readback, applied_at, and postflight proof. The next action " "depends on proposal readiness: approved_needs_apply_payload means normalize a strict payload and review it " "before any apply; contract presence does not prove the production worker is enabled or authorized. " ) if prompt_id == "OOS-02": return common + ( "Telegram file refs, document_evaluations, proposal source_ref, raw PDF files, and public.sources are " "different layers. I would preserve source provenance, deduplicate claims, retain contradictory evidence " "as caveats, and keep source/claim candidates in staging; no candidate rows enter public.* before review. " "I would stage a proposal, then apply only after approval." ) if prompt_id == "OOS-03": return common + ( "SOUL.md is a runtime artifact. Canonical identity may include personas, strategies, and public.beliefs, " "not only claims. A direct SOUL.md edit does not change canonical identity; no active general renderer " "automation is proven, so verify identity rows, render/sync, restart, and compare the artifact." ) if prompt_id == "OOS-05": return common + ( "A Telegram attachment or proposal source_ref alone does not prove canonical evidence from that " "attachment. Canonical evidence exists when public.claim_evidence links the claim to a public.sources " "row. A missing URL or storage_path leaves weak, citation-only provenance that is not traceable to the " "raw artifact; it does not undo the canonical link. Audit the document_evaluation, source_ref, source " "row, claim_evidence join, and guarded apply receipt." ) if prompt_id == "OOS-06": return common + ( "Keep factual observations as empirical claims with sources and evidence, link contradictory claims, " "place the framework in a reasoning tool, and retain the disputed interpretation as a caveat. Store the " "behavioral rule in public.behavioral_rules using agent_id, category, rank, rule, and rationale; reserve " "public.governance_gates for the evaluative gate. Current approve_claim supports neither " "behavioral_rules nor governance_gates, so each needs a separate reviewed apply capability. Record the " "correction as a superseding relationship before review and apply." ) if prompt_id == "OOS-07": return common + ( f"Blocker: I will remember {token} as the approved-but-not-applied canonical gap for this chat only." ) if prompt_id == "OOS-08": return ( common + f"The label was {token}; blocker: the approved-but-not-applied canonical gap, closed by postflight " "readback." ) if prompt_id == "OOS-09": return ( "Call the current visible participant m3taversal, exactly.\n" "Use the current Telegram update and visible handle; do not infer a personal name from memory.\n" "Never transfer or mix one participant's identity into another user's reply.\n" "I cannot claim another identity source without explicit proof." ) if prompt_id == "OOS-10": return ( "Postgres is the canonical database, but an unchanged count receipt does not prove answer behavior is " "unchanged, and unchanged counts do not prove unchanged rows; check row IDs, timestamps, or fingerprints. " "Deployed skills, runtime configuration, rendered SOUL.md, session state, and conversation context also " "affect replies. Hermes state.db and session JSONL provide durable continuity, so a restart does not " "necessarily erase every prior-session fact. Proof tiers are separate: a temporary-profile handler run, " "Telegram-visible delivery, and canonical public.* database mutation with a row receipt." ) if prompt_id == "OOS-11": return common + ( "Use one shared claim with shared sources and evidence; do not duplicate the factual claim per agent. " "Store agent-specific positions in public.beliefs using agent_id, level, statement, confidence, falsifier, " "rank, and status. It has no claim-ID foreign key, so an exact belief-to-shared-claim link is a schema gap. " "Contradictory factual conclusions may remain separate claims joined by a contradicts edge." ) if prompt_id == "OOS-12": return common + ( "Preserve the original 60% probability and history; do not overwrite it. The missing criteria make the " "outcome ambiguous, so retain that caveat. Current v1 public.claims has no forecast-resolution field and " "there is no resolves edge. Stage a separate schema proposal, then review and apply it before using a new " "resolution mechanism." ) if prompt_id == "OOS-13": return ( "No. The temporary-profile GatewayRunner handler run posted nothing to Telegram, so it is not " "Telegram-visible delivery proof. It proves handler execution and reply behavior only. Canonical database " "mutation is a third tier requiring public.* row and applied proposal receipts. The next action is one " "authorized Telegram-visible prompt with a visible reply, Telegram message ID, and timestamp readback " "while confirming no DB count change." ) if prompt_id == "OOS-14": return common + ( "The build-only local source compiler may capture a retained artifact, URL or storage path, and hash; " "extract candidates and produce a pending_review packet without canonical-apply approval. It is not yet " "an autonomous live-VPS intake path. A temporary chat label or source_ref is not real provenance. Explicit " "operator authorization begins at guarded canonical apply, after review." ) if prompt_id == "OOS-15": return common + ( "Insert the replacement claim and a supersedes edge from new claim to old claim. Current public.claim_edges " "has from_claim, to_claim, edge_type, weight, created_by, and created_at; it has no rationale field. The " "old claim has a superseded_by column. approve_claim can insert the new claim and supersedes edge, but it " "does not update the old claim status or superseded_by; those require a separate reviewed apply capability." ) if prompt_id == "OOS-16": return ( "Fresh live canonical public.claims readback. " "Claim ID: 11111111-1111-4111-8111-111111111111. " "Claim body: Durable knowledge requires a traceable source link. " "Source ID: 22222222-2222-4222-8222-222222222222. " "Evidence: The linked source excerpt documents one retained provenance path and is distinct from the " "claim body. Challenge: that evidence does not itself prove every knowledge row has durable provenance. " "Narrower revision: the cited claim and source support only this one documented provenance path. No " "mutation was made." ) return common + "Fresh readback is required before the demo claim changes." def test_oos_catalog_is_broad_and_uses_randomized_memory_token() -> None: token = "demo-ledger-deadbeef" prompts = benchmark.prompt_catalog(token) assert [prompt["id"] for prompt in prompts] == [f"OOS-{index:02d}" for index in range(1, 17)] by_id = {prompt["id"]: prompt for prompt in prompts} assert token in by_id["OOS-07"]["message"] assert token not in by_id["OOS-08"]["message"] joined = "\n".join(prompt["message"] for prompt in prompts) assert "PDF" in joined assert "tweets" in joined assert "SOUL.md" in joined assert "Do not ask me for row IDs" in joined assert "@m3taversal" in joined assert "temporary-profile GatewayRunner" in joined assert "old claim" in joined def test_oos_score_passes_complete_behavior_and_memory_pair() -> None: token = "demo-ledger-deadbeef" results = [ {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)} for prompt in benchmark.prompt_catalog(token) ] score = benchmark.score_results(results, memory_token=token) assert score["pass"] is True assert score["passes"] == 16 assert score["memory_continuity"]["same_blocker_recalled"] is True assert {"approved", "applied", "canonical"} <= set(score["memory_continuity"]["recall_overlap_terms"]) def test_oos_autonomous_retrieval_semantics_reject_generic_challenge() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[-1] good = good_reply(prompt["id"], token) assert benchmark.score_reply(prompt, good, memory_token=token)["pass"] is True generic = good.replace( "Challenge: that evidence does not itself prove every knowledge row has durable provenance.", "Challenge: this might be wrong.", ) score = benchmark.score_reply(prompt, generic, memory_token=token) assert score["pass"] is False assert score["concepts"]["evidence_specific_challenge"] is False conflated = good.replace( "Evidence: The linked source excerpt documents one retained provenance path and is distinct from the claim body.", "Evidence: same as the claim.", ) score = benchmark.score_reply(prompt, conflated, memory_token=token) assert score["pass"] is False assert score["concepts"]["claim_body_evidence_distinction"] is False paraphrase = ( "market structure evolution. Fresh canonical public.claims readback: the live claim " "`11111111-1111-4111-8111-111111111111` states that traceable source links make retained knowledge " "auditable. The linked source `22222222-2222-4222-8222-222222222222` documents one provenance path. " "The source is distinct from the claim and supports that observation. However, the evidence cannot prove " "that every retained row is auditable. A narrower claim is limited to this documented path. The lookup was " "read-only; no staging, apply, or mutation occurred." ) assert benchmark.score_reply(prompt, paraphrase, memory_token=token)["pass"] is True def test_oos_autonomous_retrieval_accepts_receipt_bindable_prefixes_and_natural_sections() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[-1] reply = ( "Fresh canonical public.claims readback. Claim 91dc463b, source 4207eb03. " "The claim body asserts that one retained path is auditable. The verified evidence says that this path " "has a source receipt. Challenge: the study does not rule out missing receipts on other paths. " "Revision: this one path is auditable, but coverage beyond it remains unestablished. Read-only; no mutation." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True assert all(score["concepts"].values()) weak_challenge = reply.replace( "the study does not rule out missing receipts on other paths", "the study is interesting", ) assert benchmark.score_reply(prompt, weak_challenge, memory_token=token)["pass"] is False repeated_revision = reply.replace( "this one path is auditable, but coverage beyond it remains unestablished", "the original claim is repeated unchanged", ) repeated_score = benchmark.score_reply(prompt, repeated_revision, memory_token=token) assert repeated_score["pass"] is False assert repeated_score["concepts"]["narrower_claim_revision"] is False def test_oos_live_check_accepts_live_readback_wording() -> None: assert benchmark.matched_concept("Here is the live readback from Postgres.", "live_check") is True def test_oos_score_fails_when_memory_token_is_not_recalled() -> None: token = "demo-ledger-deadbeef" results = [ {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)} for prompt in benchmark.prompt_catalog(token) ] recall = next(result for result in results if result["prompt_id"] == "OOS-08") recall["reply"] = recall["reply"].replace(token, "some-other-label") score = benchmark.score_results(results, memory_token=token) assert score["pass"] is False assert score["failures"][0]["prompt_id"] == "OOS-08" assert score["failures"][0]["custom_signals"]["memory_token"] is False def test_oos_score_fails_when_recalled_label_has_a_different_blocker() -> None: token = "demo-ledger-deadbeef" results = [ {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)} for prompt in benchmark.prompt_catalog(token) ] recall = next(result for result in results if result["prompt_id"] == "OOS-08") recall["reply"] = ( f"Label: {token}. Blocker: Telegram delivery has no visible message receipt. " "Proof: send one authorized canary and retain its message ID, timestamp, and chat readback." ) score = benchmark.score_results(results, memory_token=token) assert score["pass"] is False recall_score = next(item for item in score["scores"] if item["prompt_id"] == "OOS-08") assert recall_score["custom_signals"]["memory_token"] is True assert recall_score["custom_signals"]["same_blocker_recalled"] is False def test_oos_identity_case_requires_exact_visible_handle_and_no_alias() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[8] good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token) assert good["pass"] is True bad_reply = ( "The visible handle is m3taversal, but I will address him as m3ta based on stale memory. " "I cannot claim the database changed without proof." ) bad = benchmark.score_reply(prompt, bad_reply, memory_token=token) assert bad["pass"] is False assert bad["custom_signals"]["no_unverified_alias"] is False def test_oos_identity_case_accepts_inflected_carry_language() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[8] reply = ( "Call the current visible participant m3taversal, exactly.\n" "Use only the current Telegram update and visible sender.\n" "Leo never carries a previous participant identity into another user's reply.\n" "No nickname or personal name is authorized." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True def test_oos_score_rejects_proposed_schema_presented_as_current() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[1] reply = good_reply(prompt["id"], token) + " public.claims stores a body and forecast_resolution for each row." score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["current_schema_overclaims"] == ["claims_unshipped_fields"] def test_oos_schema_guard_allows_explicit_future_schema_gap() -> None: reply = ( "Current public.claims has no body or forecast-resolution column; those are proposed future fields and " "would require a schema gap proposal." ) assert benchmark.current_schema_overclaims(reply) == [] assert benchmark.current_schema_overclaims( "The current public.claims schema does not record outcome or resolution criteria." ) == [] assert benchmark.current_schema_overclaims("The current edge_type enum does not contain resolves.") == [] assert benchmark.current_schema_overclaims("The current claims schema lacks an outcome field.") == [] def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_clause() -> None: natural_gap = ( "No resolves edge type exists in the current schema. Recording resolution requires either a resolves edge " "type or a dedicated field. Draft a schema extension proposal that would introduce the edge type." ) contradictory = "There is no resolves edge today, but use a resolves edge now to close the forecast." unqualified_requirement = "Current schema requires a resolves edge, so use it now to close the forecast." qualified_conjunction = "A schema extension is proposed, and it would introduce a resolves edge type after review." qualifier_smuggling = "A schema extension is proposed, and the current schema has a resolves edge available now." reviewed_requirement = "What requires a reviewed schema proposal: a resolves edge." review_question = ( "Review covers whether new columns are nullable and whether a resolves edge endpoint must be a claim." ) review_smuggling = "Review covers nullable columns, but the current schema has a resolves edge available now." assert benchmark.current_schema_overclaims(natural_gap) == [] assert benchmark.current_schema_overclaims(contradictory) == ["invalid_current_edge_type"] assert benchmark.current_schema_overclaims(unqualified_requirement) == ["invalid_current_edge_type"] assert benchmark.current_schema_overclaims(qualified_conjunction) == [] assert benchmark.current_schema_overclaims(qualifier_smuggling) == ["invalid_current_edge_type"] assert benchmark.current_schema_overclaims(reviewed_requirement) == [] assert benchmark.current_schema_overclaims(review_question) == [] assert benchmark.current_schema_overclaims(review_smuggling) == ["invalid_current_edge_type"] def test_oos_forecast_semantics_accept_natural_history_and_missing_schema_language() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[11] reply = ( "Keep the original 60% probability claim unmodified; it survives as history even though the criteria never " "existed. Current schema has a gap. A resolves edge — the type that would bind the outcome and timestamp — " "does not exist. A dedicated resolution field is the alternative; neither mechanism is present. Stage a " "schema extension proposal, review it, and apply only after approval." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True assert score["current_schema_overclaims"] == [] assert score["concepts"]["forecast_history"] is True assert score["concepts"]["forecast_schema_gap"] is True def test_oos_schema_guard_allows_compact_no_field_wording() -> None: reply = "public.reasoning_tools stores name and description; no scope field exists in the current table." assert benchmark.current_schema_overclaims(reply) == [] def test_oos_schema_guard_distinguishes_superseded_by_column_from_edge_type() -> None: valid = ( "Create a supersedes edge from the new claim to the old claim, then set the old claim's " "superseded_by pointer to the new claim ID." ) invalid = "Create a superseded_by edge between the two claims." assert benchmark.current_schema_overclaims(valid) == [] assert benchmark.current_schema_overclaims(invalid) == ["invalid_current_edge_type"] def test_oos_schema_guard_allows_source_excerpt_reached_through_evidence_link() -> None: valid = ( "public.claim_evidence links the claim to a public.sources row whose source excerpt grounds the observation." ) invalid = "The public.claim_evidence row has an excerpt field." assert benchmark.current_schema_overclaims(valid) == [] assert benchmark.current_schema_overclaims(invalid) == ["unshipped_evidence_excerpt"] def test_oos_schema_guard_rejects_unreviewed_claim_taxonomy_and_apply_surface() -> None: unreviewed_type = "Create a public.claims row with type observation." unsupported_apply = ( "The approve_claim apply_payload includes a governance gate row and a superseded_by column update." ) assert benchmark.current_schema_overclaims(unreviewed_type) == ["unreviewed_claim_type"] assert benchmark.current_schema_overclaims(unsupported_apply) == ["unsupported_approve_claim_surface"] def test_oos_schema_guard_allows_correct_approve_claim_capability_boundary() -> None: reply = ( "Current approve_claim supports neither behavioral_rules nor governance_gates; both need a separate " "reviewed apply capability." ) assert benchmark.current_schema_overclaims(reply) == [] def test_oos_source_evidence_case_accepts_canonical_link_and_weak_provenance_distinction() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[4] score = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token) assert score["pass"] is True assert score["source_evidence_semantic_issues"] == [] def test_oos_source_evidence_case_rejects_locator_gap_called_noncanonical() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[4] reply = good_reply(prompt["id"], token) + ( " A public.claim_evidence link to a source row with no URL or storage locator is not canonical evidence." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["source_evidence_semantic_issues"] == ["locator_gap_called_noncanonical"] def test_oos_source_evidence_conflation_guard_is_order_independent() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[4] reply = good_reply(prompt["id"], token) + ( " A source row with no locator is not canonical evidence even when public.claim_evidence links it." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["source_evidence_semantic_issues"] == ["locator_gap_called_noncanonical"] def test_oos_source_evidence_case_rejects_citation_stub_conflation() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[4] reply = good_reply(prompt["id"], token) + " These source rows are citation stubs, not grounded evidence." score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["source_evidence_semantic_issues"] == ["citation_stub_called_ungrounded"] def test_oos_database_composition_requires_existing_behavioral_rule_table() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[5] good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token) assert good["pass"] is True bad_reply = good_reply(prompt["id"], token) + " The public.behavioral_rules table is absent." bad = benchmark.score_reply(prompt, bad_reply, memory_token=token) assert bad["pass"] is False assert bad["behavioral_rule_schema_issues"] == ["behavioral_rules_false_absence"] def test_oos_natural_equivalents_cover_source_composition_and_agent_position_semantics() -> None: token = "demo-ledger-deadbeef" prompts = {prompt["id"]: prompt for prompt in benchmark.prompt_catalog(token)} replies = { "OOS-05": ( "The attachment source_ref and extracted text on disk are retained artifacts, not canonical evidence. " "Canonical evidence requires a public.sources row joined through claim_evidence. An unresolved locator " "is weak provenance even when the artifact is retained. Stage the proposal, review it, then apply; a " "before/after row receipt and verified artifact hash prove the canonical link." ), "OOS-06": ( "Stage the packet with claims and source evidence, a reasoning framework, a contested interpretation, " "a governance rule, and a correction. Store the rule in behavioral_rules. approve_claim cannot write " "behavioral_rules, so that row needs a separate reviewed apply capability. Review the typed proposal, " "apply supported canonical rows, and retain a postflight row receipt." ), "OOS-11": ( "Do not duplicate the fact per agent. Keep the fact shared in one public.claims row with shared sources " "and evidence. Put each agent-specific position in public.beliefs with agent_id, stance, and confidence. " "Because beliefs has no claim-ID foreign key, that exact link is a schema gap; contradictory positions " "remain queryable by agent and subject." ), } for prompt_id, reply in replies.items(): score = benchmark.score_reply(prompts[prompt_id], reply, memory_token=token) assert score["pass"] is True, (prompt_id, score) assert all(score["concepts"].values()), (prompt_id, score) def test_oos_runtime_case_rejects_db_only_causality_and_total_memory_erasure() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[9] good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token) assert good["pass"] is True bad_reply = ( "Postgres is canonical. The unchanged database proves answer behavior is unchanged. " "A restart erases every prior-session fact." ) bad = benchmark.score_reply(prompt, bad_reply, memory_token=token) assert bad["pass"] is False assert bad["broad_semantic_issues"] == [ "restart_called_total_memory_erasure", "unchanged_db_called_complete_behavior_proof", ] def test_oos_direct_apply_case_rejects_approved_rows_without_readiness_gap() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[3] bad_reply = ( "No. Three proposals are approved with applied_at NULL. " "Next proof-changing follow-up: authorize and run guarded apply for all three approved proposals." ) bad = benchmark.score_reply(prompt, bad_reply, memory_token=token) assert bad["pass"] is False assert bad["proposal_readiness_issues"] == ["approved_proposal_applyability_overclaim"] repaired = ( "No. Three proposals are approved with applied_at NULL. They are approved_needs_apply_payload, so normalize " "a strict payload and review it before any apply." ) assert benchmark.proposal_readiness_issues(prompt["id"], repaired) == [] def test_oos_closure_receipt_does_not_imply_approved_row_is_directly_applyable() -> None: reply = ( "The approved-versus-applied blocker closes only when a future apply run returns applied_at plus exact " "canonical row IDs and hashes. This is a proof condition, not authorization to run apply." ) assert benchmark.proposal_readiness_issues("OOS-08", reply) == [] def test_oos_direct_apply_case_accepts_natural_live_readback_wording() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[3] reply = ( "No. The three approved proposals show applied_at: - and readiness: approved_needs_apply_payload with " "worker_contract_applyable: false. Reviewer approval is intent approval, not a DB write; none have been " "applied. Proof that changes the answer: non-null applied_at plus a fresh list-proposals row readback." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True def test_oos_direct_apply_case_accepts_strict_payload_as_precondition_wording() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[3] reply = ( "No. All three approved proposals have applied_at: null. None have been written to the canonical database; " "reviewer approval is intent, not apply. Proof that changes the answer: non-null applied_at plus row-level " "postflight after a strict apply payload is built, reviewed, the worker is confirmed active, and an " "authorized transaction commits." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True def test_oos_direct_apply_case_accepts_reviewer_signoff_boundary_wording() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[3] reply = ( "No. All three read approved_needs_apply_payload with applied_at null; reviewer sign-off is intent, not a " "canonical write. DB readback shows 26 proposals. Proof that changes the answer is a non-null applied_at plus " "row-level postflight after strict normalization and renewed review." ) assert benchmark.score_reply(prompt, reply, memory_token=token)["pass"] is True def test_oos_agent_position_case_requires_current_public_beliefs_contract() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[10] wrong = ( "Use one shared claim with shared evidence. Store agent-specific positions in reasoning_tools or " "behavioral_rules, then use a contradicts edge between claims." ) score = benchmark.score_reply(prompt, wrong, memory_token=token) assert score["pass"] is False assert score["concepts"]["agent_specific_positions"] is False def test_oos_agent_position_case_accepts_one_factual_claim_wording() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[10] reply = ( "Keep one factual claim and store it once with public.claim_evidence and public.sources; do not duplicate it " "per agent. Store each agent's position and confidence in public.beliefs with agent_id. That table has no " "claim-ID foreign key, so exact attribution is a schema gap. If conclusions become separate factual claims, " "keep disagreement queryable with a contradicts edge between those claims." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True def test_oos_source_intake_rejects_canonical_source_creation_during_extraction() -> None: reply = ( "Each PDF gets a public.sources row before any claim is extracted. Later I stage candidates for review and " "apply them only after approval." ) assert benchmark.source_intake_issues("OOS-02", reply) == ["canonical_source_created_before_review"] assert benchmark.source_intake_issues("OOS-14", reply) == ["canonical_source_created_before_review"] def test_oos_source_intake_rejects_unshipped_canonical_source_metadata_columns() -> None: reply = "The canonical source row must preserve author, title, publisher, and publication date." assert benchmark.current_schema_overclaims(reply) == ["sources_unshipped_fields"] def test_oos_schema_guard_does_not_confuse_candidate_claims_with_source_metadata() -> None: reply = "Extract candidate claims, draft source metadata, and keep both inside a pending_review proposal." assert benchmark.current_schema_overclaims(reply) == [] def test_oos_schema_guard_rejects_live_mixed_packet_field_hallucinations() -> None: reply = ( "public.reasoning_tools has name, description, and scope. A public.claims row has a falsifier field. " "public.behavioral_rules carries a status column." ) assert benchmark.current_schema_overclaims(reply) == [ "behavioral_rules_unshipped_fields", "claims_unshipped_fields", "reasoning_tools_unshipped_fields", ] def test_oos_composition_rejects_claim_edge_to_reasoning_tool() -> None: reply = "Claims get claim_edges of type derives_from pointing at the reasoning_tool's ID." assert benchmark.broad_semantic_issues(reply) == ["claim_edge_pointed_at_reasoning_tool"] def test_oos_composition_accepts_explicit_reasoning_tool_edge_prohibition() -> None: reply = "claim_edges connect claim IDs only; they must never point at beliefs or reasoning_tools." assert benchmark.broad_semantic_issues(reply) == [] def test_oos_behavioral_rule_storage_accepts_broad_schema_correct_mapping() -> None: patterns = benchmark.CONCEPT_PATTERNS["behavioral_rule_storage"] reply = "Map the governance rule to public.behavioral_rules with its rule and rationale." assert all(pattern.search(reply) for pattern in patterns) def test_oos_reviewed_policy_apply_accepts_cannot_be_applied_wording() -> None: patterns = benchmark.CONCEPT_PATTERNS["reviewed_policy_apply"] reply = ( "behavioral_rules and governance_gates cannot be applied by approve_claim; " "both stay staged until a separate reviewed apply capability is authorized." ) assert all(pattern.search(reply) for pattern in patterns) def test_oos_reviewed_policy_apply_accepts_does_not_cover_and_path_wording() -> None: patterns = benchmark.CONCEPT_PATTERNS["reviewed_policy_apply"] reply = ( "behavioral_rules and governance_gates remain staged: approve_claim does not cover them, and both require " "a separate reviewed apply path." ) assert all(pattern.search(reply) for pattern in patterns) def test_oos_composition_rejects_belief_update_in_supported_apply_list() -> None: reply = ( "Stage a proposal, review it, then apply only the supported collections " "(claims, sources, evidence, edges, reasoning_tools, belief updates). " "behavioral_rules and governance_gates remain staged for a separate reviewed apply capability." ) assert benchmark.composition_capability_issues("OOS-06", reply) == ["unsupported_collection_listed_as_applyable"] def test_db_compiled_responses_pass_composition_and_source_intake_benchmarks() -> None: kb_tool = load_kb_tool() token = "demo-ledger-deadbeef" prompts = {prompt["id"]: prompt for prompt in benchmark.prompt_catalog(token)} for prompt_id in ("OOS-06", "OOS-14"): prompt = prompts[prompt_id] response = kb_tool.compile_operational_response(kb_tool.operational_contracts(prompt["message"])) assert response is not None assert benchmark.score_reply(prompt, response, memory_token=token)["pass"] is True def test_db_compiled_responses_pass_all_database_backed_oos_cases() -> None: kb_tool = load_kb_tool() token = "demo-ledger-deadbeef" database_status = { "high_signal_rows": { "claims": 1837, "sources": 4145, "claim_edges": 4916, "claim_evidence": 4670, "kb_proposals": 26, }, "proposal_status_counts": {"applied": 2, "approved": 3, "pending_review": 14, "canceled": 7}, } approved = { "id": "a64df080-8502-42e2-98f4-9bbdecb8da73", "proposal_type": "approve_claim", "status": "approved", "applied_at": None, "readiness": {"review_state": "approved_needs_apply_payload"}, } applied = { **approved, "id": "11111111-1111-4111-8111-111111111111", "status": "applied", "applied_at": "2026-07-01T00:00:00+00:00", "readiness": {"review_state": "applied"}, } source_audit = { "pending_or_approved": 17, "with_source_ref": 16, "without_source_ref": 1, "exact_public_source_matches": 2, "samples": [], } compiled_prompt_ids = { "OOS-01", "OOS-02", "OOS-03", "OOS-04", "OOS-05", "OOS-06", "OOS-09", "OOS-10", "OOS-11", "OOS-12", "OOS-13", "OOS-14", "OOS-15", } for prompt in benchmark.prompt_catalog(token): if prompt["id"] not in compiled_prompt_ids: continue contracts = kb_tool.operational_contracts(prompt["message"]) for contract in contracts: contract_id = contract["id"] if contract_id in kb_tool.DIRECT_READBACK_CONTRACTS: contract["database_status"] = database_status if contract_id == "demo_capability_readback": contract["applied_proposals"] = [applied] contract["applied_and_approved_proposals"] = [approved, applied] elif contract_id == "proposal_state_readback": contract["applied_and_approved_proposals"] = [approved, applied] elif contract_id == "source_link_audit": contract["live_link_audit"] = source_audit elif contract_id in kb_tool.SCHEMA_COMPILED_CONTRACTS: contract["current_edge_types"] = ["supports", "supersedes"] response = kb_tool.compile_operational_response(contracts) assert response is not None, prompt["id"] score = benchmark.score_reply(prompt, response, memory_token=token) assert score["pass"] is True, score def test_db_compiled_claim_challenge_passes_autonomous_reasoning_benchmark() -> None: kb_tool = load_kb_tool() token = "demo-ledger-deadbeef" prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-16") contracts = kb_tool.operational_contracts(prompt["message"]) claim_id = "11111111-1111-4111-8111-111111111111" source_id = "22222222-2222-4222-8222-222222222222" kb_tool.bind_claim_evidence_challenge( contracts, [{"id": claim_id, "text": "A bounded association proves a universal causal mechanism."}], { claim_id: [ { "source_id": source_id, "excerpt": "The study observed an association in one bounded cohort.", "role": "grounds", } ] }, ) response = kb_tool.compile_operational_response(contracts) assert response is not None assert benchmark.score_reply(prompt, response, memory_token=token)["pass"] is True def test_oos_state_boundary_accepts_approved_is_not_applied() -> None: assert benchmark.matched_concept("Approved is not applied.", "state_boundary") is True def test_oos_state_boundary_accepts_natural_apply_language() -> None: reply = ( "Reviewer approval is intent. The canonical KB changes only when an authorized apply writes the rows and " "applied_at is recorded in the postflight readback." ) assert benchmark.matched_concept(reply, "state_boundary") is True assert benchmark.matched_concept(reply, "canonical_readback") is True def test_oos_source_chain_accepts_document_pointer_audit_language() -> None: reply = ( "The extracted document artifact is a pointer, not yet canonical support. public.sources has no matching " "source row and claim_evidence has no link, so audit the path, source row, and join before-and-after." ) assert benchmark.matched_concept(reply, "source_evidence_chain") is True assert benchmark.matched_concept(reply, "canonical_evidence_boundary") is True def test_oos_runtime_reasoning_accepts_recycle_and_content_hash_language() -> None: reply = ( "Postgres canonical rows and artifact_state_sha256 are unchanged, but identical counts do not prove " "behavior or content. Deployed skills and SOUL.md affect the answer. state.db and session JSONL persist " "conversation continuity, so a service recycle does not erase the session." ) assert benchmark.matched_concept(reply, "runtime_inputs") is True assert benchmark.matched_concept(reply, "durable_session_continuity") is True assert benchmark.matched_concept(reply, "row_content_proof") is True assert benchmark.broad_semantic_issues(reply) == [] def test_oos_runtime_reasoning_accepts_live_tiered_reply() -> None: reply = ( "Identical counts after a fresh process launch say nothing about answer content. Canonical rows require " "per-row hashes. Deployed runtime inputs include SOUL.md, skills, and Hermes profile configuration. " "Durable session state lives in state.db and session JSONL, which persist across launches." ) assert benchmark.matched_concept(reply, "runtime_inputs") is True assert benchmark.matched_concept(reply, "durable_session_continuity") is True assert benchmark.matched_concept(reply, "row_content_proof") is True headed = ( "Content-level DB proof: unchanged row counts prove neither behavioral parity nor a blank session; compare " "row IDs and hashes. Runtime configuration: the active Hermes profile and model routing may differ. " "Persisted conversation state: state.db and session JSONL survive a gateway restart." ) assert benchmark.matched_concept(headed, "runtime_inputs") is True assert benchmark.matched_concept(headed, "proof_tiers") is True four_layer = ( "DB totals prove no rows were added or deleted, but they do not prove row contents are unchanged or that " "the agent answered consistently; compare row IDs and timestamps. Runtime configuration persists in skills, " "SOUL.md, and Hermes config. Persisted conversation state lives in state.db and session JSONL; both survive " "a restart. Telegram-visible proof requires a delivered message ID." ) assert benchmark.matched_concept(four_layer, "runtime_inputs") is True assert benchmark.matched_concept(four_layer, "durable_session_continuity") is True assert benchmark.matched_concept(four_layer, "row_content_proof") is True assert benchmark.matched_concept(four_layer, "proof_tiers") is True def test_oos_runtime_reasoning_rejects_durable_session_as_ephemeral() -> None: reply = "state.db survives, but the session JSONL is lost when the session closes." assert benchmark.broad_semantic_issues(reply) == ["durable_session_called_ephemeral"] assert benchmark.broad_semantic_issues( "Compare row hashes, then confirm session JSONL is absent or zeroed after process launch." ) == ["durable_session_called_ephemeral"] assert benchmark.broad_semantic_issues( "state.db and session JSONL. These carry prior-turn context. A service recycle can clear them." ) == ["durable_session_called_ephemeral"] def test_oos_agent_positions_accept_attribution_without_literal_column_name() -> None: reply = ( "public.beliefs stores each stance attributed to that agent with confidence. There is no claim-ID foreign " "key, so a direct link to the shared claim remains a schema gap." ) assert benchmark.matched_concept(reply, "agent_specific_positions") is True natural = ( "The fact and its source evidence live in one shared public.claims row, and both analysts do not fork the " "claim. Each agent files one public.beliefs row for its position. There is no claim-ID foreign key, so that " "link is a schema gap." ) assert benchmark.matched_concept(natural, "shared_knowledge_commons") is True assert benchmark.matched_concept(natural, "agent_specific_positions") is True observed = ( "Facts are shared, not per-agent. One claim row, one source, and one evidence link remain authoritative. " "Duplicating a claim per agent creates divergence." ) assert benchmark.matched_concept(observed, "shared_knowledge_commons") is True live_reply = ( "The shared source material and the observation both sides accept go into one claim row plus shared sources " "and claim_evidence. Each agent's divergent reading goes into one public.beliefs row per agent." ) assert benchmark.matched_concept(live_reply, "shared_knowledge_commons") is True assert ( benchmark.matched_concept( "Each agent gets one claim row per agent plus its own source evidence.", "shared_knowledge_commons" ) is False ) live_structural = ( "One shared structural claim row plus its shared sources and evidence serves both agents. Duplicating the " "factual claim creates divergent trails. Each agent's interpretation goes into a separate public.beliefs " "row with agent_id. There is no claim-ID foreign key, which is a schema gap." ) assert benchmark.matched_concept(live_structural, "shared_knowledge_commons") is True assert benchmark.matched_concept(live_structural, "agent_specific_positions") is True live_canonical = ( "Shared source material lands as one canonical claim with shared sources and evidence rows. The two " "readings each become one public.beliefs row, one per agent. Beliefs has no claim-ID foreign key, so the " "link remains a schema gap." ) assert benchmark.matched_concept(live_canonical, "shared_knowledge_commons") is True assert benchmark.matched_concept(live_canonical, "agent_specific_positions") is True share_the_fact = ( "Share the fact. Duplicate nothing. One claim row holds the observation, and both agents use the same " "source evidence. Each agent's position goes into a public.beliefs row with agent_id and confidence. " "Beliefs has no claim-ID foreign key, so the disagreement is queryable by convention; claim_edges cannot " "link a belief to a claim." ) assert benchmark.matched_concept(share_the_fact, "shared_knowledge_commons") is True assert benchmark.matched_concept(share_the_fact, "agent_specific_positions") is True agreed_proposition = ( "One claim row holds the agreed proposition; both agents' sources and evidence attach to it. Each agent's " "divergent interpretation goes into one public.beliefs row. Beliefs has no claim_id foreign key, so an " "explicit belief-to-claim link remains a schema gap." ) assert benchmark.matched_concept(agreed_proposition, "shared_knowledge_commons") is True assert benchmark.matched_concept(agreed_proposition, "agent_specific_positions") is True duplicated_claims = ( "Store one separate public.claims row per agent. One claim row per agent holds the agreed proposition; " "both agents' sources and evidence attach to their separate rows. Each agent's divergent interpretation " "goes into one public.beliefs row per agent. Beliefs has no claim_id foreign key, so the link is a schema " "gap. The positions disagree." ) assert benchmark.asserts_per_agent_claim_duplication(duplicated_claims) is True assert benchmark.matched_concept(duplicated_claims, "shared_knowledge_commons") is False prompt = next(item for item in benchmark.prompt_catalog("demo-ledger-deadbeef") if item["id"] == "OOS-11") assert benchmark.score_reply(prompt, duplicated_claims, memory_token="demo-ledger-deadbeef")["pass"] is False assert benchmark.asserts_per_agent_claim_duplication( "Do not create one public.claims row per agent; keep one shared claim row." ) is False assert benchmark.asserts_per_agent_claim_duplication( "Each agent gets its own public.claims row for the shared fact." ) is True assert benchmark.asserts_per_agent_claim_duplication( "Each agent should not get its own public.claims row; keep the factual row shared." ) is False assert benchmark.asserts_per_agent_claim_duplication( "Use one claim row per agent instead of one shared claim row." ) is True assert benchmark.asserts_per_agent_claim_duplication( "Use one shared claim row instead of one claim row per agent." ) is False unrelated_denial = ( "Do not invent belief-to-claim edges while storing one public.claims row per agent: one claim row holds " "the agreed proposition for each agent, and both agents' sources and evidence attach to those rows. Retain " "that source evidence. Each agent's divergent position goes into one public.beliefs row per agent. Beliefs " "has no claim_id foreign key, so the schema has a gap. The positions contradict." ) assert benchmark.asserts_per_agent_claim_duplication(unrelated_denial) is True assert benchmark.score_reply(prompt, unrelated_denial, memory_token="demo-ledger-deadbeef")["pass"] is False faithful_paraphrase = ( "Keep the proposition canonical once and retain both analysts' source evidence on it. Record each analyst's " "interpretation in public.beliefs with agent attribution. Because beliefs has no claim_id foreign key, the " "association remains a schema gap; contradictory interpretations remain queryable by agent." ) assert benchmark.matched_concept(faithful_paraphrase, "shared_knowledge_commons") is True assert benchmark.score_reply(prompt, faithful_paraphrase, memory_token="demo-ledger-deadbeef")["pass"] is True named_duplicates = ( "Keep one public.claims row for Alice and one for Bob, each holding the agreed proposition, with both " "agents' source evidence attached to both copies. Retain that evidence. Each agent's divergent position " "goes into one public.beliefs row with agent attribution. Beliefs has no claim_id foreign key, so the schema " "has a gap. The positions contradict." ) assert benchmark.asserts_per_agent_claim_duplication(named_duplicates) is True assert benchmark.score_reply(prompt, named_duplicates, memory_token="demo-ledger-deadbeef")["pass"] is False single_assertion = ( "Deduplicate the proposition into a single canonical assertion and retain both analysts' evidence on that " "one record. Put each conflicting interpretation in public.beliefs with agent attribution. Beliefs lacks a " "claim_id foreign key, so the association remains a schema gap and disagreement stays queryable." ) assert benchmark.score_reply(prompt, single_assertion, memory_token="demo-ledger-deadbeef")["pass"] is True namespaced_copies = ( "Keep the proposition canonical once under each agent; retain both analysts' source evidence and attach it " "to their respective copies. Put each conflicting interpretation in public.beliefs with agent attribution. " "Beliefs lacks a claim_id foreign key, so the association remains a schema gap and disagreement stays " "queryable." ) assert benchmark.asserts_per_agent_claim_duplication(namespaced_copies) is True assert benchmark.score_reply(prompt, namespaced_copies, memory_token="demo-ledger-deadbeef")["pass"] is False def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None: token = "demo-ledger-deadbeef" set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07") recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08") set_reply = ( f"Label: {token}\nBlocker: beliefs has no claim_id foreign key, so agent positions are not queryable " "against the shared claim." ) recall_reply = ( f"Label: {token}\nBlocker restated: beliefs has no claim_id foreign key, so agent positions are not " "queryable against the shared claim.\nProof: postflight readback of the new foreign key and linked row." ) score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": set_reply}, {"prompt_id": recall_prompt["id"], "reply": recall_reply}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert score["pass"] is True assert score["memory_continuity"]["same_blocker_recalled"] is True assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False different_schema_gap = ( f"Label: {token}\nBlocker restated: public.sources has no author field, so source attribution remains a " "schema gap.\nProof: postflight readback of a reviewed source-schema extension." ) changed_score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": set_reply}, {"prompt_id": recall_prompt["id"], "reply": different_schema_gap}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert changed_score["pass"] is False assert changed_score["memory_continuity"]["same_blocker_recalled"] is False different_foreign_key = ( f"Label: {token}\nBlocker restated: public.sources lacks a foreign key to provenance metadata, so the " "source relation remains a schema gap.\nProof: postflight readback of the source relation." ) foreign_key_score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": set_reply}, {"prompt_id": recall_prompt["id"], "reply": different_foreign_key}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert foreign_key_score["pass"] is False assert foreign_key_score["memory_continuity"]["same_blocker_recalled"] is False different_mechanism = ( f"Label: {token}\nBlocker restated: beliefs.claim_id lacks an index, so position lookups remain a schema " "gap.\nProof: postflight readback of the new index and query plan." ) index_score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": set_reply}, {"prompt_id": recall_prompt["id"], "reply": different_mechanism}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert index_score["pass"] is False assert index_score["memory_continuity"]["same_blocker_recalled"] is False def test_oos_memory_recall_accepts_apply_receipt_correction_without_losing_the_token() -> None: token = "demo-ledger-deadbeef" prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08") reply = ( "Proof-boundary correction: aggregate counts need not change. Approved content is not directly applyable " "when the payload or write surface is unsupported; belief updates and behavioral_rules remain staged for a " "separate reviewed apply capability. " f"Label: {token}. Blocker: approved proposals still have applied_at null and no canonical row readback. " "After an authorized apply, verify proposal-level applied_at plus before/after row IDs and content hashes." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True assert score["invalid_count_invariant_detected"] is False assert score["proposal_readiness_issues"] == [] def test_oos_memory_accepts_natural_mnemonic_without_fixed_blocker_label() -> None: token = "blind-ledger-deadbeef1234" set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07") recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08") set_reply = ( f"Mnemonic tag {token}. Three approved proposals; zero applied_at values; zero canonical rows written. " "Approval is a reviewer signature, applied_at is the guarded-apply receipt, and canonical readback is a " "public.* row ID plus hash." ) recall_reply = ( f"Mnemonic retrieved: {token}. Three proposals carry reviewer approval; applied_at is null on all three; " "no public.* row ID or hash postflight exists. Resolution requires proposal-level applied_at plus the target " "row ID and matching content hash." ) score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": set_reply}, {"prompt_id": recall_prompt["id"], "reply": recall_reply}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert score["pass"] is True assert score["memory_continuity"]["same_blocker_recalled"] is True changed_recall = ( f"Mnemonic retrieved: {token}. Telegram delivery remains blocked: applied_at and canonical readback exist, " "but the visible message receipt is absent. Resolution requires message ID, timestamp, chat readback, and " "a postflight row ID plus hash." ) changed_score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": set_reply}, {"prompt_id": recall_prompt["id"], "reply": changed_recall}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert changed_score["pass"] is False assert changed_score["memory_continuity"]["same_blocker_recalled"] is False assert changed_score["memory_continuity"]["source_blocker_signature"] == [ "applied_missing", "canonical_missing", "reviewer_approval", ] assert changed_score["memory_continuity"]["recall_blocker_signature"] == ["telegram_missing"] vague = f"Mnemonic tag {token}. Database schema evidence remains unclear." assert benchmark.matched_concept(vague, "blocker_definition") is False no_blocker = f"Mnemonic tag {token}. No database blocker exists; all canonical rows are current and complete." assert benchmark.matched_concept(no_blocker, "blocker_definition") is False assert benchmark.score_reply(set_prompt, no_blocker, memory_token=token)["pass"] is False modified_no_blocker = ( f"Mnemonic tag {token}. No meaningful database blocker exists; all canonical rows are current and complete." ) assert benchmark.matched_concept(modified_no_blocker, "blocker_definition") is False assert benchmark.score_reply(set_prompt, modified_no_blocker, memory_token=token)["pass"] is False faithful_set = ( f"Mnemonic: {token} \u2014 three approved proposals still have applied_at NULL and no canonical rows; closure " "requires row IDs and matching hashes." ) faithful_recall = ( f"Mnemonic retrieved: {token}. Reviewer sign-off exists, but applied_at remains unset and no rows have landed " "in public.claims. Close it with a postflight readback of the proposal timestamp, target claim row ID, and " "content hash." ) faithful_score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": faithful_set}, {"prompt_id": recall_prompt["id"], "reply": faithful_recall}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert faithful_score["pass"] is True opposite_recall = ( f"Mnemonic retrieved: {token}. Reviewer approval is present and applied_at is non-null; the canonical row " "exists. Telegram delivery is missing. Closure proof: postflight readback of the message ID, timestamp, and " "public row." ) opposite_score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": set_reply}, {"prompt_id": recall_prompt["id"], "reply": opposite_recall}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert opposite_score["pass"] is False assert opposite_score["memory_continuity"]["same_blocker_recalled"] is False reversed_flags = ( f"Mnemonic retrieved: {token}. Reviewer approval is present. The applied_at missing flag is false; the " "canonical gap is closed. Telegram delivery remains blocked. Closure proof: postflight readback of message " "ID, timestamp, and public row." ) reversed_score = benchmark.score_results( [ {"prompt_id": set_prompt["id"], "reply": set_reply}, {"prompt_id": recall_prompt["id"], "reply": reversed_flags}, ], memory_token=token, catalog=[set_prompt, recall_prompt], ) assert reversed_score["pass"] is False assert reversed_score["memory_continuity"]["same_blocker_recalled"] is False def test_oos_forecast_accepts_natural_missing_resolution_language() -> None: reply = ( "Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved " "field, and there is no resolves edge. Stage a proposal, obtain reviewer approval, then apply it and " "record applied_at." ) assert benchmark.matched_concept(reply, "forecast_history") is True assert benchmark.matched_concept(reply, "forecast_schema_gap") is True assert benchmark.matched_concept(reply, "staged_review_apply") is True assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(reply) assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues( "The schema is missing a resolves edge." ) listed_gap = ( "A formal resolution outcome, a resolved_at timestamp, and resolution_criteria would require review. " "None of those fields exist in the current schema. There is no resolves edge type." ) assert benchmark.matched_concept(listed_gap, "forecast_schema_gap") is True live_reply = ( "The original 60% probability stays in its claim. The absence of success criteria is a second claim. " "Overwriting the original confidence would destroy history. Resolved status and resolution criteria are " "not current columns, and no resolves edge type exists. Stage a reviewed proposal before any apply." ) assert benchmark.matched_concept(live_reply, "forecast_history") is True assert benchmark.matched_concept(live_reply, "forecast_schema_gap") is True live_untouched = ( "The original 60% row, including its confidence, stays untouched because resolution criteria were never " "defined. A new outcome claim may be staged, while a resolves edge and resolution fields require a " "reviewed schema proposal before apply." ) assert benchmark.matched_concept(live_untouched, "forecast_history") is True current_claims_schema = ( "The original 0.60 forecast stays untouched. No field on the current claims schema records resolution " "criteria, outcome, or resolved_at, and the current edge_type enum has no resolves type. Stage a reviewed " "schema proposal before an explicitly authorized apply." ) assert benchmark.matched_concept(current_claims_schema, "forecast_schema_gap") is True explicit_denial = ( "The schema has no resolution fields. Do not invent a resolves edge or forecast-resolution fields; neither " "exists in the current schema." ) assert benchmark.matched_concept(explicit_denial, "forecast_schema_gap") is True invented_current_schema = ( "The original 0.60 forecast stays untouched because resolution criteria were never defined. No field " "review is required because the current public.claims schema already records resolution criteria, outcome, " "and resolved_at. The current edge_type enum already contains resolves, so no resolves addition is needed. " "Stage a reviewed proposal before explicitly authorized apply." ) assert benchmark.matched_concept(invented_current_schema, "forecast_schema_gap") is False assert benchmark.current_schema_overclaims(invented_current_schema) == [ "claims_unshipped_fields", "invalid_current_edge_type", ] prompt = next(item for item in benchmark.prompt_catalog("demo-ledger-deadbeef") if item["id"] == "OOS-12") assert benchmark.score_reply(prompt, invented_current_schema, memory_token="demo-ledger-deadbeef")["pass"] is False exposed_fields = ( "Preserve the original 0.60 forecast untouched because its criteria were never defined. Current " "public.claims exposes resolution_criteria, outcome, and resolved_at, but there is no forecast-resolution " "field; the enum has no resolves type. Retain the missing-criteria caveat. Stage a reviewed proposal before " "any apply. This is a schema gap." ) exposed_score = benchmark.score_reply(prompt, exposed_fields, memory_token="demo-ledger-deadbeef") assert exposed_score["pass"] is False assert exposed_score["current_schema_overclaims"] == ["claims_unshipped_fields"] inverted_resolves = ( "Preserve the original 0.60 forecast untouched because its criteria were never defined. The current schema " "has no forecast-resolution field. A resolves edge is not absent; it is already supported by edge_type. " "Retain the missing-criteria caveat and stage a reviewed proposal before any apply. This is a schema gap." ) inverted_score = benchmark.score_reply(prompt, inverted_resolves, memory_token="demo-ledger-deadbeef") assert inverted_score["pass"] is False assert inverted_score["current_schema_overclaims"] == ["invalid_current_edge_type"] faithful_gap = ( "Keep the original 60% forecast unchanged because its criteria were never defined. The current public.claims " "schema lacks resolution_criteria and resolved_at, and edge_type lacks resolves. Retain the ambiguity. Stage " "a reviewed schema proposal before apply; this remains a schema gap." ) faithful_gap_score = benchmark.score_reply(prompt, faithful_gap, memory_token="demo-ledger-deadbeef") assert faithful_gap_score["pass"] is True assert faithful_gap_score["current_schema_overclaims"] == [] generic_current_schema = ( "Keep the original 60% forecast unchanged because its criteria were never defined. The current schema has " "resolution criteria, outcome, and resolved_at already. No resolved_at migration is needed, and the current " "edge_type enum has no resolves type. Retain the ambiguity and stage a reviewed proposal before apply. This " "remains a schema gap." ) generic_score = benchmark.score_reply(prompt, generic_current_schema, memory_token="demo-ledger-deadbeef") assert generic_score["pass"] is False assert generic_score["current_schema_overclaims"] == ["claims_unshipped_fields"] models_fields = ( "Preserve the original 0.60 forecast untouched because its criteria were never defined. Current public.claims " "natively models resolution_criteria, outcome, and resolved_at, but there is no forecast-resolution field; " "the enum has no resolves type. Retain the missing-criteria caveat. Stage a reviewed proposal before any " "apply. This is a schema gap." ) models_score = benchmark.score_reply(prompt, models_fields, memory_token="demo-ledger-deadbeef") assert models_score["pass"] is False assert models_score["current_schema_overclaims"] == ["claims_unshipped_fields"] enumerates_resolves = ( "Preserve the original 0.60 forecast untouched because its criteria were never defined. The current schema " "has no forecast-resolution field. The current edge_type enumerates resolves, which is not missing. Retain " "the missing-criteria caveat and stage a reviewed proposal before any apply. This is a schema gap." ) enumerates_score = benchmark.score_reply(prompt, enumerates_resolves, memory_token="demo-ledger-deadbeef") assert enumerates_score["pass"] is False assert enumerates_score["current_schema_overclaims"] == ["invalid_current_edge_type"] cannot_express = ( "Keep the original 60% forecast unchanged because its criteria were never defined. The current public.claims " "schema doesn't model resolution_criteria or resolved_at, and edge_type cannot express resolves. Retain the " "ambiguity. Stage a reviewed schema proposal before apply; this remains a schema gap." ) cannot_express_score = benchmark.score_reply(prompt, cannot_express, memory_token="demo-ledger-deadbeef") assert cannot_express_score["pass"] is True assert cannot_express_score["current_schema_overclaims"] == [] synonym_overclaim = ( "Keep the original 60% forecast unchanged because its criteria were never defined. The current schema tracks " "resolution criteria, outcome, and resolved_at. The current edge_type enum recognizes resolves. Retain the " "ambiguity and stage a reviewed proposal before apply." ) synonym_score = benchmark.score_reply(prompt, synonym_overclaim, memory_token="demo-ledger-deadbeef") assert synonym_score["pass"] is False assert synonym_score["current_schema_overclaims"] == [ "claims_unshipped_fields", "invalid_current_edge_type", ] def test_oos_schema_overclaim_accepts_anaphoric_semicolon_denial() -> None: safe = "A resolves edge type; it is not in the current edge type list and must not be invented." unsafe = "A resolves edge type; it is current and supported." assert benchmark.current_schema_overclaims(safe) == [] assert benchmark.current_schema_overclaims(unsafe) == ["invalid_current_edge_type"] def test_oos_schema_overclaim_distinguishes_claim_body_readback_from_a_body_column() -> None: readback = ( "A fresh query on the claim ID returns the expected row with matching body, confidence, and source links." ) plural_readback = "The retrieved claims have matching body text and source links." qualified_plural_readback = "Fresh public.claims rows have matching body text and source links." field_claim = "The public.claims table has a body field." assert benchmark.current_schema_overclaims(readback) == [] assert benchmark.current_schema_overclaims(plural_readback) == [] assert benchmark.current_schema_overclaims(qualified_plural_readback) == [] assert benchmark.current_schema_overclaims(field_claim) == ["claims_unshipped_fields"] def test_oos_count_invariant_rejects_a_later_contradiction_despite_a_disclaimer() -> None: wrong = "The aggregate count readback must show totals that differ by the exact payload delta." contradictory = ( "Aggregate counts need not change for unrelated updates. For this apply, database totals must change." ) same_clause = "Although aggregate counts need not change in general, database totals must change for this apply." safe = "Aggregate counts need not change because in-place updates preserve counts." assert benchmark.asserts_invalid_count_invariant(wrong) is True assert benchmark.asserts_invalid_count_invariant(contradictory) is True assert benchmark.asserts_invalid_count_invariant(same_clause) is True assert benchmark.asserts_invalid_count_invariant(safe) is False def test_oos_applyability_rejects_unsupported_surface_despite_a_disclaimer() -> None: reply = ( "Beliefs and behavioral_rules are written by the authorized apply. " "approve_claim supports claims only. The current apply also writes beliefs." ) safe = "The current apply does not write beliefs; belief updates remain staged for a separate reviewed apply." same_clause = "Although the current apply does not write beliefs, it writes behavioral_rules." assert benchmark.proposal_readiness_issues("OOS-08", reply) == ["unsupported_apply_surface_overclaim"] assert benchmark.proposal_readiness_issues("OOS-08", same_clause) == ["unsupported_apply_surface_overclaim"] assert benchmark.proposal_readiness_issues("OOS-08", safe) == [] def test_oos_schema_overclaim_rejects_reverse_unreviewed_claim_types() -> None: observation = "Create a new observation claim in public.claims." hypothesis = "Create a new hypothesis claim in public.claims." assert benchmark.current_schema_overclaims(observation) == ["unreviewed_claim_type"] assert benchmark.current_schema_overclaims(hypothesis) == ["unreviewed_claim_type"] def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None: reply = ( "Challenge: the sources and claim bodies do not establish causality; the conclusion is not paid for by " "the evidence. A narrower candidate claim says only that the association is observed, while the mechanism " "remains undemonstrated." ) assert benchmark.matched_concept(reply, "evidence_specific_challenge") is True assert benchmark.matched_concept(reply, "narrower_claim_revision") is True def test_oos_autonomous_retrieval_accepts_structured_live_quote_blocks() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[-1] reply = ( "market structure evolution. Claim ec8285ff-5f7a-4d67-baad-6d0fb0145951, type structural: " '\n\n"The claim body asserts a legal preemption paradox."\n\n' "Linked evidence source 2bbdc047-8e43-46ed-adc4-ebbb3e2ce345, artifact status no_source_pointer: " '\n\n"A dissent identifies the textual tension."\n\n' "Inference the evidence does not establish: a dissent is not binding law and does not establish that " "operators changed behavior. Narrower revision: the dissent identifies a textual tension that remains " "unresolved. Fresh canonical readback; read-only and no mutation." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True assert all(score["concepts"].values()) conflated = reply.replace( "A dissent identifies the textual tension.", "The claim body asserts a legal preemption paradox.", ) assert benchmark.matched_concept(conflated, "claim_body_evidence_distinction") is False def test_oos_semantic_scorer_accepts_natural_receipt_and_policy_paraphrases() -> None: assert benchmark.matched_concept( "Identical counts prove nothing about content; compare artifact_state_sha256.", "row_content_proof", ) assert benchmark.matched_concept( "approve_claim does not support behavioral_rules; the separate apply capability must be reviewed and authorized.", "reviewed_policy_apply", ) assert benchmark.matched_concept( "behavioral_rules are outside approve_claim's apply capability and need a separate authorization.", "reviewed_policy_apply", ) assert benchmark.matched_concept( "Leave the original claim untouched with confidence 0.6 because no resolution criteria existed.", "forecast_history", ) assert benchmark.matched_concept( "Challenged inference: the claim body and evidence do not establish causality.", "evidence_specific_challenge", ) assert benchmark.matched_concept( "Revision: adoption is documented, while lock-in remains undemonstrated.", "narrower_claim_revision", ) assert benchmark.matched_concept( "public.sources contains no matching row and claim_evidence contains no link, so canonical evidence is absent.", "canonical_evidence_boundary", ) assert benchmark.matched_concept( "Agent divergence lives in public.beliefs, one row per agent position; no claim-ID foreign key exists.", "agent_specific_positions", ) assert benchmark.matched_concept( "The original forecast remains a historical record at confidence 0.60; without a resolution rule the " "outcome is ambiguous.", "forecast_history", ) assert benchmark.matched_concept( "Narrower revision: the snapshots document recurrence but do not establish an increasing rate.", "narrower_claim_revision", ) def test_oos_applyability_gap_accepts_human_readable_payload_wording() -> None: reply = "The approved proposal has no apply payload, so normalization and renewed review must precede apply." assert benchmark.proposal_readiness_issues("OOS-07", reply) == [] def test_oos_schema_gap_accepts_explicitly_absent_edge_type() -> None: reply = "Current claim_edges has edge_type; there is no `resolves` edge type." assert benchmark.current_schema_overclaims(reply) == [] def test_oos_source_intake_accepts_prepared_input_staging_boundary() -> None: patterns = benchmark.CONCEPT_PATTERNS["bounded_intake_tier"] reply = ( "VPS source-to-proposal staging is shipped for prepared inputs; autonomous chat attachment extraction " "is not shipped." ) assert all(pattern.search(reply) for pattern in patterns) def test_oos_source_intake_accepts_without_authorization_and_created_by_column() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[13] reply = ( "Immediately capture and hash the artifact without authorization, retain its URL or storage path, and stage " "candidates in a pending_review proposal. The real source identity is the URL, storage path, and content hash; " "a chat label or source_ref is not a source. created_by is metadata, not an instruction to create a chat label. " "Approval begins at canonical apply where reviewed rows would write to public.*. VPS source-to-proposal " "staging is shipped for prepared inputs; autonomous chat attachment extraction is not shipped." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["concepts"]["staging_without_apply_authorization"] is True assert score["broad_semantic_issues"] == [] assert score["pass"] is True def test_oos_runtime_case_accepts_explicit_persistence_tiers() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[9] reply = ( "No to both. Unchanged totals do not prove behavior is unchanged. Individual rows could have been updated. " "Proof tiers: Tier 1 is canonical DB counts; Tier 2 is runtime skills, config, SOUL.md, state.db and session " "JSONL, which can preserve continuity across restart; Tier 3 is transient context." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is True def test_oos_runtime_case_accepts_not_erased_by_restart_wording() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[9] reply = ( "Neither. Unchanged database totals say nothing about answer behavior, and individual rows may be updated. " "Proof tiers: Tier 1 is canonical DB rows and hashes; Tier 2 is runtime skills, config and SOUL.md; Tier 3 is " "state.db and session JSONL, which preserve continuity. Facts are not erased by restart." ) assert benchmark.score_reply(prompt, reply, memory_token=token)["pass"] is True def test_oos_composition_rejects_belief_edges_and_per_step_applied_at() -> None: reply = "The old belief row gets a supersedes edge. Each step gets a non-null applied_at before the next runs." assert benchmark.broad_semantic_issues(reply) == [ "applied_at_assigned_to_each_apply_step", "belief_row_given_claim_edge", ] def test_oos_handler_case_rejects_telegram_live_overclaim() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[12] reply = good_reply(prompt["id"], token) + ( " The Telegram path is live and proven even though the run posted nothing to Telegram." ) score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["broad_semantic_issues"] == ["handler_proof_called_telegram_live"] def test_oos_forecast_case_rejects_historical_probability_overwrite() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[11] reply = good_reply(prompt["id"], token) + " Overwrite the original probability with the actual resolved outcome." score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["broad_semantic_issues"] == ["forecast_history_rewrite"] def test_oos_source_intake_case_rejects_temporary_label_as_source() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[13] reply = good_reply(prompt["id"], token) + " Create the temporary chat label as a canonical source_ref." score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["broad_semantic_issues"] == ["temporary_label_promoted_to_source"] def test_oos_score_rejects_architecture_lecture_length() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[0] reply = good_reply(prompt["id"], token) + " detail" * 301 score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["response_too_long"] is True assert score["max_response_words"] == benchmark.DEFAULT_MAX_RESPONSE_WORDS def test_oos_score_rejects_blanket_all_five_counts_must_move_claim() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[3] reply = good_reply(prompt["id"], token) + " All five counts must move after every apply." score = benchmark.score_reply(prompt, reply, memory_token=token) assert score["pass"] is False assert score["invalid_count_invariant_detected"] is True def test_oos_score_rejects_blanket_all_five_counts_higher_claim() -> None: reply = "The proof is all five canonical counts higher than the baseline." assert benchmark.asserts_invalid_count_invariant(reply) is True def test_oos_score_does_not_penalize_an_explicit_retraction() -> None: reply = "The claim that all five counts must move is wrong; packet-specific row readback is required." assert benchmark.asserts_invalid_count_invariant(reply) is False def test_count_receipt_correction_requires_packet_specific_row_receipts() -> None: reply = """ No. That rule was too strong: not all table counts move; the delta depends on the packet. public.claims and public.sources stay unchanged when reused. For attach_evidence with an existing claim and existing source, public.claim_evidence gets a new row. For an edge-only packet, public.claim_edges gets a new row. kb_stage.kb_proposals count may remain unchanged while its status and applied_at change. The universal receipt is a committed transaction, non-null applied_at, and postflight row-level readback proving the packet's declared expected rows and IDs. public.claims, public.sources, public.claim_edges, and public.claim_evidence are checked table by table against that packet. """ score = correction.score_reply(reply) assert score["pass"] is True def test_count_receipt_correction_rejects_repeated_bad_invariant() -> None: reply = "All five counts must move after every apply." score = correction.score_reply(reply) assert score["pass"] is False assert score["invalid_count_invariant_detected"] is True def test_generic_handler_template_accepts_an_oos_prompt_set() -> None: script = handler.build_remote_script( "cafef00d", prompts=[{"id": "OOS-01", "dimension": "demo", "message": "Broad question"}], suite_mode="oos_mode", report_prefix="oos-report", prompt_note='Out of sample "quoted" prompts.', ) assert '"mode": "oos_mode"' in script assert 'RUN_ID = "cafef00d"' in script assert 'REPORT_PREFIX = "oos-report"' in script assert 'Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")' in script assert 'Out of sample \\"quoted\\" prompts.' in script assert '"id": "OOS-01"' in script assert "__PROMPTS_JSON__" not in script def test_generic_handler_template_rejects_unsafe_report_prefix() -> None: try: handler.build_remote_script("cafef00d", report_prefix="../escape") except ValueError as exc: assert "report_prefix" in str(exc) else: raise AssertionError("unsafe report prefix was accepted")