#!/usr/bin/env python3
"""Out-of-sample m3taversal-style benchmark for the live Leo handler.

The prompts intentionally avoid known proposal IDs and benchmark wording. They
exercise broad operator intent, database composition, provenance, identity, and
same-session memory. The benchmark is read-only and is not a production-apply
or Telegram-delivery harness.
"""

from __future__ import annotations

import argparse
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import working_leo_open_ended_benchmark as base

SCENARIOS: list[dict[str, Any]] = [
    {
        "id": "OOS-01",
        "dimension": "demo_pressure_without_ids",
        "message": (
            "m3taversal walks in now and says, 'Leo's database has not learned anything useful since yesterday and I "
            "cannot demo this.' Do not ask me for row IDs. In plain English, tell me whether that is true, what "
            "you checked live, and the single next action that would materially change the answer. Keep this "
            "read-only: do not stage, approve, apply, or edit anything."
        ),
        "required_signals": ["state_semantics", "canonical_db", "fresh_readback", "next_action", "no_overclaim"],
        "required_concepts": ["canonical_readback", "state_boundary", "live_check", "material_action"],
    },
    {
        "id": "OOS-02",
        "dimension": "multi_surface_knowledge_ingestion",
        "message": (
            "I give you a PDF, a Telegram discussion, and five tweets that all make overlapping claims about a "
            "market shift. Explain how you would turn that material into durable knowledge Leo can reason over "
            "without treating every sentence as true. Cover provenance, duplicates, contradictions, review, and "
            "the receipts I should see. Explain only; do not stage or mutate the KB."
        ),
        "required_signals": [
            "document_artifact_linking",
            "staging_or_review",
            "canonical_db",
            "caveat_retention",
            "next_action",
            "no_overclaim",
        ],
        "required_concepts": [
            "provenance",
            "deduplication",
            "contradiction",
            "candidate_before_canonical",
            "staged_review_apply",
            "receipt",
        ],
    },
    {
        "id": "OOS-03",
        "dimension": "identity_restart_truth",
        "message": (
            "I corrected Leo's worldview in chat and someone also edited SOUL.md. Will that correction be part of "
            "Leo's canonical identity tomorrow after a restart? Give me the exact truth test from database row to "
            "runtime artifact. Do not change either surface."
        ),
        "required_signals": ["identity_rendering", "canonical_db", "row_level_proof", "no_overclaim"],
        "required_concepts": ["identity_chain", "identity_rows_beyond_claims", "state_boundary", "receipt"],
    },
    {
        "id": "OOS-04",
        "dimension": "partner_demo_claim_ceiling",
        "message": (
            "Three changes have reviewer approval and the partner demo starts in an hour. Can I honestly say the "
            "knowledge database is updated? Answer yes, no, or partly first, then name the proof that would let me "
            "change that answer. Do not apply anything."
        ),
        "required_signals": ["state_semantics", "canonical_db", "row_level_proof", "artifact", "no_overclaim"],
        "required_concepts": ["canonical_readback", "state_boundary", "receipt"],
    },
    {
        "id": "OOS-05",
        "dimension": "source_pointer_needs_canonical_link_audit",
        "message": (
            "A proposal points to a Telegram attachment and its extracted text exists on disk. Does that mean the "
            "claim already has canonical evidence? Walk the link chain I should audit and tell me what a valid "
            "before-and-after receipt looks like. Read-only only."
        ),
        "required_signals": [
            "document_artifact_linking",
            "canonical_db",
            "staging_or_review",
            "row_level_proof",
            "no_overclaim",
        ],
        "required_concepts": [
            "source_evidence_chain",
            "canonical_evidence_boundary",
            "evidence_provenance_quality",
            "staged_review_apply",
            "receipt",
        ],
    },
    {
        "id": "OOS-06",
        "dimension": "database_composition_judgment",
        "message": (
            "Suppose a new research packet contains factual observations, a strategic framework, a disputed "
            "interpretation, a governance rule, and a correction to an old belief. How should Leo compose the "
            "database so it can reason over the packet later without flattening it all into claims? Give the "
            "review and apply sequence too, but do not write anything now."
        ),
        "required_signals": ["canonical_db", "staging_or_review", "caveat_retention", "next_action", "no_overclaim"],
        "required_concepts": [
            "heterogeneous_types",
            "behavioral_rule_storage",
            "reviewed_policy_apply",
            "staged_review_apply",
            "receipt",
        ],
    },
    {
        "id": "OOS-07",
        "dimension": "same_session_memory_set",
        "message": (
            "From everything you checked in this conversation, pick the single highest-impact demo blocker. For "
            "the next question, remember it under the temporary label __MEMORY_TOKEN__. State the label and blocker "
            "now. This is chat memory only; do not write it to the KB."
        ),
        "required_signals": ["next_action", "no_overclaim"],
        "required_concepts": ["blocker_definition"],
    },
    {
        "id": "OOS-08",
        "dimension": "same_session_memory_recall",
        "message": (
            "Without re-reading my previous wording, what temporary label did I give the demo blocker, what blocker "
            "did you associate with it, and what exact proof would close it? Do not mutate anything."
        ),
        "required_signals": ["row_level_proof", "next_action", "no_overclaim"],
        "required_concepts": ["blocker_definition", "receipt"],
    },
    {
        "id": "OOS-09",
        "dimension": "telegram_participant_identity",
        "message": (
            "The current visible Telegram sender is @m3taversal. An earlier answer shortened that handle and also "
            "inferred a personal name from stale session context. What should Leo call this participant, which "
            "identity sources are allowed, and how should Leo avoid mixing identities when another user replies? "
            "Answer in four lines. Do not write to memory or the KB."
        ),
        "required_signals": ["no_overclaim"],
        "required_concepts": [],
    },
    {
        "id": "OOS-10",
        "dimension": "runtime_vs_canonical_causality",
        "message": (
            "The five database totals are unchanged after a restart. Does that prove Leo's answer behavior is "
            "unchanged and that every fact from the prior session was erased? Explain which persisted and deployed "
            "surfaces can affect the answer, and separate the proof tiers. Read-only; keep it under 180 words."
        ),
        "required_signals": ["canonical_db", "no_overclaim"],
        "required_concepts": [
            "runtime_inputs",
            "durable_session_continuity",
            "proof_tiers",
            "row_content_proof",
        ],
    },
    {
        "id": "OOS-11",
        "dimension": "shared_claims_agent_positions",
        "message": (
            "Two agents read the same evidence and reach different conclusions. Should Leo duplicate the factual "
            "claim once per agent, or keep shared knowledge and represent agent-specific positions elsewhere? Give "
            "the current-schema answer and how disagreement remains queryable. Do not write anything."
        ),
        "required_signals": ["canonical_db", "caveat_retention", "no_overclaim"],
        "required_concepts": ["shared_knowledge_commons", "agent_specific_positions", "contradiction"],
    },
    {
        "id": "OOS-12",
        "dimension": "forecast_resolution_without_rewriting_history",
        "message": (
            "An old claim recorded a 60% forecast but never defined resolution criteria. The event is now over. "
            "What may Leo record in the current database without rewriting history, and what needs a schema "
            "proposal? Do not apply anything."
        ),
        "required_signals": ["canonical_db", "staging_or_review", "caveat_retention", "no_overclaim"],
        "required_concepts": ["forecast_history", "forecast_schema_gap", "staged_review_apply"],
    },
    {
        "id": "OOS-13",
        "dimension": "handler_is_not_telegram_delivery",
        "message": (
            "A temporary-profile GatewayRunner answered every test prompt, but the run explicitly posted nothing "
            "to Telegram. Can I tell a partner the Telegram path is proven live? Answer first, then state exactly "
            "what this run proves and the smallest test that closes the gap."
        ),
        "required_signals": ["artifact", "next_action", "no_overclaim"],
        "required_concepts": ["handler_not_telegram", "delivery_receipt"],
    },
    {
        "id": "OOS-14",
        "dimension": "autonomous_source_intake_boundary",
        "message": (
            "I hand Leo a document and say: absorb this as far as safely possible without making me approve every "
            "mechanical step. What can Leo capture and stage immediately, what real source identity must be retained, "
            "and where does explicit approval begin? Explain only; do not ingest this prompt."
        ),
        "required_signals": ["authorization", "staging_or_review", "artifact", "no_overclaim"],
        "required_concepts": [
            "staging_without_apply_authorization",
            "real_source_identity",
            "bounded_intake_tier",
            "staged_review_apply",
        ],
    },
    {
        "id": "OOS-15",
        "dimension": "schema_valid_supersession",
        "message": (
            "A canonical claim is wrong. I want the replacement, an explanation, and the old claim visibly retired. "
            "In current v1, which exact claim and edge fields exist, which requested writes fit approve_claim, and "
            "which require a separate reviewed apply capability? Do not mutate anything."
        ),
        "required_signals": ["canonical_db", "staging_or_review", "row_level_proof", "no_overclaim"],
        "required_concepts": ["valid_supersession", "current_edge_schema", "apply_capability_boundary"],
    },
]


CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
    "canonical_readback": (re.compile(r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)", re.I),),
    "state_boundary": (
        re.compile(
            r"approved is not the same as applied|applied_at:\s*(?:none|null)|not canonical|"
            r"proposal does not commit|no receipt,? no durable knowledge",
            re.I,
        ),
    ),
    "live_check": (
        re.compile(r"checked live|I ran `?teleo-kb|fresh readback|live readback|current canonical row counts", re.I),
    ),
    "material_action": (
        re.compile(
            r"next .*action|rebuild .*apply_payload|operator .*authoriz|review .*apply|apply sequence|postflight",
            re.I,
        ),
    ),
    "provenance": (re.compile(r"provenance|stable (?:reference|ref)|source row|author/channel|file hash", re.I),),
    "deduplication": (re.compile(r"deduplic|duplicate|existing claim|overlap", re.I),),
    "contradiction": (re.compile(r"contradict|divergence|competing interpretation|disagree", re.I),),
    "staged_review_apply": (
        re.compile(r"staging|stage|proposal", re.I),
        re.compile(r"review|approve|reviewer", re.I),
        re.compile(r"apply|canonical", re.I),
    ),
    "receipt": (
        re.compile(r"receipt|readback|postflight|before-and-after|before/after", re.I),
        re.compile(r"row|count|applied_at|public\.", re.I),
    ),
    "identity_chain": (
        re.compile(r"SOUL\.md|identity", re.I),
        re.compile(r"Postgres|database|canonical", re.I),
        re.compile(r"render|sync", re.I),
        re.compile(r"restart|session start|runtime injection", re.I),
    ),
    "identity_rows_beyond_claims": (
        re.compile(r"personas?|strateg(?:y|ies)|beliefs?", re.I),
        re.compile(r"identity|SOUL\.md", re.I),
        re.compile(r"renderer|render automation|render/sync", re.I),
    ),
    "candidate_before_canonical": (
        re.compile(r"candidate|proposal|staging", re.I),
        re.compile(
            r"(?:nothing|no (?:candidate )?rows?).{0,80}public\.\*|"
            r"not canonical.{0,80}(?:review|apply)|apply only after (?:review|approval)",
            re.I | re.S,
        ),
    ),
    "source_evidence_chain": (
        re.compile(r"file|attachment|source_ref", re.I),
        re.compile(r"public\.sources", re.I),
        re.compile(r"claim_evidence", re.I),
        re.compile(r"audit|link|join|row chain|before-and-after|before/after", re.I),
    ),
    "canonical_evidence_boundary": (
        re.compile(r"canonical evidence", re.I),
        re.compile(r"claim_evidence", re.I),
        re.compile(r"public\.sources|source row", re.I),
        re.compile(
            r"attachment.{0,160}(?:does not|doesn't|is not|isn't|cannot|can't|alone|until|unless)|"
            r"(?:does not|doesn't|is not|isn't|cannot|can't).{0,100}canonical evidence from (?:that|the) attachment",
            re.I | re.S,
        ),
    ),
    "evidence_provenance_quality": (
        re.compile(r"(?:missing|no|without).{0,50}(?:url|storage(?:_path)?|locator)", re.I | re.S),
        re.compile(r"weak|citation-only|citation only|not traceable|raw artifact", re.I),
        re.compile(r"canonical evidence|canonical link", re.I),
    ),
    "heterogeneous_types": (
        re.compile(r"claim", re.I),
        re.compile(r"source|evidence", re.I),
        re.compile(r"framework|reasoning tool|concept map", re.I),
        re.compile(r"governance", re.I),
        re.compile(r"correction|supersed", re.I),
        re.compile(r"disput|contradict", re.I),
    ),
    "behavioral_rule_storage": (
        re.compile(r"public\.behavioral_rules", re.I),
        re.compile(r"\bagent_id\b", re.I),
        re.compile(r"\bcategory\b", re.I),
        re.compile(r"\brank\b", re.I),
        re.compile(r"\brule\b", re.I),
        re.compile(r"\brationale\b", re.I),
    ),
    "reviewed_policy_apply": (
        re.compile(r"approve_claim", re.I),
        re.compile(r"behavioral_rules", re.I),
        re.compile(r"governance_gates", re.I),
        re.compile(r"does not (?:accept|insert|support)|supports neither|neither.{0,80}nor", re.I | re.S),
        re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply capability", re.I | re.S),
    ),
    "runtime_inputs": (
        re.compile(r"Postgres|canonical (?:DB|database|counts?)", re.I),
        re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I),
        re.compile(r"session|conversation context", re.I),
        re.compile(r"unchanged.{0,80}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)", re.I | re.S),
    ),
    "durable_session_continuity": (
        re.compile(r"state\.db|session JSONL", re.I),
        re.compile(r"persist|durable|continuity", re.I),
        re.compile(r"restart.{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)", re.I | re.S),
    ),
    "row_content_proof": (
        re.compile(r"unchanged (?:counts?|totals?)", re.I),
        re.compile(r"does not prove|doesn't prove|do not prove", re.I),
        re.compile(r"row (?:IDs?|hashes?)|fingerprints?|timestamps?|balanced (?:insert|write|change)", re.I),
    ),
    "proof_tiers": (
        re.compile(r"handler|temporary[- ]profile|GatewayRunner", re.I),
        re.compile(r"Telegram", re.I),
        re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I),
    ),
    "shared_knowledge_commons": (
        re.compile(r"shared (?:claim|knowledge|commons)|claims.{0,40}shared", re.I | re.S),
        re.compile(r"source|evidence", re.I),
        re.compile(r"do not duplicate|don'?t duplicate|one shared claim|single shared claim", re.I),
    ),
    "agent_specific_positions": (
        re.compile(r"public\.beliefs", re.I),
        re.compile(r"agent_id", re.I),
        re.compile(r"belief|position|stance|confidence", re.I),
        re.compile(r"no.{0,40}claim(?:-ID|_id).{0,30}(?:foreign key|link)|schema gap", re.I | re.S),
    ),
    "forecast_history": (
        re.compile(r"original (?:probability|confidence)|60%|history", re.I),
        re.compile(r"preserve|retain|do not overwrite|don'?t overwrite", re.I),
        re.compile(r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria", re.I | re.S),
    ),
    "forecast_schema_gap": (
        re.compile(r"current (?:v1|schema)|public\.claims", re.I),
        re.compile(
            r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution",
            re.I | re.S,
        ),
        re.compile(r"resolves.{0,30}(?:not|isn'?t|does not)|no.{0,30}resolves", re.I | re.S),
    ),
    "handler_not_telegram": (
        re.compile(r"no|not", re.I),
        re.compile(r"handler|GatewayRunner|temporary[- ]profile", re.I),
        re.compile(r"did not post|posted nothing|not Telegram-visible|does not prove Telegram", re.I),
    ),
    "delivery_receipt": (
        re.compile(r"visible reply|Telegram-visible reply", re.I),
        re.compile(r"message ID|timestamp|readback", re.I),
    ),
    "staging_without_apply_authorization": (
        re.compile(r"capture|hash|archive", re.I),
        re.compile(r"stage|pending_review|proposal", re.I),
        re.compile(
            r"does not require.{0,50}(?:apply )?approval|without.{0,50}(?:apply )?approval|approval begins.{0,80}apply",
            re.I | re.S,
        ),
    ),
    "real_source_identity": (
        re.compile(r"URL|storage path|file hash|content hash|retained artifact", re.I),
        re.compile(r"temporary label|chat label|proposal pointer|source_ref", re.I),
        re.compile(r"not.{0,40}(?:a )?(?:source|provenance)|must not.{0,40}source|do not manufacture", re.I | re.S),
    ),
    "bounded_intake_tier": (
        re.compile(r"build-only|local(?:ly)?|clone", re.I),
        re.compile(r"not yet.{0,80}(?:live|autonomous|production)|not.{0,80}live-VPS", re.I | re.S),
    ),
    "valid_supersession": (
        re.compile(r"new|replacement", re.I),
        re.compile(r"supersedes", re.I),
        re.compile(r"superseded_by", re.I),
        re.compile(r"old claim", re.I),
    ),
    "current_edge_schema": (
        re.compile(r"claim_edges", re.I),
        re.compile(r"from_claim", re.I),
        re.compile(r"to_claim", re.I),
        re.compile(r"edge_type", re.I),
        re.compile(r"no.{0,30}rationale|does not.{0,30}rationale", re.I | re.S),
    ),
    "apply_capability_boundary": (
        re.compile(r"approve_claim", re.I),
        re.compile(r"insert.{0,50}(?:new|replacement) claim|new claim.{0,50}insert", re.I | re.S),
        re.compile(r"supersedes edge", re.I),
        re.compile(r"separate.{0,50}reviewed apply|does not update.{0,80}(?:status|superseded_by)", re.I | re.S),
    ),
    "blocker_definition": (
        re.compile(r"blocker", re.I),
        re.compile(r"approved|applied_at", re.I),
        re.compile(r"canonical|public\.\*|row counts", re.I),
    ),
}

INVALID_COUNT_INVARIANT_RE = re.compile(
    r"\ball five (?:canonical )?(?:numbers|counts) (?:must|need to) move\b|"
    r"\bneed all five (?:canonical )?(?:numbers|counts) to move\b|"
    r"\ball five (?:canonical )?(?:numbers|counts).{0,40}\b(?:higher|increase|increased|rise|change|changed)\b",
    re.I,
)
COUNT_INVARIANT_REJECTION_RE = re.compile(
    r"(?:all five (?:numbers|counts)|count invariant).{0,50}(?:wrong|false|invalid|not valid|too strong)|"
    r"(?:wrong|false|invalid|not valid|too strong).{0,50}all five (?:numbers|counts)",
    re.I | re.S,
)

SCHEMA_GAP_QUALIFIER_RE = re.compile(
    r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|"
    r"has no|have no|no column|not an edge|would require|schema gap|must be added|"
    r"does not support|doesn't support|supports neither|not supported|must not|do not invent)\b",
    re.I,
)
CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
    "claims_unshipped_fields": re.compile(
        r"(?:public\.)?claims?.{0,100}\b(?:body|metadata|forecast[_ -]resolution|resolved_at)\b",
        re.I,
    ),
    "sources_unshipped_fields": re.compile(
        r"(?:public\.)?sources?.{0,100}\b(?:author|channel|published_at|source_date)\b",
        re.I,
    ),
    "invalid_current_edge_type": re.compile(
        r"(?:\b(?:claim_edges?|edge type|edge)\b\s*(?:is|=|:|named|called|of)?\s*[`'\"]?"
        r"(?:superseded_by|relates_to|resolves|derived_from)\b|"
        r"\b(?:superseded_by|relates_to|resolves|derived_from)\b[`'\"]?\s+(?:claim_)?edge\b)",
        re.I,
    ),
    "unshipped_edge_rationale": re.compile(r"\bclaim_edges?\b.{0,100}\brationale\b", re.I),
    "unshipped_evidence_excerpt": re.compile(
        r"\bclaim_evidence\b.{0,100}\bwith\s+(?:an?\s+)?(?:excerpt|excerpt_anchor)\b|"
        r"\bclaim_evidence\b.{0,100}\b(?:has|contains|stores|carries)\b.{0,30}\bexcerpt\b|"
        r"\bclaim_evidence\b.{0,100}\b(?:excerpt|excerpt_anchor)\s+(?:column|field)\b",
        re.I,
    ),
    "unreviewed_claim_type": re.compile(
        r"(?:public\.)?claims?.{0,100}\b(?:type|typed)\s*[`'\":=]?\s*(?:observation|hypothesis|belief)\b",
        re.I,
    ),
    "reasoning_tools_unshipped_fields": re.compile(
        r"\breasoning_tools?\b.{0,120}\b(?:structured criteria|structured steps|criteria or steps)\b",
        re.I,
    ),
    "unsupported_approve_claim_surface": re.compile(
        r"\b(?:apply_payload|approve_claim)\b.{0,500}\b(?:governance gate row|governance_gates|"
        r"superseded_by column update|update the existing old claim)\b",
        re.I,
    ),
}
UNVERIFIED_M3TAVERSAL_ALIAS_RE = re.compile(r"\b(?:Cory|m3ta)\b", re.I)

SOURCE_EVIDENCE_CANONICAL_OBJECT_RE = re.compile(r"claim_evidence|public\.sources|source rows?", re.I)
SOURCE_EVIDENCE_LOCATOR_GAP_RE = re.compile(
    r"(?:missing|without|no).{0,50}(?:url|storage(?:_path)?|locator)", re.I | re.S
)
SOURCE_EVIDENCE_DENIAL_RE = re.compile(
    r"(?:not|isn't|is not|doesn't count as|does not count as).{0,40}(?:canonical|grounded) evidence",
    re.I | re.S,
)
SOURCE_EVIDENCE_CITATION_STUB_DENIAL_RE = re.compile(
    r"citation stubs?.{0,50}(?:not|isn't|is not).{0,30}(?:canonical|grounded) evidence",
    re.I | re.S,
)

BEHAVIORAL_RULES_FALSE_ABSENCE_RE = re.compile(
    r"(?:public\.)?behavioral_rules(?: table)?.{0,50}(?:does not exist|doesn't exist|is absent|is missing|not shipped)|"
    r"(?:no|missing) (?:public\.)?behavioral_rules table",
    re.I | re.S,
)

DB_ONLY_CAUSALITY_RE = re.compile(
    r"(?:unchanged|same) (?:canonical )?(?:DB|database|counts?).{0,80}"
    r"(?:proves?|means|therefore).{0,80}(?:behavior|answer).{0,30}(?:unchanged|cannot change)",
    re.I | re.S,
)
RESTART_ERASES_ALL_RE = re.compile(
    r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S
)
HANDLER_TELEGRAM_OVERCLAIM_RE = re.compile(
    r"(?:Telegram (?:path|delivery).{0,40}(?:is|was) (?:live|proven)|"
    r"live Telegram (?:path|delivery)).{0,160}(?:posted nothing|did not post|no Telegram post)",
    re.I | re.S,
)
FORECAST_HISTORY_REWRITE_RE = re.compile(
    r"(?<!not )(?<!don't )(?<!dont )(?:overwrite|update|replace).{0,40}"
    r"(?:original )?(?:probability|confidence).{0,80}"
    r"(?:resolved|actual|outcome)",
    re.I | re.S,
)
TEMP_LABEL_AS_SOURCE_RE = re.compile(
    r"(?:use|write|create|store|promote).{0,50}(?:temporary|memory|chat) (?:label|token).{0,100}"
    r"(?:source_ref|public\.sources|canonical source)",
    re.I | re.S,
)
APPROVED_APPLY_ACTION_RE = re.compile(
    r"approved.{0,240}(?:guarded )?apply|(?:guarded )?apply.{0,240}approved",
    re.I | re.S,
)
CANONICAL_SOURCE_BEFORE_REVIEW_RE = re.compile(
    r"(?:each|the|a) (?:artifact|PDF|document|tweet).{0,100}(?:gets|creates?|writes?).{0,40}"
    r"public\.sources row.{0,80}before (?:any )?(?:claim is )?extract",
    re.I | re.S,
)
APPLYABILITY_GAP_RE = re.compile(
    r"approved_needs_apply_payload|worker_(?:contract_)?applyable(?:_count)?\s*[:=]?\s*(?:false|0)|"
    r"(?:no|missing|without).{0,50}(?:strict )?apply_payload|"
    r"normaliz(?:e|ation).{0,100}(?:before|then).{0,60}(?:review|apply)|"
    r"not (?:directly )?applyable",
    re.I | re.S,
)

DEFAULT_MAX_RESPONSE_WORDS = 220
MAX_RESPONSE_WORDS = {"OOS-09": 100, "OOS-10": 180}


def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
    return [
        {
            **scenario,
            "message": scenario["message"].replace("__MEMORY_TOKEN__", memory_token),
        }
        for scenario in SCENARIOS
    ]


def matched_concept(reply: str, concept: str) -> bool:
    return all(pattern.search(reply) for pattern in CONCEPT_PATTERNS[concept])


def asserts_invalid_count_invariant(reply: str) -> bool:
    return bool(INVALID_COUNT_INVARIANT_RE.search(reply) and not COUNT_INVARIANT_REJECTION_RE.search(reply))


def current_schema_overclaims(reply: str) -> list[str]:
    """Return proposed-v3-as-current assertions that are not explicitly qualified."""

    findings: list[str] = []
    for segment in re.split(r"(?<=[.!?])\s+|\n+", reply):
        if SCHEMA_GAP_QUALIFIER_RE.search(segment):
            continue
        for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items():
            if pattern.search(segment):
                findings.append(label)
    return sorted(set(findings))


def source_evidence_semantic_issues(reply: str) -> list[str]:
    """Reject answers that confuse canonical linkage with locator quality."""

    findings: set[str] = set()
    for segment in re.split(r"(?<=[.!?])\s+|\n+", reply):
        if (
            SOURCE_EVIDENCE_CANONICAL_OBJECT_RE.search(segment)
            and SOURCE_EVIDENCE_LOCATOR_GAP_RE.search(segment)
            and SOURCE_EVIDENCE_DENIAL_RE.search(segment)
        ):
            findings.add("locator_gap_called_noncanonical")
    if SOURCE_EVIDENCE_CITATION_STUB_DENIAL_RE.search(reply):
        findings.add("citation_stub_called_ungrounded")
    return sorted(findings)


def behavioral_rule_schema_issues(reply: str) -> list[str]:
    """Reject claims that the current behavioral-rules table is absent."""

    return ["behavioral_rules_false_absence"] if BEHAVIORAL_RULES_FALSE_ABSENCE_RE.search(reply) else []


def broad_semantic_issues(reply: str) -> list[str]:
    """Reject high-impact false claims that can hide inside otherwise complete prose."""

    findings: set[str] = set()
    if DB_ONLY_CAUSALITY_RE.search(reply) and not re.search(
        r"does not prove|doesn't prove|do not prove|does not mean|doesn't mean|do not mean|cannot prove", reply, re.I
    ):
        findings.add("unchanged_db_called_complete_behavior_proof")
    if RESTART_ERASES_ALL_RE.search(reply) and not re.search(
        r"restart.{0,100}(?:does not|doesn't|need not|not necessarily).{0,60}(?:erase|forget|lose)",
        reply,
        re.I | re.S,
    ):
        findings.add("restart_called_total_memory_erasure")
    if HANDLER_TELEGRAM_OVERCLAIM_RE.search(reply):
        findings.add("handler_proof_called_telegram_live")
    if FORECAST_HISTORY_REWRITE_RE.search(reply):
        findings.add("forecast_history_rewrite")
    if TEMP_LABEL_AS_SOURCE_RE.search(reply):
        findings.add("temporary_label_promoted_to_source")
    return sorted(findings)


def proposal_readiness_issues(prompt_id: str, reply: str) -> list[str]:
    if prompt_id not in {"OOS-01", "OOS-04", "OOS-07", "OOS-08"}:
        return []
    if APPROVED_APPLY_ACTION_RE.search(reply) and not APPLYABILITY_GAP_RE.search(reply):
        return ["approved_proposal_applyability_overclaim"]
    return []


def source_intake_issues(prompt_id: str, reply: str) -> list[str]:
    if prompt_id == "OOS-02" and CANONICAL_SOURCE_BEFORE_REVIEW_RE.search(reply):
        return ["canonical_source_created_before_review"]
    return []


def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dict[str, Any]:
    legacy_score = base.score_reply(prompt, reply)
    concepts = {concept: matched_concept(reply, concept) for concept in prompt["required_concepts"]}
    custom_signals: dict[str, bool] = {}
    if prompt["id"] in {"OOS-07", "OOS-08"}:
        custom_signals["memory_token"] = memory_token.lower() in reply.lower()
    if prompt["id"] == "OOS-08":
        lowered = reply.lower()
        custom_signals["closure_proof"] = any(
            phrase in lowered
            for phrase in ("readback", "before/after", "before-and-after", "postflight", "canonical row", "applied_at")
        )
    if prompt["id"] == "OOS-09":
        custom_signals["exact_participant_handle"] = "m3taversal" in reply.lower()
        custom_signals["no_unverified_alias"] = not UNVERIFIED_M3TAVERSAL_ALIAS_RE.search(reply)
        custom_signals["current_update_identity_boundary"] = bool(
            re.search(r"current (?:Telegram )?(?:update|sender|handle)|visible handle", reply, re.I)
        )
        custom_signals["no_cross_participant_identity_bleed"] = bool(
            re.search(
                r"(?:do(?:es)? not|never)\s+(?:\w+\s+){0,3}"
                r"(?:transfer|transfers|carry|carries|mix|mixes|reuse|reuses)",
                reply,
                re.I,
            )
        )
    invalid_count_invariant = asserts_invalid_count_invariant(reply)
    schema_overclaims = current_schema_overclaims(reply)
    source_evidence_issues = source_evidence_semantic_issues(reply) if prompt["id"] == "OOS-05" else []
    behavioral_rule_issues = behavioral_rule_schema_issues(reply) if prompt["id"] == "OOS-06" else []
    semantic_issues = broad_semantic_issues(reply)
    readiness_issues = proposal_readiness_issues(prompt["id"], reply)
    intake_issues = source_intake_issues(prompt["id"], reply)
    word_count = len(re.findall(r"\b\w+(?:[-']\w+)*\b", reply))
    max_response_words = MAX_RESPONSE_WORDS.get(prompt["id"], DEFAULT_MAX_RESPONSE_WORDS)
    response_too_long = word_count > max_response_words
    return {
        "prompt_id": prompt["id"],
        "dimension": prompt["dimension"],
        "concepts": concepts,
        "custom_signals": custom_signals,
        "legacy_signals": legacy_score["signals"],
        "legacy_pass": legacy_score["pass"],
        "overclaim_detected": legacy_score["overclaim_detected"],
        "response_issue_detected": legacy_score["response_issue_detected"],
        "invalid_count_invariant_detected": invalid_count_invariant,
        "current_schema_overclaims": schema_overclaims,
        "source_evidence_semantic_issues": source_evidence_issues,
        "behavioral_rule_schema_issues": behavioral_rule_issues,
        "broad_semantic_issues": semantic_issues,
        "proposal_readiness_issues": readiness_issues,
        "source_intake_issues": intake_issues,
        "word_count": word_count,
        "max_response_words": max_response_words,
        "response_too_long": response_too_long,
        "pass": bool(
            all(concepts.values())
            and all(custom_signals.values())
            and not legacy_score["overclaim_detected"]
            and not invalid_count_invariant
            and not schema_overclaims
            and not source_evidence_issues
            and not behavioral_rule_issues
            and not semantic_issues
            and not readiness_issues
            and not intake_issues
            and not response_too_long
        ),
    }


def score_results(results: list[dict[str, Any]], *, memory_token: str) -> dict[str, Any]:
    catalog = prompt_catalog(memory_token)
    expected_ids = [prompt["id"] for prompt in catalog]
    by_prompt = {prompt["id"]: prompt for prompt in catalog}
    by_result = {str(result.get("prompt_id")): result for result in results if result.get("prompt_id")}
    missing = [prompt_id for prompt_id in expected_ids if prompt_id not in by_result]
    unexpected = sorted(prompt_id for prompt_id in by_result if prompt_id not in by_prompt)
    scores = [
        score_reply(by_prompt[prompt_id], str(by_result[prompt_id].get("reply") or ""), memory_token=memory_token)
        for prompt_id in expected_ids
        if prompt_id in by_result
    ]
    return {
        "expected_prompt_count": len(expected_ids),
        "expected_prompt_ids": expected_ids,
        "missing_prompt_ids": missing,
        "unexpected_prompt_ids": unexpected,
        "prompt_count": len(scores),
        "passes": sum(1 for score in scores if score["pass"]),
        "failures": [score for score in scores if not score["pass"]],
        "scores": scores,
        "pass": not missing
        and not unexpected
        and len(scores) == len(expected_ids)
        and all(score["pass"] for score in scores),
    }


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--results-json", type=Path, required=True)
    parser.add_argument("--memory-token", required=True)
    parser.add_argument("--out", type=Path)
    args = parser.parse_args()
    payload = json.loads(args.results_json.read_text(encoding="utf-8"))
    results = payload.get("results", payload) if isinstance(payload, dict) else payload
    if not isinstance(results, list):
        raise SystemExit("results JSON must contain a list or a top-level results list")
    report = {
        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
        "mode": "working_leo_m3taversal_out_of_sample_score",
        "source_results_json": str(args.results_json),
        "memory_token": args.memory_token,
        "score": score_results(results, memory_token=args.memory_token),
    }
    if args.out:
        args.out.parent.mkdir(parents=True, exist_ok=True)
        args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    print(json.dumps(report, indent=2, sort_keys=True))
    return 0 if report["score"]["pass"] else 1


if __name__ == "__main__":
    raise SystemExit(main())