Harden blind scoring for natural Leo replies

scripts/working_leo_m3taversal_oos_benchmark.py

@@ -426,13 +426,14 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
         re.compile(
             r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|"
             r"keep the factual claim once|store it once|one public\.claims row|"
-            r"one shared public\.claims row|fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
+            r"one shared public\.claims row|facts? (?:are |remain )?shared|"
+            r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
             re.I | re.S,
         ),
         re.compile(r"source|evidence", re.I),
         re.compile(
             r"do not duplicate|don'?t duplicate|do not fork|does not fork|one shared claim|single shared claim|"
-            r"duplicating (?:the )?claim",
+            r"duplicating (?:(?:a|the) )?claim",
             re.I,
         ),
     ),
@@ -475,7 +476,9 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
             r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
             r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
             r"(?:does not exist|is absent|is not present|isn't present)|"
-            r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present",
+            r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present|"
+            r"(?:resolution outcome|resolved_at|resolution_criteria).{0,180}"
+            r"none of those fields.{0,50}(?:exist|present)",
             re.I | re.S,
         ),
         re.compile(
@@ -541,8 +544,6 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
     ),
     "blocker_definition": (
         re.compile(r"blocker", re.I),
-        re.compile(r"approved|applied_at", re.I),
-        re.compile(r"canonical|public\.\*|row counts", re.I),
     ),
     "live_claim_evidence_ids": (
         re.compile(
@@ -619,7 +620,7 @@ SCHEMA_GAP_QUALIFIER_RE = re.compile(
     r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
     r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|"
     r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
-    r"schema extension|extension proposal)\b|"
+    r"schema extension|extension proposal|reviewed schema proposal)\b|"
     r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?"
     r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
     r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?"
@@ -805,6 +806,11 @@ BLOCKER_STOPWORDS = {
     "will",
     "with",
 }
+BLOCKER_DIAGNOSTIC_RE = re.compile(
+    r"\b(?:applied_at|canonical|public\.\*|rows?|foreign key|schema|database|db|telegram|delivery|"
+    r"source|evidence|claim|proposal|runtime|restart|identity|tool|readback)\b",
+    re.I,
+)
 
 
 def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
@@ -819,7 +825,7 @@ def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
 
 def extract_blocker_clause(reply: str) -> str | None:
     for pattern in (
-        re.compile(r"\bblocker\s*:\s*(?P<value>[^\n]+)", re.I),
+        re.compile(r"\bblocker(?:\s+restated)?\s*:\s*(?P<value>[^\n]+)", re.I),
         re.compile(r"\bblocker\s+(?:is|was)\s+(?P<value>[^\n]+)", re.I),
     ):
         match = pattern.search(reply)
@@ -840,6 +846,10 @@ def blocker_terms(value: str | None, *, memory_token: str) -> set[str]:
 
 
 def matched_concept(reply: str, concept: str) -> bool:
+    if concept == "blocker_definition":
+        blocker = extract_blocker_clause(reply)
+        terms = blocker_terms(blocker, memory_token="")
+        return bool(blocker and len(terms) >= 3 and BLOCKER_DIAGNOSTIC_RE.search(blocker))
     if concept == "claim_body_evidence_distinction" and CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
         return False
     if concept == "narrower_claim_revision":
@@ -865,9 +875,18 @@ def current_schema_overclaims(reply: str) -> list[str]:
         # Negation and future-schema qualifiers apply to their clause, not to a
         # contradictory assertion in a later conjunction in the same sentence.
         clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence)
+        review_context = bool(
+            re.match(r"^\s*(?:[-*]\s*)?review (?:covers|includes|asks|considers|checks)\b", sentence, re.I)
+        )
         for clause in clauses:
             if SCHEMA_GAP_QUALIFIER_RE.search(clause):
                 continue
+            if review_context and re.search(r"\b(?:whether|if|how)\b", clause, re.I) and not re.search(
+                r"\bcurrent schema\b.{0,80}\b(?:has|supports|includes|provides|allows)\b",
+                clause,
+                re.I | re.S,
+            ):
+                continue
             for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items():
                 if pattern.search(clause):
                     findings.append(label)
@@ -967,6 +986,10 @@ def composition_capability_issues(prompt_id: str, reply: str) -> list[str]:
 def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dict[str, Any]:
     scorer_prompt_id = str(prompt.get("scorer_id") or prompt["id"])
     legacy_prompt = {**prompt, "id": scorer_prompt_id}
+    if scorer_prompt_id == "OOS-07":
+        legacy_prompt["required_signals"] = []
+    elif scorer_prompt_id == "OOS-08":
+        legacy_prompt["required_signals"] = ["row_level_proof"]
     legacy_score = base.score_reply(legacy_prompt, reply)
     concepts = {concept: matched_concept(reply, concept) for concept in prompt["required_concepts"]}
     custom_signals: dict[str, bool] = {}

scripts/working_leo_m3taversal_oos_protocol.py

@@ -324,7 +324,7 @@ BLINDED_FAMILIES: tuple[dict[str, Any], ...] = (
         "scorer_id": "OOS-07",
         "dimension": "clean_session_memory_set",
         "subjects": ("demo blocker", "proof blocker", "operator blocker"),
-        "subject_anchors": ("blocker", "approved", "applied", "canonical"),
+        "subject_anchors": ("blocker", "label", "remember", "chat"),
         "expected_follow_up": "bind one evidence-based blocker to a temporary chat-only label",
         "variants": (
             (
@@ -349,7 +349,7 @@ BLINDED_FAMILIES: tuple[dict[str, Any], ...] = (
         "scorer_id": "OOS-08",
         "dimension": "clean_session_memory_recall",
         "subjects": ("demo blocker", "proof blocker", "operator blocker"),
-        "subject_anchors": ("blocker", "approved", "applied", "readback"),
+        "subject_anchors": ("blocker", "label", "proof", "readback"),
         "expected_follow_up": "recall the label, same blocker, and exact closure proof",
         "variants": (
             (

tests/test_working_leo_m3taversal_oos_benchmark.py

@@ -324,12 +324,20 @@ def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_
     unqualified_requirement = "Current schema requires a resolves edge, so use it now to close the forecast."
     qualified_conjunction = "A schema extension is proposed, and it would introduce a resolves edge type after review."
     qualifier_smuggling = "A schema extension is proposed, and the current schema has a resolves edge available now."
+    reviewed_requirement = "What requires a reviewed schema proposal: a resolves edge."
+    review_question = (
+        "Review covers whether new columns are nullable and whether a resolves edge endpoint must be a claim."
+    )
+    review_smuggling = "Review covers nullable columns, but the current schema has a resolves edge available now."
 
     assert benchmark.current_schema_overclaims(natural_gap) == []
     assert benchmark.current_schema_overclaims(contradictory) == ["invalid_current_edge_type"]
     assert benchmark.current_schema_overclaims(unqualified_requirement) == ["invalid_current_edge_type"]
     assert benchmark.current_schema_overclaims(qualified_conjunction) == []
     assert benchmark.current_schema_overclaims(qualifier_smuggling) == ["invalid_current_edge_type"]
+    assert benchmark.current_schema_overclaims(reviewed_requirement) == []
+    assert benchmark.current_schema_overclaims(review_question) == []
+    assert benchmark.current_schema_overclaims(review_smuggling) == ["invalid_current_edge_type"]
 
 
 def test_oos_forecast_semantics_accept_natural_history_and_missing_schema_language() -> None:
@@ -839,6 +847,39 @@ def test_oos_agent_positions_accept_attribution_without_literal_column_name() ->
     assert benchmark.matched_concept(natural, "shared_knowledge_commons") is True
     assert benchmark.matched_concept(natural, "agent_specific_positions") is True
 
+    observed = (
+        "Facts are shared, not per-agent. One claim row, one source, and one evidence link remain authoritative. "
+        "Duplicating a claim per agent creates divergence."
+    )
+    assert benchmark.matched_concept(observed, "shared_knowledge_commons") is True
+
+
+def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None:
+    token = "demo-ledger-deadbeef"
+    set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07")
+    recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08")
+    set_reply = (
+        f"Label: {token}\nBlocker: beliefs has no claim_id foreign key, so agent positions are not queryable "
+        "against the shared claim."
+    )
+    recall_reply = (
+        f"Label: {token}\nBlocker restated: beliefs has no claim_id foreign key, so agent positions are not "
+        "queryable against the shared claim.\nProof: postflight readback of the new foreign key and linked row."
+    )
+
+    score = benchmark.score_results(
+        [
+            {"prompt_id": set_prompt["id"], "reply": set_reply},
+            {"prompt_id": recall_prompt["id"], "reply": recall_reply},
+        ],
+        memory_token=token,
+        catalog=[set_prompt, recall_prompt],
+    )
+
+    assert score["pass"] is True
+    assert score["memory_continuity"]["same_blocker_recalled"] is True
+    assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False
+
 
 def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
     reply = (
@@ -855,6 +896,12 @@ def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
         "The schema is missing a resolves edge."
     )
 
+    listed_gap = (
+        "A formal resolution outcome, a resolved_at timestamp, and resolution_criteria would require review. "
+        "None of those fields exist in the current schema. There is no resolves edge type."
+    )
+    assert benchmark.matched_concept(listed_gap, "forecast_schema_gap") is True
+
 
 def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None:
     reply = (

tests/test_working_leo_m3taversal_oos_protocol.py

@@ -1201,6 +1201,12 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words
     )
     assert protocol_lib._subject_alignment(retrieval_prompt, good + f" {retrieval_sibling}.") is False
 
+    memory_prompt = next(
+        item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "session_memory_set"
+    )
+    memory_reply = f"Subject: {memory_prompt['subject']}\nLabel: temporary\nBlocker: database schema gap."
+    assert protocol_lib._subject_alignment(memory_prompt, memory_reply) is True
+
 
 def test_evidence_probe_four_line_reply_binds_exact_subject_and_receipt() -> None:
     protocol = frozen_protocol()