diff --git a/scripts/working_leo_m3taversal_oos_benchmark.py b/scripts/working_leo_m3taversal_oos_benchmark.py index fe11f99..2afaf28 100755 --- a/scripts/working_leo_m3taversal_oos_benchmark.py +++ b/scripts/working_leo_m3taversal_oos_benchmark.py @@ -426,13 +426,14 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile( r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|" r"keep the factual claim once|store it once|one public\.claims row|" - r"one shared public\.claims row|fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)", + r"one shared public\.claims row|facts? (?:are |remain )?shared|" + r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)", re.I | re.S, ), re.compile(r"source|evidence", re.I), re.compile( r"do not duplicate|don'?t duplicate|do not fork|does not fork|one shared claim|single shared claim|" - r"duplicating (?:the )?claim", + r"duplicating (?:(?:a|the) )?claim", re.I, ), ), @@ -475,7 +476,9 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|" r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}" r"(?:does not exist|is absent|is not present|isn't present)|" - r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present", + r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present|" + r"(?:resolution outcome|resolved_at|resolution_criteria).{0,180}" + r"none of those fields.{0,50}(?:exist|present)", re.I | re.S, ), re.compile( @@ -541,8 +544,6 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { ), "blocker_definition": ( re.compile(r"blocker", re.I), - re.compile(r"approved|applied_at", re.I), - re.compile(r"canonical|public\.\*|row counts", re.I), ), "live_claim_evidence_ids": ( re.compile( @@ -619,7 +620,7 @@ SCHEMA_GAP_QUALIFIER_RE = re.compile( r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|" r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|" r"does not support|doesn't support|supports neither|not supported|must not|do not invent|" - r"schema extension|extension proposal)\b|" + r"schema extension|extension proposal|reviewed schema proposal)\b|" r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?" r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|" r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?" @@ -805,6 +806,11 @@ BLOCKER_STOPWORDS = { "will", "with", } +BLOCKER_DIAGNOSTIC_RE = re.compile( + r"\b(?:applied_at|canonical|public\.\*|rows?|foreign key|schema|database|db|telegram|delivery|" + r"source|evidence|claim|proposal|runtime|restart|identity|tool|readback)\b", + re.I, +) def prompt_catalog(memory_token: str) -> list[dict[str, Any]]: @@ -819,7 +825,7 @@ def prompt_catalog(memory_token: str) -> list[dict[str, Any]]: def extract_blocker_clause(reply: str) -> str | None: for pattern in ( - re.compile(r"\bblocker\s*:\s*(?P[^\n]+)", re.I), + re.compile(r"\bblocker(?:\s+restated)?\s*:\s*(?P[^\n]+)", re.I), re.compile(r"\bblocker\s+(?:is|was)\s+(?P[^\n]+)", re.I), ): match = pattern.search(reply) @@ -840,6 +846,10 @@ def blocker_terms(value: str | None, *, memory_token: str) -> set[str]: def matched_concept(reply: str, concept: str) -> bool: + if concept == "blocker_definition": + blocker = extract_blocker_clause(reply) + terms = blocker_terms(blocker, memory_token="") + return bool(blocker and len(terms) >= 3 and BLOCKER_DIAGNOSTIC_RE.search(blocker)) if concept == "claim_body_evidence_distinction" and CLAIM_EVIDENCE_CONFLATION_RE.search(reply): return False if concept == "narrower_claim_revision": @@ -865,9 +875,18 @@ def current_schema_overclaims(reply: str) -> list[str]: # Negation and future-schema qualifiers apply to their clause, not to a # contradictory assertion in a later conjunction in the same sentence. clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence) + review_context = bool( + re.match(r"^\s*(?:[-*]\s*)?review (?:covers|includes|asks|considers|checks)\b", sentence, re.I) + ) for clause in clauses: if SCHEMA_GAP_QUALIFIER_RE.search(clause): continue + if review_context and re.search(r"\b(?:whether|if|how)\b", clause, re.I) and not re.search( + r"\bcurrent schema\b.{0,80}\b(?:has|supports|includes|provides|allows)\b", + clause, + re.I | re.S, + ): + continue for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items(): if pattern.search(clause): findings.append(label) @@ -967,6 +986,10 @@ def composition_capability_issues(prompt_id: str, reply: str) -> list[str]: def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dict[str, Any]: scorer_prompt_id = str(prompt.get("scorer_id") or prompt["id"]) legacy_prompt = {**prompt, "id": scorer_prompt_id} + if scorer_prompt_id == "OOS-07": + legacy_prompt["required_signals"] = [] + elif scorer_prompt_id == "OOS-08": + legacy_prompt["required_signals"] = ["row_level_proof"] legacy_score = base.score_reply(legacy_prompt, reply) concepts = {concept: matched_concept(reply, concept) for concept in prompt["required_concepts"]} custom_signals: dict[str, bool] = {} diff --git a/scripts/working_leo_m3taversal_oos_protocol.py b/scripts/working_leo_m3taversal_oos_protocol.py index 3d2cf03..ccde259 100644 --- a/scripts/working_leo_m3taversal_oos_protocol.py +++ b/scripts/working_leo_m3taversal_oos_protocol.py @@ -324,7 +324,7 @@ BLINDED_FAMILIES: tuple[dict[str, Any], ...] = ( "scorer_id": "OOS-07", "dimension": "clean_session_memory_set", "subjects": ("demo blocker", "proof blocker", "operator blocker"), - "subject_anchors": ("blocker", "approved", "applied", "canonical"), + "subject_anchors": ("blocker", "label", "remember", "chat"), "expected_follow_up": "bind one evidence-based blocker to a temporary chat-only label", "variants": ( ( @@ -349,7 +349,7 @@ BLINDED_FAMILIES: tuple[dict[str, Any], ...] = ( "scorer_id": "OOS-08", "dimension": "clean_session_memory_recall", "subjects": ("demo blocker", "proof blocker", "operator blocker"), - "subject_anchors": ("blocker", "approved", "applied", "readback"), + "subject_anchors": ("blocker", "label", "proof", "readback"), "expected_follow_up": "recall the label, same blocker, and exact closure proof", "variants": ( ( diff --git a/tests/test_working_leo_m3taversal_oos_benchmark.py b/tests/test_working_leo_m3taversal_oos_benchmark.py index 25cb687..62d7397 100644 --- a/tests/test_working_leo_m3taversal_oos_benchmark.py +++ b/tests/test_working_leo_m3taversal_oos_benchmark.py @@ -324,12 +324,20 @@ def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_ unqualified_requirement = "Current schema requires a resolves edge, so use it now to close the forecast." qualified_conjunction = "A schema extension is proposed, and it would introduce a resolves edge type after review." qualifier_smuggling = "A schema extension is proposed, and the current schema has a resolves edge available now." + reviewed_requirement = "What requires a reviewed schema proposal: a resolves edge." + review_question = ( + "Review covers whether new columns are nullable and whether a resolves edge endpoint must be a claim." + ) + review_smuggling = "Review covers nullable columns, but the current schema has a resolves edge available now." assert benchmark.current_schema_overclaims(natural_gap) == [] assert benchmark.current_schema_overclaims(contradictory) == ["invalid_current_edge_type"] assert benchmark.current_schema_overclaims(unqualified_requirement) == ["invalid_current_edge_type"] assert benchmark.current_schema_overclaims(qualified_conjunction) == [] assert benchmark.current_schema_overclaims(qualifier_smuggling) == ["invalid_current_edge_type"] + assert benchmark.current_schema_overclaims(reviewed_requirement) == [] + assert benchmark.current_schema_overclaims(review_question) == [] + assert benchmark.current_schema_overclaims(review_smuggling) == ["invalid_current_edge_type"] def test_oos_forecast_semantics_accept_natural_history_and_missing_schema_language() -> None: @@ -839,6 +847,39 @@ def test_oos_agent_positions_accept_attribution_without_literal_column_name() -> assert benchmark.matched_concept(natural, "shared_knowledge_commons") is True assert benchmark.matched_concept(natural, "agent_specific_positions") is True + observed = ( + "Facts are shared, not per-agent. One claim row, one source, and one evidence link remain authoritative. " + "Duplicating a claim per agent creates divergence." + ) + assert benchmark.matched_concept(observed, "shared_knowledge_commons") is True + + +def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None: + token = "demo-ledger-deadbeef" + set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07") + recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08") + set_reply = ( + f"Label: {token}\nBlocker: beliefs has no claim_id foreign key, so agent positions are not queryable " + "against the shared claim." + ) + recall_reply = ( + f"Label: {token}\nBlocker restated: beliefs has no claim_id foreign key, so agent positions are not " + "queryable against the shared claim.\nProof: postflight readback of the new foreign key and linked row." + ) + + score = benchmark.score_results( + [ + {"prompt_id": set_prompt["id"], "reply": set_reply}, + {"prompt_id": recall_prompt["id"], "reply": recall_reply}, + ], + memory_token=token, + catalog=[set_prompt, recall_prompt], + ) + + assert score["pass"] is True + assert score["memory_continuity"]["same_blocker_recalled"] is True + assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False + def test_oos_forecast_accepts_natural_missing_resolution_language() -> None: reply = ( @@ -855,6 +896,12 @@ def test_oos_forecast_accepts_natural_missing_resolution_language() -> None: "The schema is missing a resolves edge." ) + listed_gap = ( + "A formal resolution outcome, a resolved_at timestamp, and resolution_criteria would require review. " + "None of those fields exist in the current schema. There is no resolves edge type." + ) + assert benchmark.matched_concept(listed_gap, "forecast_schema_gap") is True + def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None: reply = ( diff --git a/tests/test_working_leo_m3taversal_oos_protocol.py b/tests/test_working_leo_m3taversal_oos_protocol.py index 561887e..55c4b5a 100644 --- a/tests/test_working_leo_m3taversal_oos_protocol.py +++ b/tests/test_working_leo_m3taversal_oos_protocol.py @@ -1201,6 +1201,12 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words ) assert protocol_lib._subject_alignment(retrieval_prompt, good + f" {retrieval_sibling}.") is False + memory_prompt = next( + item for item in protocol["trials"][0]["prompts"] if item["family_id"] == "session_memory_set" + ) + memory_reply = f"Subject: {memory_prompt['subject']}\nLabel: temporary\nBlocker: database schema gap." + assert protocol_lib._subject_alignment(memory_prompt, memory_reply) is True + def test_evidence_probe_four_line_reply_binds_exact_subject_and_receipt() -> None: protocol = frozen_protocol()