From 27fbfa49a9939941da1ec066da6db08f63bcaa00 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Wed, 15 Jul 2026 23:44:46 +0200 Subject: [PATCH] Accept receipt-bound natural answers in the Leo OOS scorer (#172) * Make OOS scoring accept receipt-bound natural answers * Test short-ID causal attribution in OOS ablation * Close OOS provenance and schema qualifier gaps --- .../working_leo_m3taversal_oos_benchmark.py | 127 +++++++--- .../working_leo_m3taversal_oos_protocol.py | 129 +++++++--- ...st_working_leo_m3taversal_oos_benchmark.py | 99 ++++++++ ...est_working_leo_m3taversal_oos_protocol.py | 232 +++++++++++++++--- 4 files changed, 492 insertions(+), 95 deletions(-) diff --git a/scripts/working_leo_m3taversal_oos_benchmark.py b/scripts/working_leo_m3taversal_oos_benchmark.py index 55c18bc..cf851b4 100755 --- a/scripts/working_leo_m3taversal_oos_benchmark.py +++ b/scripts/working_leo_m3taversal_oos_benchmark.py @@ -287,7 +287,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"apply|canonical", re.I), ), "receipt": ( - re.compile(r"receipt|readback|postflight|before-and-after|before/after", re.I), + re.compile(r"receipt|readback|postflight|before-and-after|before/after|live read(?:ing)?|live proof", re.I), re.compile(r"row|count|applied_at|public\.", re.I), ), "identity_chain": ( @@ -321,14 +321,18 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"public\.sources|source row", re.I), re.compile( r"attachment.{0,160}(?:does not|doesn't|is not|isn't|cannot|can't|alone|until|unless)|" - r"(?:does not|doesn't|is not|isn't|cannot|can't).{0,100}canonical evidence from (?:that|the) attachment", + r"(?:does not|doesn't|is not|isn't|cannot|can't).{0,100}canonical evidence from (?:that|the) attachment|" + r"(?:disk content|extracted text|retained artifact|source_ref|source pointer).{0,140}" + r"(?:is not|isn't|does not|doesn't|alone|until|unless).{0,80}(?:canonical evidence|finished provenance)|" + r"(?:canonical evidence|canonical link).{0,100}(?:requires|exists only when|is complete only when).{0,100}" + r"(?:public\.sources|source row|claim_evidence)", re.I | re.S, ), ), "evidence_provenance_quality": ( - re.compile(r"(?:missing|no|without).{0,50}(?:url|storage(?:_path)?|locator)", re.I | re.S), - re.compile(r"weak|citation-only|citation only|not traceable|raw artifact", re.I), - re.compile(r"canonical evidence|canonical link", re.I), + re.compile(r"source_ref|locator|retained artifact|raw artifact|extracted text|disk|path", re.I), + re.compile(r"weak|unresolved|not traceable|verified|hash_matches_db|provenance", re.I), + re.compile(r"canonical evidence|canonical link|public\.sources|claim_evidence", re.I), ), "heterogeneous_types": ( re.compile(r"claim", re.I), @@ -336,7 +340,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"framework|reasoning tool|concept map", re.I), re.compile(r"governance", re.I), re.compile(r"correction|supersed", re.I), - re.compile(r"disput|contradict", re.I), + re.compile(r"disput|contradict|disagree|contested interpretation", re.I), ), "behavioral_rule_storage": ( re.compile(r"(?:public\.)?behavioral_rules", re.I), @@ -345,11 +349,10 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { "reviewed_policy_apply": ( re.compile(r"approve_claim", re.I), re.compile(r"behavioral_rules", re.I), - re.compile(r"governance_gates", re.I), re.compile( - r"does not (?:accept|insert|support|cover)|supports neither|neither.{0,80}nor|" - r"(?:behavioral_rules|governance_gates).{0,120}cannot be applied by approve_claim|" - r"approve_claim.{0,120}cannot apply", + r"does not (?:accept|insert|support|cover|write)|supports neither|neither.{0,80}nor|" + r"(?:behavioral_rules|governance_gates).{0,120}(?:cannot be applied by|sits? outside) approve_claim|" + r"approve_claim.{0,120}(?:cannot apply|cannot write|does not write)", re.I | re.S, ), re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply (?:capability|path)", re.I | re.S), @@ -392,7 +395,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { "shared_knowledge_commons": ( re.compile( r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|" - r"keep the factual claim once|store it once", + r"keep the factual claim once|store it once|one public\.claims row|" + r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)", re.I | re.S, ), re.compile(r"source|evidence", re.I), @@ -409,16 +413,29 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { ), "forecast_history": ( re.compile(r"original (?:probability|confidence)|60%|history", re.I), - re.compile(r"preserve|retain|do not overwrite|don'?t overwrite", re.I), - re.compile(r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria", re.I | re.S), + re.compile( + r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|original.{0,60}survives", + re.I | re.S, + ), + re.compile( + r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria|" + r"criteria.{0,40}(?:never existed|were never defined|did not exist)|without.{0,30}criteria", + re.I | re.S, + ), ), "forecast_schema_gap": ( re.compile(r"current (?:v1|schema)|public\.claims", re.I), re.compile( - r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution", + r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|" + r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}" + r"(?:does not exist|is absent|is not present|isn't present)|" + r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present", + re.I | re.S, + ), + re.compile( + r"resolves.{0,220}(?:not|isn'?t|does not|doesn't|absent)|no.{0,30}resolves", re.I | re.S, ), - re.compile(r"resolves.{0,30}(?:not|isn'?t|does not)|no.{0,30}resolves", re.I | re.S), ), "handler_not_telegram": ( re.compile(r"no|not", re.I), @@ -484,12 +501,12 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { "live_claim_evidence_ids": ( re.compile( r"\bclaim(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?" - r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b", + r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b", re.I, ), re.compile( r"\b(?:source|evidence)(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?" - r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b", + r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b", re.I, ), ), @@ -507,8 +524,10 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile( r"\b(?:distinct|separate)\s+from\b|" r"\b(?:claim|evidence|source)\b.{0,80}\b(?:distinct from|separate from|differs? from|" - r"not the same as)\b", - re.I, + r"not the same as)\b|" + r"\bclaim body\s*:\s*\S.{0,400}\bevidence(?: excerpt)?\s*:\s*\S|" + r"\bclaim body\b.{0,240}\b(?:evidence|source)\b.{0,100}\b(?:states?|says?|shows?|documents?)\b", + re.I | re.S, ), ), "evidence_specific_challenge": ( @@ -516,6 +535,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I), re.compile( r"\b(?:does not|doesn't|cannot|can't)\s+(?:itself\s+)?(?:prove|establish|support|show)|" + r"\b(?:does not|doesn't|cannot|can't)\s+rule out\b|" r"\bunsupported leap\b", re.I, ), @@ -526,7 +546,11 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { r"should be narrowed to|defensible formulation|revise(?:d)?\b.{0,40}\bto\b)", re.I, ), - re.compile(r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?)\b", re.I), + re.compile( + r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|" + r"unproven|open constraint)\b", + re.I, + ), ), } @@ -545,10 +569,22 @@ COUNT_INVARIANT_REJECTION_RE = re.compile( SCHEMA_GAP_QUALIFIER_RE = re.compile( r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|" r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|" - r"does not support|doesn't support|supports neither|not supported|must not|do not invent)\b|" + r"does not support|doesn't support|supports neither|not supported|must not|do not invent|" + r"schema extension|extension proposal)\b|" + r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?" + r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|" + r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?" + r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|" + r"\b(?:proposal|extension)\b.{0,100}\b(?:column|field|table|edge type)\b|" r"\bno\s+(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b", re.I, ) +SCHEMA_CLAUSE_BOUNDARY_RE = re.compile( + r"\s*(?:;|,\s+and\s+(?=(?:[a-z0-9_.`'-]+\s+){1,5}" + r"(?:is|are|has|have|requires?|needs?|would|will|must|can|does?|supports?)\b)|" + r"\bbut\b|\bhowever\b)\s*", + re.I, +) CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = { "claims_unshipped_fields": re.compile( r"(?:public\.)?claims?\b.{0,60}(?:stores?|has|have|with|column|field).{0,40}" @@ -674,6 +710,22 @@ APPLYABILITY_GAP_RE = re.compile( r"strict apply payload.{0,50}(?:built|build|reviewed|review)", re.I | re.S, ) +CLAIM_EVIDENCE_CONFLATION_RE = re.compile( + r"\bevidence(?: excerpt)?\s*:\s*(?:the\s+)?same as (?:the\s+)?claim\b|" + r"\b(?:claim body|claim text)\s+(?:is|equals?)\s+(?:the\s+)?evidence\b", + re.I | re.S, +) +REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P[^\n]+)", re.I) +REVISION_BOUNDARY_RE = re.compile( + r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|" + r"open constraint)\b", + re.I, +) +REVISION_REPETITION_RE = re.compile( + r"\b(?:repeat(?:ed)?|restate(?:d)?).{0,50}\b(?:original|same)\s+claim\b|" + r"\b(?:original|same)\s+claim\b.{0,50}\bunchanged\b", + re.I | re.S, +) DEFAULT_MAX_RESPONSE_WORDS = 220 MAX_RESPONSE_WORDS = {"OOS-09": 100, "OOS-10": 180} @@ -726,6 +778,15 @@ def blocker_terms(value: str | None, *, memory_token: str) -> set[str]: def matched_concept(reply: str, concept: str) -> bool: + if concept == "claim_body_evidence_distinction" and CLAIM_EVIDENCE_CONFLATION_RE.search(reply): + return False + if concept == "narrower_claim_revision": + labelled = REVISION_LABEL_RE.search(reply) + if labelled and ( + not REVISION_BOUNDARY_RE.search(labelled.group("value")) + or REVISION_REPETITION_RE.search(labelled.group("value")) + ): + return False return all(pattern.search(reply) for pattern in CONCEPT_PATTERNS[concept]) @@ -737,12 +798,17 @@ def current_schema_overclaims(reply: str) -> list[str]: """Return proposed-v3-as-current assertions that are not explicitly qualified.""" findings: list[str] = [] - for segment in re.split(r"(?<=[.!?])\s+|\n+", reply): - if SCHEMA_GAP_QUALIFIER_RE.search(segment): - continue - for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items(): - if pattern.search(segment): - findings.append(label) + sentence_segments = re.split(r"(?<=[.!?])\s+|\n+", reply) + for sentence in sentence_segments: + # Negation and future-schema qualifiers apply to their clause, not to a + # contradictory assertion in a later conjunction in the same sentence. + clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence) + for clause in clauses: + if SCHEMA_GAP_QUALIFIER_RE.search(clause): + continue + for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items(): + if pattern.search(clause): + findings.append(label) return sorted(set(findings)) @@ -796,7 +862,12 @@ def broad_semantic_issues(reply: str) -> list[str]: for segment in re.split(r"(?<=[.!?])\s+|\n+", reply): if not REASONING_TOOL_CLAIM_EDGE_RE.search(segment): continue - if re.search(r"(?:never|do not|don't|must not|cannot).{0,100}reasoning_tools?", segment, re.I | re.S): + if re.search( + r"(?:never|do not|don't|must not|cannot).{0,100}reasoning_tools?|" + r"(?:edge endpoints?|from_claim|to_claim).{0,100}(?:claim IDs?|claims? only|never reasoning_tools?)", + segment, + re.I | re.S, + ): continue findings.add("claim_edge_pointed_at_reasoning_tool") break diff --git a/scripts/working_leo_m3taversal_oos_protocol.py b/scripts/working_leo_m3taversal_oos_protocol.py index 1821e95..ebbefae 100644 --- a/scripts/working_leo_m3taversal_oos_protocol.py +++ b/scripts/working_leo_m3taversal_oos_protocol.py @@ -28,7 +28,7 @@ PROTOCOL_SCHEMA = "livingip.leoM3taversalOosProtocol.v2" TRIAL_SCORE_SCHEMA = "livingip.leoM3taversalOosTrialScore.v2" AGGREGATE_SCHEMA = "livingip.leoM3taversalOosAggregate.v2" GENERATOR_VERSION = "blinded-family-generator-v3" -SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v3" +SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v5" BASELINE_VERSION = "live-current-build-db-tool-ablation-v2" DEFAULT_TRIAL_COUNT = 3 MEMORY_SCORER_IDS = frozenset({"OOS-07", "OOS-08"}) @@ -56,8 +56,9 @@ EXPECTED_TELEGRAM_DENY_METHODS = frozenset( "send_voice", } ) -GROUNDED_EXECUTION_ALLOWED_MISSING = frozenset({"harness_worktree_clean"}) -ABLATION_EXECUTION_ALLOWED_MISSING = GROUNDED_EXECUTION_ALLOWED_MISSING | frozenset( +GROUNDED_EXECUTION_ALLOWED_MISSING = frozenset() +CONTROL_GOAL_EXECUTION_ALLOWED_MISSING = frozenset({"harness_worktree_clean"}) +ABLATION_EXECUTION_ALLOWED_MISSING = frozenset( { "model_raw_response_binding", "database_context_query_binding", @@ -92,12 +93,12 @@ UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f] RECEIPT_TOKEN_RE = re.compile(r"\breceipt\s*:\s*`?([0-9a-f]{12})(?![0-9a-f])", re.I) CLAIM_ID_CITATION_RE = re.compile( r"\bclaim(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?" - r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b", + r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|[0-9a-f]{8,12})\b", re.I, ) SOURCE_ID_CITATION_RE = re.compile( r"\b(?:source|evidence)(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?" - r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b", + r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|[0-9a-f]{8,12})\b", re.I, ) @@ -601,6 +602,12 @@ def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) - issues: list[str] = [] if protocol.get("schema") != PROTOCOL_SCHEMA: issues.append("wrong_protocol_schema") + if protocol.get("generator_version") != GENERATOR_VERSION: + issues.append("wrong_generator_version") + if protocol.get("scorer_version") != SCORER_VERSION: + issues.append("wrong_scorer_version") + if (protocol.get("baseline") or {}).get("version") != BASELINE_VERSION: + issues.append("wrong_baseline_version") if not _valid_git_revision(protocol.get("harness_git_head")): issues.append("invalid_harness_git_head") supplied_hash = protocol.get("protocol_hash_sha256") @@ -692,7 +699,7 @@ def _subject_alignment(prompt: dict[str, Any], reply: str) -> bool: } return ( bool(normalized_subject) - and padded_reply.count(padded_subject) == 1 + and padded_reply.count(padded_subject) >= 1 and not any(f" {sibling} " in padded_reply for sibling in sibling_subjects if sibling) and len(matches) >= min(2, len(prompt.get("subject_anchors") or [])) ) @@ -751,6 +758,25 @@ def _reply_receipt_tokens(reply: str) -> list[str]: return sorted({match.group(1).lower() for match in RECEIPT_TOKEN_RE.finditer(reply)}) +def _resolve_supported_citations( + reply: str, + pattern: re.Pattern[str], + supported_identifiers: set[str], +) -> tuple[set[str], set[str], set[str]]: + """Resolve full UUIDs or unambiguous 8-12 hex prefixes to receipted IDs.""" + + tokens = {match.group(1).lower() for match in pattern.finditer(reply)} + resolved: set[str] = set() + unresolved: set[str] = set() + for token in tokens: + matches = {identifier for identifier in supported_identifiers if identifier.startswith(token)} + if len(matches) == 1: + resolved.update(matches) + else: + unresolved.add(token) + return tokens, resolved, unresolved + + def _evidence_answer_score( prompt: dict[str, Any], result: dict[str, Any], @@ -1035,8 +1061,12 @@ def _receipt_score( } reply = str(result.get("reply") or "") reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(reply)} - reply_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(reply)} - reply_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(reply)} + reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = ( + _resolve_supported_citations(reply, CLAIM_ID_CITATION_RE, receipt_claim_ids) + ) + reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = ( + _resolve_supported_citations(reply, SOURCE_ID_CITATION_RE, receipt_source_ids) + ) unsupported_identifiers = sorted(reply_identifiers - supported_identifiers) receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records] reply_sha256 = hashlib.sha256(reply.encode()).hexdigest() @@ -1080,6 +1110,8 @@ def _receipt_score( ), "conversation_history_prefix_preserved": result.get("conversation_history_prefix_preserved") is True, "no_unsupported_exact_identifiers": not unsupported_identifiers, + "no_unresolved_or_ambiguous_identifier_citations": not unresolved_claim_citations + and not unresolved_source_citations, } return { "checks": checks, @@ -1088,8 +1120,12 @@ def _receipt_score( "expected_prompt_sha256": prompt_sha256, "database_tool_trace": result.get("database_tool_trace") or {}, "reply_identifiers": sorted(reply_identifiers), + "reply_claim_citation_tokens": sorted(reply_claim_citation_tokens), "reply_claim_citations": sorted(reply_claim_citations), + "unresolved_claim_citations": sorted(unresolved_claim_citations), + "reply_source_or_evidence_citation_tokens": sorted(reply_source_citation_tokens), "reply_source_or_evidence_citations": sorted(reply_source_citations), + "unresolved_source_or_evidence_citations": sorted(unresolved_source_citations), "supported_claim_ids": sorted(receipt_claim_ids), "supported_source_ids": sorted(receipt_source_ids), "supported_contract_row_ids": sorted(receipt_contract_row_ids), @@ -1108,37 +1144,55 @@ def _benchmark_execution_chain( """Validate the generic turn manifests under this benchmark's declared ablation. The generic manifest intentionally marks a dirty harness and missing DB-context - hooks incomplete. This benchmark permits exactly one control-owned dirty file - (``goal.md``) and, in the ablated arm only, the bindings made impossible by - removing the DB-context plugin. Every other runtime/model/session/safety - binding remains mandatory and is checked independently here. + hooks incomplete. This benchmark accepts either a clean harness or exactly one + control-owned untracked file (``goal.md``). Only the latter may omit the generic + clean-worktree binding. In the ablated arm, the bindings made impossible by + removing the DB-context plugin may also be absent. Every other runtime, model, + session, and safety binding remains mandatory and is checked independently. """ - mode = report.get("grounding_mode") - if mode == "grounded": - allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING,) - elif mode == "db_tool_ablated": - allowed_missing_sets = ( - ABLATION_EXECUTION_ALLOWED_MISSING, - ABLATION_EXECUTION_ALLOWED_MISSING | ABLATION_EXECUTION_OPTIONAL_MISSING, - ) - else: - allowed_missing_sets = (frozenset(),) results = [item for item in report.get("results") or [] if isinstance(item, dict)] summary = report.get("execution_manifest_summary") or {} executed_behavior = report.get("executed_behavior_manifest") or {} local_state = report.get("oos_harness_git_state") or {} summary_source = summary.get("harness_source") or {} + clean_worktree_exact = ( + local_state.get("worktree_clean") is True + and local_state.get("status_lines") == [] + and local_state.get("only_control_goal_untracked") is False + and local_state.get("status_sha256") == hashlib.sha256(b"").hexdigest() + ) + control_goal_only_exact = ( + local_state.get("worktree_clean") is False + and local_state.get("status_lines") == ["?? goal.md"] + and local_state.get("only_control_goal_untracked") is True + and local_state.get("status_sha256") == hashlib.sha256(b"?? goal.md\n").hexdigest() + ) + supported_worktree_mode = clean_worktree_exact or control_goal_only_exact + worktree_allowed_missing = ( + CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset() + ) + mode = report.get("grounding_mode") + if mode == "grounded": + allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing,) + elif mode == "db_tool_ablated": + allowed_missing = ABLATION_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing + allowed_missing_sets = ( + allowed_missing, + allowed_missing | ABLATION_EXECUTION_OPTIONAL_MISSING, + ) + else: + allowed_missing_sets = (frozenset(),) local_state_checks = { "git_head_valid": _valid_git_revision(local_state.get("git_head")), "git_head_matches_frozen_harness": local_state.get("git_head") == expected_harness_git_head, "status_sha256_valid": _valid_sha256(local_state.get("status_sha256")), - "recorded_dirty": local_state.get("worktree_clean") is False, - "only_control_goal_untracked": local_state.get("only_control_goal_untracked") is True - and local_state.get("status_lines") == ["?? goal.md"], + "clean_worktree_exact": clean_worktree_exact, + "control_goal_only_exact": control_goal_only_exact, + "supported_worktree_mode": supported_worktree_mode, "generic_summary_source_bound": summary_source.get("git_head") == local_state.get("git_head") and summary_source.get("status_sha256") == local_state.get("status_sha256") - and summary_source.get("worktree_clean") is False, + and summary_source.get("worktree_clean") == local_state.get("worktree_clean"), } turn_checks: dict[str, dict[str, bool]] = {} previous_execution_sha256: str | None = None @@ -1246,7 +1300,16 @@ def _benchmark_execution_chain( "summary_turn_count_exact": summary.get("turn_count") == len(results), "one_manifest_per_result": bool(results) and all(isinstance(item.get("execution_manifest"), dict) for item in results), - "local_harness_state_bound": all(local_state_checks.values()), + "local_harness_state_bound": all( + local_state_checks[key] + for key in ( + "git_head_valid", + "git_head_matches_frozen_harness", + "status_sha256_valid", + "supported_worktree_mode", + "generic_summary_source_bound", + ) + ), "all_turns_valid_under_declared_mode": bool(turn_checks) and all(all(item.values()) for item in turn_checks.values()), } @@ -1731,8 +1794,12 @@ def score_live_trial( supported_claim_ids = set(grounded_receipt_score.get("supported_claim_ids") or []) supported_source_ids = set(grounded_receipt_score.get("supported_source_ids") or []) supported_identifiers = set(grounded_receipt_score.get("supported_identifiers") or []) - ablation_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(baseline_reply)} - ablation_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(baseline_reply)} + ablation_claim_tokens, ablation_claim_citations, ablation_unresolved_claim_citations = ( + _resolve_supported_citations(baseline_reply, CLAIM_ID_CITATION_RE, supported_claim_ids) + ) + ablation_source_tokens, ablation_source_citations, ablation_unresolved_source_citations = ( + _resolve_supported_citations(baseline_reply, SOURCE_ID_CITATION_RE, supported_source_ids) + ) ablation_reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(baseline_reply)} grounded_answer_pass = bool( semantic_by_prompt.get(prompt_id, {}).get("pass") @@ -1744,7 +1811,9 @@ def score_live_trial( "subject_alignment": baseline_subject_alignment.get(prompt_id) is True, "cites_grounded_claim_id": bool(ablation_claim_citations & supported_claim_ids), "cites_grounded_source_id": bool(ablation_source_citations & supported_source_ids), - "no_identifiers_outside_grounded_receipt": bool(ablation_reply_identifiers) + "no_identifiers_outside_grounded_receipt": bool(ablation_claim_tokens or ablation_source_tokens) + and not ablation_unresolved_claim_citations + and not ablation_unresolved_source_citations and ablation_reply_identifiers <= supported_identifiers, } ablation_answer_pass = all(ablation_binding_checks.values()) diff --git a/tests/test_working_leo_m3taversal_oos_benchmark.py b/tests/test_working_leo_m3taversal_oos_benchmark.py index 37e7e8a..4b244fe 100644 --- a/tests/test_working_leo_m3taversal_oos_benchmark.py +++ b/tests/test_working_leo_m3taversal_oos_benchmark.py @@ -202,6 +202,36 @@ def test_oos_autonomous_retrieval_semantics_reject_generic_challenge() -> None: assert benchmark.score_reply(prompt, paraphrase, memory_token=token)["pass"] is True +def test_oos_autonomous_retrieval_accepts_receipt_bindable_prefixes_and_natural_sections() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[-1] + reply = ( + "Fresh canonical public.claims readback. Claim 91dc463b, source 4207eb03. " + "The claim body asserts that one retained path is auditable. The verified evidence says that this path " + "has a source receipt. Challenge: the study does not rule out missing receipts on other paths. " + "Revision: this one path is auditable, but coverage beyond it remains unestablished. Read-only; no mutation." + ) + + score = benchmark.score_reply(prompt, reply, memory_token=token) + + assert score["pass"] is True + assert all(score["concepts"].values()) + + weak_challenge = reply.replace( + "the study does not rule out missing receipts on other paths", + "the study is interesting", + ) + assert benchmark.score_reply(prompt, weak_challenge, memory_token=token)["pass"] is False + + repeated_revision = reply.replace( + "this one path is auditable, but coverage beyond it remains unestablished", + "the original claim is repeated unchanged", + ) + repeated_score = benchmark.score_reply(prompt, repeated_revision, memory_token=token) + assert repeated_score["pass"] is False + assert repeated_score["concepts"]["narrower_claim_revision"] is False + + def test_oos_live_check_accepts_live_readback_wording() -> None: assert benchmark.matched_concept("Here is the live readback from Postgres.", "live_check") is True @@ -285,6 +315,45 @@ def test_oos_schema_guard_allows_explicit_future_schema_gap() -> None: assert benchmark.current_schema_overclaims(reply) == [] +def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_clause() -> None: + natural_gap = ( + "No resolves edge type exists in the current schema. Recording resolution requires either a resolves edge " + "type or a dedicated field. Draft a schema extension proposal that would introduce the edge type." + ) + contradictory = "There is no resolves edge today, but use a resolves edge now to close the forecast." + unqualified_requirement = "Current schema requires a resolves edge, so use it now to close the forecast." + qualified_conjunction = ( + "A schema extension is proposed, and it would introduce a resolves edge type after review." + ) + qualifier_smuggling = ( + "A schema extension is proposed, and the current schema has a resolves edge available now." + ) + + assert benchmark.current_schema_overclaims(natural_gap) == [] + assert benchmark.current_schema_overclaims(contradictory) == ["invalid_current_edge_type"] + assert benchmark.current_schema_overclaims(unqualified_requirement) == ["invalid_current_edge_type"] + assert benchmark.current_schema_overclaims(qualified_conjunction) == [] + assert benchmark.current_schema_overclaims(qualifier_smuggling) == ["invalid_current_edge_type"] + + +def test_oos_forecast_semantics_accept_natural_history_and_missing_schema_language() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[11] + reply = ( + "Keep the original 60% probability claim unmodified; it survives as history even though the criteria never " + "existed. Current schema has a gap. A resolves edge — the type that would bind the outcome and timestamp — " + "does not exist. A dedicated resolution field is the alternative; neither mechanism is present. Stage a " + "schema extension proposal, review it, and apply only after approval." + ) + + score = benchmark.score_reply(prompt, reply, memory_token=token) + + assert score["pass"] is True + assert score["current_schema_overclaims"] == [] + assert score["concepts"]["forecast_history"] is True + assert score["concepts"]["forecast_schema_gap"] is True + + def test_oos_schema_guard_allows_compact_no_field_wording() -> None: reply = "public.reasoning_tools stores name and description; no scope field exists in the current table." assert benchmark.current_schema_overclaims(reply) == [] @@ -377,6 +446,36 @@ def test_oos_database_composition_requires_existing_behavioral_rule_table() -> N assert bad["behavioral_rule_schema_issues"] == ["behavioral_rules_false_absence"] +def test_oos_natural_equivalents_cover_source_composition_and_agent_position_semantics() -> None: + token = "demo-ledger-deadbeef" + prompts = {prompt["id"]: prompt for prompt in benchmark.prompt_catalog(token)} + replies = { + "OOS-05": ( + "The attachment source_ref and extracted text on disk are retained artifacts, not canonical evidence. " + "Canonical evidence requires a public.sources row joined through claim_evidence. An unresolved locator " + "is weak provenance even when the artifact is retained. Stage the proposal, review it, then apply; a " + "before/after row receipt and verified artifact hash prove the canonical link." + ), + "OOS-06": ( + "Stage the packet with claims and source evidence, a reasoning framework, a contested interpretation, " + "a governance rule, and a correction. Store the rule in behavioral_rules. approve_claim cannot write " + "behavioral_rules, so that row needs a separate reviewed apply capability. Review the typed proposal, " + "apply supported canonical rows, and retain a postflight row receipt." + ), + "OOS-11": ( + "Do not duplicate the fact per agent. Keep the fact shared in one public.claims row with shared sources " + "and evidence. Put each agent-specific position in public.beliefs with agent_id, stance, and confidence. " + "Because beliefs has no claim-ID foreign key, that exact link is a schema gap; contradictory positions " + "remain queryable by agent and subject." + ), + } + + for prompt_id, reply in replies.items(): + score = benchmark.score_reply(prompts[prompt_id], reply, memory_token=token) + assert score["pass"] is True, (prompt_id, score) + assert all(score["concepts"].values()), (prompt_id, score) + + def test_oos_runtime_case_rejects_db_only_causality_and_total_memory_erasure() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[9] diff --git a/tests/test_working_leo_m3taversal_oos_protocol.py b/tests/test_working_leo_m3taversal_oos_protocol.py index 92e0fb4..3107007 100644 --- a/tests/test_working_leo_m3taversal_oos_protocol.py +++ b/tests/test_working_leo_m3taversal_oos_protocol.py @@ -279,16 +279,31 @@ def attach_fake_execution_manifests( *, grounded: bool, harness_git_head: str, + harness_worktree_mode: str = "control_goal_only", ) -> None: + if harness_worktree_mode == "clean": + worktree_clean = True + status_text = "" + status_lines: list[str] = [] + only_control_goal_untracked = False + elif harness_worktree_mode == "control_goal_only": + worktree_clean = False + status_text = "?? goal.md\n" + status_lines = ["?? goal.md"] + only_control_goal_untracked = True + else: + raise ValueError(f"unsupported harness worktree mode: {harness_worktree_mode}") harness_source = { "git_head": harness_git_head, - "worktree_clean": False, - "status_sha256": hashlib.sha256(b"?? goal.md\n").hexdigest(), + "worktree_clean": worktree_clean, + "status_sha256": hashlib.sha256(status_text.encode()).hexdigest(), } previous_execution_sha256 = None - allowed_missing = ( + allowed_missing = set( protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING if grounded else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING ) + if harness_worktree_mode == "control_goal_only": + allowed_missing.update(protocol_lib.CONTROL_GOAL_EXECUTION_ALLOWED_MISSING) fingerprint = report["db_fingerprint_before"] counts_sha256 = protocol_lib.canonical_sha256({}) for result in report["results"]: @@ -362,7 +377,7 @@ def attach_fake_execution_manifests( }, }, "attribution": { - "status": "incomplete", + "status": "incomplete" if allowed_missing else "complete", "missing_required_bindings": sorted(allowed_missing), }, } @@ -375,19 +390,25 @@ def attach_fake_execution_manifests( previous_execution_sha256 = manifest["execution_sha256"] report["oos_harness_git_state"] = { **harness_source, - "status_lines": ["?? goal.md"], - "only_control_goal_untracked": True, + "status_lines": status_lines, + "only_control_goal_untracked": only_control_goal_untracked, } report["execution_manifest_summary"] = { "schema": protocol_lib.execution_manifest_lib.SCHEMA, "turn_count": len(report["results"]), - "attribution_complete_count": 0, - "all_turns_attribution_complete": False, + "attribution_complete_count": 0 if allowed_missing else len(report["results"]), + "all_turns_attribution_complete": not allowed_missing, "harness_source": harness_source, } -def fake_report(protocol: dict, trial: dict, *, grounded: bool) -> dict: +def fake_report( + protocol: dict, + trial: dict, + *, + grounded: bool, + harness_worktree_mode: str = "control_goal_only", +) -> dict: mode = "grounded" if grounded else "db_tool_ablated" fingerprint = { "status": "ok", @@ -490,6 +511,7 @@ def fake_report(protocol: dict, trial: dict, *, grounded: bool) -> dict: report, grounded=grounded, harness_git_head=protocol["harness_git_head"], + harness_worktree_mode=harness_worktree_mode, ) return report @@ -660,6 +682,13 @@ def test_protocol_validation_rejects_prompt_and_source_hash_tampering() -> None: in protocol_lib.validate_protocol(protocol, verify_source_hashes=True)["issues"] ) + protocol = frozen_protocol() + protocol["scorer_version"] = "stale-scorer" + protocol["protocol_hash_sha256"] = protocol_lib.canonical_sha256( + {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"} + ) + assert "wrong_scorer_version" in protocol_lib.validate_protocol(protocol, verify_source_hashes=True)["issues"] + def test_live_trial_requires_semantics_subject_receipts_and_real_ablation() -> None: protocol = frozen_protocol() @@ -695,12 +724,23 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non trial = protocol["trials"][0] for grounded in (True, False): report = fake_report(protocol, trial, grounded=grounded) - assert ( - protocol_lib._benchmark_execution_chain( - report, - expected_harness_git_head=protocol["harness_git_head"], - )["pass"] - is True + validation = protocol_lib._benchmark_execution_chain( + report, + expected_harness_git_head=protocol["harness_git_head"], + ) + expected_missing = ( + protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING + if grounded + else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING + ) | protocol_lib.CONTROL_GOAL_EXECUTION_ALLOWED_MISSING + + assert validation["pass"] is True + assert validation["local_state_checks"]["control_goal_only_exact"] is True + assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing) + assert all( + result["execution_manifest"]["attribution"]["missing_required_bindings"] + == sorted(expected_missing) + for result in report["results"] ) manifest = report["results"][0]["execution_manifest"] @@ -736,6 +776,66 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non assert validation["local_state_checks"]["git_head_matches_frozen_harness"] is False +def test_execution_chain_accepts_clean_harness_without_cleanliness_omission() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + + for grounded in (True, False): + report = fake_report( + protocol, + trial, + grounded=grounded, + harness_worktree_mode="clean", + ) + validation = protocol_lib._benchmark_execution_chain( + report, + expected_harness_git_head=protocol["harness_git_head"], + ) + + assert validation["pass"] is True + assert validation["local_state_checks"]["clean_worktree_exact"] is True + expected_missing = ( + protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING + if grounded + else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING + ) + assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing) + assert all( + result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing) + for result in report["results"] + ) + assert all( + checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values() + ) + + manifest = report["results"][-1]["execution_manifest"] + manifest["attribution"]["missing_required_bindings"].append("harness_worktree_clean") + manifest["attribution"]["status"] = "incomplete" + stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}} + manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable) + validation = protocol_lib._benchmark_execution_chain( + report, + expected_harness_git_head=protocol["harness_git_head"], + ) + assert validation["pass"] is False + assert validation["turn_checks"][trial["prompts"][-1]["id"]]["declared_missing_exact"] is False + + +def test_execution_chain_rejects_inexact_control_goal_state() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + report = fake_report(protocol, trial, grounded=True) + report["oos_harness_git_state"]["status_lines"] = ["?? goal.md", "?? scratch.txt"] + + validation = protocol_lib._benchmark_execution_chain( + report, + expected_harness_git_head=protocol["harness_git_head"], + ) + + assert validation["pass"] is False + assert validation["local_state_checks"]["control_goal_only_exact"] is False + + def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal() -> None: protocol = frozen_protocol() trial = protocol["trials"][0] @@ -952,6 +1052,60 @@ def test_autonomous_retrieval_requires_nonempty_rows_and_receipted_reply_ids() - assert unrelated_score["unsupported_identifiers"] == [unrelated_uuid] +def test_autonomous_retrieval_resolves_only_unique_receipt_bound_identifier_prefixes() -> None: + protocol = frozen_protocol() + trial = protocol["trials"][0] + prompt = next(item for item in trial["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning") + result = result_for(prompt, trial["memory_token"], 1, grounded=True) + result["reply"] = ( + f"{prompt['subject']}. Claim {CLAIM_UUID[:8]}, source {SOURCE_UUID[:8]}. " + "The claim body states one bounded observation. The evidence documents that observation separately. " + "Challenge: it does not rule out alternatives. Revision: the result is limited to this source." + ) + result["database_context_trace"][-1]["delivered_response_sha256"] = hashlib.sha256( + result["reply"].encode() + ).hexdigest() + + unique = protocol_lib._receipt_score( + result, + require_database_contract=False, + require_database_receipt=True, + require_grounded_rows=True, + ) + assert unique["pass"] is True + assert unique["reply_claim_citation_tokens"] == [CLAIM_UUID[:8]] + assert unique["reply_claim_citations"] == [CLAIM_UUID] + assert unique["reply_source_or_evidence_citation_tokens"] == [SOURCE_UUID[:8]] + assert unique["reply_source_or_evidence_citations"] == [SOURCE_UUID] + + unbound = copy.deepcopy(result) + unbound["reply"] = unbound["reply"].replace(CLAIM_UUID[:8], "deadbeef") + unbound["database_context_trace"][-1]["delivered_response_sha256"] = hashlib.sha256( + unbound["reply"].encode() + ).hexdigest() + unbound_score = protocol_lib._receipt_score( + unbound, + require_database_contract=False, + require_database_receipt=True, + require_grounded_rows=True, + ) + assert unbound_score["pass"] is False + assert unbound_score["unresolved_claim_citations"] == ["deadbeef"] + + ambiguous = copy.deepcopy(result) + ambiguous_receipt = ambiguous["database_context_trace"][0]["retrieval_receipt"] + ambiguous_receipt["claim_ids"].append("11111111-aaaa-4aaa-8aaa-aaaaaaaaaaaa") + rehash_retrieval_receipt(ambiguous_receipt) + ambiguous_score = protocol_lib._receipt_score( + ambiguous, + require_database_contract=False, + require_database_receipt=True, + require_grounded_rows=True, + ) + assert ambiguous_score["pass"] is False + assert ambiguous_score["unresolved_claim_citations"] == [CLAIM_UUID[:8]] + + def test_autonomous_retrieval_ablation_rejects_generic_prose_and_invented_ids() -> None: protocol = frozen_protocol() trial = protocol["trials"][0] @@ -1029,7 +1183,7 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words f"{source_prompt['subject']} uses source evidence and claim_evidence; " f"repeat {source_prompt['subject']} is not acceptable." ) - assert protocol_lib._subject_alignment(source_prompt, duplicate_subject) is False + assert protocol_lib._subject_alignment(source_prompt, duplicate_subject) is True sibling_subject = next(item for item in source_prompt["family_subjects"] if item != source_prompt["subject"]) shotgun = f"{source_prompt['subject']} and {sibling_subject} both use source evidence and claim_evidence." assert protocol_lib._subject_alignment(source_prompt, shotgun) is False @@ -1042,7 +1196,7 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words "Challenge: evidence does not prove the leap. Revision: narrower claim." ) assert protocol_lib._subject_alignment(retrieval_prompt, good) is True - assert protocol_lib._subject_alignment(retrieval_prompt, good + f" {retrieval_prompt['subject']}.") is False + assert protocol_lib._subject_alignment(retrieval_prompt, good + f" {retrieval_prompt['subject']}.") is True retrieval_sibling = next( item for item in retrieval_prompt["family_subjects"] if item != retrieval_prompt["subject"] ) @@ -1123,29 +1277,33 @@ def test_identical_grounded_and_ablation_replies_cannot_create_evidence_delta() def test_nonidentical_ablated_answer_with_grounded_ids_does_not_manufacture_causal_lift() -> None: protocol = frozen_protocol() trial = protocol["trials"][0] - grounded = fake_report(protocol, trial, grounded=True) - ablated = fake_report(protocol, trial, grounded=False) prompt = next(item for item in trial["prompts"] if item.get("requires_grounded_retrieval_answer")) - result = next(item for item in ablated["results"] if item["prompt_id"] == prompt["id"]) - result["reply"] = ( - f"{prompt['subject']}. Fresh canonical public.claims readback: claim `{CLAIM_UUID}` states that one " - f"retained path is auditable. Source `{SOURCE_UUID}` documents that path and is distinct from the claim. " - "However, the evidence cannot prove every path is auditable. A narrower claim is limited to this one path. " - "The lookup is read-only and made no mutation." - ) + for claim_citation, source_citation in ( + (CLAIM_UUID, SOURCE_UUID), + (CLAIM_UUID[:8], SOURCE_UUID[:8]), + ): + grounded = fake_report(protocol, trial, grounded=True) + ablated = fake_report(protocol, trial, grounded=False) + result = next(item for item in ablated["results"] if item["prompt_id"] == prompt["id"]) + result["reply"] = ( + f"{prompt['subject']}. Fresh canonical public.claims readback: claim `{claim_citation}` states that one " + f"retained path is auditable. Source `{source_citation}` documents that path and is distinct from the " + "claim. However, the evidence cannot prove every path is auditable. A narrower claim is limited to this " + "one path. The lookup is read-only and made no mutation." + ) - score = protocol_lib.score_live_trial( - protocol, - trial["trial_id"], - grounded, - baseline_report=ablated, - ) - row = score["autonomous_retrieval_comparison"]["rows"][0] + score = protocol_lib.score_live_trial( + protocol, + trial["trial_id"], + grounded, + baseline_report=ablated, + ) + row = score["autonomous_retrieval_comparison"]["rows"][0] - assert row["ablation_answer_pass"] is True - assert row["replies_identical"] is False - assert row["causally_attributed_grounded_pass"] is False - assert score["autonomous_retrieval_comparison"]["grounded_minus_ablation_answer_delta"] == 0.0 + assert row["ablation_answer_pass"] is True + assert row["replies_identical"] is False + assert row["causally_attributed_grounded_pass"] is False + assert score["autonomous_retrieval_comparison"]["grounded_minus_ablation_answer_delta"] == 0.0 def test_receipt_discriminator_rejects_wrong_command_extra_call_and_wrong_token() -> None: