Harden Leo benchmark semantic invariants

This commit is contained in:
twentyOne2x 2026-07-16 11:38:51 +02:00
parent 3b0e1bc296
commit 2712e7fbb5
5 changed files with 618 additions and 17 deletions

View file

@ -444,7 +444,11 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
r"(?:shared source material|observation).{0,100}(?:one|single) claim row|" r"(?:shared source material|observation).{0,100}(?:one|single) claim row|"
r"shared (?:source material|evidence).{0,100}one canonical claim|" r"shared (?:source material|evidence).{0,100}one canonical claim|"
r"one shared (?:structural |canonical )?claim row|" r"one shared (?:structural |canonical )?claim row|"
r"(?:single|one) canonical assertion|"
r"deduplicate.{0,80}(?:single|one) canonical (?:assertion|record)|"
r"share the fact.{0,100}one claim row|" r"share the fact.{0,100}one claim row|"
r"keep (?:the )?(?:fact|claim|proposition) (?:canonical )?once|"
r"one claim row.{0,80}(?:holds|contains|records).{0,60}(?:agreed|shared|factual) (?:fact|claim|proposition)|"
r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)", r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
re.I | re.S, re.I | re.S,
), ),
@ -454,7 +458,9 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
r"one shared claim|single shared claim|" r"one shared claim|single shared claim|"
r"duplicating (?:(?:a|the) )?(?:factual )?claim|" r"duplicating (?:(?:a|the) )?(?:factual )?claim|"
r"one (?:canonical |structural )?claim.{0,100}shared (?:sources?|evidence|claim_evidence)|" r"one (?:canonical |structural )?claim.{0,100}shared (?:sources?|evidence|claim_evidence)|"
r"one claim row.{0,100}shared (?:sources?|evidence|claim_evidence)", r"one claim row.{0,180}(?:(?:both|each) agents?'|shared) (?:sources?|evidence|claim_evidence)|"
r"(?:both|each) (?:agents?|analysts?)' (?:sources?|source evidence|evidence)|"
r"both (?:agents?|analysts?)' evidence.{0,80}(?:one|single) (?:record|assertion|claim)",
re.I | re.S, re.I | re.S,
), ),
), ),
@ -495,11 +501,20 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
), ),
"forecast_schema_gap": ( "forecast_schema_gap": (
re.compile( re.compile(
r"current (?:v1|schema|columns?|fields?|edge types?)|public\.claims|the schema|schema has|claims table", r"current (?:v1|schema|columns?|fields?|edge types?|(?:public\.)?claims schema|edge_type enum)|"
r"public\.claims|the schema|schema has|claims table",
re.I, re.I,
), ),
re.compile( re.compile(
r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|" r"(?:there is |the schema has |(?:public\.)?claims has )?no "
r"(?:forecast[- ]resolution|resolution fields?|resolved_at)"
r"(?!\s+(?:migration|review|proposal|addition|change|work)\b)|"
r"does not have.{0,50}resolution|"
r"(?:schema|claims).{0,80}lacks?.{0,60}(?:resolution criteria|resolution_criteria|outcome|resolved_at)|"
r"(?:schema|claims).{0,80}(?:does not|doesn't|cannot) (?:model|express|represent|record|store)"
r".{0,60}(?:resolution criteria|resolution_criteria|outcome|resolved_at)|"
r"no (?:column|field) (?:on|in) .{0,100}(?:records?|stores?|contains?).{0,100}"
r"(?:resolution criteria|outcome|resolved_at)|"
r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|" r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}" r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
r"(?:does not exist|is absent|is not present|isn't present)|" r"(?:does not exist|is absent|is not present|isn't present)|"
@ -511,7 +526,15 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.I | re.S, re.I | re.S,
), ),
re.compile( re.compile(
r"resolves.{0,220}(?:not|isn'?t|does not|doesn't|absent)|no.{0,30}resolves", r"resolves (?:edge|type).{0,100}(?:does not exist|is absent|is missing|is not present|isn't present|"
r"is not current|isn't current|is unsupported)|"
r"(?:there is|schema has|enum has|edge types? (?:has|have)|(?:current )?edge_type enum has) "
r"no `?resolves`?(?: (?:edge|type))?|"
r"no `?resolves`? (?:edge|type)(?: exists| is present)?|"
r"(?:edge_type|edge type|enum) lacks? resolves|"
r"(?:edge_type|edge type|enum) (?:cannot|does not|doesn't) (?:express|represent|support) resolves|"
r"(?:do not|don't|must not) invent (?:an? )?resolves (?:edge|type)|"
r"resolves (?:edge|type).{0,120}neither (?:exists|is present)",
re.I | re.S, re.I | re.S,
), ),
), ),
@ -655,15 +678,18 @@ COUNT_INVARIANT_REJECTION_RE = re.compile(
SCHEMA_GAP_QUALIFIER_RE = re.compile( SCHEMA_GAP_QUALIFIER_RE = re.compile(
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|" r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|" r"has no|have no|lacks?|without|there is no|no column|not an edge|would require|schema gap|must be added|"
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|" r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
r"schema extension|extension proposal|reviewed schema proposal|not (?:in|part of|among) the current)\b|" r"schema extension|extension proposal|reviewed schema proposal|not (?:in|part of|among) the current)\b|"
r"\b(?:does not|doesn't|cannot)\s+"
r"(?:record|store|contain|include|have|expose|provide|support|model|express|represent)\b|"
r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?" r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?"
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|" r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?" r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?"
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|" r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
r"\b(?:proposal|extension)\b.{0,100}\b(?:column|field|table|edge type)\b|" r"\b(?:proposal|extension)\b.{0,100}\b(?:column|field|table|edge type)\b|"
r"\bno\s+(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b", r"\bno\s+(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b"
r"(?!\s+(?:review|proposal|change|addition|work)\b)",
re.I, re.I,
) )
SCHEMA_CLAUSE_BOUNDARY_RE = re.compile( SCHEMA_CLAUSE_BOUNDARY_RE = re.compile(
@ -674,12 +700,23 @@ SCHEMA_CLAUSE_BOUNDARY_RE = re.compile(
) )
CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = { CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
"claims_unshipped_fields": re.compile( "claims_unshipped_fields": re.compile(
r"claims\s+(?:table|schema)\b.{0,80}\b(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b|" r"(?:the )?current schema\b.{0,120}\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|"
r"public\.claims\s+(?:stores?|has|contains?)\s+(?:an?\s+)?" r"resolved_at|falsifier)\b|"
r"(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b|" r"public\.claims\b.{0,120}\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|"
r"falsifier)\b|"
r"(?:the )?current schema\s+(?:already\s+)?"
r"(?:stores?|has|contains?|records?|exposes?|provides?|supports?|includes?|models?|represents?)\s+.{0,80}"
r"\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
r"claims\s+(?:table|schema)\b.{0,100}\b(?:body|metadata|forecast[_ -]resolution|"
r"resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
r"public\.claims\s+(?:(?:natively|already|currently)\s+)?"
r"(?:stores?|has|contains?|records?|exposes?|provides?|supports?|includes?|models?|represents?)\s+"
r"(?:an?\s+)?"
r"(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
r"public\.claims\b.{0,80}\b(?:column|field)\b.{0,30}" r"public\.claims\b.{0,80}\b(?:column|field)\b.{0,30}"
r"\b(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b|" r"\b(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
r"public\.claims\b.{0,80}\b(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b.{0,30}" r"public\.claims\b.{0,80}\b(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|"
r"outcome|resolved_at|falsifier)\b.{0,30}"
r"\b(?:column|field)\b", r"\b(?:column|field)\b",
re.I, re.I,
), ),
@ -689,9 +726,15 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
re.I, re.I,
), ),
"invalid_current_edge_type": re.compile( "invalid_current_edge_type": re.compile(
r"\b(?:current )?edge_type enum\b.{0,80}\bresolves\b|"
r"\bcurrent edge_type\b.{0,80}\bresolves\b|"
r"(?:\b(?:claim_edges?|edge type|edge)\b\s*(?:is|=|:|named|called|of)?\s*[`'\"]?" r"(?:\b(?:claim_edges?|edge type|edge)\b\s*(?:is|=|:|named|called|of)?\s*[`'\"]?"
r"(?:superseded_by|relates_to|resolves|derived_from)\b|" r"(?:superseded_by|relates_to|resolves|derived_from)\b|"
r"\b(?:superseded_by|relates_to|resolves|derived_from)\b[`'\"]?\s+(?:claim_)?edge\b)", r"\b(?:superseded_by|relates_to|resolves|derived_from)\b[`'\"]?\s+(?:claim_)?edge\b|"
r"\b(?:current )?edge_type enum\b.{0,50}\b(?:contains|has|includes|supports|enumerates)\b.{0,20}"
r"\bresolves\b|"
r"\b(?:current )?edge_type\b.{0,50}\b(?:contains|has|includes|supports|enumerates)\b.{0,20}"
r"\bresolves\b)",
re.I, re.I,
), ),
"unshipped_edge_rationale": re.compile(r"\bclaim_edges?\b.{0,100}\brationale\b", re.I), "unshipped_edge_rationale": re.compile(r"\bclaim_edges?\b.{0,100}\brationale\b", re.I),
@ -721,6 +764,17 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
), ),
} }
UNVERIFIED_M3TAVERSAL_ALIAS_RE = re.compile(r"\b(?:Cory|m3ta)\b", re.I) UNVERIFIED_M3TAVERSAL_ALIAS_RE = re.compile(r"\b(?:Cory|m3ta)\b", re.I)
RESOLVES_PRESENCE_ASSERTION_RE = re.compile(
r"\bresolves (?:edge|type)\b.{0,120}\b(?:is not (?:absent|missing)|already (?:exists|supported|present)|"
r"is (?:already )?supported by (?:the )?(?:current )?edge_type)\b|"
r"\b(?:current )?edge_type\b.{0,60}\b(?:already )?(?:contains|has|includes|supports|enumerates)\b.{0,20}"
r"\bresolves\b",
re.I | re.S,
)
EDGE_TYPE_RESOLVES_DENIAL_RE = re.compile(
r"\b(?:current )?edge_type\b.{0,50}\b(?:does not|doesn't|cannot|lacks?|has no)\b.{0,40}\bresolves\b",
re.I | re.S,
)
SOURCE_EVIDENCE_CANONICAL_OBJECT_RE = re.compile(r"claim_evidence|public\.sources|source rows?", re.I) SOURCE_EVIDENCE_CANONICAL_OBJECT_RE = re.compile(r"claim_evidence|public\.sources|source rows?", re.I)
SOURCE_EVIDENCE_LOCATOR_GAP_RE = re.compile( SOURCE_EVIDENCE_LOCATOR_GAP_RE = re.compile(
@ -886,6 +940,41 @@ BLOCKER_DIAGNOSTIC_RE = re.compile(
r"source|evidence|claim|proposal|runtime|restart|identity|tool|readback)\b", r"source|evidence|claim|proposal|runtime|restart|identity|tool|readback)\b",
re.I, re.I,
) )
BLOCKER_FAILURE_STATE_RE = re.compile(
r"\b(?:no|none|zero|missing|absent|null|unavailable|unsupported|unapplied|not|cannot|can't|blocked|"
r"fails?|stale|wrong|gap|lacks?|without)\b",
re.I,
)
BLOCKER_ABSENCE_RE = re.compile(
r"\b(?:there is |there's )?no (?:[a-z][a-z-]*\s+){0,4}blocker "
r"(?:exists|remains|is present)\b|\bnothing (?:is )?blocked\b",
re.I,
)
PER_AGENT_CLAIM_DUPLICATION_RE = re.compile(
r"\b(?:(?:one|separate|distinct|individual)\s+)?(?:public\.)?claims?(?:\s+row)?\s+per\s+agent\b|"
r"\b(?:claim|public\.claims)(?:\s+row)?(?:\s+once)?\s+(?:for|to)\s+each\s+agent\b|"
r"\beach\s+agent\s+(?:gets?|receives?|has|stores?|uses?)\s+"
r"(?:(?:its|their)\s+own|an?\s+separate|one)\s+(?:public\.)?claims?(?:\s+row)?\b|"
r"\b(?:give|assign)\s+each\s+agent\s+(?:(?:its|their)\s+own|an?\s+separate|one)\s+"
r"(?:public\.)?claims?(?:\s+row)?\b|"
r"\bone\s+(?:public\.)?claims?(?:\s+row)?\s+for\s+[A-Z][a-z0-9_-]+\s+and\s+one\s+"
r"(?:(?:public\.)?claims?(?:\s+row)?\s+)?for\s+[A-Z][a-z0-9_-]+\b|"
r"\b(?:fact|proposition|claim|assertion)\b.{0,80}\b(?:under|within|inside)\s+each\s+agent\b|"
r"\b(?:their|the agents?'?)\s+respective\s+(?:copies|claim rows?|records?)\b",
re.I,
)
PER_AGENT_DUPLICATION_PREFIX_DENIAL_RE = re.compile(
r"(?:\b(?:do not|don't|does not|doesn't|should not|shouldn't|must not|never|avoid)\b\s*"
r"(?:(?:create|store|write|give|assign|duplicate|fork|make|keep|use|add)(?:ing)?\s+)?"
r"(?:(?:the|an?|one)\s+)?(?:(?:factual|canonical|shared|separate|distinct)\s+)?|"
r"\b(?:instead of|rather than|not)\s+)$",
re.I,
)
PER_AGENT_DUPLICATION_SUFFIX_DENIAL_RE = re.compile(
r"^.{0,80}\b(?:is|would be|remains?)\s+(?:wrong|incorrect|invalid|duplicative)\b|"
r"^.{0,80}\b(?:creates?|causes?|would create)\s+(?:data\s+)?(?:divergence|fragmentation)\b",
re.I,
)
def prompt_catalog(memory_token: str) -> list[dict[str, Any]]: def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
@ -902,6 +991,11 @@ def extract_blocker_clause(reply: str) -> str | None:
for pattern in ( for pattern in (
re.compile(r"\bblocker(?:\s+restated)?\s*:\s*(?P<value>[^\n]+)", re.I), re.compile(r"\bblocker(?:\s+restated)?\s*:\s*(?P<value>[^\n]+)", re.I),
re.compile(r"\bblocker\s+(?:is|was)\s+(?P<value>[^\n]+)", re.I), re.compile(r"\bblocker\s+(?:is|was)\s+(?P<value>[^\n]+)", re.I),
re.compile(
r"\b(?:mnemonic|label)(?:\s+(?:tag|retrieved))?\s*:?[^.\n]{0,140}?"
r"(?:[.]|\s+(?:-|\u2014)\s+)\s*(?P<value>[^\n]+)",
re.I,
),
): ):
match = pattern.search(reply) match = pattern.search(reply)
if match: if match:
@ -920,11 +1014,120 @@ def blocker_terms(value: str | None, *, memory_token: str) -> set[str]:
} }
def blocker_signature(value: str | None) -> set[str]:
"""Extract polarity-aware blocker categories for same-session recall."""
if not value:
return set()
signatures: set[str] = set()
clauses = [
clause.strip()
for clause in re.split(r"(?<=[.!?;])\s+|\n+|,\s+(?:but|and|so)\s+", value, flags=re.I)
if clause.strip()
]
if re.search(r"\b(?:approved|reviewer approval|reviewer signature|reviewer sign[- ]off)\b", value, re.I):
signatures.add("reviewer_approval")
applied_missing = re.compile(
r"\bapproved[- ]but[- ]not[- ]applied\b|\b(?:not applied|unapplied)\b|"
r"\b(?:zero|no|missing|absent)\s+applied_at(?:\s+values?)?\b|"
r"\bapplied_at\b.{0,20}\b(?:unset|missing|absent|zero|none|null|not set)\b|"
r"\b(?:unset|missing|absent|zero|none|null)\b.{0,20}\bapplied_at\b",
re.I,
)
applied_present = re.compile(
r"\bapplied_at\b.{0,30}\b(?:non[- ]null|not null|exists|present|is set|has a value|"
r"missing (?:flag )?(?:is|=) false|is not missing)\b",
re.I,
)
if any(applied_missing.search(clause) and not applied_present.search(clause) for clause in clauses):
signatures.add("applied_missing")
canonical_missing = re.compile(
r"\bcanonical (?:gap|row|readback)\b.{0,30}\b(?:missing|absent|none|null|zero|not (?:live|written)|"
r"unavailable)\b|"
r"\b(?:no|zero|missing|absent)\b.{0,25}\b(?:canonical|public\.\*)\b.{0,25}\brows?\b|"
r"\b(?:no|zero|missing|absent)\s+public\.\*\s+row(?: ID)?\b|"
r"\bno rows?\b.{0,30}\b(?:landed|written|applied)\b.{0,20}\bpublic\.(?:claims|sources|\*)\b|"
r"\bapproved[- ]but[- ]not[- ]applied canonical gap\b|\bcanonical gap\b",
re.I,
)
canonical_present = re.compile(
r"\bcanonical (?:row|readback)\b.{0,20}\b(?:exists|present|current|complete|is live)\b|"
r"\bcanonical rows?\b.{0,20}\b(?:exist|are present|are current|are complete)\b|"
r"\bcanonical gap\b.{0,20}\b(?:is|was|has been|now)\s+(?:closed|resolved|cleared|fixed)\b",
re.I,
)
if any(canonical_missing.search(clause) and not canonical_present.search(clause) for clause in clauses):
signatures.add("canonical_missing")
telegram_missing = re.compile(
r"\b(?:telegram|delivery|visible message|message receipt)\b.{0,50}"
r"\b(?:no|missing|absent|blocked|unproven|not proven|unavailable)\b|"
r"\b(?:no|missing|absent|blocked|unproven|not proven|unavailable)\b.{0,50}"
r"\b(?:telegram|delivery|visible message|message receipt)\b",
re.I,
)
if any(telegram_missing.search(clause) for clause in clauses):
signatures.add("telegram_missing")
if re.search(
r"\b(?:beliefs?\b.{0,40})?(?:has|have|with)\s+no\s+claim[_ -]?id\s+foreign key\b|"
r"\b(?:missing|absent|no)\b.{0,50}\b(?:foreign key|belief[- ]to[- ]claim link)\b|"
r"\bschema gap\b",
value,
re.I | re.S,
):
signatures.add("schema_link_missing")
if (
re.search(r"\bbeliefs?(?:\.claim[_ -]?id)?\b", value, re.I)
and re.search(r"\bclaim[_ -]?id\b", value, re.I)
and re.search(r"\b(?:foreign key|FK)\b", value, re.I)
and BLOCKER_FAILURE_STATE_RE.search(value)
):
signatures.add("belief_claim_foreign_key_missing")
if (
re.search(r"\bbeliefs?(?:\.claim[_ -]?id)?\b", value, re.I)
and re.search(r"\bclaim[_ -]?id\b", value, re.I)
and re.search(r"\bindex\b", value, re.I)
and BLOCKER_FAILURE_STATE_RE.search(value)
):
signatures.add("belief_claim_index_missing")
if re.search(
r"\bpublic\.sources\b.{0,80}\b(?:no|missing|absent|lacks?)\b.{0,40}\bforeign key\b|"
r"\b(?:no|missing|absent)\b.{0,40}\bforeign key\b.{0,80}\bpublic\.sources\b",
value,
re.I | re.S,
):
signatures.add("source_foreign_key_missing")
return signatures
def asserts_per_agent_claim_duplication(reply: str) -> bool:
"""Reject affirmative per-agent duplication while allowing explicit denials."""
clauses = re.split(r"(?<=[.!?;])\s+|\n+|,\s+(?:but|and|so)\s+", reply, flags=re.I)
for clause in clauses:
for match in PER_AGENT_CLAIM_DUPLICATION_RE.finditer(clause):
prefix = clause[max(0, match.start() - 80) : match.start()]
suffix = clause[match.end() : match.end() + 100]
if PER_AGENT_DUPLICATION_PREFIX_DENIAL_RE.search(prefix):
continue
if PER_AGENT_DUPLICATION_SUFFIX_DENIAL_RE.search(suffix):
continue
return True
return False
def matched_concept(reply: str, concept: str) -> bool: def matched_concept(reply: str, concept: str) -> bool:
if concept == "blocker_definition": if concept == "blocker_definition":
blocker = extract_blocker_clause(reply) blocker = extract_blocker_clause(reply)
terms = blocker_terms(blocker, memory_token="") terms = blocker_terms(blocker, memory_token="")
return bool(blocker and len(terms) >= 3 and BLOCKER_DIAGNOSTIC_RE.search(blocker)) return bool(
blocker
and len(terms) >= 3
and BLOCKER_DIAGNOSTIC_RE.search(blocker)
and BLOCKER_FAILURE_STATE_RE.search(blocker)
and not BLOCKER_ABSENCE_RE.search(blocker)
)
if concept == "shared_knowledge_commons" and asserts_per_agent_claim_duplication(reply):
return False
if concept == "claim_body_evidence_distinction": if concept == "claim_body_evidence_distinction":
if CLAIM_EVIDENCE_CONFLATION_RE.search(reply): if CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
return False return False
@ -963,6 +1166,8 @@ def current_schema_overclaims(reply: str) -> list[str]:
findings: list[str] = [] findings: list[str] = []
sentence_segments = re.split(r"(?<=[.!?])\s+|\n+", reply) sentence_segments = re.split(r"(?<=[.!?])\s+|\n+", reply)
for sentence in sentence_segments: for sentence in sentence_segments:
if RESOLVES_PRESENCE_ASSERTION_RE.search(sentence) and not EDGE_TYPE_RESOLVES_DENIAL_RE.search(sentence):
findings.append("invalid_current_edge_type")
# Negation and future-schema qualifiers apply to their clause, not to a # Negation and future-schema qualifiers apply to their clause, not to a
# contradictory assertion in a later conjunction in the same sentence. # contradictory assertion in a later conjunction in the same sentence.
clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence) clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence)
@ -1211,11 +1416,25 @@ def score_results(
memory_source_reply = str((by_result.get(memory_source_id) or {}).get("reply") or "") memory_source_reply = str((by_result.get(memory_source_id) or {}).get("reply") or "")
memory_recall_reply = str((by_result.get(memory_recall_id) or {}).get("reply") or "") memory_recall_reply = str((by_result.get(memory_recall_id) or {}).get("reply") or "")
source_clause = extract_blocker_clause(memory_source_reply) source_clause = extract_blocker_clause(memory_source_reply)
recall_clause = extract_blocker_clause(memory_recall_reply)
source_terms = blocker_terms(source_clause, memory_token=memory_token) source_terms = blocker_terms(source_clause, memory_token=memory_token)
recall_terms = blocker_terms(extract_blocker_clause(memory_recall_reply), memory_token=memory_token) recall_terms = blocker_terms(recall_clause, memory_token=memory_token)
overlap_terms = source_terms & recall_terms overlap_terms = source_terms & recall_terms
required_overlap = min(3, max(1, (len(source_terms) + 2) // 3)) if source_terms else 1 required_overlap = min(3, max(1, (len(source_terms) + 2) // 3)) if source_terms else 1
same_blocker_recalled = bool(source_clause and len(overlap_terms) >= required_overlap) source_signature = blocker_signature(source_clause)
recall_signature = blocker_signature(recall_clause)
source_failure_signature = source_signature - {"reviewer_approval"}
recall_failure_signature = recall_signature - {"reviewer_approval"}
signature_overlap = source_failure_signature & recall_failure_signature
required_signature_overlap = min(2, len(source_failure_signature)) if source_failure_signature else 0
same_blocker_recalled = bool(
source_clause
and (
len(signature_overlap) >= required_signature_overlap and len(overlap_terms) >= required_overlap
if source_failure_signature
else len(overlap_terms) >= required_overlap
)
)
for score in scores: for score in scores:
if score["prompt_id"] != memory_recall_id: if score["prompt_id"] != memory_recall_id:
continue continue
@ -1227,6 +1446,10 @@ def score_results(
"source_blocker_terms": sorted(source_terms), "source_blocker_terms": sorted(source_terms),
"recall_overlap_terms": sorted(overlap_terms), "recall_overlap_terms": sorted(overlap_terms),
"required_overlap": required_overlap, "required_overlap": required_overlap,
"source_blocker_signature": sorted(source_signature),
"recall_blocker_signature": sorted(recall_signature),
"signature_overlap": sorted(signature_overlap),
"required_signature_overlap": required_signature_overlap,
"same_blocker_recalled": same_blocker_recalled, "same_blocker_recalled": same_blocker_recalled,
} }
return { return {

View file

@ -29,7 +29,7 @@ PROTOCOL_SCHEMA = "livingip.leoM3taversalOosProtocol.v3"
TRIAL_SCORE_SCHEMA = "livingip.leoM3taversalOosTrialScore.v3" TRIAL_SCORE_SCHEMA = "livingip.leoM3taversalOosTrialScore.v3"
AGGREGATE_SCHEMA = "livingip.leoM3taversalOosAggregate.v3" AGGREGATE_SCHEMA = "livingip.leoM3taversalOosAggregate.v3"
GENERATOR_VERSION = "blinded-family-generator-v5" GENERATOR_VERSION = "blinded-family-generator-v5"
SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v9" SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v10"
BASELINE_VERSION = "live-current-build-db-tool-ablation-v3" BASELINE_VERSION = "live-current-build-db-tool-ablation-v3"
DEFAULT_TRIAL_COUNT = 3 DEFAULT_TRIAL_COUNT = 3
MEMORY_SCORER_IDS = frozenset({"OOS-07", "OOS-08"}) MEMORY_SCORER_IDS = frozenset({"OOS-07", "OOS-08"})
@ -403,6 +403,14 @@ def harness_git_head() -> str:
).strip() ).strip()
def harness_worktree_clean() -> bool:
return not subprocess.check_output(
["git", "status", "--porcelain", "--untracked-files=no"],
cwd=ROOT,
text=True,
).strip()
def instrument_db_context_plugin_source(source: str) -> str: def instrument_db_context_plugin_source(source: str) -> str:
marker = """ safe["receipt_sha256"] = hashlib.sha256( marker = """ safe["receipt_sha256"] = hashlib.sha256(
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
@ -562,6 +570,8 @@ def freeze_protocol(
) -> dict[str, Any]: ) -> dict[str, Any]:
if trial_count < 3: if trial_count < 3:
raise ValueError("at least three trials are required for clean/restart variance") raise ValueError("at least three trials are required for clean/restart variance")
if not harness_worktree_clean():
raise RuntimeError("refusing to freeze a protocol from a dirty tracked worktree")
modes = ["clean_session"] * (trial_count - 1) + ["post_restart_clean_session"] modes = ["clean_session"] * (trial_count - 1) + ["post_restart_clean_session"]
trials = [build_blinded_trial(seed, index, session_mode=modes[index]) for index in range(trial_count)] trials = [build_blinded_trial(seed, index, session_mode=modes[index]) for index in range(trial_count)]
protocol: dict[str, Any] = { protocol: dict[str, Any] = {
@ -610,12 +620,15 @@ def freeze_protocol(
"no_supplied_row_ids": True, "no_supplied_row_ids": True,
"prompt_variants_per_family": min(len(family["variants"]) for family in BLINDED_FAMILIES), "prompt_variants_per_family": min(len(family["variants"]) for family in BLINDED_FAMILIES),
}, },
"harness_worktree_clean_at_freeze": True,
"harness_git_head": harness_git_head(), "harness_git_head": harness_git_head(),
"source_hashes": {key: file_sha256(path) for key, path in source_paths().items()}, "source_hashes": {key: file_sha256(path) for key, path in source_paths().items()},
"trials": trials, "trials": trials,
} }
protocol["protocol_hash_sha256"] = canonical_sha256(protocol) protocol["protocol_hash_sha256"] = canonical_sha256(protocol)
validate_protocol(protocol, verify_source_hashes=True) validation = validate_protocol(protocol, verify_source_hashes=True)
if not validation["pass"]:
raise RuntimeError("frozen protocol failed validation: " + ", ".join(validation["issues"]))
return protocol return protocol
@ -631,6 +644,8 @@ def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) -
issues.append("wrong_baseline_version") issues.append("wrong_baseline_version")
if not _valid_git_revision(protocol.get("harness_git_head")): if not _valid_git_revision(protocol.get("harness_git_head")):
issues.append("invalid_harness_git_head") issues.append("invalid_harness_git_head")
if protocol.get("harness_worktree_clean_at_freeze") is not True:
issues.append("harness_worktree_not_clean_at_freeze")
supplied_hash = protocol.get("protocol_hash_sha256") supplied_hash = protocol.get("protocol_hash_sha256")
unhashed = {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"} unhashed = {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"}
if supplied_hash != canonical_sha256(unhashed): if supplied_hash != canonical_sha256(unhashed):
@ -700,6 +715,8 @@ def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) -
if len(seen) < min(3, len(trials)): if len(seen) < min(3, len(trials)):
issues.append(f"variant_repetition:{family_id}") issues.append(f"variant_repetition:{family_id}")
if verify_source_hashes: if verify_source_hashes:
if not harness_worktree_clean():
issues.append("harness_worktree_dirty")
if protocol.get("harness_git_head") != harness_git_head(): if protocol.get("harness_git_head") != harness_git_head():
issues.append("harness_git_head_changed_after_freeze") issues.append("harness_git_head_changed_after_freeze")
source_hashes = protocol.get("source_hashes") or {} source_hashes = protocol.get("source_hashes") or {}

View file

@ -14,6 +14,11 @@ import run_leo_m3taversal_oos_handler_suite as suite # noqa: E402
import working_leo_m3taversal_oos_protocol as protocol_lib # noqa: E402 import working_leo_m3taversal_oos_protocol as protocol_lib # noqa: E402
@pytest.fixture(autouse=True)
def clean_protocol_worktree(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(protocol_lib, "harness_worktree_clean", lambda: True)
def protocol() -> dict: def protocol() -> dict:
return protocol_lib.freeze_protocol( return protocol_lib.freeze_protocol(
"runner-test-seed", "runner-test-seed",

View file

@ -313,6 +313,11 @@ def test_oos_schema_guard_allows_explicit_future_schema_gap() -> None:
"would require a schema gap proposal." "would require a schema gap proposal."
) )
assert benchmark.current_schema_overclaims(reply) == [] assert benchmark.current_schema_overclaims(reply) == []
assert benchmark.current_schema_overclaims(
"The current public.claims schema does not record outcome or resolution criteria."
) == []
assert benchmark.current_schema_overclaims("The current edge_type enum does not contain resolves.") == []
assert benchmark.current_schema_overclaims("The current claims schema lacks an outcome field.") == []
def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_clause() -> None: def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_clause() -> None:
@ -925,6 +930,82 @@ def test_oos_agent_positions_accept_attribution_without_literal_column_name() ->
assert benchmark.matched_concept(share_the_fact, "shared_knowledge_commons") is True assert benchmark.matched_concept(share_the_fact, "shared_knowledge_commons") is True
assert benchmark.matched_concept(share_the_fact, "agent_specific_positions") is True assert benchmark.matched_concept(share_the_fact, "agent_specific_positions") is True
agreed_proposition = (
"One claim row holds the agreed proposition; both agents' sources and evidence attach to it. Each agent's "
"divergent interpretation goes into one public.beliefs row. Beliefs has no claim_id foreign key, so an "
"explicit belief-to-claim link remains a schema gap."
)
assert benchmark.matched_concept(agreed_proposition, "shared_knowledge_commons") is True
assert benchmark.matched_concept(agreed_proposition, "agent_specific_positions") is True
duplicated_claims = (
"Store one separate public.claims row per agent. One claim row per agent holds the agreed proposition; "
"both agents' sources and evidence attach to their separate rows. Each agent's divergent interpretation "
"goes into one public.beliefs row per agent. Beliefs has no claim_id foreign key, so the link is a schema "
"gap. The positions disagree."
)
assert benchmark.asserts_per_agent_claim_duplication(duplicated_claims) is True
assert benchmark.matched_concept(duplicated_claims, "shared_knowledge_commons") is False
prompt = next(item for item in benchmark.prompt_catalog("demo-ledger-deadbeef") if item["id"] == "OOS-11")
assert benchmark.score_reply(prompt, duplicated_claims, memory_token="demo-ledger-deadbeef")["pass"] is False
assert benchmark.asserts_per_agent_claim_duplication(
"Do not create one public.claims row per agent; keep one shared claim row."
) is False
assert benchmark.asserts_per_agent_claim_duplication(
"Each agent gets its own public.claims row for the shared fact."
) is True
assert benchmark.asserts_per_agent_claim_duplication(
"Each agent should not get its own public.claims row; keep the factual row shared."
) is False
assert benchmark.asserts_per_agent_claim_duplication(
"Use one claim row per agent instead of one shared claim row."
) is True
assert benchmark.asserts_per_agent_claim_duplication(
"Use one shared claim row instead of one claim row per agent."
) is False
unrelated_denial = (
"Do not invent belief-to-claim edges while storing one public.claims row per agent: one claim row holds "
"the agreed proposition for each agent, and both agents' sources and evidence attach to those rows. Retain "
"that source evidence. Each agent's divergent position goes into one public.beliefs row per agent. Beliefs "
"has no claim_id foreign key, so the schema has a gap. The positions contradict."
)
assert benchmark.asserts_per_agent_claim_duplication(unrelated_denial) is True
assert benchmark.score_reply(prompt, unrelated_denial, memory_token="demo-ledger-deadbeef")["pass"] is False
faithful_paraphrase = (
"Keep the proposition canonical once and retain both analysts' source evidence on it. Record each analyst's "
"interpretation in public.beliefs with agent attribution. Because beliefs has no claim_id foreign key, the "
"association remains a schema gap; contradictory interpretations remain queryable by agent."
)
assert benchmark.matched_concept(faithful_paraphrase, "shared_knowledge_commons") is True
assert benchmark.score_reply(prompt, faithful_paraphrase, memory_token="demo-ledger-deadbeef")["pass"] is True
named_duplicates = (
"Keep one public.claims row for Alice and one for Bob, each holding the agreed proposition, with both "
"agents' source evidence attached to both copies. Retain that evidence. Each agent's divergent position "
"goes into one public.beliefs row with agent attribution. Beliefs has no claim_id foreign key, so the schema "
"has a gap. The positions contradict."
)
assert benchmark.asserts_per_agent_claim_duplication(named_duplicates) is True
assert benchmark.score_reply(prompt, named_duplicates, memory_token="demo-ledger-deadbeef")["pass"] is False
single_assertion = (
"Deduplicate the proposition into a single canonical assertion and retain both analysts' evidence on that "
"one record. Put each conflicting interpretation in public.beliefs with agent attribution. Beliefs lacks a "
"claim_id foreign key, so the association remains a schema gap and disagreement stays queryable."
)
assert benchmark.score_reply(prompt, single_assertion, memory_token="demo-ledger-deadbeef")["pass"] is True
namespaced_copies = (
"Keep the proposition canonical once under each agent; retain both analysts' source evidence and attach it "
"to their respective copies. Put each conflicting interpretation in public.beliefs with agent attribution. "
"Beliefs lacks a claim_id foreign key, so the association remains a schema gap and disagreement stays "
"queryable."
)
assert benchmark.asserts_per_agent_claim_duplication(namespaced_copies) is True
assert benchmark.score_reply(prompt, namespaced_copies, memory_token="demo-ledger-deadbeef")["pass"] is False
def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None: def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None:
token = "demo-ledger-deadbeef" token = "demo-ledger-deadbeef"
@ -952,6 +1033,51 @@ def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> N
assert score["memory_continuity"]["same_blocker_recalled"] is True assert score["memory_continuity"]["same_blocker_recalled"] is True
assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False
different_schema_gap = (
f"Label: {token}\nBlocker restated: public.sources has no author field, so source attribution remains a "
"schema gap.\nProof: postflight readback of a reviewed source-schema extension."
)
changed_score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": set_reply},
{"prompt_id": recall_prompt["id"], "reply": different_schema_gap},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert changed_score["pass"] is False
assert changed_score["memory_continuity"]["same_blocker_recalled"] is False
different_foreign_key = (
f"Label: {token}\nBlocker restated: public.sources lacks a foreign key to provenance metadata, so the "
"source relation remains a schema gap.\nProof: postflight readback of the source relation."
)
foreign_key_score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": set_reply},
{"prompt_id": recall_prompt["id"], "reply": different_foreign_key},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert foreign_key_score["pass"] is False
assert foreign_key_score["memory_continuity"]["same_blocker_recalled"] is False
different_mechanism = (
f"Label: {token}\nBlocker restated: beliefs.claim_id lacks an index, so position lookups remain a schema "
"gap.\nProof: postflight readback of the new index and query plan."
)
index_score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": set_reply},
{"prompt_id": recall_prompt["id"], "reply": different_mechanism},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert index_score["pass"] is False
assert index_score["memory_continuity"]["same_blocker_recalled"] is False
def test_oos_memory_recall_accepts_apply_receipt_correction_without_losing_the_token() -> None: def test_oos_memory_recall_accepts_apply_receipt_correction_without_losing_the_token() -> None:
token = "demo-ledger-deadbeef" token = "demo-ledger-deadbeef"
@ -971,6 +1097,119 @@ def test_oos_memory_recall_accepts_apply_receipt_correction_without_losing_the_t
assert score["proposal_readiness_issues"] == [] assert score["proposal_readiness_issues"] == []
def test_oos_memory_accepts_natural_mnemonic_without_fixed_blocker_label() -> None:
token = "blind-ledger-deadbeef1234"
set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07")
recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08")
set_reply = (
f"Mnemonic tag {token}. Three approved proposals; zero applied_at values; zero canonical rows written. "
"Approval is a reviewer signature, applied_at is the guarded-apply receipt, and canonical readback is a "
"public.* row ID plus hash."
)
recall_reply = (
f"Mnemonic retrieved: {token}. Three proposals carry reviewer approval; applied_at is null on all three; "
"no public.* row ID or hash postflight exists. Resolution requires proposal-level applied_at plus the target "
"row ID and matching content hash."
)
score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": set_reply},
{"prompt_id": recall_prompt["id"], "reply": recall_reply},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert score["pass"] is True
assert score["memory_continuity"]["same_blocker_recalled"] is True
changed_recall = (
f"Mnemonic retrieved: {token}. Telegram delivery remains blocked: applied_at and canonical readback exist, "
"but the visible message receipt is absent. Resolution requires message ID, timestamp, chat readback, and "
"a postflight row ID plus hash."
)
changed_score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": set_reply},
{"prompt_id": recall_prompt["id"], "reply": changed_recall},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert changed_score["pass"] is False
assert changed_score["memory_continuity"]["same_blocker_recalled"] is False
assert changed_score["memory_continuity"]["source_blocker_signature"] == [
"applied_missing",
"canonical_missing",
"reviewer_approval",
]
assert changed_score["memory_continuity"]["recall_blocker_signature"] == ["telegram_missing"]
vague = f"Mnemonic tag {token}. Database schema evidence remains unclear."
assert benchmark.matched_concept(vague, "blocker_definition") is False
no_blocker = f"Mnemonic tag {token}. No database blocker exists; all canonical rows are current and complete."
assert benchmark.matched_concept(no_blocker, "blocker_definition") is False
assert benchmark.score_reply(set_prompt, no_blocker, memory_token=token)["pass"] is False
modified_no_blocker = (
f"Mnemonic tag {token}. No meaningful database blocker exists; all canonical rows are current and complete."
)
assert benchmark.matched_concept(modified_no_blocker, "blocker_definition") is False
assert benchmark.score_reply(set_prompt, modified_no_blocker, memory_token=token)["pass"] is False
faithful_set = (
f"Mnemonic: {token} \u2014 three approved proposals still have applied_at NULL and no canonical rows; closure "
"requires row IDs and matching hashes."
)
faithful_recall = (
f"Mnemonic retrieved: {token}. Reviewer sign-off exists, but applied_at remains unset and no rows have landed "
"in public.claims. Close it with a postflight readback of the proposal timestamp, target claim row ID, and "
"content hash."
)
faithful_score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": faithful_set},
{"prompt_id": recall_prompt["id"], "reply": faithful_recall},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert faithful_score["pass"] is True
opposite_recall = (
f"Mnemonic retrieved: {token}. Reviewer approval is present and applied_at is non-null; the canonical row "
"exists. Telegram delivery is missing. Closure proof: postflight readback of the message ID, timestamp, and "
"public row."
)
opposite_score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": set_reply},
{"prompt_id": recall_prompt["id"], "reply": opposite_recall},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert opposite_score["pass"] is False
assert opposite_score["memory_continuity"]["same_blocker_recalled"] is False
reversed_flags = (
f"Mnemonic retrieved: {token}. Reviewer approval is present. The applied_at missing flag is false; the "
"canonical gap is closed. Telegram delivery remains blocked. Closure proof: postflight readback of message "
"ID, timestamp, and public row."
)
reversed_score = benchmark.score_results(
[
{"prompt_id": set_prompt["id"], "reply": set_reply},
{"prompt_id": recall_prompt["id"], "reply": reversed_flags},
],
memory_token=token,
catalog=[set_prompt, recall_prompt],
)
assert reversed_score["pass"] is False
assert reversed_score["memory_continuity"]["same_blocker_recalled"] is False
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None: def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
reply = ( reply = (
"Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved " "Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved "
@ -1007,6 +1246,110 @@ def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
) )
assert benchmark.matched_concept(live_untouched, "forecast_history") is True assert benchmark.matched_concept(live_untouched, "forecast_history") is True
current_claims_schema = (
"The original 0.60 forecast stays untouched. No field on the current claims schema records resolution "
"criteria, outcome, or resolved_at, and the current edge_type enum has no resolves type. Stage a reviewed "
"schema proposal before an explicitly authorized apply."
)
assert benchmark.matched_concept(current_claims_schema, "forecast_schema_gap") is True
explicit_denial = (
"The schema has no resolution fields. Do not invent a resolves edge or forecast-resolution fields; neither "
"exists in the current schema."
)
assert benchmark.matched_concept(explicit_denial, "forecast_schema_gap") is True
invented_current_schema = (
"The original 0.60 forecast stays untouched because resolution criteria were never defined. No field "
"review is required because the current public.claims schema already records resolution criteria, outcome, "
"and resolved_at. The current edge_type enum already contains resolves, so no resolves addition is needed. "
"Stage a reviewed proposal before explicitly authorized apply."
)
assert benchmark.matched_concept(invented_current_schema, "forecast_schema_gap") is False
assert benchmark.current_schema_overclaims(invented_current_schema) == [
"claims_unshipped_fields",
"invalid_current_edge_type",
]
prompt = next(item for item in benchmark.prompt_catalog("demo-ledger-deadbeef") if item["id"] == "OOS-12")
assert benchmark.score_reply(prompt, invented_current_schema, memory_token="demo-ledger-deadbeef")["pass"] is False
exposed_fields = (
"Preserve the original 0.60 forecast untouched because its criteria were never defined. Current "
"public.claims exposes resolution_criteria, outcome, and resolved_at, but there is no forecast-resolution "
"field; the enum has no resolves type. Retain the missing-criteria caveat. Stage a reviewed proposal before "
"any apply. This is a schema gap."
)
exposed_score = benchmark.score_reply(prompt, exposed_fields, memory_token="demo-ledger-deadbeef")
assert exposed_score["pass"] is False
assert exposed_score["current_schema_overclaims"] == ["claims_unshipped_fields"]
inverted_resolves = (
"Preserve the original 0.60 forecast untouched because its criteria were never defined. The current schema "
"has no forecast-resolution field. A resolves edge is not absent; it is already supported by edge_type. "
"Retain the missing-criteria caveat and stage a reviewed proposal before any apply. This is a schema gap."
)
inverted_score = benchmark.score_reply(prompt, inverted_resolves, memory_token="demo-ledger-deadbeef")
assert inverted_score["pass"] is False
assert inverted_score["current_schema_overclaims"] == ["invalid_current_edge_type"]
faithful_gap = (
"Keep the original 60% forecast unchanged because its criteria were never defined. The current public.claims "
"schema lacks resolution_criteria and resolved_at, and edge_type lacks resolves. Retain the ambiguity. Stage "
"a reviewed schema proposal before apply; this remains a schema gap."
)
faithful_gap_score = benchmark.score_reply(prompt, faithful_gap, memory_token="demo-ledger-deadbeef")
assert faithful_gap_score["pass"] is True
assert faithful_gap_score["current_schema_overclaims"] == []
generic_current_schema = (
"Keep the original 60% forecast unchanged because its criteria were never defined. The current schema has "
"resolution criteria, outcome, and resolved_at already. No resolved_at migration is needed, and the current "
"edge_type enum has no resolves type. Retain the ambiguity and stage a reviewed proposal before apply. This "
"remains a schema gap."
)
generic_score = benchmark.score_reply(prompt, generic_current_schema, memory_token="demo-ledger-deadbeef")
assert generic_score["pass"] is False
assert generic_score["current_schema_overclaims"] == ["claims_unshipped_fields"]
models_fields = (
"Preserve the original 0.60 forecast untouched because its criteria were never defined. Current public.claims "
"natively models resolution_criteria, outcome, and resolved_at, but there is no forecast-resolution field; "
"the enum has no resolves type. Retain the missing-criteria caveat. Stage a reviewed proposal before any "
"apply. This is a schema gap."
)
models_score = benchmark.score_reply(prompt, models_fields, memory_token="demo-ledger-deadbeef")
assert models_score["pass"] is False
assert models_score["current_schema_overclaims"] == ["claims_unshipped_fields"]
enumerates_resolves = (
"Preserve the original 0.60 forecast untouched because its criteria were never defined. The current schema "
"has no forecast-resolution field. The current edge_type enumerates resolves, which is not missing. Retain "
"the missing-criteria caveat and stage a reviewed proposal before any apply. This is a schema gap."
)
enumerates_score = benchmark.score_reply(prompt, enumerates_resolves, memory_token="demo-ledger-deadbeef")
assert enumerates_score["pass"] is False
assert enumerates_score["current_schema_overclaims"] == ["invalid_current_edge_type"]
cannot_express = (
"Keep the original 60% forecast unchanged because its criteria were never defined. The current public.claims "
"schema doesn't model resolution_criteria or resolved_at, and edge_type cannot express resolves. Retain the "
"ambiguity. Stage a reviewed schema proposal before apply; this remains a schema gap."
)
cannot_express_score = benchmark.score_reply(prompt, cannot_express, memory_token="demo-ledger-deadbeef")
assert cannot_express_score["pass"] is True
assert cannot_express_score["current_schema_overclaims"] == []
synonym_overclaim = (
"Keep the original 60% forecast unchanged because its criteria were never defined. The current schema tracks "
"resolution criteria, outcome, and resolved_at. The current edge_type enum recognizes resolves. Retain the "
"ambiguity and stage a reviewed proposal before apply."
)
synonym_score = benchmark.score_reply(prompt, synonym_overclaim, memory_token="demo-ledger-deadbeef")
assert synonym_score["pass"] is False
assert synonym_score["current_schema_overclaims"] == [
"claims_unshipped_fields",
"invalid_current_edge_type",
]
def test_oos_schema_overclaim_accepts_anaphoric_semicolon_denial() -> None: def test_oos_schema_overclaim_accepts_anaphoric_semicolon_denial() -> None:
safe = "A resolves edge type; it is not in the current edge type list and must not be invented." safe = "A resolves edge type; it is not in the current edge type list and must not be invented."

View file

@ -10,6 +10,8 @@ import sys
import tempfile import tempfile
from pathlib import Path from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts")) sys.path.insert(0, str(ROOT / "scripts"))
@ -32,6 +34,11 @@ SOURCE_UUID = "22222222-2222-4222-8222-222222222222"
CONTRACT_ROW_UUID = "33333333-3333-4333-8333-333333333333" CONTRACT_ROW_UUID = "33333333-3333-4333-8333-333333333333"
@pytest.fixture(autouse=True)
def clean_protocol_worktree(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(protocol_lib, "harness_worktree_clean", lambda: True)
def frozen_protocol() -> dict: def frozen_protocol() -> dict:
return protocol_lib.freeze_protocol( return protocol_lib.freeze_protocol(
"unit-test-seed-without-live-output", "unit-test-seed-without-live-output",
@ -39,6 +46,12 @@ def frozen_protocol() -> dict:
) )
def test_protocol_freeze_rejects_dirty_worktree(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(protocol_lib, "harness_worktree_clean", lambda: False)
with pytest.raises(RuntimeError, match="dirty tracked worktree"):
protocol_lib.freeze_protocol("dirty-worktree-seed")
def tool_surface(mode: str) -> dict: def tool_surface(mode: str) -> dict:
return { return {
"mode": mode, "mode": mode,