Harden Leo benchmark semantic invariants
This commit is contained in:
parent
3b0e1bc296
commit
2712e7fbb5
5 changed files with 618 additions and 17 deletions
|
|
@ -444,7 +444,11 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||||
r"(?:shared source material|observation).{0,100}(?:one|single) claim row|"
|
r"(?:shared source material|observation).{0,100}(?:one|single) claim row|"
|
||||||
r"shared (?:source material|evidence).{0,100}one canonical claim|"
|
r"shared (?:source material|evidence).{0,100}one canonical claim|"
|
||||||
r"one shared (?:structural |canonical )?claim row|"
|
r"one shared (?:structural |canonical )?claim row|"
|
||||||
|
r"(?:single|one) canonical assertion|"
|
||||||
|
r"deduplicate.{0,80}(?:single|one) canonical (?:assertion|record)|"
|
||||||
r"share the fact.{0,100}one claim row|"
|
r"share the fact.{0,100}one claim row|"
|
||||||
|
r"keep (?:the )?(?:fact|claim|proposition) (?:canonical )?once|"
|
||||||
|
r"one claim row.{0,80}(?:holds|contains|records).{0,60}(?:agreed|shared|factual) (?:fact|claim|proposition)|"
|
||||||
r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
|
r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
|
||||||
re.I | re.S,
|
re.I | re.S,
|
||||||
),
|
),
|
||||||
|
|
@ -454,7 +458,9 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||||
r"one shared claim|single shared claim|"
|
r"one shared claim|single shared claim|"
|
||||||
r"duplicating (?:(?:a|the) )?(?:factual )?claim|"
|
r"duplicating (?:(?:a|the) )?(?:factual )?claim|"
|
||||||
r"one (?:canonical |structural )?claim.{0,100}shared (?:sources?|evidence|claim_evidence)|"
|
r"one (?:canonical |structural )?claim.{0,100}shared (?:sources?|evidence|claim_evidence)|"
|
||||||
r"one claim row.{0,100}shared (?:sources?|evidence|claim_evidence)",
|
r"one claim row.{0,180}(?:(?:both|each) agents?'|shared) (?:sources?|evidence|claim_evidence)|"
|
||||||
|
r"(?:both|each) (?:agents?|analysts?)' (?:sources?|source evidence|evidence)|"
|
||||||
|
r"both (?:agents?|analysts?)' evidence.{0,80}(?:one|single) (?:record|assertion|claim)",
|
||||||
re.I | re.S,
|
re.I | re.S,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
|
@ -495,11 +501,20 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||||
),
|
),
|
||||||
"forecast_schema_gap": (
|
"forecast_schema_gap": (
|
||||||
re.compile(
|
re.compile(
|
||||||
r"current (?:v1|schema|columns?|fields?|edge types?)|public\.claims|the schema|schema has|claims table",
|
r"current (?:v1|schema|columns?|fields?|edge types?|(?:public\.)?claims schema|edge_type enum)|"
|
||||||
|
r"public\.claims|the schema|schema has|claims table",
|
||||||
re.I,
|
re.I,
|
||||||
),
|
),
|
||||||
re.compile(
|
re.compile(
|
||||||
r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|"
|
r"(?:there is |the schema has |(?:public\.)?claims has )?no "
|
||||||
|
r"(?:forecast[- ]resolution|resolution fields?|resolved_at)"
|
||||||
|
r"(?!\s+(?:migration|review|proposal|addition|change|work)\b)|"
|
||||||
|
r"does not have.{0,50}resolution|"
|
||||||
|
r"(?:schema|claims).{0,80}lacks?.{0,60}(?:resolution criteria|resolution_criteria|outcome|resolved_at)|"
|
||||||
|
r"(?:schema|claims).{0,80}(?:does not|doesn't|cannot) (?:model|express|represent|record|store)"
|
||||||
|
r".{0,60}(?:resolution criteria|resolution_criteria|outcome|resolved_at)|"
|
||||||
|
r"no (?:column|field) (?:on|in) .{0,100}(?:records?|stores?|contains?).{0,100}"
|
||||||
|
r"(?:resolution criteria|outcome|resolved_at)|"
|
||||||
r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
|
r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
|
||||||
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
|
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
|
||||||
r"(?:does not exist|is absent|is not present|isn't present)|"
|
r"(?:does not exist|is absent|is not present|isn't present)|"
|
||||||
|
|
@ -511,7 +526,15 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||||
re.I | re.S,
|
re.I | re.S,
|
||||||
),
|
),
|
||||||
re.compile(
|
re.compile(
|
||||||
r"resolves.{0,220}(?:not|isn'?t|does not|doesn't|absent)|no.{0,30}resolves",
|
r"resolves (?:edge|type).{0,100}(?:does not exist|is absent|is missing|is not present|isn't present|"
|
||||||
|
r"is not current|isn't current|is unsupported)|"
|
||||||
|
r"(?:there is|schema has|enum has|edge types? (?:has|have)|(?:current )?edge_type enum has) "
|
||||||
|
r"no `?resolves`?(?: (?:edge|type))?|"
|
||||||
|
r"no `?resolves`? (?:edge|type)(?: exists| is present)?|"
|
||||||
|
r"(?:edge_type|edge type|enum) lacks? resolves|"
|
||||||
|
r"(?:edge_type|edge type|enum) (?:cannot|does not|doesn't) (?:express|represent|support) resolves|"
|
||||||
|
r"(?:do not|don't|must not) invent (?:an? )?resolves (?:edge|type)|"
|
||||||
|
r"resolves (?:edge|type).{0,120}neither (?:exists|is present)",
|
||||||
re.I | re.S,
|
re.I | re.S,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
|
@ -655,15 +678,18 @@ COUNT_INVARIANT_REJECTION_RE = re.compile(
|
||||||
|
|
||||||
SCHEMA_GAP_QUALIFIER_RE = re.compile(
|
SCHEMA_GAP_QUALIFIER_RE = re.compile(
|
||||||
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
|
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
|
||||||
r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|"
|
r"has no|have no|lacks?|without|there is no|no column|not an edge|would require|schema gap|must be added|"
|
||||||
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
|
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
|
||||||
r"schema extension|extension proposal|reviewed schema proposal|not (?:in|part of|among) the current)\b|"
|
r"schema extension|extension proposal|reviewed schema proposal|not (?:in|part of|among) the current)\b|"
|
||||||
|
r"\b(?:does not|doesn't|cannot)\s+"
|
||||||
|
r"(?:record|store|contain|include|have|expose|provide|support|model|express|represent)\b|"
|
||||||
r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?"
|
r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?"
|
||||||
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
|
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
|
||||||
r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?"
|
r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?"
|
||||||
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
|
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
|
||||||
r"\b(?:proposal|extension)\b.{0,100}\b(?:column|field|table|edge type)\b|"
|
r"\b(?:proposal|extension)\b.{0,100}\b(?:column|field|table|edge type)\b|"
|
||||||
r"\bno\s+(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b",
|
r"\bno\s+(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b"
|
||||||
|
r"(?!\s+(?:review|proposal|change|addition|work)\b)",
|
||||||
re.I,
|
re.I,
|
||||||
)
|
)
|
||||||
SCHEMA_CLAUSE_BOUNDARY_RE = re.compile(
|
SCHEMA_CLAUSE_BOUNDARY_RE = re.compile(
|
||||||
|
|
@ -674,12 +700,23 @@ SCHEMA_CLAUSE_BOUNDARY_RE = re.compile(
|
||||||
)
|
)
|
||||||
CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
||||||
"claims_unshipped_fields": re.compile(
|
"claims_unshipped_fields": re.compile(
|
||||||
r"claims\s+(?:table|schema)\b.{0,80}\b(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b|"
|
r"(?:the )?current schema\b.{0,120}\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|"
|
||||||
r"public\.claims\s+(?:stores?|has|contains?)\s+(?:an?\s+)?"
|
r"resolved_at|falsifier)\b|"
|
||||||
r"(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b|"
|
r"public\.claims\b.{0,120}\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|"
|
||||||
|
r"falsifier)\b|"
|
||||||
|
r"(?:the )?current schema\s+(?:already\s+)?"
|
||||||
|
r"(?:stores?|has|contains?|records?|exposes?|provides?|supports?|includes?|models?|represents?)\s+.{0,80}"
|
||||||
|
r"\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
|
||||||
|
r"claims\s+(?:table|schema)\b.{0,100}\b(?:body|metadata|forecast[_ -]resolution|"
|
||||||
|
r"resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
|
||||||
|
r"public\.claims\s+(?:(?:natively|already|currently)\s+)?"
|
||||||
|
r"(?:stores?|has|contains?|records?|exposes?|provides?|supports?|includes?|models?|represents?)\s+"
|
||||||
|
r"(?:an?\s+)?"
|
||||||
|
r"(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
|
||||||
r"public\.claims\b.{0,80}\b(?:column|field)\b.{0,30}"
|
r"public\.claims\b.{0,80}\b(?:column|field)\b.{0,30}"
|
||||||
r"\b(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b|"
|
r"\b(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
|
||||||
r"public\.claims\b.{0,80}\b(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b.{0,30}"
|
r"public\.claims\b.{0,80}\b(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|"
|
||||||
|
r"outcome|resolved_at|falsifier)\b.{0,30}"
|
||||||
r"\b(?:column|field)\b",
|
r"\b(?:column|field)\b",
|
||||||
re.I,
|
re.I,
|
||||||
),
|
),
|
||||||
|
|
@ -689,9 +726,15 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
||||||
re.I,
|
re.I,
|
||||||
),
|
),
|
||||||
"invalid_current_edge_type": re.compile(
|
"invalid_current_edge_type": re.compile(
|
||||||
|
r"\b(?:current )?edge_type enum\b.{0,80}\bresolves\b|"
|
||||||
|
r"\bcurrent edge_type\b.{0,80}\bresolves\b|"
|
||||||
r"(?:\b(?:claim_edges?|edge type|edge)\b\s*(?:is|=|:|named|called|of)?\s*[`'\"]?"
|
r"(?:\b(?:claim_edges?|edge type|edge)\b\s*(?:is|=|:|named|called|of)?\s*[`'\"]?"
|
||||||
r"(?:superseded_by|relates_to|resolves|derived_from)\b|"
|
r"(?:superseded_by|relates_to|resolves|derived_from)\b|"
|
||||||
r"\b(?:superseded_by|relates_to|resolves|derived_from)\b[`'\"]?\s+(?:claim_)?edge\b)",
|
r"\b(?:superseded_by|relates_to|resolves|derived_from)\b[`'\"]?\s+(?:claim_)?edge\b|"
|
||||||
|
r"\b(?:current )?edge_type enum\b.{0,50}\b(?:contains|has|includes|supports|enumerates)\b.{0,20}"
|
||||||
|
r"\bresolves\b|"
|
||||||
|
r"\b(?:current )?edge_type\b.{0,50}\b(?:contains|has|includes|supports|enumerates)\b.{0,20}"
|
||||||
|
r"\bresolves\b)",
|
||||||
re.I,
|
re.I,
|
||||||
),
|
),
|
||||||
"unshipped_edge_rationale": re.compile(r"\bclaim_edges?\b.{0,100}\brationale\b", re.I),
|
"unshipped_edge_rationale": re.compile(r"\bclaim_edges?\b.{0,100}\brationale\b", re.I),
|
||||||
|
|
@ -721,6 +764,17 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
UNVERIFIED_M3TAVERSAL_ALIAS_RE = re.compile(r"\b(?:Cory|m3ta)\b", re.I)
|
UNVERIFIED_M3TAVERSAL_ALIAS_RE = re.compile(r"\b(?:Cory|m3ta)\b", re.I)
|
||||||
|
RESOLVES_PRESENCE_ASSERTION_RE = re.compile(
|
||||||
|
r"\bresolves (?:edge|type)\b.{0,120}\b(?:is not (?:absent|missing)|already (?:exists|supported|present)|"
|
||||||
|
r"is (?:already )?supported by (?:the )?(?:current )?edge_type)\b|"
|
||||||
|
r"\b(?:current )?edge_type\b.{0,60}\b(?:already )?(?:contains|has|includes|supports|enumerates)\b.{0,20}"
|
||||||
|
r"\bresolves\b",
|
||||||
|
re.I | re.S,
|
||||||
|
)
|
||||||
|
EDGE_TYPE_RESOLVES_DENIAL_RE = re.compile(
|
||||||
|
r"\b(?:current )?edge_type\b.{0,50}\b(?:does not|doesn't|cannot|lacks?|has no)\b.{0,40}\bresolves\b",
|
||||||
|
re.I | re.S,
|
||||||
|
)
|
||||||
|
|
||||||
SOURCE_EVIDENCE_CANONICAL_OBJECT_RE = re.compile(r"claim_evidence|public\.sources|source rows?", re.I)
|
SOURCE_EVIDENCE_CANONICAL_OBJECT_RE = re.compile(r"claim_evidence|public\.sources|source rows?", re.I)
|
||||||
SOURCE_EVIDENCE_LOCATOR_GAP_RE = re.compile(
|
SOURCE_EVIDENCE_LOCATOR_GAP_RE = re.compile(
|
||||||
|
|
@ -886,6 +940,41 @@ BLOCKER_DIAGNOSTIC_RE = re.compile(
|
||||||
r"source|evidence|claim|proposal|runtime|restart|identity|tool|readback)\b",
|
r"source|evidence|claim|proposal|runtime|restart|identity|tool|readback)\b",
|
||||||
re.I,
|
re.I,
|
||||||
)
|
)
|
||||||
|
BLOCKER_FAILURE_STATE_RE = re.compile(
|
||||||
|
r"\b(?:no|none|zero|missing|absent|null|unavailable|unsupported|unapplied|not|cannot|can't|blocked|"
|
||||||
|
r"fails?|stale|wrong|gap|lacks?|without)\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
BLOCKER_ABSENCE_RE = re.compile(
|
||||||
|
r"\b(?:there is |there's )?no (?:[a-z][a-z-]*\s+){0,4}blocker "
|
||||||
|
r"(?:exists|remains|is present)\b|\bnothing (?:is )?blocked\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
PER_AGENT_CLAIM_DUPLICATION_RE = re.compile(
|
||||||
|
r"\b(?:(?:one|separate|distinct|individual)\s+)?(?:public\.)?claims?(?:\s+row)?\s+per\s+agent\b|"
|
||||||
|
r"\b(?:claim|public\.claims)(?:\s+row)?(?:\s+once)?\s+(?:for|to)\s+each\s+agent\b|"
|
||||||
|
r"\beach\s+agent\s+(?:gets?|receives?|has|stores?|uses?)\s+"
|
||||||
|
r"(?:(?:its|their)\s+own|an?\s+separate|one)\s+(?:public\.)?claims?(?:\s+row)?\b|"
|
||||||
|
r"\b(?:give|assign)\s+each\s+agent\s+(?:(?:its|their)\s+own|an?\s+separate|one)\s+"
|
||||||
|
r"(?:public\.)?claims?(?:\s+row)?\b|"
|
||||||
|
r"\bone\s+(?:public\.)?claims?(?:\s+row)?\s+for\s+[A-Z][a-z0-9_-]+\s+and\s+one\s+"
|
||||||
|
r"(?:(?:public\.)?claims?(?:\s+row)?\s+)?for\s+[A-Z][a-z0-9_-]+\b|"
|
||||||
|
r"\b(?:fact|proposition|claim|assertion)\b.{0,80}\b(?:under|within|inside)\s+each\s+agent\b|"
|
||||||
|
r"\b(?:their|the agents?'?)\s+respective\s+(?:copies|claim rows?|records?)\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
PER_AGENT_DUPLICATION_PREFIX_DENIAL_RE = re.compile(
|
||||||
|
r"(?:\b(?:do not|don't|does not|doesn't|should not|shouldn't|must not|never|avoid)\b\s*"
|
||||||
|
r"(?:(?:create|store|write|give|assign|duplicate|fork|make|keep|use|add)(?:ing)?\s+)?"
|
||||||
|
r"(?:(?:the|an?|one)\s+)?(?:(?:factual|canonical|shared|separate|distinct)\s+)?|"
|
||||||
|
r"\b(?:instead of|rather than|not)\s+)$",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
PER_AGENT_DUPLICATION_SUFFIX_DENIAL_RE = re.compile(
|
||||||
|
r"^.{0,80}\b(?:is|would be|remains?)\s+(?:wrong|incorrect|invalid|duplicative)\b|"
|
||||||
|
r"^.{0,80}\b(?:creates?|causes?|would create)\s+(?:data\s+)?(?:divergence|fragmentation)\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
|
def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
|
||||||
|
|
@ -902,6 +991,11 @@ def extract_blocker_clause(reply: str) -> str | None:
|
||||||
for pattern in (
|
for pattern in (
|
||||||
re.compile(r"\bblocker(?:\s+restated)?\s*:\s*(?P<value>[^\n]+)", re.I),
|
re.compile(r"\bblocker(?:\s+restated)?\s*:\s*(?P<value>[^\n]+)", re.I),
|
||||||
re.compile(r"\bblocker\s+(?:is|was)\s+(?P<value>[^\n]+)", re.I),
|
re.compile(r"\bblocker\s+(?:is|was)\s+(?P<value>[^\n]+)", re.I),
|
||||||
|
re.compile(
|
||||||
|
r"\b(?:mnemonic|label)(?:\s+(?:tag|retrieved))?\s*:?[^.\n]{0,140}?"
|
||||||
|
r"(?:[.]|\s+(?:-|\u2014)\s+)\s*(?P<value>[^\n]+)",
|
||||||
|
re.I,
|
||||||
|
),
|
||||||
):
|
):
|
||||||
match = pattern.search(reply)
|
match = pattern.search(reply)
|
||||||
if match:
|
if match:
|
||||||
|
|
@ -920,11 +1014,120 @@ def blocker_terms(value: str | None, *, memory_token: str) -> set[str]:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def blocker_signature(value: str | None) -> set[str]:
|
||||||
|
"""Extract polarity-aware blocker categories for same-session recall."""
|
||||||
|
|
||||||
|
if not value:
|
||||||
|
return set()
|
||||||
|
signatures: set[str] = set()
|
||||||
|
clauses = [
|
||||||
|
clause.strip()
|
||||||
|
for clause in re.split(r"(?<=[.!?;])\s+|\n+|,\s+(?:but|and|so)\s+", value, flags=re.I)
|
||||||
|
if clause.strip()
|
||||||
|
]
|
||||||
|
if re.search(r"\b(?:approved|reviewer approval|reviewer signature|reviewer sign[- ]off)\b", value, re.I):
|
||||||
|
signatures.add("reviewer_approval")
|
||||||
|
applied_missing = re.compile(
|
||||||
|
r"\bapproved[- ]but[- ]not[- ]applied\b|\b(?:not applied|unapplied)\b|"
|
||||||
|
r"\b(?:zero|no|missing|absent)\s+applied_at(?:\s+values?)?\b|"
|
||||||
|
r"\bapplied_at\b.{0,20}\b(?:unset|missing|absent|zero|none|null|not set)\b|"
|
||||||
|
r"\b(?:unset|missing|absent|zero|none|null)\b.{0,20}\bapplied_at\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
applied_present = re.compile(
|
||||||
|
r"\bapplied_at\b.{0,30}\b(?:non[- ]null|not null|exists|present|is set|has a value|"
|
||||||
|
r"missing (?:flag )?(?:is|=) false|is not missing)\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
if any(applied_missing.search(clause) and not applied_present.search(clause) for clause in clauses):
|
||||||
|
signatures.add("applied_missing")
|
||||||
|
canonical_missing = re.compile(
|
||||||
|
r"\bcanonical (?:gap|row|readback)\b.{0,30}\b(?:missing|absent|none|null|zero|not (?:live|written)|"
|
||||||
|
r"unavailable)\b|"
|
||||||
|
r"\b(?:no|zero|missing|absent)\b.{0,25}\b(?:canonical|public\.\*)\b.{0,25}\brows?\b|"
|
||||||
|
r"\b(?:no|zero|missing|absent)\s+public\.\*\s+row(?: ID)?\b|"
|
||||||
|
r"\bno rows?\b.{0,30}\b(?:landed|written|applied)\b.{0,20}\bpublic\.(?:claims|sources|\*)\b|"
|
||||||
|
r"\bapproved[- ]but[- ]not[- ]applied canonical gap\b|\bcanonical gap\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
canonical_present = re.compile(
|
||||||
|
r"\bcanonical (?:row|readback)\b.{0,20}\b(?:exists|present|current|complete|is live)\b|"
|
||||||
|
r"\bcanonical rows?\b.{0,20}\b(?:exist|are present|are current|are complete)\b|"
|
||||||
|
r"\bcanonical gap\b.{0,20}\b(?:is|was|has been|now)\s+(?:closed|resolved|cleared|fixed)\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
if any(canonical_missing.search(clause) and not canonical_present.search(clause) for clause in clauses):
|
||||||
|
signatures.add("canonical_missing")
|
||||||
|
telegram_missing = re.compile(
|
||||||
|
r"\b(?:telegram|delivery|visible message|message receipt)\b.{0,50}"
|
||||||
|
r"\b(?:no|missing|absent|blocked|unproven|not proven|unavailable)\b|"
|
||||||
|
r"\b(?:no|missing|absent|blocked|unproven|not proven|unavailable)\b.{0,50}"
|
||||||
|
r"\b(?:telegram|delivery|visible message|message receipt)\b",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
|
if any(telegram_missing.search(clause) for clause in clauses):
|
||||||
|
signatures.add("telegram_missing")
|
||||||
|
if re.search(
|
||||||
|
r"\b(?:beliefs?\b.{0,40})?(?:has|have|with)\s+no\s+claim[_ -]?id\s+foreign key\b|"
|
||||||
|
r"\b(?:missing|absent|no)\b.{0,50}\b(?:foreign key|belief[- ]to[- ]claim link)\b|"
|
||||||
|
r"\bschema gap\b",
|
||||||
|
value,
|
||||||
|
re.I | re.S,
|
||||||
|
):
|
||||||
|
signatures.add("schema_link_missing")
|
||||||
|
if (
|
||||||
|
re.search(r"\bbeliefs?(?:\.claim[_ -]?id)?\b", value, re.I)
|
||||||
|
and re.search(r"\bclaim[_ -]?id\b", value, re.I)
|
||||||
|
and re.search(r"\b(?:foreign key|FK)\b", value, re.I)
|
||||||
|
and BLOCKER_FAILURE_STATE_RE.search(value)
|
||||||
|
):
|
||||||
|
signatures.add("belief_claim_foreign_key_missing")
|
||||||
|
if (
|
||||||
|
re.search(r"\bbeliefs?(?:\.claim[_ -]?id)?\b", value, re.I)
|
||||||
|
and re.search(r"\bclaim[_ -]?id\b", value, re.I)
|
||||||
|
and re.search(r"\bindex\b", value, re.I)
|
||||||
|
and BLOCKER_FAILURE_STATE_RE.search(value)
|
||||||
|
):
|
||||||
|
signatures.add("belief_claim_index_missing")
|
||||||
|
if re.search(
|
||||||
|
r"\bpublic\.sources\b.{0,80}\b(?:no|missing|absent|lacks?)\b.{0,40}\bforeign key\b|"
|
||||||
|
r"\b(?:no|missing|absent)\b.{0,40}\bforeign key\b.{0,80}\bpublic\.sources\b",
|
||||||
|
value,
|
||||||
|
re.I | re.S,
|
||||||
|
):
|
||||||
|
signatures.add("source_foreign_key_missing")
|
||||||
|
return signatures
|
||||||
|
|
||||||
|
|
||||||
|
def asserts_per_agent_claim_duplication(reply: str) -> bool:
|
||||||
|
"""Reject affirmative per-agent duplication while allowing explicit denials."""
|
||||||
|
|
||||||
|
clauses = re.split(r"(?<=[.!?;])\s+|\n+|,\s+(?:but|and|so)\s+", reply, flags=re.I)
|
||||||
|
for clause in clauses:
|
||||||
|
for match in PER_AGENT_CLAIM_DUPLICATION_RE.finditer(clause):
|
||||||
|
prefix = clause[max(0, match.start() - 80) : match.start()]
|
||||||
|
suffix = clause[match.end() : match.end() + 100]
|
||||||
|
if PER_AGENT_DUPLICATION_PREFIX_DENIAL_RE.search(prefix):
|
||||||
|
continue
|
||||||
|
if PER_AGENT_DUPLICATION_SUFFIX_DENIAL_RE.search(suffix):
|
||||||
|
continue
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def matched_concept(reply: str, concept: str) -> bool:
|
def matched_concept(reply: str, concept: str) -> bool:
|
||||||
if concept == "blocker_definition":
|
if concept == "blocker_definition":
|
||||||
blocker = extract_blocker_clause(reply)
|
blocker = extract_blocker_clause(reply)
|
||||||
terms = blocker_terms(blocker, memory_token="")
|
terms = blocker_terms(blocker, memory_token="")
|
||||||
return bool(blocker and len(terms) >= 3 and BLOCKER_DIAGNOSTIC_RE.search(blocker))
|
return bool(
|
||||||
|
blocker
|
||||||
|
and len(terms) >= 3
|
||||||
|
and BLOCKER_DIAGNOSTIC_RE.search(blocker)
|
||||||
|
and BLOCKER_FAILURE_STATE_RE.search(blocker)
|
||||||
|
and not BLOCKER_ABSENCE_RE.search(blocker)
|
||||||
|
)
|
||||||
|
if concept == "shared_knowledge_commons" and asserts_per_agent_claim_duplication(reply):
|
||||||
|
return False
|
||||||
if concept == "claim_body_evidence_distinction":
|
if concept == "claim_body_evidence_distinction":
|
||||||
if CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
|
if CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
|
||||||
return False
|
return False
|
||||||
|
|
@ -963,6 +1166,8 @@ def current_schema_overclaims(reply: str) -> list[str]:
|
||||||
findings: list[str] = []
|
findings: list[str] = []
|
||||||
sentence_segments = re.split(r"(?<=[.!?])\s+|\n+", reply)
|
sentence_segments = re.split(r"(?<=[.!?])\s+|\n+", reply)
|
||||||
for sentence in sentence_segments:
|
for sentence in sentence_segments:
|
||||||
|
if RESOLVES_PRESENCE_ASSERTION_RE.search(sentence) and not EDGE_TYPE_RESOLVES_DENIAL_RE.search(sentence):
|
||||||
|
findings.append("invalid_current_edge_type")
|
||||||
# Negation and future-schema qualifiers apply to their clause, not to a
|
# Negation and future-schema qualifiers apply to their clause, not to a
|
||||||
# contradictory assertion in a later conjunction in the same sentence.
|
# contradictory assertion in a later conjunction in the same sentence.
|
||||||
clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence)
|
clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence)
|
||||||
|
|
@ -1211,11 +1416,25 @@ def score_results(
|
||||||
memory_source_reply = str((by_result.get(memory_source_id) or {}).get("reply") or "")
|
memory_source_reply = str((by_result.get(memory_source_id) or {}).get("reply") or "")
|
||||||
memory_recall_reply = str((by_result.get(memory_recall_id) or {}).get("reply") or "")
|
memory_recall_reply = str((by_result.get(memory_recall_id) or {}).get("reply") or "")
|
||||||
source_clause = extract_blocker_clause(memory_source_reply)
|
source_clause = extract_blocker_clause(memory_source_reply)
|
||||||
|
recall_clause = extract_blocker_clause(memory_recall_reply)
|
||||||
source_terms = blocker_terms(source_clause, memory_token=memory_token)
|
source_terms = blocker_terms(source_clause, memory_token=memory_token)
|
||||||
recall_terms = blocker_terms(extract_blocker_clause(memory_recall_reply), memory_token=memory_token)
|
recall_terms = blocker_terms(recall_clause, memory_token=memory_token)
|
||||||
overlap_terms = source_terms & recall_terms
|
overlap_terms = source_terms & recall_terms
|
||||||
required_overlap = min(3, max(1, (len(source_terms) + 2) // 3)) if source_terms else 1
|
required_overlap = min(3, max(1, (len(source_terms) + 2) // 3)) if source_terms else 1
|
||||||
same_blocker_recalled = bool(source_clause and len(overlap_terms) >= required_overlap)
|
source_signature = blocker_signature(source_clause)
|
||||||
|
recall_signature = blocker_signature(recall_clause)
|
||||||
|
source_failure_signature = source_signature - {"reviewer_approval"}
|
||||||
|
recall_failure_signature = recall_signature - {"reviewer_approval"}
|
||||||
|
signature_overlap = source_failure_signature & recall_failure_signature
|
||||||
|
required_signature_overlap = min(2, len(source_failure_signature)) if source_failure_signature else 0
|
||||||
|
same_blocker_recalled = bool(
|
||||||
|
source_clause
|
||||||
|
and (
|
||||||
|
len(signature_overlap) >= required_signature_overlap and len(overlap_terms) >= required_overlap
|
||||||
|
if source_failure_signature
|
||||||
|
else len(overlap_terms) >= required_overlap
|
||||||
|
)
|
||||||
|
)
|
||||||
for score in scores:
|
for score in scores:
|
||||||
if score["prompt_id"] != memory_recall_id:
|
if score["prompt_id"] != memory_recall_id:
|
||||||
continue
|
continue
|
||||||
|
|
@ -1227,6 +1446,10 @@ def score_results(
|
||||||
"source_blocker_terms": sorted(source_terms),
|
"source_blocker_terms": sorted(source_terms),
|
||||||
"recall_overlap_terms": sorted(overlap_terms),
|
"recall_overlap_terms": sorted(overlap_terms),
|
||||||
"required_overlap": required_overlap,
|
"required_overlap": required_overlap,
|
||||||
|
"source_blocker_signature": sorted(source_signature),
|
||||||
|
"recall_blocker_signature": sorted(recall_signature),
|
||||||
|
"signature_overlap": sorted(signature_overlap),
|
||||||
|
"required_signature_overlap": required_signature_overlap,
|
||||||
"same_blocker_recalled": same_blocker_recalled,
|
"same_blocker_recalled": same_blocker_recalled,
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ PROTOCOL_SCHEMA = "livingip.leoM3taversalOosProtocol.v3"
|
||||||
TRIAL_SCORE_SCHEMA = "livingip.leoM3taversalOosTrialScore.v3"
|
TRIAL_SCORE_SCHEMA = "livingip.leoM3taversalOosTrialScore.v3"
|
||||||
AGGREGATE_SCHEMA = "livingip.leoM3taversalOosAggregate.v3"
|
AGGREGATE_SCHEMA = "livingip.leoM3taversalOosAggregate.v3"
|
||||||
GENERATOR_VERSION = "blinded-family-generator-v5"
|
GENERATOR_VERSION = "blinded-family-generator-v5"
|
||||||
SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v9"
|
SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v10"
|
||||||
BASELINE_VERSION = "live-current-build-db-tool-ablation-v3"
|
BASELINE_VERSION = "live-current-build-db-tool-ablation-v3"
|
||||||
DEFAULT_TRIAL_COUNT = 3
|
DEFAULT_TRIAL_COUNT = 3
|
||||||
MEMORY_SCORER_IDS = frozenset({"OOS-07", "OOS-08"})
|
MEMORY_SCORER_IDS = frozenset({"OOS-07", "OOS-08"})
|
||||||
|
|
@ -403,6 +403,14 @@ def harness_git_head() -> str:
|
||||||
).strip()
|
).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def harness_worktree_clean() -> bool:
|
||||||
|
return not subprocess.check_output(
|
||||||
|
["git", "status", "--porcelain", "--untracked-files=no"],
|
||||||
|
cwd=ROOT,
|
||||||
|
text=True,
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
|
||||||
def instrument_db_context_plugin_source(source: str) -> str:
|
def instrument_db_context_plugin_source(source: str) -> str:
|
||||||
marker = """ safe["receipt_sha256"] = hashlib.sha256(
|
marker = """ safe["receipt_sha256"] = hashlib.sha256(
|
||||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
|
@ -562,6 +570,8 @@ def freeze_protocol(
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
if trial_count < 3:
|
if trial_count < 3:
|
||||||
raise ValueError("at least three trials are required for clean/restart variance")
|
raise ValueError("at least three trials are required for clean/restart variance")
|
||||||
|
if not harness_worktree_clean():
|
||||||
|
raise RuntimeError("refusing to freeze a protocol from a dirty tracked worktree")
|
||||||
modes = ["clean_session"] * (trial_count - 1) + ["post_restart_clean_session"]
|
modes = ["clean_session"] * (trial_count - 1) + ["post_restart_clean_session"]
|
||||||
trials = [build_blinded_trial(seed, index, session_mode=modes[index]) for index in range(trial_count)]
|
trials = [build_blinded_trial(seed, index, session_mode=modes[index]) for index in range(trial_count)]
|
||||||
protocol: dict[str, Any] = {
|
protocol: dict[str, Any] = {
|
||||||
|
|
@ -610,12 +620,15 @@ def freeze_protocol(
|
||||||
"no_supplied_row_ids": True,
|
"no_supplied_row_ids": True,
|
||||||
"prompt_variants_per_family": min(len(family["variants"]) for family in BLINDED_FAMILIES),
|
"prompt_variants_per_family": min(len(family["variants"]) for family in BLINDED_FAMILIES),
|
||||||
},
|
},
|
||||||
|
"harness_worktree_clean_at_freeze": True,
|
||||||
"harness_git_head": harness_git_head(),
|
"harness_git_head": harness_git_head(),
|
||||||
"source_hashes": {key: file_sha256(path) for key, path in source_paths().items()},
|
"source_hashes": {key: file_sha256(path) for key, path in source_paths().items()},
|
||||||
"trials": trials,
|
"trials": trials,
|
||||||
}
|
}
|
||||||
protocol["protocol_hash_sha256"] = canonical_sha256(protocol)
|
protocol["protocol_hash_sha256"] = canonical_sha256(protocol)
|
||||||
validate_protocol(protocol, verify_source_hashes=True)
|
validation = validate_protocol(protocol, verify_source_hashes=True)
|
||||||
|
if not validation["pass"]:
|
||||||
|
raise RuntimeError("frozen protocol failed validation: " + ", ".join(validation["issues"]))
|
||||||
return protocol
|
return protocol
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -631,6 +644,8 @@ def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) -
|
||||||
issues.append("wrong_baseline_version")
|
issues.append("wrong_baseline_version")
|
||||||
if not _valid_git_revision(protocol.get("harness_git_head")):
|
if not _valid_git_revision(protocol.get("harness_git_head")):
|
||||||
issues.append("invalid_harness_git_head")
|
issues.append("invalid_harness_git_head")
|
||||||
|
if protocol.get("harness_worktree_clean_at_freeze") is not True:
|
||||||
|
issues.append("harness_worktree_not_clean_at_freeze")
|
||||||
supplied_hash = protocol.get("protocol_hash_sha256")
|
supplied_hash = protocol.get("protocol_hash_sha256")
|
||||||
unhashed = {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"}
|
unhashed = {key: value for key, value in protocol.items() if key != "protocol_hash_sha256"}
|
||||||
if supplied_hash != canonical_sha256(unhashed):
|
if supplied_hash != canonical_sha256(unhashed):
|
||||||
|
|
@ -700,6 +715,8 @@ def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) -
|
||||||
if len(seen) < min(3, len(trials)):
|
if len(seen) < min(3, len(trials)):
|
||||||
issues.append(f"variant_repetition:{family_id}")
|
issues.append(f"variant_repetition:{family_id}")
|
||||||
if verify_source_hashes:
|
if verify_source_hashes:
|
||||||
|
if not harness_worktree_clean():
|
||||||
|
issues.append("harness_worktree_dirty")
|
||||||
if protocol.get("harness_git_head") != harness_git_head():
|
if protocol.get("harness_git_head") != harness_git_head():
|
||||||
issues.append("harness_git_head_changed_after_freeze")
|
issues.append("harness_git_head_changed_after_freeze")
|
||||||
source_hashes = protocol.get("source_hashes") or {}
|
source_hashes = protocol.get("source_hashes") or {}
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,11 @@ import run_leo_m3taversal_oos_handler_suite as suite # noqa: E402
|
||||||
import working_leo_m3taversal_oos_protocol as protocol_lib # noqa: E402
|
import working_leo_m3taversal_oos_protocol as protocol_lib # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def clean_protocol_worktree(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(protocol_lib, "harness_worktree_clean", lambda: True)
|
||||||
|
|
||||||
|
|
||||||
def protocol() -> dict:
|
def protocol() -> dict:
|
||||||
return protocol_lib.freeze_protocol(
|
return protocol_lib.freeze_protocol(
|
||||||
"runner-test-seed",
|
"runner-test-seed",
|
||||||
|
|
|
||||||
|
|
@ -313,6 +313,11 @@ def test_oos_schema_guard_allows_explicit_future_schema_gap() -> None:
|
||||||
"would require a schema gap proposal."
|
"would require a schema gap proposal."
|
||||||
)
|
)
|
||||||
assert benchmark.current_schema_overclaims(reply) == []
|
assert benchmark.current_schema_overclaims(reply) == []
|
||||||
|
assert benchmark.current_schema_overclaims(
|
||||||
|
"The current public.claims schema does not record outcome or resolution criteria."
|
||||||
|
) == []
|
||||||
|
assert benchmark.current_schema_overclaims("The current edge_type enum does not contain resolves.") == []
|
||||||
|
assert benchmark.current_schema_overclaims("The current claims schema lacks an outcome field.") == []
|
||||||
|
|
||||||
|
|
||||||
def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_clause() -> None:
|
def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_clause() -> None:
|
||||||
|
|
@ -925,6 +930,82 @@ def test_oos_agent_positions_accept_attribution_without_literal_column_name() ->
|
||||||
assert benchmark.matched_concept(share_the_fact, "shared_knowledge_commons") is True
|
assert benchmark.matched_concept(share_the_fact, "shared_knowledge_commons") is True
|
||||||
assert benchmark.matched_concept(share_the_fact, "agent_specific_positions") is True
|
assert benchmark.matched_concept(share_the_fact, "agent_specific_positions") is True
|
||||||
|
|
||||||
|
agreed_proposition = (
|
||||||
|
"One claim row holds the agreed proposition; both agents' sources and evidence attach to it. Each agent's "
|
||||||
|
"divergent interpretation goes into one public.beliefs row. Beliefs has no claim_id foreign key, so an "
|
||||||
|
"explicit belief-to-claim link remains a schema gap."
|
||||||
|
)
|
||||||
|
assert benchmark.matched_concept(agreed_proposition, "shared_knowledge_commons") is True
|
||||||
|
assert benchmark.matched_concept(agreed_proposition, "agent_specific_positions") is True
|
||||||
|
|
||||||
|
duplicated_claims = (
|
||||||
|
"Store one separate public.claims row per agent. One claim row per agent holds the agreed proposition; "
|
||||||
|
"both agents' sources and evidence attach to their separate rows. Each agent's divergent interpretation "
|
||||||
|
"goes into one public.beliefs row per agent. Beliefs has no claim_id foreign key, so the link is a schema "
|
||||||
|
"gap. The positions disagree."
|
||||||
|
)
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(duplicated_claims) is True
|
||||||
|
assert benchmark.matched_concept(duplicated_claims, "shared_knowledge_commons") is False
|
||||||
|
prompt = next(item for item in benchmark.prompt_catalog("demo-ledger-deadbeef") if item["id"] == "OOS-11")
|
||||||
|
assert benchmark.score_reply(prompt, duplicated_claims, memory_token="demo-ledger-deadbeef")["pass"] is False
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(
|
||||||
|
"Do not create one public.claims row per agent; keep one shared claim row."
|
||||||
|
) is False
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(
|
||||||
|
"Each agent gets its own public.claims row for the shared fact."
|
||||||
|
) is True
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(
|
||||||
|
"Each agent should not get its own public.claims row; keep the factual row shared."
|
||||||
|
) is False
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(
|
||||||
|
"Use one claim row per agent instead of one shared claim row."
|
||||||
|
) is True
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(
|
||||||
|
"Use one shared claim row instead of one claim row per agent."
|
||||||
|
) is False
|
||||||
|
|
||||||
|
unrelated_denial = (
|
||||||
|
"Do not invent belief-to-claim edges while storing one public.claims row per agent: one claim row holds "
|
||||||
|
"the agreed proposition for each agent, and both agents' sources and evidence attach to those rows. Retain "
|
||||||
|
"that source evidence. Each agent's divergent position goes into one public.beliefs row per agent. Beliefs "
|
||||||
|
"has no claim_id foreign key, so the schema has a gap. The positions contradict."
|
||||||
|
)
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(unrelated_denial) is True
|
||||||
|
assert benchmark.score_reply(prompt, unrelated_denial, memory_token="demo-ledger-deadbeef")["pass"] is False
|
||||||
|
|
||||||
|
faithful_paraphrase = (
|
||||||
|
"Keep the proposition canonical once and retain both analysts' source evidence on it. Record each analyst's "
|
||||||
|
"interpretation in public.beliefs with agent attribution. Because beliefs has no claim_id foreign key, the "
|
||||||
|
"association remains a schema gap; contradictory interpretations remain queryable by agent."
|
||||||
|
)
|
||||||
|
assert benchmark.matched_concept(faithful_paraphrase, "shared_knowledge_commons") is True
|
||||||
|
assert benchmark.score_reply(prompt, faithful_paraphrase, memory_token="demo-ledger-deadbeef")["pass"] is True
|
||||||
|
|
||||||
|
named_duplicates = (
|
||||||
|
"Keep one public.claims row for Alice and one for Bob, each holding the agreed proposition, with both "
|
||||||
|
"agents' source evidence attached to both copies. Retain that evidence. Each agent's divergent position "
|
||||||
|
"goes into one public.beliefs row with agent attribution. Beliefs has no claim_id foreign key, so the schema "
|
||||||
|
"has a gap. The positions contradict."
|
||||||
|
)
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(named_duplicates) is True
|
||||||
|
assert benchmark.score_reply(prompt, named_duplicates, memory_token="demo-ledger-deadbeef")["pass"] is False
|
||||||
|
|
||||||
|
single_assertion = (
|
||||||
|
"Deduplicate the proposition into a single canonical assertion and retain both analysts' evidence on that "
|
||||||
|
"one record. Put each conflicting interpretation in public.beliefs with agent attribution. Beliefs lacks a "
|
||||||
|
"claim_id foreign key, so the association remains a schema gap and disagreement stays queryable."
|
||||||
|
)
|
||||||
|
assert benchmark.score_reply(prompt, single_assertion, memory_token="demo-ledger-deadbeef")["pass"] is True
|
||||||
|
|
||||||
|
namespaced_copies = (
|
||||||
|
"Keep the proposition canonical once under each agent; retain both analysts' source evidence and attach it "
|
||||||
|
"to their respective copies. Put each conflicting interpretation in public.beliefs with agent attribution. "
|
||||||
|
"Beliefs lacks a claim_id foreign key, so the association remains a schema gap and disagreement stays "
|
||||||
|
"queryable."
|
||||||
|
)
|
||||||
|
assert benchmark.asserts_per_agent_claim_duplication(namespaced_copies) is True
|
||||||
|
assert benchmark.score_reply(prompt, namespaced_copies, memory_token="demo-ledger-deadbeef")["pass"] is False
|
||||||
|
|
||||||
|
|
||||||
def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None:
|
def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> None:
|
||||||
token = "demo-ledger-deadbeef"
|
token = "demo-ledger-deadbeef"
|
||||||
|
|
@ -952,6 +1033,51 @@ def test_oos_memory_accepts_concrete_noncanonical_blocker_and_restatement() -> N
|
||||||
assert score["memory_continuity"]["same_blocker_recalled"] is True
|
assert score["memory_continuity"]["same_blocker_recalled"] is True
|
||||||
assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False
|
assert benchmark.matched_concept("Blocker: something is wrong.", "blocker_definition") is False
|
||||||
|
|
||||||
|
different_schema_gap = (
|
||||||
|
f"Label: {token}\nBlocker restated: public.sources has no author field, so source attribution remains a "
|
||||||
|
"schema gap.\nProof: postflight readback of a reviewed source-schema extension."
|
||||||
|
)
|
||||||
|
changed_score = benchmark.score_results(
|
||||||
|
[
|
||||||
|
{"prompt_id": set_prompt["id"], "reply": set_reply},
|
||||||
|
{"prompt_id": recall_prompt["id"], "reply": different_schema_gap},
|
||||||
|
],
|
||||||
|
memory_token=token,
|
||||||
|
catalog=[set_prompt, recall_prompt],
|
||||||
|
)
|
||||||
|
assert changed_score["pass"] is False
|
||||||
|
assert changed_score["memory_continuity"]["same_blocker_recalled"] is False
|
||||||
|
|
||||||
|
different_foreign_key = (
|
||||||
|
f"Label: {token}\nBlocker restated: public.sources lacks a foreign key to provenance metadata, so the "
|
||||||
|
"source relation remains a schema gap.\nProof: postflight readback of the source relation."
|
||||||
|
)
|
||||||
|
foreign_key_score = benchmark.score_results(
|
||||||
|
[
|
||||||
|
{"prompt_id": set_prompt["id"], "reply": set_reply},
|
||||||
|
{"prompt_id": recall_prompt["id"], "reply": different_foreign_key},
|
||||||
|
],
|
||||||
|
memory_token=token,
|
||||||
|
catalog=[set_prompt, recall_prompt],
|
||||||
|
)
|
||||||
|
assert foreign_key_score["pass"] is False
|
||||||
|
assert foreign_key_score["memory_continuity"]["same_blocker_recalled"] is False
|
||||||
|
|
||||||
|
different_mechanism = (
|
||||||
|
f"Label: {token}\nBlocker restated: beliefs.claim_id lacks an index, so position lookups remain a schema "
|
||||||
|
"gap.\nProof: postflight readback of the new index and query plan."
|
||||||
|
)
|
||||||
|
index_score = benchmark.score_results(
|
||||||
|
[
|
||||||
|
{"prompt_id": set_prompt["id"], "reply": set_reply},
|
||||||
|
{"prompt_id": recall_prompt["id"], "reply": different_mechanism},
|
||||||
|
],
|
||||||
|
memory_token=token,
|
||||||
|
catalog=[set_prompt, recall_prompt],
|
||||||
|
)
|
||||||
|
assert index_score["pass"] is False
|
||||||
|
assert index_score["memory_continuity"]["same_blocker_recalled"] is False
|
||||||
|
|
||||||
|
|
||||||
def test_oos_memory_recall_accepts_apply_receipt_correction_without_losing_the_token() -> None:
|
def test_oos_memory_recall_accepts_apply_receipt_correction_without_losing_the_token() -> None:
|
||||||
token = "demo-ledger-deadbeef"
|
token = "demo-ledger-deadbeef"
|
||||||
|
|
@ -971,6 +1097,119 @@ def test_oos_memory_recall_accepts_apply_receipt_correction_without_losing_the_t
|
||||||
assert score["proposal_readiness_issues"] == []
|
assert score["proposal_readiness_issues"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_oos_memory_accepts_natural_mnemonic_without_fixed_blocker_label() -> None:
|
||||||
|
token = "blind-ledger-deadbeef1234"
|
||||||
|
set_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-07")
|
||||||
|
recall_prompt = next(item for item in benchmark.prompt_catalog(token) if item["id"] == "OOS-08")
|
||||||
|
set_reply = (
|
||||||
|
f"Mnemonic tag {token}. Three approved proposals; zero applied_at values; zero canonical rows written. "
|
||||||
|
"Approval is a reviewer signature, applied_at is the guarded-apply receipt, and canonical readback is a "
|
||||||
|
"public.* row ID plus hash."
|
||||||
|
)
|
||||||
|
recall_reply = (
|
||||||
|
f"Mnemonic retrieved: {token}. Three proposals carry reviewer approval; applied_at is null on all three; "
|
||||||
|
"no public.* row ID or hash postflight exists. Resolution requires proposal-level applied_at plus the target "
|
||||||
|
"row ID and matching content hash."
|
||||||
|
)
|
||||||
|
|
||||||
|
score = benchmark.score_results(
|
||||||
|
[
|
||||||
|
{"prompt_id": set_prompt["id"], "reply": set_reply},
|
||||||
|
{"prompt_id": recall_prompt["id"], "reply": recall_reply},
|
||||||
|
],
|
||||||
|
memory_token=token,
|
||||||
|
catalog=[set_prompt, recall_prompt],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert score["pass"] is True
|
||||||
|
assert score["memory_continuity"]["same_blocker_recalled"] is True
|
||||||
|
|
||||||
|
changed_recall = (
|
||||||
|
f"Mnemonic retrieved: {token}. Telegram delivery remains blocked: applied_at and canonical readback exist, "
|
||||||
|
"but the visible message receipt is absent. Resolution requires message ID, timestamp, chat readback, and "
|
||||||
|
"a postflight row ID plus hash."
|
||||||
|
)
|
||||||
|
changed_score = benchmark.score_results(
|
||||||
|
[
|
||||||
|
{"prompt_id": set_prompt["id"], "reply": set_reply},
|
||||||
|
{"prompt_id": recall_prompt["id"], "reply": changed_recall},
|
||||||
|
],
|
||||||
|
memory_token=token,
|
||||||
|
catalog=[set_prompt, recall_prompt],
|
||||||
|
)
|
||||||
|
assert changed_score["pass"] is False
|
||||||
|
assert changed_score["memory_continuity"]["same_blocker_recalled"] is False
|
||||||
|
assert changed_score["memory_continuity"]["source_blocker_signature"] == [
|
||||||
|
"applied_missing",
|
||||||
|
"canonical_missing",
|
||||||
|
"reviewer_approval",
|
||||||
|
]
|
||||||
|
assert changed_score["memory_continuity"]["recall_blocker_signature"] == ["telegram_missing"]
|
||||||
|
|
||||||
|
vague = f"Mnemonic tag {token}. Database schema evidence remains unclear."
|
||||||
|
assert benchmark.matched_concept(vague, "blocker_definition") is False
|
||||||
|
|
||||||
|
no_blocker = f"Mnemonic tag {token}. No database blocker exists; all canonical rows are current and complete."
|
||||||
|
assert benchmark.matched_concept(no_blocker, "blocker_definition") is False
|
||||||
|
assert benchmark.score_reply(set_prompt, no_blocker, memory_token=token)["pass"] is False
|
||||||
|
modified_no_blocker = (
|
||||||
|
f"Mnemonic tag {token}. No meaningful database blocker exists; all canonical rows are current and complete."
|
||||||
|
)
|
||||||
|
assert benchmark.matched_concept(modified_no_blocker, "blocker_definition") is False
|
||||||
|
assert benchmark.score_reply(set_prompt, modified_no_blocker, memory_token=token)["pass"] is False
|
||||||
|
|
||||||
|
faithful_set = (
|
||||||
|
f"Mnemonic: {token} \u2014 three approved proposals still have applied_at NULL and no canonical rows; closure "
|
||||||
|
"requires row IDs and matching hashes."
|
||||||
|
)
|
||||||
|
faithful_recall = (
|
||||||
|
f"Mnemonic retrieved: {token}. Reviewer sign-off exists, but applied_at remains unset and no rows have landed "
|
||||||
|
"in public.claims. Close it with a postflight readback of the proposal timestamp, target claim row ID, and "
|
||||||
|
"content hash."
|
||||||
|
)
|
||||||
|
faithful_score = benchmark.score_results(
|
||||||
|
[
|
||||||
|
{"prompt_id": set_prompt["id"], "reply": faithful_set},
|
||||||
|
{"prompt_id": recall_prompt["id"], "reply": faithful_recall},
|
||||||
|
],
|
||||||
|
memory_token=token,
|
||||||
|
catalog=[set_prompt, recall_prompt],
|
||||||
|
)
|
||||||
|
assert faithful_score["pass"] is True
|
||||||
|
|
||||||
|
opposite_recall = (
|
||||||
|
f"Mnemonic retrieved: {token}. Reviewer approval is present and applied_at is non-null; the canonical row "
|
||||||
|
"exists. Telegram delivery is missing. Closure proof: postflight readback of the message ID, timestamp, and "
|
||||||
|
"public row."
|
||||||
|
)
|
||||||
|
opposite_score = benchmark.score_results(
|
||||||
|
[
|
||||||
|
{"prompt_id": set_prompt["id"], "reply": set_reply},
|
||||||
|
{"prompt_id": recall_prompt["id"], "reply": opposite_recall},
|
||||||
|
],
|
||||||
|
memory_token=token,
|
||||||
|
catalog=[set_prompt, recall_prompt],
|
||||||
|
)
|
||||||
|
assert opposite_score["pass"] is False
|
||||||
|
assert opposite_score["memory_continuity"]["same_blocker_recalled"] is False
|
||||||
|
|
||||||
|
reversed_flags = (
|
||||||
|
f"Mnemonic retrieved: {token}. Reviewer approval is present. The applied_at missing flag is false; the "
|
||||||
|
"canonical gap is closed. Telegram delivery remains blocked. Closure proof: postflight readback of message "
|
||||||
|
"ID, timestamp, and public row."
|
||||||
|
)
|
||||||
|
reversed_score = benchmark.score_results(
|
||||||
|
[
|
||||||
|
{"prompt_id": set_prompt["id"], "reply": set_reply},
|
||||||
|
{"prompt_id": recall_prompt["id"], "reply": reversed_flags},
|
||||||
|
],
|
||||||
|
memory_token=token,
|
||||||
|
catalog=[set_prompt, recall_prompt],
|
||||||
|
)
|
||||||
|
assert reversed_score["pass"] is False
|
||||||
|
assert reversed_score["memory_continuity"]["same_blocker_recalled"] is False
|
||||||
|
|
||||||
|
|
||||||
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
|
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
|
||||||
reply = (
|
reply = (
|
||||||
"Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved "
|
"Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved "
|
||||||
|
|
@ -1007,6 +1246,110 @@ def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
|
||||||
)
|
)
|
||||||
assert benchmark.matched_concept(live_untouched, "forecast_history") is True
|
assert benchmark.matched_concept(live_untouched, "forecast_history") is True
|
||||||
|
|
||||||
|
current_claims_schema = (
|
||||||
|
"The original 0.60 forecast stays untouched. No field on the current claims schema records resolution "
|
||||||
|
"criteria, outcome, or resolved_at, and the current edge_type enum has no resolves type. Stage a reviewed "
|
||||||
|
"schema proposal before an explicitly authorized apply."
|
||||||
|
)
|
||||||
|
assert benchmark.matched_concept(current_claims_schema, "forecast_schema_gap") is True
|
||||||
|
explicit_denial = (
|
||||||
|
"The schema has no resolution fields. Do not invent a resolves edge or forecast-resolution fields; neither "
|
||||||
|
"exists in the current schema."
|
||||||
|
)
|
||||||
|
assert benchmark.matched_concept(explicit_denial, "forecast_schema_gap") is True
|
||||||
|
|
||||||
|
invented_current_schema = (
|
||||||
|
"The original 0.60 forecast stays untouched because resolution criteria were never defined. No field "
|
||||||
|
"review is required because the current public.claims schema already records resolution criteria, outcome, "
|
||||||
|
"and resolved_at. The current edge_type enum already contains resolves, so no resolves addition is needed. "
|
||||||
|
"Stage a reviewed proposal before explicitly authorized apply."
|
||||||
|
)
|
||||||
|
assert benchmark.matched_concept(invented_current_schema, "forecast_schema_gap") is False
|
||||||
|
assert benchmark.current_schema_overclaims(invented_current_schema) == [
|
||||||
|
"claims_unshipped_fields",
|
||||||
|
"invalid_current_edge_type",
|
||||||
|
]
|
||||||
|
prompt = next(item for item in benchmark.prompt_catalog("demo-ledger-deadbeef") if item["id"] == "OOS-12")
|
||||||
|
assert benchmark.score_reply(prompt, invented_current_schema, memory_token="demo-ledger-deadbeef")["pass"] is False
|
||||||
|
|
||||||
|
exposed_fields = (
|
||||||
|
"Preserve the original 0.60 forecast untouched because its criteria were never defined. Current "
|
||||||
|
"public.claims exposes resolution_criteria, outcome, and resolved_at, but there is no forecast-resolution "
|
||||||
|
"field; the enum has no resolves type. Retain the missing-criteria caveat. Stage a reviewed proposal before "
|
||||||
|
"any apply. This is a schema gap."
|
||||||
|
)
|
||||||
|
exposed_score = benchmark.score_reply(prompt, exposed_fields, memory_token="demo-ledger-deadbeef")
|
||||||
|
assert exposed_score["pass"] is False
|
||||||
|
assert exposed_score["current_schema_overclaims"] == ["claims_unshipped_fields"]
|
||||||
|
|
||||||
|
inverted_resolves = (
|
||||||
|
"Preserve the original 0.60 forecast untouched because its criteria were never defined. The current schema "
|
||||||
|
"has no forecast-resolution field. A resolves edge is not absent; it is already supported by edge_type. "
|
||||||
|
"Retain the missing-criteria caveat and stage a reviewed proposal before any apply. This is a schema gap."
|
||||||
|
)
|
||||||
|
inverted_score = benchmark.score_reply(prompt, inverted_resolves, memory_token="demo-ledger-deadbeef")
|
||||||
|
assert inverted_score["pass"] is False
|
||||||
|
assert inverted_score["current_schema_overclaims"] == ["invalid_current_edge_type"]
|
||||||
|
|
||||||
|
faithful_gap = (
|
||||||
|
"Keep the original 60% forecast unchanged because its criteria were never defined. The current public.claims "
|
||||||
|
"schema lacks resolution_criteria and resolved_at, and edge_type lacks resolves. Retain the ambiguity. Stage "
|
||||||
|
"a reviewed schema proposal before apply; this remains a schema gap."
|
||||||
|
)
|
||||||
|
faithful_gap_score = benchmark.score_reply(prompt, faithful_gap, memory_token="demo-ledger-deadbeef")
|
||||||
|
assert faithful_gap_score["pass"] is True
|
||||||
|
assert faithful_gap_score["current_schema_overclaims"] == []
|
||||||
|
|
||||||
|
generic_current_schema = (
|
||||||
|
"Keep the original 60% forecast unchanged because its criteria were never defined. The current schema has "
|
||||||
|
"resolution criteria, outcome, and resolved_at already. No resolved_at migration is needed, and the current "
|
||||||
|
"edge_type enum has no resolves type. Retain the ambiguity and stage a reviewed proposal before apply. This "
|
||||||
|
"remains a schema gap."
|
||||||
|
)
|
||||||
|
generic_score = benchmark.score_reply(prompt, generic_current_schema, memory_token="demo-ledger-deadbeef")
|
||||||
|
assert generic_score["pass"] is False
|
||||||
|
assert generic_score["current_schema_overclaims"] == ["claims_unshipped_fields"]
|
||||||
|
|
||||||
|
models_fields = (
|
||||||
|
"Preserve the original 0.60 forecast untouched because its criteria were never defined. Current public.claims "
|
||||||
|
"natively models resolution_criteria, outcome, and resolved_at, but there is no forecast-resolution field; "
|
||||||
|
"the enum has no resolves type. Retain the missing-criteria caveat. Stage a reviewed proposal before any "
|
||||||
|
"apply. This is a schema gap."
|
||||||
|
)
|
||||||
|
models_score = benchmark.score_reply(prompt, models_fields, memory_token="demo-ledger-deadbeef")
|
||||||
|
assert models_score["pass"] is False
|
||||||
|
assert models_score["current_schema_overclaims"] == ["claims_unshipped_fields"]
|
||||||
|
|
||||||
|
enumerates_resolves = (
|
||||||
|
"Preserve the original 0.60 forecast untouched because its criteria were never defined. The current schema "
|
||||||
|
"has no forecast-resolution field. The current edge_type enumerates resolves, which is not missing. Retain "
|
||||||
|
"the missing-criteria caveat and stage a reviewed proposal before any apply. This is a schema gap."
|
||||||
|
)
|
||||||
|
enumerates_score = benchmark.score_reply(prompt, enumerates_resolves, memory_token="demo-ledger-deadbeef")
|
||||||
|
assert enumerates_score["pass"] is False
|
||||||
|
assert enumerates_score["current_schema_overclaims"] == ["invalid_current_edge_type"]
|
||||||
|
|
||||||
|
cannot_express = (
|
||||||
|
"Keep the original 60% forecast unchanged because its criteria were never defined. The current public.claims "
|
||||||
|
"schema doesn't model resolution_criteria or resolved_at, and edge_type cannot express resolves. Retain the "
|
||||||
|
"ambiguity. Stage a reviewed schema proposal before apply; this remains a schema gap."
|
||||||
|
)
|
||||||
|
cannot_express_score = benchmark.score_reply(prompt, cannot_express, memory_token="demo-ledger-deadbeef")
|
||||||
|
assert cannot_express_score["pass"] is True
|
||||||
|
assert cannot_express_score["current_schema_overclaims"] == []
|
||||||
|
|
||||||
|
synonym_overclaim = (
|
||||||
|
"Keep the original 60% forecast unchanged because its criteria were never defined. The current schema tracks "
|
||||||
|
"resolution criteria, outcome, and resolved_at. The current edge_type enum recognizes resolves. Retain the "
|
||||||
|
"ambiguity and stage a reviewed proposal before apply."
|
||||||
|
)
|
||||||
|
synonym_score = benchmark.score_reply(prompt, synonym_overclaim, memory_token="demo-ledger-deadbeef")
|
||||||
|
assert synonym_score["pass"] is False
|
||||||
|
assert synonym_score["current_schema_overclaims"] == [
|
||||||
|
"claims_unshipped_fields",
|
||||||
|
"invalid_current_edge_type",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_oos_schema_overclaim_accepts_anaphoric_semicolon_denial() -> None:
|
def test_oos_schema_overclaim_accepts_anaphoric_semicolon_denial() -> None:
|
||||||
safe = "A resolves edge type; it is not in the current edge type list and must not be invented."
|
safe = "A resolves edge type; it is not in the current edge type list and must not be invented."
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,8 @@ import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
sys.path.insert(0, str(ROOT / "scripts"))
|
sys.path.insert(0, str(ROOT / "scripts"))
|
||||||
|
|
||||||
|
|
@ -32,6 +34,11 @@ SOURCE_UUID = "22222222-2222-4222-8222-222222222222"
|
||||||
CONTRACT_ROW_UUID = "33333333-3333-4333-8333-333333333333"
|
CONTRACT_ROW_UUID = "33333333-3333-4333-8333-333333333333"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def clean_protocol_worktree(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(protocol_lib, "harness_worktree_clean", lambda: True)
|
||||||
|
|
||||||
|
|
||||||
def frozen_protocol() -> dict:
|
def frozen_protocol() -> dict:
|
||||||
return protocol_lib.freeze_protocol(
|
return protocol_lib.freeze_protocol(
|
||||||
"unit-test-seed-without-live-output",
|
"unit-test-seed-without-live-output",
|
||||||
|
|
@ -39,6 +46,12 @@ def frozen_protocol() -> dict:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_protocol_freeze_rejects_dirty_worktree(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(protocol_lib, "harness_worktree_clean", lambda: False)
|
||||||
|
with pytest.raises(RuntimeError, match="dirty tracked worktree"):
|
||||||
|
protocol_lib.freeze_protocol("dirty-worktree-seed")
|
||||||
|
|
||||||
|
|
||||||
def tool_surface(mode: str) -> dict:
|
def tool_surface(mode: str) -> dict:
|
||||||
return {
|
return {
|
||||||
"mode": mode,
|
"mode": mode,
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue