Accept receipt-bound natural answers in the Leo OOS scorer (#172)
* Make OOS scoring accept receipt-bound natural answers * Test short-ID causal attribution in OOS ablation * Close OOS provenance and schema qualifier gaps
This commit is contained in:
twentyOne2x
GitHub
parent
b6efa6c4a3
commit
27fbfa49a9
4 changed files with 492 additions and 95 deletions
|
|
@ -287,7 +287,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(r"apply|canonical", re.I),
|
||||
),
|
||||
"receipt": (
|
||||
re.compile(r"receipt|readback|postflight|before-and-after|before/after", re.I),
|
||||
re.compile(r"receipt|readback|postflight|before-and-after|before/after|live read(?:ing)?|live proof", re.I),
|
||||
re.compile(r"row|count|applied_at|public\.", re.I),
|
||||
),
|
||||
"identity_chain": (
|
||||
|
|
@ -321,14 +321,18 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(r"public\.sources|source row", re.I),
|
||||
re.compile(
|
||||
r"attachment.{0,160}(?:does not|doesn't|is not|isn't|cannot|can't|alone|until|unless)|"
|
||||
r"(?:does not|doesn't|is not|isn't|cannot|can't).{0,100}canonical evidence from (?:that|the) attachment",
|
||||
r"(?:does not|doesn't|is not|isn't|cannot|can't).{0,100}canonical evidence from (?:that|the) attachment|"
|
||||
r"(?:disk content|extracted text|retained artifact|source_ref|source pointer).{0,140}"
|
||||
r"(?:is not|isn't|does not|doesn't|alone|until|unless).{0,80}(?:canonical evidence|finished provenance)|"
|
||||
r"(?:canonical evidence|canonical link).{0,100}(?:requires|exists only when|is complete only when).{0,100}"
|
||||
r"(?:public\.sources|source row|claim_evidence)",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"evidence_provenance_quality": (
|
||||
re.compile(r"(?:missing|no|without).{0,50}(?:url|storage(?:_path)?|locator)", re.I | re.S),
|
||||
re.compile(r"weak|citation-only|citation only|not traceable|raw artifact", re.I),
|
||||
re.compile(r"canonical evidence|canonical link", re.I),
|
||||
re.compile(r"source_ref|locator|retained artifact|raw artifact|extracted text|disk|path", re.I),
|
||||
re.compile(r"weak|unresolved|not traceable|verified|hash_matches_db|provenance", re.I),
|
||||
re.compile(r"canonical evidence|canonical link|public\.sources|claim_evidence", re.I),
|
||||
),
|
||||
"heterogeneous_types": (
|
||||
re.compile(r"claim", re.I),
|
||||
|
|
@ -336,7 +340,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(r"framework|reasoning tool|concept map", re.I),
|
||||
re.compile(r"governance", re.I),
|
||||
re.compile(r"correction|supersed", re.I),
|
||||
re.compile(r"disput|contradict", re.I),
|
||||
re.compile(r"disput|contradict|disagree|contested interpretation", re.I),
|
||||
),
|
||||
"behavioral_rule_storage": (
|
||||
re.compile(r"(?:public\.)?behavioral_rules", re.I),
|
||||
|
|
@ -345,11 +349,10 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
"reviewed_policy_apply": (
|
||||
re.compile(r"approve_claim", re.I),
|
||||
re.compile(r"behavioral_rules", re.I),
|
||||
re.compile(r"governance_gates", re.I),
|
||||
re.compile(
|
||||
r"does not (?:accept|insert|support|cover)|supports neither|neither.{0,80}nor|"
|
||||
r"(?:behavioral_rules|governance_gates).{0,120}cannot be applied by approve_claim|"
|
||||
r"approve_claim.{0,120}cannot apply",
|
||||
r"does not (?:accept|insert|support|cover|write)|supports neither|neither.{0,80}nor|"
|
||||
r"(?:behavioral_rules|governance_gates).{0,120}(?:cannot be applied by|sits? outside) approve_claim|"
|
||||
r"approve_claim.{0,120}(?:cannot apply|cannot write|does not write)",
|
||||
re.I | re.S,
|
||||
),
|
||||
re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply (?:capability|path)", re.I | re.S),
|
||||
|
|
@ -392,7 +395,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
"shared_knowledge_commons": (
|
||||
re.compile(
|
||||
r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|"
|
||||
r"keep the factual claim once|store it once",
|
||||
r"keep the factual claim once|store it once|one public\.claims row|"
|
||||
r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
|
||||
re.I | re.S,
|
||||
),
|
||||
re.compile(r"source|evidence", re.I),
|
||||
|
|
@ -409,16 +413,29 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
),
|
||||
"forecast_history": (
|
||||
re.compile(r"original (?:probability|confidence)|60%|history", re.I),
|
||||
re.compile(r"preserve|retain|do not overwrite|don'?t overwrite", re.I),
|
||||
re.compile(r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria", re.I | re.S),
|
||||
re.compile(
|
||||
r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|original.{0,60}survives",
|
||||
re.I | re.S,
|
||||
),
|
||||
re.compile(
|
||||
r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria|"
|
||||
r"criteria.{0,40}(?:never existed|were never defined|did not exist)|without.{0,30}criteria",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"forecast_schema_gap": (
|
||||
re.compile(r"current (?:v1|schema)|public\.claims", re.I),
|
||||
re.compile(
|
||||
r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution",
|
||||
r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|"
|
||||
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
|
||||
r"(?:does not exist|is absent|is not present|isn't present)|"
|
||||
r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present",
|
||||
re.I | re.S,
|
||||
),
|
||||
re.compile(
|
||||
r"resolves.{0,220}(?:not|isn'?t|does not|doesn't|absent)|no.{0,30}resolves",
|
||||
re.I | re.S,
|
||||
),
|
||||
re.compile(r"resolves.{0,30}(?:not|isn'?t|does not)|no.{0,30}resolves", re.I | re.S),
|
||||
),
|
||||
"handler_not_telegram": (
|
||||
re.compile(r"no|not", re.I),
|
||||
|
|
@ -484,12 +501,12 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
"live_claim_evidence_ids": (
|
||||
re.compile(
|
||||
r"\bclaim(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?"
|
||||
r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b",
|
||||
r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b",
|
||||
re.I,
|
||||
),
|
||||
re.compile(
|
||||
r"\b(?:source|evidence)(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?"
|
||||
r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b",
|
||||
r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
|
|
@ -507,8 +524,10 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(
|
||||
r"\b(?:distinct|separate)\s+from\b|"
|
||||
r"\b(?:claim|evidence|source)\b.{0,80}\b(?:distinct from|separate from|differs? from|"
|
||||
r"not the same as)\b",
|
||||
re.I,
|
||||
r"not the same as)\b|"
|
||||
r"\bclaim body\s*:\s*\S.{0,400}\bevidence(?: excerpt)?\s*:\s*\S|"
|
||||
r"\bclaim body\b.{0,240}\b(?:evidence|source)\b.{0,100}\b(?:states?|says?|shows?|documents?)\b",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"evidence_specific_challenge": (
|
||||
|
|
@ -516,6 +535,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I),
|
||||
re.compile(
|
||||
r"\b(?:does not|doesn't|cannot|can't)\s+(?:itself\s+)?(?:prove|establish|support|show)|"
|
||||
r"\b(?:does not|doesn't|cannot|can't)\s+rule out\b|"
|
||||
r"\bunsupported leap\b",
|
||||
re.I,
|
||||
),
|
||||
|
|
@ -526,7 +546,11 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
r"should be narrowed to|defensible formulation|revise(?:d)?\b.{0,40}\bto\b)",
|
||||
re.I,
|
||||
),
|
||||
re.compile(r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?)\b", re.I),
|
||||
re.compile(
|
||||
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|"
|
||||
r"unproven|open constraint)\b",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
|
@ -545,10 +569,22 @@ COUNT_INVARIANT_REJECTION_RE = re.compile(
|
|||
SCHEMA_GAP_QUALIFIER_RE = re.compile(
|
||||
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|"
|
||||
r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|"
|
||||
r"does not support|doesn't support|supports neither|not supported|must not|do not invent)\b|"
|
||||
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
|
||||
r"schema extension|extension proposal)\b|"
|
||||
r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?"
|
||||
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
|
||||
r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?"
|
||||
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
|
||||
r"\b(?:proposal|extension)\b.{0,100}\b(?:column|field|table|edge type)\b|"
|
||||
r"\bno\s+(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b",
|
||||
re.I,
|
||||
)
|
||||
SCHEMA_CLAUSE_BOUNDARY_RE = re.compile(
|
||||
r"\s*(?:;|,\s+and\s+(?=(?:[a-z0-9_.`'-]+\s+){1,5}"
|
||||
r"(?:is|are|has|have|requires?|needs?|would|will|must|can|does?|supports?)\b)|"
|
||||
r"\bbut\b|\bhowever\b)\s*",
|
||||
re.I,
|
||||
)
|
||||
CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
||||
"claims_unshipped_fields": re.compile(
|
||||
r"(?:public\.)?claims?\b.{0,60}(?:stores?|has|have|with|column|field).{0,40}"
|
||||
|
|
@ -674,6 +710,22 @@ APPLYABILITY_GAP_RE = re.compile(
|
|||
r"strict apply payload.{0,50}(?:built|build|reviewed|review)",
|
||||
re.I | re.S,
|
||||
)
|
||||
CLAIM_EVIDENCE_CONFLATION_RE = re.compile(
|
||||
r"\bevidence(?: excerpt)?\s*:\s*(?:the\s+)?same as (?:the\s+)?claim\b|"
|
||||
r"\b(?:claim body|claim text)\s+(?:is|equals?)\s+(?:the\s+)?evidence\b",
|
||||
re.I | re.S,
|
||||
)
|
||||
REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P<value>[^\n]+)", re.I)
|
||||
REVISION_BOUNDARY_RE = re.compile(
|
||||
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|"
|
||||
r"open constraint)\b",
|
||||
re.I,
|
||||
)
|
||||
REVISION_REPETITION_RE = re.compile(
|
||||
r"\b(?:repeat(?:ed)?|restate(?:d)?).{0,50}\b(?:original|same)\s+claim\b|"
|
||||
r"\b(?:original|same)\s+claim\b.{0,50}\bunchanged\b",
|
||||
re.I | re.S,
|
||||
)
|
||||
|
||||
DEFAULT_MAX_RESPONSE_WORDS = 220
|
||||
MAX_RESPONSE_WORDS = {"OOS-09": 100, "OOS-10": 180}
|
||||
|
|
@ -726,6 +778,15 @@ def blocker_terms(value: str | None, *, memory_token: str) -> set[str]:
|
|||
|
||||
|
||||
def matched_concept(reply: str, concept: str) -> bool:
|
||||
if concept == "claim_body_evidence_distinction" and CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
|
||||
return False
|
||||
if concept == "narrower_claim_revision":
|
||||
labelled = REVISION_LABEL_RE.search(reply)
|
||||
if labelled and (
|
||||
not REVISION_BOUNDARY_RE.search(labelled.group("value"))
|
||||
or REVISION_REPETITION_RE.search(labelled.group("value"))
|
||||
):
|
||||
return False
|
||||
return all(pattern.search(reply) for pattern in CONCEPT_PATTERNS[concept])
|
||||
|
||||
|
||||
|
|
@ -737,12 +798,17 @@ def current_schema_overclaims(reply: str) -> list[str]:
|
|||
"""Return proposed-v3-as-current assertions that are not explicitly qualified."""
|
||||
|
||||
findings: list[str] = []
|
||||
for segment in re.split(r"(?<=[.!?])\s+|\n+", reply):
|
||||
if SCHEMA_GAP_QUALIFIER_RE.search(segment):
|
||||
continue
|
||||
for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items():
|
||||
if pattern.search(segment):
|
||||
findings.append(label)
|
||||
sentence_segments = re.split(r"(?<=[.!?])\s+|\n+", reply)
|
||||
for sentence in sentence_segments:
|
||||
# Negation and future-schema qualifiers apply to their clause, not to a
|
||||
# contradictory assertion in a later conjunction in the same sentence.
|
||||
clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence)
|
||||
for clause in clauses:
|
||||
if SCHEMA_GAP_QUALIFIER_RE.search(clause):
|
||||
continue
|
||||
for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items():
|
||||
if pattern.search(clause):
|
||||
findings.append(label)
|
||||
return sorted(set(findings))
|
||||
|
||||
|
||||
|
|
@ -796,7 +862,12 @@ def broad_semantic_issues(reply: str) -> list[str]:
|
|||
for segment in re.split(r"(?<=[.!?])\s+|\n+", reply):
|
||||
if not REASONING_TOOL_CLAIM_EDGE_RE.search(segment):
|
||||
continue
|
||||
if re.search(r"(?:never|do not|don't|must not|cannot).{0,100}reasoning_tools?", segment, re.I | re.S):
|
||||
if re.search(
|
||||
r"(?:never|do not|don't|must not|cannot).{0,100}reasoning_tools?|"
|
||||
r"(?:edge endpoints?|from_claim|to_claim).{0,100}(?:claim IDs?|claims? only|never reasoning_tools?)",
|
||||
segment,
|
||||
re.I | re.S,
|
||||
):
|
||||
continue
|
||||
findings.add("claim_edge_pointed_at_reasoning_tool")
|
||||
break
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ PROTOCOL_SCHEMA = "livingip.leoM3taversalOosProtocol.v2"
|
|||
TRIAL_SCORE_SCHEMA = "livingip.leoM3taversalOosTrialScore.v2"
|
||||
AGGREGATE_SCHEMA = "livingip.leoM3taversalOosAggregate.v2"
|
||||
GENERATOR_VERSION = "blinded-family-generator-v3"
|
||||
SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v3"
|
||||
SCORER_VERSION = "invariant-reasoning-live-receipts-and-factual-ablation-v5"
|
||||
BASELINE_VERSION = "live-current-build-db-tool-ablation-v2"
|
||||
DEFAULT_TRIAL_COUNT = 3
|
||||
MEMORY_SCORER_IDS = frozenset({"OOS-07", "OOS-08"})
|
||||
|
|
@ -56,8 +56,9 @@ EXPECTED_TELEGRAM_DENY_METHODS = frozenset(
|
|||
"send_voice",
|
||||
}
|
||||
)
|
||||
GROUNDED_EXECUTION_ALLOWED_MISSING = frozenset({"harness_worktree_clean"})
|
||||
ABLATION_EXECUTION_ALLOWED_MISSING = GROUNDED_EXECUTION_ALLOWED_MISSING | frozenset(
|
||||
GROUNDED_EXECUTION_ALLOWED_MISSING = frozenset()
|
||||
CONTROL_GOAL_EXECUTION_ALLOWED_MISSING = frozenset({"harness_worktree_clean"})
|
||||
ABLATION_EXECUTION_ALLOWED_MISSING = frozenset(
|
||||
{
|
||||
"model_raw_response_binding",
|
||||
"database_context_query_binding",
|
||||
|
|
@ -92,12 +93,12 @@ UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]
|
|||
RECEIPT_TOKEN_RE = re.compile(r"\breceipt\s*:\s*`?([0-9a-f]{12})(?![0-9a-f])", re.I)
|
||||
CLAIM_ID_CITATION_RE = re.compile(
|
||||
r"\bclaim(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?"
|
||||
r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b",
|
||||
r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|[0-9a-f]{8,12})\b",
|
||||
re.I,
|
||||
)
|
||||
SOURCE_ID_CITATION_RE = re.compile(
|
||||
r"\b(?:source|evidence)(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?"
|
||||
r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b",
|
||||
r"([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|[0-9a-f]{8,12})\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
|
@ -601,6 +602,12 @@ def validate_protocol(protocol: dict[str, Any], *, verify_source_hashes: bool) -
|
|||
issues: list[str] = []
|
||||
if protocol.get("schema") != PROTOCOL_SCHEMA:
|
||||
issues.append("wrong_protocol_schema")
|
||||
if protocol.get("generator_version") != GENERATOR_VERSION:
|
||||
issues.append("wrong_generator_version")
|
||||
if protocol.get("scorer_version") != SCORER_VERSION:
|
||||
issues.append("wrong_scorer_version")
|
||||
if (protocol.get("baseline") or {}).get("version") != BASELINE_VERSION:
|
||||
issues.append("wrong_baseline_version")
|
||||
if not _valid_git_revision(protocol.get("harness_git_head")):
|
||||
issues.append("invalid_harness_git_head")
|
||||
supplied_hash = protocol.get("protocol_hash_sha256")
|
||||
|
|
@ -692,7 +699,7 @@ def _subject_alignment(prompt: dict[str, Any], reply: str) -> bool:
|
|||
}
|
||||
return (
|
||||
bool(normalized_subject)
|
||||
and padded_reply.count(padded_subject) == 1
|
||||
and padded_reply.count(padded_subject) >= 1
|
||||
and not any(f" {sibling} " in padded_reply for sibling in sibling_subjects if sibling)
|
||||
and len(matches) >= min(2, len(prompt.get("subject_anchors") or []))
|
||||
)
|
||||
|
|
@ -751,6 +758,25 @@ def _reply_receipt_tokens(reply: str) -> list[str]:
|
|||
return sorted({match.group(1).lower() for match in RECEIPT_TOKEN_RE.finditer(reply)})
|
||||
|
||||
|
||||
def _resolve_supported_citations(
|
||||
reply: str,
|
||||
pattern: re.Pattern[str],
|
||||
supported_identifiers: set[str],
|
||||
) -> tuple[set[str], set[str], set[str]]:
|
||||
"""Resolve full UUIDs or unambiguous 8-12 hex prefixes to receipted IDs."""
|
||||
|
||||
tokens = {match.group(1).lower() for match in pattern.finditer(reply)}
|
||||
resolved: set[str] = set()
|
||||
unresolved: set[str] = set()
|
||||
for token in tokens:
|
||||
matches = {identifier for identifier in supported_identifiers if identifier.startswith(token)}
|
||||
if len(matches) == 1:
|
||||
resolved.update(matches)
|
||||
else:
|
||||
unresolved.add(token)
|
||||
return tokens, resolved, unresolved
|
||||
|
||||
|
||||
def _evidence_answer_score(
|
||||
prompt: dict[str, Any],
|
||||
result: dict[str, Any],
|
||||
|
|
@ -1035,8 +1061,12 @@ def _receipt_score(
|
|||
}
|
||||
reply = str(result.get("reply") or "")
|
||||
reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(reply)}
|
||||
reply_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(reply)}
|
||||
reply_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(reply)}
|
||||
reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = (
|
||||
_resolve_supported_citations(reply, CLAIM_ID_CITATION_RE, receipt_claim_ids)
|
||||
)
|
||||
reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = (
|
||||
_resolve_supported_citations(reply, SOURCE_ID_CITATION_RE, receipt_source_ids)
|
||||
)
|
||||
unsupported_identifiers = sorted(reply_identifiers - supported_identifiers)
|
||||
receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records]
|
||||
reply_sha256 = hashlib.sha256(reply.encode()).hexdigest()
|
||||
|
|
@ -1080,6 +1110,8 @@ def _receipt_score(
|
|||
),
|
||||
"conversation_history_prefix_preserved": result.get("conversation_history_prefix_preserved") is True,
|
||||
"no_unsupported_exact_identifiers": not unsupported_identifiers,
|
||||
"no_unresolved_or_ambiguous_identifier_citations": not unresolved_claim_citations
|
||||
and not unresolved_source_citations,
|
||||
}
|
||||
return {
|
||||
"checks": checks,
|
||||
|
|
@ -1088,8 +1120,12 @@ def _receipt_score(
|
|||
"expected_prompt_sha256": prompt_sha256,
|
||||
"database_tool_trace": result.get("database_tool_trace") or {},
|
||||
"reply_identifiers": sorted(reply_identifiers),
|
||||
"reply_claim_citation_tokens": sorted(reply_claim_citation_tokens),
|
||||
"reply_claim_citations": sorted(reply_claim_citations),
|
||||
"unresolved_claim_citations": sorted(unresolved_claim_citations),
|
||||
"reply_source_or_evidence_citation_tokens": sorted(reply_source_citation_tokens),
|
||||
"reply_source_or_evidence_citations": sorted(reply_source_citations),
|
||||
"unresolved_source_or_evidence_citations": sorted(unresolved_source_citations),
|
||||
"supported_claim_ids": sorted(receipt_claim_ids),
|
||||
"supported_source_ids": sorted(receipt_source_ids),
|
||||
"supported_contract_row_ids": sorted(receipt_contract_row_ids),
|
||||
|
|
@ -1108,37 +1144,55 @@ def _benchmark_execution_chain(
|
|||
"""Validate the generic turn manifests under this benchmark's declared ablation.
|
||||
|
||||
The generic manifest intentionally marks a dirty harness and missing DB-context
|
||||
hooks incomplete. This benchmark permits exactly one control-owned dirty file
|
||||
(``goal.md``) and, in the ablated arm only, the bindings made impossible by
|
||||
removing the DB-context plugin. Every other runtime/model/session/safety
|
||||
binding remains mandatory and is checked independently here.
|
||||
hooks incomplete. This benchmark accepts either a clean harness or exactly one
|
||||
control-owned untracked file (``goal.md``). Only the latter may omit the generic
|
||||
clean-worktree binding. In the ablated arm, the bindings made impossible by
|
||||
removing the DB-context plugin may also be absent. Every other runtime, model,
|
||||
session, and safety binding remains mandatory and is checked independently.
|
||||
"""
|
||||
|
||||
mode = report.get("grounding_mode")
|
||||
if mode == "grounded":
|
||||
allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING,)
|
||||
elif mode == "db_tool_ablated":
|
||||
allowed_missing_sets = (
|
||||
ABLATION_EXECUTION_ALLOWED_MISSING,
|
||||
ABLATION_EXECUTION_ALLOWED_MISSING | ABLATION_EXECUTION_OPTIONAL_MISSING,
|
||||
)
|
||||
else:
|
||||
allowed_missing_sets = (frozenset(),)
|
||||
results = [item for item in report.get("results") or [] if isinstance(item, dict)]
|
||||
summary = report.get("execution_manifest_summary") or {}
|
||||
executed_behavior = report.get("executed_behavior_manifest") or {}
|
||||
local_state = report.get("oos_harness_git_state") or {}
|
||||
summary_source = summary.get("harness_source") or {}
|
||||
clean_worktree_exact = (
|
||||
local_state.get("worktree_clean") is True
|
||||
and local_state.get("status_lines") == []
|
||||
and local_state.get("only_control_goal_untracked") is False
|
||||
and local_state.get("status_sha256") == hashlib.sha256(b"").hexdigest()
|
||||
)
|
||||
control_goal_only_exact = (
|
||||
local_state.get("worktree_clean") is False
|
||||
and local_state.get("status_lines") == ["?? goal.md"]
|
||||
and local_state.get("only_control_goal_untracked") is True
|
||||
and local_state.get("status_sha256") == hashlib.sha256(b"?? goal.md\n").hexdigest()
|
||||
)
|
||||
supported_worktree_mode = clean_worktree_exact or control_goal_only_exact
|
||||
worktree_allowed_missing = (
|
||||
CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset()
|
||||
)
|
||||
mode = report.get("grounding_mode")
|
||||
if mode == "grounded":
|
||||
allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing,)
|
||||
elif mode == "db_tool_ablated":
|
||||
allowed_missing = ABLATION_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing
|
||||
allowed_missing_sets = (
|
||||
allowed_missing,
|
||||
allowed_missing | ABLATION_EXECUTION_OPTIONAL_MISSING,
|
||||
)
|
||||
else:
|
||||
allowed_missing_sets = (frozenset(),)
|
||||
local_state_checks = {
|
||||
"git_head_valid": _valid_git_revision(local_state.get("git_head")),
|
||||
"git_head_matches_frozen_harness": local_state.get("git_head") == expected_harness_git_head,
|
||||
"status_sha256_valid": _valid_sha256(local_state.get("status_sha256")),
|
||||
"recorded_dirty": local_state.get("worktree_clean") is False,
|
||||
"only_control_goal_untracked": local_state.get("only_control_goal_untracked") is True
|
||||
and local_state.get("status_lines") == ["?? goal.md"],
|
||||
"clean_worktree_exact": clean_worktree_exact,
|
||||
"control_goal_only_exact": control_goal_only_exact,
|
||||
"supported_worktree_mode": supported_worktree_mode,
|
||||
"generic_summary_source_bound": summary_source.get("git_head") == local_state.get("git_head")
|
||||
and summary_source.get("status_sha256") == local_state.get("status_sha256")
|
||||
and summary_source.get("worktree_clean") is False,
|
||||
and summary_source.get("worktree_clean") == local_state.get("worktree_clean"),
|
||||
}
|
||||
turn_checks: dict[str, dict[str, bool]] = {}
|
||||
previous_execution_sha256: str | None = None
|
||||
|
|
@ -1246,7 +1300,16 @@ def _benchmark_execution_chain(
|
|||
"summary_turn_count_exact": summary.get("turn_count") == len(results),
|
||||
"one_manifest_per_result": bool(results)
|
||||
and all(isinstance(item.get("execution_manifest"), dict) for item in results),
|
||||
"local_harness_state_bound": all(local_state_checks.values()),
|
||||
"local_harness_state_bound": all(
|
||||
local_state_checks[key]
|
||||
for key in (
|
||||
"git_head_valid",
|
||||
"git_head_matches_frozen_harness",
|
||||
"status_sha256_valid",
|
||||
"supported_worktree_mode",
|
||||
"generic_summary_source_bound",
|
||||
)
|
||||
),
|
||||
"all_turns_valid_under_declared_mode": bool(turn_checks)
|
||||
and all(all(item.values()) for item in turn_checks.values()),
|
||||
}
|
||||
|
|
@ -1731,8 +1794,12 @@ def score_live_trial(
|
|||
supported_claim_ids = set(grounded_receipt_score.get("supported_claim_ids") or [])
|
||||
supported_source_ids = set(grounded_receipt_score.get("supported_source_ids") or [])
|
||||
supported_identifiers = set(grounded_receipt_score.get("supported_identifiers") or [])
|
||||
ablation_claim_citations = {match.group(1).lower() for match in CLAIM_ID_CITATION_RE.finditer(baseline_reply)}
|
||||
ablation_source_citations = {match.group(1).lower() for match in SOURCE_ID_CITATION_RE.finditer(baseline_reply)}
|
||||
ablation_claim_tokens, ablation_claim_citations, ablation_unresolved_claim_citations = (
|
||||
_resolve_supported_citations(baseline_reply, CLAIM_ID_CITATION_RE, supported_claim_ids)
|
||||
)
|
||||
ablation_source_tokens, ablation_source_citations, ablation_unresolved_source_citations = (
|
||||
_resolve_supported_citations(baseline_reply, SOURCE_ID_CITATION_RE, supported_source_ids)
|
||||
)
|
||||
ablation_reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(baseline_reply)}
|
||||
grounded_answer_pass = bool(
|
||||
semantic_by_prompt.get(prompt_id, {}).get("pass")
|
||||
|
|
@ -1744,7 +1811,9 @@ def score_live_trial(
|
|||
"subject_alignment": baseline_subject_alignment.get(prompt_id) is True,
|
||||
"cites_grounded_claim_id": bool(ablation_claim_citations & supported_claim_ids),
|
||||
"cites_grounded_source_id": bool(ablation_source_citations & supported_source_ids),
|
||||
"no_identifiers_outside_grounded_receipt": bool(ablation_reply_identifiers)
|
||||
"no_identifiers_outside_grounded_receipt": bool(ablation_claim_tokens or ablation_source_tokens)
|
||||
and not ablation_unresolved_claim_citations
|
||||
and not ablation_unresolved_source_citations
|
||||
and ablation_reply_identifiers <= supported_identifiers,
|
||||
}
|
||||
ablation_answer_pass = all(ablation_binding_checks.values())
|
||||
|
|
|
|||
|
|
@ -202,6 +202,36 @@ def test_oos_autonomous_retrieval_semantics_reject_generic_challenge() -> None:
|
|||
assert benchmark.score_reply(prompt, paraphrase, memory_token=token)["pass"] is True
|
||||
|
||||
|
||||
def test_oos_autonomous_retrieval_accepts_receipt_bindable_prefixes_and_natural_sections() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[-1]
|
||||
reply = (
|
||||
"Fresh canonical public.claims readback. Claim 91dc463b, source 4207eb03. "
|
||||
"The claim body asserts that one retained path is auditable. The verified evidence says that this path "
|
||||
"has a source receipt. Challenge: the study does not rule out missing receipts on other paths. "
|
||||
"Revision: this one path is auditable, but coverage beyond it remains unestablished. Read-only; no mutation."
|
||||
)
|
||||
|
||||
score = benchmark.score_reply(prompt, reply, memory_token=token)
|
||||
|
||||
assert score["pass"] is True
|
||||
assert all(score["concepts"].values())
|
||||
|
||||
weak_challenge = reply.replace(
|
||||
"the study does not rule out missing receipts on other paths",
|
||||
"the study is interesting",
|
||||
)
|
||||
assert benchmark.score_reply(prompt, weak_challenge, memory_token=token)["pass"] is False
|
||||
|
||||
repeated_revision = reply.replace(
|
||||
"this one path is auditable, but coverage beyond it remains unestablished",
|
||||
"the original claim is repeated unchanged",
|
||||
)
|
||||
repeated_score = benchmark.score_reply(prompt, repeated_revision, memory_token=token)
|
||||
assert repeated_score["pass"] is False
|
||||
assert repeated_score["concepts"]["narrower_claim_revision"] is False
|
||||
|
||||
|
||||
def test_oos_live_check_accepts_live_readback_wording() -> None:
|
||||
assert benchmark.matched_concept("Here is the live readback from Postgres.", "live_check") is True
|
||||
|
||||
|
|
@ -285,6 +315,45 @@ def test_oos_schema_guard_allows_explicit_future_schema_gap() -> None:
|
|||
assert benchmark.current_schema_overclaims(reply) == []
|
||||
|
||||
|
||||
def test_oos_schema_guard_scopes_negation_and_schema_extension_language_to_each_clause() -> None:
|
||||
natural_gap = (
|
||||
"No resolves edge type exists in the current schema. Recording resolution requires either a resolves edge "
|
||||
"type or a dedicated field. Draft a schema extension proposal that would introduce the edge type."
|
||||
)
|
||||
contradictory = "There is no resolves edge today, but use a resolves edge now to close the forecast."
|
||||
unqualified_requirement = "Current schema requires a resolves edge, so use it now to close the forecast."
|
||||
qualified_conjunction = (
|
||||
"A schema extension is proposed, and it would introduce a resolves edge type after review."
|
||||
)
|
||||
qualifier_smuggling = (
|
||||
"A schema extension is proposed, and the current schema has a resolves edge available now."
|
||||
)
|
||||
|
||||
assert benchmark.current_schema_overclaims(natural_gap) == []
|
||||
assert benchmark.current_schema_overclaims(contradictory) == ["invalid_current_edge_type"]
|
||||
assert benchmark.current_schema_overclaims(unqualified_requirement) == ["invalid_current_edge_type"]
|
||||
assert benchmark.current_schema_overclaims(qualified_conjunction) == []
|
||||
assert benchmark.current_schema_overclaims(qualifier_smuggling) == ["invalid_current_edge_type"]
|
||||
|
||||
|
||||
def test_oos_forecast_semantics_accept_natural_history_and_missing_schema_language() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[11]
|
||||
reply = (
|
||||
"Keep the original 60% probability claim unmodified; it survives as history even though the criteria never "
|
||||
"existed. Current schema has a gap. A resolves edge — the type that would bind the outcome and timestamp — "
|
||||
"does not exist. A dedicated resolution field is the alternative; neither mechanism is present. Stage a "
|
||||
"schema extension proposal, review it, and apply only after approval."
|
||||
)
|
||||
|
||||
score = benchmark.score_reply(prompt, reply, memory_token=token)
|
||||
|
||||
assert score["pass"] is True
|
||||
assert score["current_schema_overclaims"] == []
|
||||
assert score["concepts"]["forecast_history"] is True
|
||||
assert score["concepts"]["forecast_schema_gap"] is True
|
||||
|
||||
|
||||
def test_oos_schema_guard_allows_compact_no_field_wording() -> None:
|
||||
reply = "public.reasoning_tools stores name and description; no scope field exists in the current table."
|
||||
assert benchmark.current_schema_overclaims(reply) == []
|
||||
|
|
@ -377,6 +446,36 @@ def test_oos_database_composition_requires_existing_behavioral_rule_table() -> N
|
|||
assert bad["behavioral_rule_schema_issues"] == ["behavioral_rules_false_absence"]
|
||||
|
||||
|
||||
def test_oos_natural_equivalents_cover_source_composition_and_agent_position_semantics() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompts = {prompt["id"]: prompt for prompt in benchmark.prompt_catalog(token)}
|
||||
replies = {
|
||||
"OOS-05": (
|
||||
"The attachment source_ref and extracted text on disk are retained artifacts, not canonical evidence. "
|
||||
"Canonical evidence requires a public.sources row joined through claim_evidence. An unresolved locator "
|
||||
"is weak provenance even when the artifact is retained. Stage the proposal, review it, then apply; a "
|
||||
"before/after row receipt and verified artifact hash prove the canonical link."
|
||||
),
|
||||
"OOS-06": (
|
||||
"Stage the packet with claims and source evidence, a reasoning framework, a contested interpretation, "
|
||||
"a governance rule, and a correction. Store the rule in behavioral_rules. approve_claim cannot write "
|
||||
"behavioral_rules, so that row needs a separate reviewed apply capability. Review the typed proposal, "
|
||||
"apply supported canonical rows, and retain a postflight row receipt."
|
||||
),
|
||||
"OOS-11": (
|
||||
"Do not duplicate the fact per agent. Keep the fact shared in one public.claims row with shared sources "
|
||||
"and evidence. Put each agent-specific position in public.beliefs with agent_id, stance, and confidence. "
|
||||
"Because beliefs has no claim-ID foreign key, that exact link is a schema gap; contradictory positions "
|
||||
"remain queryable by agent and subject."
|
||||
),
|
||||
}
|
||||
|
||||
for prompt_id, reply in replies.items():
|
||||
score = benchmark.score_reply(prompts[prompt_id], reply, memory_token=token)
|
||||
assert score["pass"] is True, (prompt_id, score)
|
||||
assert all(score["concepts"].values()), (prompt_id, score)
|
||||
|
||||
|
||||
def test_oos_runtime_case_rejects_db_only_causality_and_total_memory_erasure() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[9]
|
||||
|
|
|
|||
|
|
@ -279,16 +279,31 @@ def attach_fake_execution_manifests(
|
|||
*,
|
||||
grounded: bool,
|
||||
harness_git_head: str,
|
||||
harness_worktree_mode: str = "control_goal_only",
|
||||
) -> None:
|
||||
if harness_worktree_mode == "clean":
|
||||
worktree_clean = True
|
||||
status_text = ""
|
||||
status_lines: list[str] = []
|
||||
only_control_goal_untracked = False
|
||||
elif harness_worktree_mode == "control_goal_only":
|
||||
worktree_clean = False
|
||||
status_text = "?? goal.md\n"
|
||||
status_lines = ["?? goal.md"]
|
||||
only_control_goal_untracked = True
|
||||
else:
|
||||
raise ValueError(f"unsupported harness worktree mode: {harness_worktree_mode}")
|
||||
harness_source = {
|
||||
"git_head": harness_git_head,
|
||||
"worktree_clean": False,
|
||||
"status_sha256": hashlib.sha256(b"?? goal.md\n").hexdigest(),
|
||||
"worktree_clean": worktree_clean,
|
||||
"status_sha256": hashlib.sha256(status_text.encode()).hexdigest(),
|
||||
}
|
||||
previous_execution_sha256 = None
|
||||
allowed_missing = (
|
||||
allowed_missing = set(
|
||||
protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING if grounded else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING
|
||||
)
|
||||
if harness_worktree_mode == "control_goal_only":
|
||||
allowed_missing.update(protocol_lib.CONTROL_GOAL_EXECUTION_ALLOWED_MISSING)
|
||||
fingerprint = report["db_fingerprint_before"]
|
||||
counts_sha256 = protocol_lib.canonical_sha256({})
|
||||
for result in report["results"]:
|
||||
|
|
@ -362,7 +377,7 @@ def attach_fake_execution_manifests(
|
|||
},
|
||||
},
|
||||
"attribution": {
|
||||
"status": "incomplete",
|
||||
"status": "incomplete" if allowed_missing else "complete",
|
||||
"missing_required_bindings": sorted(allowed_missing),
|
||||
},
|
||||
}
|
||||
|
|
@ -375,19 +390,25 @@ def attach_fake_execution_manifests(
|
|||
previous_execution_sha256 = manifest["execution_sha256"]
|
||||
report["oos_harness_git_state"] = {
|
||||
**harness_source,
|
||||
"status_lines": ["?? goal.md"],
|
||||
"only_control_goal_untracked": True,
|
||||
"status_lines": status_lines,
|
||||
"only_control_goal_untracked": only_control_goal_untracked,
|
||||
}
|
||||
report["execution_manifest_summary"] = {
|
||||
"schema": protocol_lib.execution_manifest_lib.SCHEMA,
|
||||
"turn_count": len(report["results"]),
|
||||
"attribution_complete_count": 0,
|
||||
"all_turns_attribution_complete": False,
|
||||
"attribution_complete_count": 0 if allowed_missing else len(report["results"]),
|
||||
"all_turns_attribution_complete": not allowed_missing,
|
||||
"harness_source": harness_source,
|
||||
}
|
||||
|
||||
|
||||
def fake_report(protocol: dict, trial: dict, *, grounded: bool) -> dict:
|
||||
def fake_report(
|
||||
protocol: dict,
|
||||
trial: dict,
|
||||
*,
|
||||
grounded: bool,
|
||||
harness_worktree_mode: str = "control_goal_only",
|
||||
) -> dict:
|
||||
mode = "grounded" if grounded else "db_tool_ablated"
|
||||
fingerprint = {
|
||||
"status": "ok",
|
||||
|
|
@ -490,6 +511,7 @@ def fake_report(protocol: dict, trial: dict, *, grounded: bool) -> dict:
|
|||
report,
|
||||
grounded=grounded,
|
||||
harness_git_head=protocol["harness_git_head"],
|
||||
harness_worktree_mode=harness_worktree_mode,
|
||||
)
|
||||
return report
|
||||
|
||||
|
|
@ -660,6 +682,13 @@ def test_protocol_validation_rejects_prompt_and_source_hash_tampering() -> None:
|
|||
in protocol_lib.validate_protocol(protocol, verify_source_hashes=True)["issues"]
|
||||
)
|
||||
|
||||
protocol = frozen_protocol()
|
||||
protocol["scorer_version"] = "stale-scorer"
|
||||
protocol["protocol_hash_sha256"] = protocol_lib.canonical_sha256(
|
||||
{key: value for key, value in protocol.items() if key != "protocol_hash_sha256"}
|
||||
)
|
||||
assert "wrong_scorer_version" in protocol_lib.validate_protocol(protocol, verify_source_hashes=True)["issues"]
|
||||
|
||||
|
||||
def test_live_trial_requires_semantics_subject_receipts_and_real_ablation() -> None:
|
||||
protocol = frozen_protocol()
|
||||
|
|
@ -695,12 +724,23 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
|
|||
trial = protocol["trials"][0]
|
||||
for grounded in (True, False):
|
||||
report = fake_report(protocol, trial, grounded=grounded)
|
||||
assert (
|
||||
protocol_lib._benchmark_execution_chain(
|
||||
report,
|
||||
expected_harness_git_head=protocol["harness_git_head"],
|
||||
)["pass"]
|
||||
is True
|
||||
validation = protocol_lib._benchmark_execution_chain(
|
||||
report,
|
||||
expected_harness_git_head=protocol["harness_git_head"],
|
||||
)
|
||||
expected_missing = (
|
||||
protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING
|
||||
if grounded
|
||||
else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING
|
||||
) | protocol_lib.CONTROL_GOAL_EXECUTION_ALLOWED_MISSING
|
||||
|
||||
assert validation["pass"] is True
|
||||
assert validation["local_state_checks"]["control_goal_only_exact"] is True
|
||||
assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing)
|
||||
assert all(
|
||||
result["execution_manifest"]["attribution"]["missing_required_bindings"]
|
||||
== sorted(expected_missing)
|
||||
for result in report["results"]
|
||||
)
|
||||
|
||||
manifest = report["results"][0]["execution_manifest"]
|
||||
|
|
@ -736,6 +776,66 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
|
|||
assert validation["local_state_checks"]["git_head_matches_frozen_harness"] is False
|
||||
|
||||
|
||||
def test_execution_chain_accepts_clean_harness_without_cleanliness_omission() -> None:
|
||||
protocol = frozen_protocol()
|
||||
trial = protocol["trials"][0]
|
||||
|
||||
for grounded in (True, False):
|
||||
report = fake_report(
|
||||
protocol,
|
||||
trial,
|
||||
grounded=grounded,
|
||||
harness_worktree_mode="clean",
|
||||
)
|
||||
validation = protocol_lib._benchmark_execution_chain(
|
||||
report,
|
||||
expected_harness_git_head=protocol["harness_git_head"],
|
||||
)
|
||||
|
||||
assert validation["pass"] is True
|
||||
assert validation["local_state_checks"]["clean_worktree_exact"] is True
|
||||
expected_missing = (
|
||||
protocol_lib.GROUNDED_EXECUTION_ALLOWED_MISSING
|
||||
if grounded
|
||||
else protocol_lib.ABLATION_EXECUTION_ALLOWED_MISSING
|
||||
)
|
||||
assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing)
|
||||
assert all(
|
||||
result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing)
|
||||
for result in report["results"]
|
||||
)
|
||||
assert all(
|
||||
checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values()
|
||||
)
|
||||
|
||||
manifest = report["results"][-1]["execution_manifest"]
|
||||
manifest["attribution"]["missing_required_bindings"].append("harness_worktree_clean")
|
||||
manifest["attribution"]["status"] = "incomplete"
|
||||
stable = {key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"}}
|
||||
manifest["execution_sha256"] = protocol_lib.execution_manifest_lib.canonical_sha256(stable)
|
||||
validation = protocol_lib._benchmark_execution_chain(
|
||||
report,
|
||||
expected_harness_git_head=protocol["harness_git_head"],
|
||||
)
|
||||
assert validation["pass"] is False
|
||||
assert validation["turn_checks"][trial["prompts"][-1]["id"]]["declared_missing_exact"] is False
|
||||
|
||||
|
||||
def test_execution_chain_rejects_inexact_control_goal_state() -> None:
|
||||
protocol = frozen_protocol()
|
||||
trial = protocol["trials"][0]
|
||||
report = fake_report(protocol, trial, grounded=True)
|
||||
report["oos_harness_git_state"]["status_lines"] = ["?? goal.md", "?? scratch.txt"]
|
||||
|
||||
validation = protocol_lib._benchmark_execution_chain(
|
||||
report,
|
||||
expected_harness_git_head=protocol["harness_git_head"],
|
||||
)
|
||||
|
||||
assert validation["pass"] is False
|
||||
assert validation["local_state_checks"]["control_goal_only_exact"] is False
|
||||
|
||||
|
||||
def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal() -> None:
|
||||
protocol = frozen_protocol()
|
||||
trial = protocol["trials"][0]
|
||||
|
|
@ -952,6 +1052,60 @@ def test_autonomous_retrieval_requires_nonempty_rows_and_receipted_reply_ids() -
|
|||
assert unrelated_score["unsupported_identifiers"] == [unrelated_uuid]
|
||||
|
||||
|
||||
def test_autonomous_retrieval_resolves_only_unique_receipt_bound_identifier_prefixes() -> None:
|
||||
protocol = frozen_protocol()
|
||||
trial = protocol["trials"][0]
|
||||
prompt = next(item for item in trial["prompts"] if item["family_id"] == "autonomous_retrieval_reasoning")
|
||||
result = result_for(prompt, trial["memory_token"], 1, grounded=True)
|
||||
result["reply"] = (
|
||||
f"{prompt['subject']}. Claim {CLAIM_UUID[:8]}, source {SOURCE_UUID[:8]}. "
|
||||
"The claim body states one bounded observation. The evidence documents that observation separately. "
|
||||
"Challenge: it does not rule out alternatives. Revision: the result is limited to this source."
|
||||
)
|
||||
result["database_context_trace"][-1]["delivered_response_sha256"] = hashlib.sha256(
|
||||
result["reply"].encode()
|
||||
).hexdigest()
|
||||
|
||||
unique = protocol_lib._receipt_score(
|
||||
result,
|
||||
require_database_contract=False,
|
||||
require_database_receipt=True,
|
||||
require_grounded_rows=True,
|
||||
)
|
||||
assert unique["pass"] is True
|
||||
assert unique["reply_claim_citation_tokens"] == [CLAIM_UUID[:8]]
|
||||
assert unique["reply_claim_citations"] == [CLAIM_UUID]
|
||||
assert unique["reply_source_or_evidence_citation_tokens"] == [SOURCE_UUID[:8]]
|
||||
assert unique["reply_source_or_evidence_citations"] == [SOURCE_UUID]
|
||||
|
||||
unbound = copy.deepcopy(result)
|
||||
unbound["reply"] = unbound["reply"].replace(CLAIM_UUID[:8], "deadbeef")
|
||||
unbound["database_context_trace"][-1]["delivered_response_sha256"] = hashlib.sha256(
|
||||
unbound["reply"].encode()
|
||||
).hexdigest()
|
||||
unbound_score = protocol_lib._receipt_score(
|
||||
unbound,
|
||||
require_database_contract=False,
|
||||
require_database_receipt=True,
|
||||
require_grounded_rows=True,
|
||||
)
|
||||
assert unbound_score["pass"] is False
|
||||
assert unbound_score["unresolved_claim_citations"] == ["deadbeef"]
|
||||
|
||||
ambiguous = copy.deepcopy(result)
|
||||
ambiguous_receipt = ambiguous["database_context_trace"][0]["retrieval_receipt"]
|
||||
ambiguous_receipt["claim_ids"].append("11111111-aaaa-4aaa-8aaa-aaaaaaaaaaaa")
|
||||
rehash_retrieval_receipt(ambiguous_receipt)
|
||||
ambiguous_score = protocol_lib._receipt_score(
|
||||
ambiguous,
|
||||
require_database_contract=False,
|
||||
require_database_receipt=True,
|
||||
require_grounded_rows=True,
|
||||
)
|
||||
assert ambiguous_score["pass"] is False
|
||||
assert ambiguous_score["unresolved_claim_citations"] == [CLAIM_UUID[:8]]
|
||||
|
||||
|
||||
def test_autonomous_retrieval_ablation_rejects_generic_prose_and_invented_ids() -> None:
|
||||
protocol = frozen_protocol()
|
||||
trial = protocol["trials"][0]
|
||||
|
|
@ -1029,7 +1183,7 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words
|
|||
f"{source_prompt['subject']} uses source evidence and claim_evidence; "
|
||||
f"repeat {source_prompt['subject']} is not acceptable."
|
||||
)
|
||||
assert protocol_lib._subject_alignment(source_prompt, duplicate_subject) is False
|
||||
assert protocol_lib._subject_alignment(source_prompt, duplicate_subject) is True
|
||||
sibling_subject = next(item for item in source_prompt["family_subjects"] if item != source_prompt["subject"])
|
||||
shotgun = f"{source_prompt['subject']} and {sibling_subject} both use source evidence and claim_evidence."
|
||||
assert protocol_lib._subject_alignment(source_prompt, shotgun) is False
|
||||
|
|
@ -1042,7 +1196,7 @@ def test_subject_alignment_rejects_a_different_family_even_with_generic_db_words
|
|||
"Challenge: evidence does not prove the leap. Revision: narrower claim."
|
||||
)
|
||||
assert protocol_lib._subject_alignment(retrieval_prompt, good) is True
|
||||
assert protocol_lib._subject_alignment(retrieval_prompt, good + f" {retrieval_prompt['subject']}.") is False
|
||||
assert protocol_lib._subject_alignment(retrieval_prompt, good + f" {retrieval_prompt['subject']}.") is True
|
||||
retrieval_sibling = next(
|
||||
item for item in retrieval_prompt["family_subjects"] if item != retrieval_prompt["subject"]
|
||||
)
|
||||
|
|
@ -1123,29 +1277,33 @@ def test_identical_grounded_and_ablation_replies_cannot_create_evidence_delta()
|
|||
def test_nonidentical_ablated_answer_with_grounded_ids_does_not_manufacture_causal_lift() -> None:
|
||||
protocol = frozen_protocol()
|
||||
trial = protocol["trials"][0]
|
||||
grounded = fake_report(protocol, trial, grounded=True)
|
||||
ablated = fake_report(protocol, trial, grounded=False)
|
||||
prompt = next(item for item in trial["prompts"] if item.get("requires_grounded_retrieval_answer"))
|
||||
result = next(item for item in ablated["results"] if item["prompt_id"] == prompt["id"])
|
||||
result["reply"] = (
|
||||
f"{prompt['subject']}. Fresh canonical public.claims readback: claim `{CLAIM_UUID}` states that one "
|
||||
f"retained path is auditable. Source `{SOURCE_UUID}` documents that path and is distinct from the claim. "
|
||||
"However, the evidence cannot prove every path is auditable. A narrower claim is limited to this one path. "
|
||||
"The lookup is read-only and made no mutation."
|
||||
)
|
||||
for claim_citation, source_citation in (
|
||||
(CLAIM_UUID, SOURCE_UUID),
|
||||
(CLAIM_UUID[:8], SOURCE_UUID[:8]),
|
||||
):
|
||||
grounded = fake_report(protocol, trial, grounded=True)
|
||||
ablated = fake_report(protocol, trial, grounded=False)
|
||||
result = next(item for item in ablated["results"] if item["prompt_id"] == prompt["id"])
|
||||
result["reply"] = (
|
||||
f"{prompt['subject']}. Fresh canonical public.claims readback: claim `{claim_citation}` states that one "
|
||||
f"retained path is auditable. Source `{source_citation}` documents that path and is distinct from the "
|
||||
"claim. However, the evidence cannot prove every path is auditable. A narrower claim is limited to this "
|
||||
"one path. The lookup is read-only and made no mutation."
|
||||
)
|
||||
|
||||
score = protocol_lib.score_live_trial(
|
||||
protocol,
|
||||
trial["trial_id"],
|
||||
grounded,
|
||||
baseline_report=ablated,
|
||||
)
|
||||
row = score["autonomous_retrieval_comparison"]["rows"][0]
|
||||
score = protocol_lib.score_live_trial(
|
||||
protocol,
|
||||
trial["trial_id"],
|
||||
grounded,
|
||||
baseline_report=ablated,
|
||||
)
|
||||
row = score["autonomous_retrieval_comparison"]["rows"][0]
|
||||
|
||||
assert row["ablation_answer_pass"] is True
|
||||
assert row["replies_identical"] is False
|
||||
assert row["causally_attributed_grounded_pass"] is False
|
||||
assert score["autonomous_retrieval_comparison"]["grounded_minus_ablation_answer_delta"] == 0.0
|
||||
assert row["ablation_answer_pass"] is True
|
||||
assert row["replies_identical"] is False
|
||||
assert row["causally_attributed_grounded_pass"] is False
|
||||
assert score["autonomous_retrieval_comparison"]["grounded_minus_ablation_answer_delta"] == 0.0
|
||||
|
||||
|
||||
def test_receipt_discriminator_rejects_wrong_command_extra_call_and_wrong_token() -> None:
|
||||
|
|
|
|||
Loading…
Reference in a new issue