Harden blinded Leo reasoning repair gate

This commit is contained in:
twentyOne2x 2026-07-16 05:19:50 +02:00
parent 4daa7853fe
commit 813d4cd91d
10 changed files with 370 additions and 52 deletions

View file

@ -855,11 +855,17 @@ def operational_contracts(
mixed_markers = ( mixed_markers = (
"packet", "packet",
"briefing",
"factual observation", "factual observation",
"evidence-backed fact",
"strategic framework", "strategic framework",
"reasoning framework",
"disputed interpretation", "disputed interpretation",
"contested position",
"governance rule", "governance rule",
"operating rule",
"old belief", "old belief",
"correction",
"compose the database", "compose the database",
) )
if sum(marker in lowered for marker in mixed_markers) >= 2: if sum(marker in lowered for marker in mixed_markers) >= 2:

View file

@ -568,6 +568,13 @@ def _mixed_packet_issues(response: str) -> list[str]:
re.I | re.S, re.I | re.S,
): ):
issues.append("unsupported_collection_listed_in_apply") issues.append("unsupported_collection_listed_in_apply")
if re.search(
r"(?:reasoning|strategic) framework.{0,80}"
r"(?:is|as|into|maps? to|mapped to|classif(?:y|ied) as) (?:an? )?(?:normative )?claim",
response,
re.I | re.S,
):
issues.append("framework_mapped_to_claim")
for segment in re.split(r"(?<=[.!?])\s+|\n+", response): for segment in re.split(r"(?<=[.!?])\s+|\n+", response):
negative = re.search(r"\b(?:no|not|never|without|must not|do not|does not|cannot)\b", segment, re.I) negative = re.search(r"\b(?:no|not|never|without|must not|do not|does not|cannot)\b", segment, re.I)
@ -807,7 +814,7 @@ def _schema_contract_issues(response: str, contracts: list[dict[str, Any]]) -> l
if "runtime_persistence" in contract_ids and affirmative( if "runtime_persistence" in contract_ids and affirmative(
r"(?:state\.db|session\s+jsonl).{0,100}" r"(?:state\.db|session\s+jsonl).{0,100}"
r"(?:in[- ]memory|ephemeral|disappears?|vanishes?|erased|discarded)" r"(?:in[- ]memory|ephemeral|disappears?|vanishes?|erased|discarded|gone|lost|deleted)"
): ):
issues.append("runtime_persistence_contradiction") issues.append("runtime_persistence_contradiction")
@ -833,6 +840,12 @@ def _schema_contract_issues(response: str, contracts: list[dict[str, Any]]) -> l
issues.append("forecast_resolution_edge_contradiction") issues.append("forecast_resolution_edge_contradiction")
if affirmative(r"public\.claims.{0,100}(?:has|stores?|contains?).{0,80}(?:forecast_resolution|resolved_at)"): if affirmative(r"public\.claims.{0,100}(?:has|stores?|contains?).{0,80}(?:forecast_resolution|resolved_at)"):
issues.append("forecast_resolution_field_contradiction") issues.append("forecast_resolution_field_contradiction")
if not (
re.search(r"\b(?:stage|proposal)\b", response, re.I)
and re.search(r"\breview(?:ed|er|ing)?\b", response, re.I)
and re.search(r"\bappl(?:y|ied|ication)\b", response, re.I)
):
issues.append("forecast_review_apply_incomplete")
return sorted(set(issues)) return sorted(set(issues))

View file

@ -345,21 +345,26 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
) )
tool_call_count = trace.get("database_tool_call_count", 0) tool_call_count = trace.get("database_tool_call_count", 0)
completed_count = trace.get("database_tool_completed_count", 0) completed_count = trace.get("database_tool_completed_count", 0)
all_results_bound = bool( all_results_content_bound = bool(
_non_negative_int(tool_call_count) _non_negative_int(tool_call_count)
and _non_negative_int(completed_count) and _non_negative_int(completed_count)
and completed_count == tool_call_count and completed_count <= tool_call_count
and len(calls) == tool_call_count and len(calls) == tool_call_count
and all( and all(
_is_sha256(call.get("tool_call_id_sha256")) _is_sha256(call.get("tool_call_id_sha256"))
and _is_sha256(call.get("arguments_sha256")) and _is_sha256(call.get("arguments_sha256"))
and _is_sha256(call.get("result_content_sha256")) and _is_sha256(call.get("result_content_sha256"))
and _non_negative_int(call.get("result_content_bytes")) and _non_negative_int(call.get("result_content_bytes"))
and call.get("result_error_detected") is False and isinstance(call.get("result_error_detected"), bool)
and call.get("result_nonempty") is True and call.get("result_nonempty") is True
for call in calls for call in calls
) )
) )
all_calls_succeeded = bool(
all_results_content_bound
and completed_count == tool_call_count
and all(call.get("result_error_detected") is False for call in calls)
)
all_calls_read_only = bool( all_calls_read_only = bool(
trace.get("database_tool_calls_read_only") is True trace.get("database_tool_calls_read_only") is True
and all( and all(
@ -376,7 +381,8 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
"tool_call_count": tool_call_count, "tool_call_count": tool_call_count,
"completed_count": completed_count, "completed_count": completed_count,
"all_calls_read_only": all_calls_read_only if tool_call_count else True, "all_calls_read_only": all_calls_read_only if tool_call_count else True,
"all_results_content_bound": all_results_bound, "all_results_content_bound": all_results_content_bound,
"all_calls_succeeded": all_calls_succeeded,
"calls": calls, "calls": calls,
"trace_sha256": canonical_sha256(trace), "trace_sha256": canonical_sha256(trace),
} }

View file

@ -254,7 +254,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
"canonical_readback": ( "canonical_readback": (
re.compile( re.compile(
r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)|" r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)|"
r"canonical (?:DB|database)|applied_at.{0,100}readiness", r"canonical (?:DB|database|KB)|(?:KB|knowledge database).{0,50}(?:changed|updated)|"
r"applied_at.{0,100}(?:readiness|postflight|canonical)",
re.I | re.S, re.I | re.S,
), ),
), ),
@ -263,7 +264,12 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
r"approved is not (?:the same as )?applied|applied_at:\s*(?:none|null)|not canonical|" r"approved is not (?:the same as )?applied|applied_at:\s*(?:none|null)|not canonical|"
r"proposal does not commit|no receipt,? no durable knowledge|" r"proposal does not commit|no receipt,? no durable knowledge|"
r"approval.{0,60}not (?:a )?(?:DB|database|canonical) write|" r"approval.{0,60}not (?:a )?(?:DB|database|canonical) write|"
r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied", r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied|"
r"(?:reviewer )?approval is intent.{0,100}canonical (?:update|changes?).{0,80}(?:apply|applied_at)|"
r"(?:reviewer )?approval is intent.{0,100}(?:canonical )?KB changes? only when.{0,60}"
r"(?:apply|applied_at)|"
r"(?:reviewer )?approval.{0,100}(?:canonical|KB).{0,60}(?:requires|happens only when).{0,50}"
r"(?:apply|applied_at)",
re.I, re.I,
), ),
), ),
@ -280,11 +286,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(r"provenance|stable (?:reference|ref)|source row|author/channel|(?:file|content) hash", re.I), re.compile(r"provenance|stable (?:reference|ref)|source row|author/channel|(?:file|content) hash", re.I),
), ),
"deduplication": (re.compile(r"deduplic|duplicate|existing claim|overlap", re.I),), "deduplication": (re.compile(r"deduplic|duplicate|existing claim|overlap", re.I),),
"contradiction": (re.compile(r"contradict|divergence|competing interpretation|disagree", re.I),), "contradiction": (
re.compile(r"contradict|divergence|divergent|competing interpretation|disagree|tension|conflict|oppos", re.I),
),
"staged_review_apply": ( "staged_review_apply": (
re.compile(r"staging|stage|proposal", re.I), re.compile(r"staging|stage|proposal", re.I),
re.compile(r"review|approve|reviewer", re.I), re.compile(r"review|approve|reviewer", re.I),
re.compile(r"apply|canonical", re.I), re.compile(r"appl(?:y|ied|ication)|canonical", re.I),
), ),
"receipt": ( "receipt": (
re.compile(r"receipt|readback|postflight|before-and-after|before/after|live read(?:ing)?|live proof", re.I), re.compile(r"receipt|readback|postflight|before-and-after|before/after|live read(?:ing)?|live proof", re.I),
@ -310,13 +318,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
), ),
), ),
"source_evidence_chain": ( "source_evidence_chain": (
re.compile(r"file|attachment|source_ref", re.I), re.compile(r"file|attachment|source_ref|document|artifact|extracted (?:text|markdown)|path|pointer", re.I),
re.compile(r"public\.sources", re.I), re.compile(r"public\.sources", re.I),
re.compile(r"claim_evidence", re.I), re.compile(r"claim_evidence", re.I),
re.compile(r"audit|link|join|row chain|before-and-after|before/after", re.I), re.compile(r"audit|link|join|row chain|before-and-after|before/after", re.I),
), ),
"canonical_evidence_boundary": ( "canonical_evidence_boundary": (
re.compile(r"canonical evidence", re.I), re.compile(r"canonical (?:evidence|support|linkage)|not yet canonical|not canonical", re.I),
re.compile(r"claim_evidence", re.I), re.compile(r"claim_evidence", re.I),
re.compile(r"public\.sources|source row", re.I), re.compile(r"public\.sources|source row", re.I),
re.compile( re.compile(
@ -325,13 +333,19 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
r"(?:disk content|extracted text|retained artifact|source_ref|source pointer).{0,140}" r"(?:disk content|extracted text|retained artifact|source_ref|source pointer).{0,140}"
r"(?:is not|isn't|does not|doesn't|alone|until|unless).{0,80}(?:canonical evidence|finished provenance)|" r"(?:is not|isn't|does not|doesn't|alone|until|unless).{0,80}(?:canonical evidence|finished provenance)|"
r"(?:canonical evidence|canonical link).{0,100}(?:requires|exists only when|is complete only when).{0,100}" r"(?:canonical evidence|canonical link).{0,100}(?:requires|exists only when|is complete only when).{0,100}"
r"(?:public\.sources|source row|claim_evidence)", r"(?:public\.sources|source row|claim_evidence)|"
r"(?:not yet canonical|staging-only).{0,300}(?:public\.sources|source row).{0,200}claim_evidence|"
r"(?:public\.sources|source row).{0,120}(?:absent|missing|requires?).{0,200}claim_evidence",
re.I | re.S, re.I | re.S,
), ),
), ),
"evidence_provenance_quality": ( "evidence_provenance_quality": (
re.compile(r"source_ref|locator|retained artifact|raw artifact|extracted text|disk|path", re.I), re.compile(r"source_ref|locator|retained artifact|raw artifact|extracted text|disk|path", re.I),
re.compile(r"weak|unresolved|not traceable|verified|hash_matches_db|provenance", re.I), re.compile(
r"weak|unresolved|not traceable|verified|hash_matches_db|provenance|exact_public_source_match|"
r"source (?:match|audit)|link audit",
re.I,
),
re.compile(r"canonical evidence|canonical link|public\.sources|claim_evidence", re.I), re.compile(r"canonical evidence|canonical link|public\.sources|claim_evidence", re.I),
), ),
"heterogeneous_types": ( "heterogeneous_types": (
@ -352,19 +366,28 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile( re.compile(
r"does not (?:accept|insert|support|cover|write)|supports neither|neither.{0,80}nor|" r"does not (?:accept|insert|support|cover|write)|supports neither|neither.{0,80}nor|"
r"(?:behavioral_rules|governance_gates).{0,120}(?:cannot be applied by|sits? outside) approve_claim|" r"(?:behavioral_rules|governance_gates).{0,120}(?:cannot be applied by|sits? outside) approve_claim|"
r"outside approve_claim(?:'s)? apply capability|approve_claim applies only|"
r"approve_claim.{0,120}(?:cannot apply|cannot write|does not write)", r"approve_claim.{0,120}(?:cannot apply|cannot write|does not write)",
re.I | re.S, re.I | re.S,
), ),
re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply (?:capability|path)", re.I | re.S), re.compile(
r"separate.{0,60}(?:reviewed|authorized|authorization).{0,60}(?:apply )?(?:capability|path|write)|"
r"separate (?:reviewed )?(?:apply )?(?:capability|authorization)|"
r"separate.{0,50}apply capability.{0,60}(?:reviewed|authorized)|"
r"reviewed.{0,50}apply (?:capability|path)",
re.I | re.S,
),
), ),
"runtime_inputs": ( "runtime_inputs": (
re.compile(r"Postgres|canonical (?:DB|database|counts?)|(?:DB|database) totals?", re.I), re.compile(r"Postgres|canonical (?:DB|database|counts?|rows?)|(?:DB|database) totals?", re.I),
re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I), re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I),
re.compile(r"session|conversation context", re.I), re.compile(r"session|conversation context", re.I),
re.compile( re.compile(
r"unchanged.{0,120}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)|" r"unchanged.{0,120}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)|"
r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)|" r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)|"
r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)", r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)|"
r"(?:identical|unchanged).{0,40}(?:counts|totals).{0,60}(?:prove nothing about content|"
r"not proof of content)",
re.I | re.S, re.I | re.S,
), ),
), ),
@ -372,23 +395,28 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(r"state\.db|session JSONL", re.I), re.compile(r"state\.db|session JSONL", re.I),
re.compile(r"persist|durable|continuity", re.I), re.compile(r"persist|durable|continuity", re.I),
re.compile( re.compile(
r"restart.{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|" r"(?:restart|recycle).{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|"
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}restart|" r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}(?:restart|recycle)|"
r"(?:facts?|context).{0,80}not erased by restart", r"(?:facts?|context).{0,80}not erased by (?:restart|recycle)",
re.I | re.S, re.I | re.S,
), ),
), ),
"row_content_proof": ( "row_content_proof": (
re.compile(r"unchanged (?:database )?(?:counts?|totals?)", re.I), re.compile(r"(?:unchanged|identical) (?:database )?(?:counts?|totals?)", re.I),
re.compile(r"does not prove|doesn't prove|do not prove|say nothing", re.I), re.compile(r"does not prove|doesn't prove|do not prove|prove(?:s)? nothing|say nothing", re.I),
re.compile( re.compile(
r"row (?:IDs?|hashes?)|fingerprints?|timestamps?|balanced (?:insert|write|change)|" r"row (?:IDs?|hashes?)|fingerprints?|timestamps?|balanced (?:insert|write|change)|"
r"individual rows?.{0,60}(?:mutated|changed|updated)", r"individual rows?.{0,60}(?:mutated|changed|updated)|artifact_state_sha256|content hashes?|"
r"row[- ]level (?:hash|readback)",
re.I | re.S, re.I | re.S,
), ),
), ),
"proof_tiers": ( "proof_tiers": (
re.compile(r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3", re.I | re.S), re.compile(
r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3|"
r"canonical rows.{0,600}deployed runtime inputs.{0,600}durable session state",
re.I | re.S,
),
re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I), re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I),
re.compile(r"runtime|skills?|session|SOUL\.md", re.I), re.compile(r"runtime|skills?|session|SOUL\.md", re.I),
), ),
@ -407,26 +435,29 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
), ),
"agent_specific_positions": ( "agent_specific_positions": (
re.compile(r"public\.beliefs", re.I), re.compile(r"public\.beliefs", re.I),
re.compile(r"agent_id", re.I), re.compile(r"agent_id|agent attribution|attributed to (?:that|each|an?) agent", re.I),
re.compile(r"belief|position|stance|confidence", re.I), re.compile(r"belief|position|stance|confidence", re.I),
re.compile(r"no.{0,40}claim(?:-ID|_id).{0,30}(?:foreign key|link)|schema gap", re.I | re.S), re.compile(r"no.{0,40}claim(?:-ID|_id).{0,30}(?:foreign key|link)|schema gap", re.I | re.S),
), ),
"forecast_history": ( "forecast_history": (
re.compile(r"original (?:probability|confidence)|60%|history", re.I), re.compile(r"original (?:probability|confidence)|original.{0,120}(?:probability|confidence)|60%|history", re.I),
re.compile( re.compile(
r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|original.{0,60}survives", r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|"
r"leave.{0,60}(?:untouched|unmodified)|original.{0,60}survives",
re.I | re.S, re.I | re.S,
), ),
re.compile( re.compile(
r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria|" r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria|"
r"lack(?:ed|s|ing).{0,30}(?:success|resolution) criteria|"
r"criteria.{0,40}(?:never existed|were never defined|did not exist)|without.{0,30}criteria", r"criteria.{0,40}(?:never existed|were never defined|did not exist)|without.{0,30}criteria",
re.I | re.S, re.I | re.S,
), ),
), ),
"forecast_schema_gap": ( "forecast_schema_gap": (
re.compile(r"current (?:v1|schema)|public\.claims", re.I), re.compile(r"current (?:v1|schema)|public\.claims|the schema|schema has|claims table", re.I),
re.compile( re.compile(
r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|" r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|"
r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}" r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
r"(?:does not exist|is absent|is not present|isn't present)|" r"(?:does not exist|is absent|is not present|isn't present)|"
r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present", r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present",
@ -531,24 +562,25 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
), ),
), ),
"evidence_specific_challenge": ( "evidence_specific_challenge": (
re.compile(r"\b(?:challenge|limitation|however|unsupported leap|caveat)\b", re.I), re.compile(r"\b(?:challeng(?:e|ed|ing)|limitation|however|unsupported leap|caveat)\b", re.I),
re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I), re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I),
re.compile( re.compile(
r"\b(?:does not|doesn't|cannot|can't)\s+(?:itself\s+)?(?:prove|establish|support|show)|" r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+(?:itself\s+)?"
r"\b(?:does not|doesn't|cannot|can't)\s+rule out\b|" r"(?:prove|establish|support|show)|"
r"\bunsupported leap\b", r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+rule out\b|"
r"\b(?:not paid for|outruns?).{0,40}\b(?:the )?evidence\b|\bunsupported leap\b",
re.I, re.I,
), ),
), ),
"narrower_claim_revision": ( "narrower_claim_revision": (
re.compile( re.compile(
r"\b(?:narrower\s+(?:revision|claim|formulation)|revision\s*:|" r"\b(?:narrower\s+(?:revision|claim|formulation|candidate(?: claim)?)|revision\s*:|"
r"should be narrowed to|defensible formulation|revise(?:d)?\b.{0,40}\bto\b)", r"should be narrowed to|defensible formulation|revise(?:d)?\b.{0,40}\bto\b)",
re.I, re.I,
), ),
re.compile( re.compile(
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|" r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|"
r"unproven|open constraint)\b", r"undemonstrated|unproven|not established|not demonstrated|remains unknown|open constraint)\b",
re.I, re.I,
), ),
), ),
@ -567,7 +599,7 @@ COUNT_INVARIANT_REJECTION_RE = re.compile(
) )
SCHEMA_GAP_QUALIFIER_RE = re.compile( SCHEMA_GAP_QUALIFIER_RE = re.compile(
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|" r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|" r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|"
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|" r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
r"schema extension|extension proposal)\b|" r"schema extension|extension proposal)\b|"
@ -656,6 +688,11 @@ DB_ONLY_CAUSALITY_RE = re.compile(
RESTART_ERASES_ALL_RE = re.compile( RESTART_ERASES_ALL_RE = re.compile(
r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S
) )
DURABLE_SESSION_EPHEMERAL_RE = re.compile(
r"session JSONL.{0,100}(?:is|are|gets?|becomes?).{0,30}(?:gone|lost|deleted|discarded)|"
r"session JSONL.{0,100}(?:gone|lost|deleted|discarded).{0,50}(?:session closes?|restart)",
re.I | re.S,
)
REASONING_TOOL_CLAIM_EDGE_RE = re.compile( REASONING_TOOL_CLAIM_EDGE_RE = re.compile(
r"claims?.{0,120}(?:get|have|create|use).{0,50}(?:claim_)?edges?.{0,100}" r"claims?.{0,120}(?:get|have|create|use).{0,50}(?:claim_)?edges?.{0,100}"
r"(?:reasoning_tool|tool(?:'s)? ID)|" r"(?:reasoning_tool|tool(?:'s)? ID)|"
@ -718,7 +755,7 @@ CLAIM_EVIDENCE_CONFLATION_RE = re.compile(
REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P<value>[^\n]+)", re.I) REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P<value>[^\n]+)", re.I)
REVISION_BOUNDARY_RE = re.compile( REVISION_BOUNDARY_RE = re.compile(
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|" r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|"
r"open constraint)\b", r"undemonstrated|not established|not demonstrated|remains unknown|open constraint)\b",
re.I, re.I,
) )
REVISION_REPETITION_RE = re.compile( REVISION_REPETITION_RE = re.compile(
@ -849,6 +886,8 @@ def broad_semantic_issues(reply: str) -> list[str]:
re.I | re.S, re.I | re.S,
): ):
findings.add("restart_called_total_memory_erasure") findings.add("restart_called_total_memory_erasure")
if DURABLE_SESSION_EPHEMERAL_RE.search(reply):
findings.add("durable_session_called_ephemeral")
if HANDLER_TELEGRAM_OVERCLAIM_RE.search(reply): if HANDLER_TELEGRAM_OVERCLAIM_RE.search(reply):
findings.add("handler_proof_called_telegram_live") findings.add("handler_proof_called_telegram_live")
if FORECAST_HISTORY_REWRITE_RE.search(reply): if FORECAST_HISTORY_REWRITE_RE.search(reply):

View file

@ -866,6 +866,7 @@ def _executed_behavior_ablation(grounded_report: dict[str, Any], baseline_report
stable_keys = { stable_keys = {
"schema", "schema",
"model_runtime", "model_runtime",
"python_runtime",
"hermes_runtime", "hermes_runtime",
"teleo_infrastructure_runtime", "teleo_infrastructure_runtime",
"components", "components",
@ -1061,11 +1062,11 @@ def _receipt_score(
} }
reply = str(result.get("reply") or "") reply = str(result.get("reply") or "")
reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(reply)} reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(reply)}
reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = ( reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = _resolve_supported_citations(
_resolve_supported_citations(reply, CLAIM_ID_CITATION_RE, receipt_claim_ids) reply, CLAIM_ID_CITATION_RE, receipt_claim_ids
) )
reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = ( reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = _resolve_supported_citations(
_resolve_supported_citations(reply, SOURCE_ID_CITATION_RE, receipt_source_ids) reply, SOURCE_ID_CITATION_RE, receipt_source_ids
) )
unsupported_identifiers = sorted(reply_identifiers - supported_identifiers) unsupported_identifiers = sorted(reply_identifiers - supported_identifiers)
receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records] receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records]
@ -1169,9 +1170,7 @@ def _benchmark_execution_chain(
and local_state.get("status_sha256") == hashlib.sha256(b"?? goal.md\n").hexdigest() and local_state.get("status_sha256") == hashlib.sha256(b"?? goal.md\n").hexdigest()
) )
supported_worktree_mode = clean_worktree_exact or control_goal_only_exact supported_worktree_mode = clean_worktree_exact or control_goal_only_exact
worktree_allowed_missing = ( worktree_allowed_missing = CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset()
CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset()
)
mode = report.get("grounding_mode") mode = report.get("grounding_mode")
if mode == "grounded": if mode == "grounded":
allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing,) allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing,)

View file

@ -28,6 +28,53 @@ def test_plugin_registers_generation_and_delivery_hooks() -> None:
assert [name for name, _callback in registered] == ["pre_llm_call", "post_llm_call"] assert [name for name, _callback in registered] == ["pre_llm_call", "post_llm_call"]
def test_mixed_packet_contract_rejects_framework_mapped_to_claim() -> None:
module = load_plugin()
contracts = [{"id": "mixed_packet_composition"}]
issues = module.response_contract_issues(
"The evidence-backed fact maps to a claim, and the reasoning framework maps to a normative claim.",
contracts,
)
assert "framework_mapped_to_claim" in issues
def test_runtime_persistence_contract_rejects_lost_session_jsonl() -> None:
module = load_plugin()
contracts = [{"id": "runtime_persistence"}]
bad = module.response_contract_issues(
"state.db persists, but the session JSONL is lost when the session closes.",
contracts,
)
good = module.response_contract_issues(
"state.db and the session JSONL persist on disk and survive a service restart.",
contracts,
)
assert "runtime_persistence_contradiction" in bad
assert "runtime_persistence_contradiction" not in good
def test_forecast_contract_requires_reviewed_apply_boundary() -> None:
module = load_plugin()
contracts = [{"id": "forecast_resolution_schema"}]
incomplete = module.response_contract_issues(
"Preserve the original forecast and stage a reviewed schema proposal.",
contracts,
)
complete = module.response_contract_issues(
"Preserve the original forecast because no criteria existed. Stage a reviewed proposal, but do not apply "
"or invent the new fields in the current schema.",
contracts,
)
assert "forecast_review_apply_incomplete" in incomplete
assert "forecast_review_apply_incomplete" not in complete
def test_plugin_injects_bounded_retrieval_rows_and_writes_body_redacted_trace(tmp_path: Path, monkeypatch) -> None: def test_plugin_injects_bounded_retrieval_rows_and_writes_body_redacted_trace(tmp_path: Path, monkeypatch) -> None:
module = load_plugin() module = load_plugin()
home = tmp_path / "profile" home = tmp_path / "profile"
@ -591,7 +638,9 @@ def test_plugin_rejects_concrete_runtime_schema_contradictions_without_requiring
assert ( assert (
module.response_contract_issues( module.response_contract_issues(
"There is no resolves edge or resolved_at field; stage a reviewed schema proposal.", forecast "There is no resolves edge or resolved_at field; stage a reviewed schema proposal and do not apply "
"it until approved.",
forecast,
) )
== [] == []
) )
@ -609,7 +658,7 @@ def test_plugin_replaces_schema_contradiction_with_current_compiled_contract() -
contracts = [{"id": "forecast_resolution_schema"}] contracts = [{"id": "forecast_resolution_schema"}]
compiled = ( compiled = (
"Preserve the original forecast. There is no resolves edge or resolved_at field in the current schema; " "Preserve the original forecast. There is no resolves edge or resolved_at field in the current schema; "
"stage a reviewed schema proposal for explicit resolution semantics." "stage a reviewed schema proposal for explicit resolution semantics and do not apply it until approved."
) )
module._store_snapshot( module._store_snapshot(
"forecast-contract", "forecast-contract",

View file

@ -38,6 +38,16 @@ def test_leoclean_bridge_files_are_present_and_parseable() -> None:
subprocess.run(["bash", "-n", str(wrapper)], check=True) subprocess.run(["bash", "-n", str(wrapper)], check=True)
def test_vps_bridge_recognizes_natural_mixed_briefing_contract() -> None:
module = _load_module(BRIDGE_DIR / "kb_tool.py")
contracts = module.operational_contracts(
"Compose this briefing from evidence-backed facts, a reasoning framework, a contested position, "
"an operating rule, and a correction."
)
assert "mixed_packet_composition" in {item["id"] for item in contracts}
def test_vps_bridge_global_options_do_not_accept_untraced_abbreviations(monkeypatch) -> None: def test_vps_bridge_global_options_do_not_accept_untraced_abbreviations(monkeypatch) -> None:
module = _load_module(BRIDGE_DIR / "kb_tool.py") module = _load_module(BRIDGE_DIR / "kb_tool.py")
monkeypatch.setattr( monkeypatch.setattr(

View file

@ -339,9 +339,7 @@ def test_valid_response_transform_binds_raw_and_delivered_hashes() -> None:
def test_dirty_or_untracked_harness_cannot_produce_complete_manifest() -> None: def test_dirty_or_untracked_harness_cannot_produce_complete_manifest() -> None:
value = build( value = build(harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64})
harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64}
)
assert "harness_worktree_clean" in value["attribution"]["missing_required_bindings"] assert "harness_worktree_clean" in value["attribution"]["missing_required_bindings"]
@ -382,6 +380,103 @@ def test_write_capable_database_tool_trace_fails_closed() -> None:
assert "database_tools_read_only" in value["attribution"]["missing_required_bindings"] assert "database_tools_read_only" in value["attribution"]["missing_required_bindings"]
def test_hash_bound_read_only_tool_error_remains_attributable() -> None:
result = turn_result()
result["database_tool_trace"] = {
"schema": "livingip.leoKbToolTrace.v1",
"database_tool_call_count": 2,
"database_tool_completed_count": 1,
"database_tool_calls_read_only": True,
"calls": [
{
"tool_call_id_sha256": "1" * 64,
"arguments_sha256": "2" * 64,
"database_invocations": [
{
"access_mode": "read_only",
"command_sha256": "3" * 64,
"executable": "teleo-kb",
"subcommand": "context",
}
],
"result": {
"content_sha256": "4" * 64,
"content_bytes": 17,
"error_detected": True,
"nonempty": True,
"row_ids": [],
"sha256_values": [],
},
},
{
"tool_call_id_sha256": "5" * 64,
"arguments_sha256": "6" * 64,
"database_invocations": [
{
"access_mode": "read_only",
"command_sha256": "7" * 64,
"executable": "teleo-kb",
"subcommand": "context",
}
],
"result": {
"content_sha256": "8" * 64,
"content_bytes": 23,
"error_detected": False,
"nonempty": True,
"row_ids": [],
"sha256_values": [],
},
},
],
}
value = build(result)
binding = value["canonical_database"]["database_tool_binding"]
assert binding["all_results_content_bound"] is True
assert binding["all_calls_succeeded"] is False
assert "database_tool_results" not in value["attribution"]["missing_required_bindings"]
assert value["attribution"]["status"] == "complete"
def test_unbound_database_tool_error_remains_incomplete() -> None:
result = turn_result()
result["database_tool_trace"] = {
"schema": "livingip.leoKbToolTrace.v1",
"database_tool_call_count": 1,
"database_tool_completed_count": 0,
"database_tool_calls_read_only": True,
"calls": [
{
"tool_call_id_sha256": "1" * 64,
"arguments_sha256": "2" * 64,
"database_invocations": [
{
"access_mode": "read_only",
"command_sha256": "3" * 64,
"executable": "teleo-kb",
"subcommand": "context",
}
],
"result": {
"content_sha256": None,
"content_bytes": 0,
"error_detected": True,
"nonempty": False,
"row_ids": [],
"sha256_values": [],
},
}
],
}
value = build(result)
assert value["canonical_database"]["database_tool_binding"]["all_results_content_bound"] is False
assert "database_tool_results" in value["attribution"]["missing_required_bindings"]
def test_later_turn_binds_previous_execution_and_conversation_state() -> None: def test_later_turn_binds_previous_execution_and_conversation_state() -> None:
first_result = turn_result() first_result = turn_result()
first = build(first_result) first = build(first_result)

View file

@ -745,6 +745,108 @@ def test_oos_state_boundary_accepts_approved_is_not_applied() -> None:
assert benchmark.matched_concept("Approved is not applied.", "state_boundary") is True assert benchmark.matched_concept("Approved is not applied.", "state_boundary") is True
def test_oos_state_boundary_accepts_natural_apply_language() -> None:
reply = (
"Reviewer approval is intent. The canonical KB changes only when an authorized apply writes the rows and "
"applied_at is recorded in the postflight readback."
)
assert benchmark.matched_concept(reply, "state_boundary") is True
assert benchmark.matched_concept(reply, "canonical_readback") is True
def test_oos_source_chain_accepts_document_pointer_audit_language() -> None:
reply = (
"The extracted document artifact is a pointer, not yet canonical support. public.sources has no matching "
"source row and claim_evidence has no link, so audit the path, source row, and join before-and-after."
)
assert benchmark.matched_concept(reply, "source_evidence_chain") is True
assert benchmark.matched_concept(reply, "canonical_evidence_boundary") is True
def test_oos_runtime_reasoning_accepts_recycle_and_content_hash_language() -> None:
reply = (
"Postgres canonical rows and artifact_state_sha256 are unchanged, but identical counts do not prove "
"behavior or content. Deployed skills and SOUL.md affect the answer. state.db and session JSONL persist "
"conversation continuity, so a service recycle does not erase the session."
)
assert benchmark.matched_concept(reply, "runtime_inputs") is True
assert benchmark.matched_concept(reply, "durable_session_continuity") is True
assert benchmark.matched_concept(reply, "row_content_proof") is True
assert benchmark.broad_semantic_issues(reply) == []
def test_oos_runtime_reasoning_rejects_durable_session_as_ephemeral() -> None:
reply = "state.db survives, but the session JSONL is lost when the session closes."
assert benchmark.broad_semantic_issues(reply) == ["durable_session_called_ephemeral"]
def test_oos_agent_positions_accept_attribution_without_literal_column_name() -> None:
reply = (
"public.beliefs stores each stance attributed to that agent with confidence. There is no claim-ID foreign "
"key, so a direct link to the shared claim remains a schema gap."
)
assert benchmark.matched_concept(reply, "agent_specific_positions") is True
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
reply = (
"Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved "
"field, and there is no resolves edge. Stage a proposal, obtain reviewer approval, then apply it and "
"record applied_at."
)
assert benchmark.matched_concept(reply, "forecast_history") is True
assert benchmark.matched_concept(reply, "forecast_schema_gap") is True
assert benchmark.matched_concept(reply, "staged_review_apply") is True
assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(reply)
assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(
"The schema is missing a resolves edge."
)
def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None:
reply = (
"Challenge: the sources and claim bodies do not establish causality; the conclusion is not paid for by "
"the evidence. A narrower candidate claim says only that the association is observed, while the mechanism "
"remains undemonstrated."
)
assert benchmark.matched_concept(reply, "evidence_specific_challenge") is True
assert benchmark.matched_concept(reply, "narrower_claim_revision") is True
def test_oos_semantic_scorer_accepts_natural_receipt_and_policy_paraphrases() -> None:
assert benchmark.matched_concept(
"Identical counts prove nothing about content; compare artifact_state_sha256.",
"row_content_proof",
)
assert benchmark.matched_concept(
"approve_claim does not support behavioral_rules; the separate apply capability must be reviewed and authorized.",
"reviewed_policy_apply",
)
assert benchmark.matched_concept(
"behavioral_rules are outside approve_claim's apply capability and need a separate authorization.",
"reviewed_policy_apply",
)
assert benchmark.matched_concept(
"Leave the original claim untouched with confidence 0.6 because no resolution criteria existed.",
"forecast_history",
)
assert benchmark.matched_concept(
"Challenged inference: the claim body and evidence do not establish causality.",
"evidence_specific_challenge",
)
assert benchmark.matched_concept(
"Revision: adoption is documented, while lock-in remains undemonstrated.",
"narrower_claim_revision",
)
def test_oos_schema_gap_accepts_explicitly_absent_edge_type() -> None: def test_oos_schema_gap_accepts_explicitly_absent_edge_type() -> None:
reply = "Current claim_edges has edge_type; there is no `resolves` edge type." reply = "Current claim_edges has edge_type; there is no `resolves` edge type."
assert benchmark.current_schema_overclaims(reply) == [] assert benchmark.current_schema_overclaims(reply) == []

View file

@ -253,6 +253,7 @@ def fake_behavior_manifest(*, grounded: bool) -> dict:
stable = { stable = {
"schema": "livingip.leoBehaviorManifest.v1", "schema": "livingip.leoBehaviorManifest.v1",
"model_runtime": {"model": "test-model"}, "model_runtime": {"model": "test-model"},
"python_runtime": {"python_version": "3.11.15", "implementation": "CPython"},
"hermes_runtime": { "hermes_runtime": {
"git_head": "e" * 40, "git_head": "e" * 40,
"source_tree": {"sha256": "f" * 64, "file_count": 1}, "source_tree": {"sha256": "f" * 64, "file_count": 1},
@ -738,8 +739,7 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
assert validation["local_state_checks"]["control_goal_only_exact"] is True assert validation["local_state_checks"]["control_goal_only_exact"] is True
assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing) assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing)
assert all( assert all(
result["execution_manifest"]["attribution"]["missing_required_bindings"] result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing)
== sorted(expected_missing)
for result in report["results"] for result in report["results"]
) )
@ -804,9 +804,7 @@ def test_execution_chain_accepts_clean_harness_without_cleanliness_omission() ->
result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing) result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing)
for result in report["results"] for result in report["results"]
) )
assert all( assert all(checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values())
checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values()
)
manifest = report["results"][-1]["execution_manifest"] manifest = report["results"][-1]["execution_manifest"]
manifest["attribution"]["missing_required_bindings"].append("harness_worktree_clean") manifest["attribution"]["missing_required_bindings"].append("harness_worktree_clean")
@ -855,6 +853,7 @@ def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal
for key in ( for key in (
"schema", "schema",
"model_runtime", "model_runtime",
"python_runtime",
"hermes_runtime", "hermes_runtime",
"teleo_infrastructure_runtime", "teleo_infrastructure_runtime",
"components", "components",