Harden blinded Leo reasoning repair gate
This commit is contained in:
twentyOne2x
parent
4daa7853fe
commit
813d4cd91d
10 changed files with 370 additions and 52 deletions
|
|
@ -855,11 +855,17 @@ def operational_contracts(
|
|||
|
||||
mixed_markers = (
|
||||
"packet",
|
||||
"briefing",
|
||||
"factual observation",
|
||||
"evidence-backed fact",
|
||||
"strategic framework",
|
||||
"reasoning framework",
|
||||
"disputed interpretation",
|
||||
"contested position",
|
||||
"governance rule",
|
||||
"operating rule",
|
||||
"old belief",
|
||||
"correction",
|
||||
"compose the database",
|
||||
)
|
||||
if sum(marker in lowered for marker in mixed_markers) >= 2:
|
||||
|
|
|
|||
|
|
@ -568,6 +568,13 @@ def _mixed_packet_issues(response: str) -> list[str]:
|
|||
re.I | re.S,
|
||||
):
|
||||
issues.append("unsupported_collection_listed_in_apply")
|
||||
if re.search(
|
||||
r"(?:reasoning|strategic) framework.{0,80}"
|
||||
r"(?:is|as|into|maps? to|mapped to|classif(?:y|ied) as) (?:an? )?(?:normative )?claim",
|
||||
response,
|
||||
re.I | re.S,
|
||||
):
|
||||
issues.append("framework_mapped_to_claim")
|
||||
|
||||
for segment in re.split(r"(?<=[.!?])\s+|\n+", response):
|
||||
negative = re.search(r"\b(?:no|not|never|without|must not|do not|does not|cannot)\b", segment, re.I)
|
||||
|
|
@ -807,7 +814,7 @@ def _schema_contract_issues(response: str, contracts: list[dict[str, Any]]) -> l
|
|||
|
||||
if "runtime_persistence" in contract_ids and affirmative(
|
||||
r"(?:state\.db|session\s+jsonl).{0,100}"
|
||||
r"(?:in[- ]memory|ephemeral|disappears?|vanishes?|erased|discarded)"
|
||||
r"(?:in[- ]memory|ephemeral|disappears?|vanishes?|erased|discarded|gone|lost|deleted)"
|
||||
):
|
||||
issues.append("runtime_persistence_contradiction")
|
||||
|
||||
|
|
@ -833,6 +840,12 @@ def _schema_contract_issues(response: str, contracts: list[dict[str, Any]]) -> l
|
|||
issues.append("forecast_resolution_edge_contradiction")
|
||||
if affirmative(r"public\.claims.{0,100}(?:has|stores?|contains?).{0,80}(?:forecast_resolution|resolved_at)"):
|
||||
issues.append("forecast_resolution_field_contradiction")
|
||||
if not (
|
||||
re.search(r"\b(?:stage|proposal)\b", response, re.I)
|
||||
and re.search(r"\breview(?:ed|er|ing)?\b", response, re.I)
|
||||
and re.search(r"\bappl(?:y|ied|ication)\b", response, re.I)
|
||||
):
|
||||
issues.append("forecast_review_apply_incomplete")
|
||||
|
||||
return sorted(set(issues))
|
||||
|
||||
|
|
|
|||
|
|
@ -345,21 +345,26 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
|
|||
)
|
||||
tool_call_count = trace.get("database_tool_call_count", 0)
|
||||
completed_count = trace.get("database_tool_completed_count", 0)
|
||||
all_results_bound = bool(
|
||||
all_results_content_bound = bool(
|
||||
_non_negative_int(tool_call_count)
|
||||
and _non_negative_int(completed_count)
|
||||
and completed_count == tool_call_count
|
||||
and completed_count <= tool_call_count
|
||||
and len(calls) == tool_call_count
|
||||
and all(
|
||||
_is_sha256(call.get("tool_call_id_sha256"))
|
||||
and _is_sha256(call.get("arguments_sha256"))
|
||||
and _is_sha256(call.get("result_content_sha256"))
|
||||
and _non_negative_int(call.get("result_content_bytes"))
|
||||
and call.get("result_error_detected") is False
|
||||
and isinstance(call.get("result_error_detected"), bool)
|
||||
and call.get("result_nonempty") is True
|
||||
for call in calls
|
||||
)
|
||||
)
|
||||
all_calls_succeeded = bool(
|
||||
all_results_content_bound
|
||||
and completed_count == tool_call_count
|
||||
and all(call.get("result_error_detected") is False for call in calls)
|
||||
)
|
||||
all_calls_read_only = bool(
|
||||
trace.get("database_tool_calls_read_only") is True
|
||||
and all(
|
||||
|
|
@ -376,7 +381,8 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
|
|||
"tool_call_count": tool_call_count,
|
||||
"completed_count": completed_count,
|
||||
"all_calls_read_only": all_calls_read_only if tool_call_count else True,
|
||||
"all_results_content_bound": all_results_bound,
|
||||
"all_results_content_bound": all_results_content_bound,
|
||||
"all_calls_succeeded": all_calls_succeeded,
|
||||
"calls": calls,
|
||||
"trace_sha256": canonical_sha256(trace),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -254,7 +254,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
"canonical_readback": (
|
||||
re.compile(
|
||||
r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)|"
|
||||
r"canonical (?:DB|database)|applied_at.{0,100}readiness",
|
||||
r"canonical (?:DB|database|KB)|(?:KB|knowledge database).{0,50}(?:changed|updated)|"
|
||||
r"applied_at.{0,100}(?:readiness|postflight|canonical)",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
|
|
@ -263,7 +264,12 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
r"approved is not (?:the same as )?applied|applied_at:\s*(?:none|null)|not canonical|"
|
||||
r"proposal does not commit|no receipt,? no durable knowledge|"
|
||||
r"approval.{0,60}not (?:a )?(?:DB|database|canonical) write|"
|
||||
r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied",
|
||||
r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied|"
|
||||
r"(?:reviewer )?approval is intent.{0,100}canonical (?:update|changes?).{0,80}(?:apply|applied_at)|"
|
||||
r"(?:reviewer )?approval is intent.{0,100}(?:canonical )?KB changes? only when.{0,60}"
|
||||
r"(?:apply|applied_at)|"
|
||||
r"(?:reviewer )?approval.{0,100}(?:canonical|KB).{0,60}(?:requires|happens only when).{0,50}"
|
||||
r"(?:apply|applied_at)",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
|
|
@ -280,11 +286,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(r"provenance|stable (?:reference|ref)|source row|author/channel|(?:file|content) hash", re.I),
|
||||
),
|
||||
"deduplication": (re.compile(r"deduplic|duplicate|existing claim|overlap", re.I),),
|
||||
"contradiction": (re.compile(r"contradict|divergence|competing interpretation|disagree", re.I),),
|
||||
"contradiction": (
|
||||
re.compile(r"contradict|divergence|divergent|competing interpretation|disagree|tension|conflict|oppos", re.I),
|
||||
),
|
||||
"staged_review_apply": (
|
||||
re.compile(r"staging|stage|proposal", re.I),
|
||||
re.compile(r"review|approve|reviewer", re.I),
|
||||
re.compile(r"apply|canonical", re.I),
|
||||
re.compile(r"appl(?:y|ied|ication)|canonical", re.I),
|
||||
),
|
||||
"receipt": (
|
||||
re.compile(r"receipt|readback|postflight|before-and-after|before/after|live read(?:ing)?|live proof", re.I),
|
||||
|
|
@ -310,13 +318,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
),
|
||||
),
|
||||
"source_evidence_chain": (
|
||||
re.compile(r"file|attachment|source_ref", re.I),
|
||||
re.compile(r"file|attachment|source_ref|document|artifact|extracted (?:text|markdown)|path|pointer", re.I),
|
||||
re.compile(r"public\.sources", re.I),
|
||||
re.compile(r"claim_evidence", re.I),
|
||||
re.compile(r"audit|link|join|row chain|before-and-after|before/after", re.I),
|
||||
),
|
||||
"canonical_evidence_boundary": (
|
||||
re.compile(r"canonical evidence", re.I),
|
||||
re.compile(r"canonical (?:evidence|support|linkage)|not yet canonical|not canonical", re.I),
|
||||
re.compile(r"claim_evidence", re.I),
|
||||
re.compile(r"public\.sources|source row", re.I),
|
||||
re.compile(
|
||||
|
|
@ -325,13 +333,19 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
r"(?:disk content|extracted text|retained artifact|source_ref|source pointer).{0,140}"
|
||||
r"(?:is not|isn't|does not|doesn't|alone|until|unless).{0,80}(?:canonical evidence|finished provenance)|"
|
||||
r"(?:canonical evidence|canonical link).{0,100}(?:requires|exists only when|is complete only when).{0,100}"
|
||||
r"(?:public\.sources|source row|claim_evidence)",
|
||||
r"(?:public\.sources|source row|claim_evidence)|"
|
||||
r"(?:not yet canonical|staging-only).{0,300}(?:public\.sources|source row).{0,200}claim_evidence|"
|
||||
r"(?:public\.sources|source row).{0,120}(?:absent|missing|requires?).{0,200}claim_evidence",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"evidence_provenance_quality": (
|
||||
re.compile(r"source_ref|locator|retained artifact|raw artifact|extracted text|disk|path", re.I),
|
||||
re.compile(r"weak|unresolved|not traceable|verified|hash_matches_db|provenance", re.I),
|
||||
re.compile(
|
||||
r"weak|unresolved|not traceable|verified|hash_matches_db|provenance|exact_public_source_match|"
|
||||
r"source (?:match|audit)|link audit",
|
||||
re.I,
|
||||
),
|
||||
re.compile(r"canonical evidence|canonical link|public\.sources|claim_evidence", re.I),
|
||||
),
|
||||
"heterogeneous_types": (
|
||||
|
|
@ -352,19 +366,28 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(
|
||||
r"does not (?:accept|insert|support|cover|write)|supports neither|neither.{0,80}nor|"
|
||||
r"(?:behavioral_rules|governance_gates).{0,120}(?:cannot be applied by|sits? outside) approve_claim|"
|
||||
r"outside approve_claim(?:'s)? apply capability|approve_claim applies only|"
|
||||
r"approve_claim.{0,120}(?:cannot apply|cannot write|does not write)",
|
||||
re.I | re.S,
|
||||
),
|
||||
re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply (?:capability|path)", re.I | re.S),
|
||||
re.compile(
|
||||
r"separate.{0,60}(?:reviewed|authorized|authorization).{0,60}(?:apply )?(?:capability|path|write)|"
|
||||
r"separate (?:reviewed )?(?:apply )?(?:capability|authorization)|"
|
||||
r"separate.{0,50}apply capability.{0,60}(?:reviewed|authorized)|"
|
||||
r"reviewed.{0,50}apply (?:capability|path)",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"runtime_inputs": (
|
||||
re.compile(r"Postgres|canonical (?:DB|database|counts?)|(?:DB|database) totals?", re.I),
|
||||
re.compile(r"Postgres|canonical (?:DB|database|counts?|rows?)|(?:DB|database) totals?", re.I),
|
||||
re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I),
|
||||
re.compile(r"session|conversation context", re.I),
|
||||
re.compile(
|
||||
r"unchanged.{0,120}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)|"
|
||||
r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)|"
|
||||
r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)",
|
||||
r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)|"
|
||||
r"(?:identical|unchanged).{0,40}(?:counts|totals).{0,60}(?:prove nothing about content|"
|
||||
r"not proof of content)",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
|
|
@ -372,23 +395,28 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(r"state\.db|session JSONL", re.I),
|
||||
re.compile(r"persist|durable|continuity", re.I),
|
||||
re.compile(
|
||||
r"restart.{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|"
|
||||
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}restart|"
|
||||
r"(?:facts?|context).{0,80}not erased by restart",
|
||||
r"(?:restart|recycle).{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|"
|
||||
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}(?:restart|recycle)|"
|
||||
r"(?:facts?|context).{0,80}not erased by (?:restart|recycle)",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"row_content_proof": (
|
||||
re.compile(r"unchanged (?:database )?(?:counts?|totals?)", re.I),
|
||||
re.compile(r"does not prove|doesn't prove|do not prove|say nothing", re.I),
|
||||
re.compile(r"(?:unchanged|identical) (?:database )?(?:counts?|totals?)", re.I),
|
||||
re.compile(r"does not prove|doesn't prove|do not prove|prove(?:s)? nothing|say nothing", re.I),
|
||||
re.compile(
|
||||
r"row (?:IDs?|hashes?)|fingerprints?|timestamps?|balanced (?:insert|write|change)|"
|
||||
r"individual rows?.{0,60}(?:mutated|changed|updated)",
|
||||
r"individual rows?.{0,60}(?:mutated|changed|updated)|artifact_state_sha256|content hashes?|"
|
||||
r"row[- ]level (?:hash|readback)",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"proof_tiers": (
|
||||
re.compile(r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3", re.I | re.S),
|
||||
re.compile(
|
||||
r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3|"
|
||||
r"canonical rows.{0,600}deployed runtime inputs.{0,600}durable session state",
|
||||
re.I | re.S,
|
||||
),
|
||||
re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I),
|
||||
re.compile(r"runtime|skills?|session|SOUL\.md", re.I),
|
||||
),
|
||||
|
|
@ -407,26 +435,29 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
),
|
||||
"agent_specific_positions": (
|
||||
re.compile(r"public\.beliefs", re.I),
|
||||
re.compile(r"agent_id", re.I),
|
||||
re.compile(r"agent_id|agent attribution|attributed to (?:that|each|an?) agent", re.I),
|
||||
re.compile(r"belief|position|stance|confidence", re.I),
|
||||
re.compile(r"no.{0,40}claim(?:-ID|_id).{0,30}(?:foreign key|link)|schema gap", re.I | re.S),
|
||||
),
|
||||
"forecast_history": (
|
||||
re.compile(r"original (?:probability|confidence)|60%|history", re.I),
|
||||
re.compile(r"original (?:probability|confidence)|original.{0,120}(?:probability|confidence)|60%|history", re.I),
|
||||
re.compile(
|
||||
r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|original.{0,60}survives",
|
||||
r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|"
|
||||
r"leave.{0,60}(?:untouched|unmodified)|original.{0,60}survives",
|
||||
re.I | re.S,
|
||||
),
|
||||
re.compile(
|
||||
r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria|"
|
||||
r"lack(?:ed|s|ing).{0,30}(?:success|resolution) criteria|"
|
||||
r"criteria.{0,40}(?:never existed|were never defined|did not exist)|without.{0,30}criteria",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"forecast_schema_gap": (
|
||||
re.compile(r"current (?:v1|schema)|public\.claims", re.I),
|
||||
re.compile(r"current (?:v1|schema)|public\.claims|the schema|schema has|claims table", re.I),
|
||||
re.compile(
|
||||
r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|"
|
||||
r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
|
||||
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
|
||||
r"(?:does not exist|is absent|is not present|isn't present)|"
|
||||
r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present",
|
||||
|
|
@ -531,24 +562,25 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
),
|
||||
),
|
||||
"evidence_specific_challenge": (
|
||||
re.compile(r"\b(?:challenge|limitation|however|unsupported leap|caveat)\b", re.I),
|
||||
re.compile(r"\b(?:challeng(?:e|ed|ing)|limitation|however|unsupported leap|caveat)\b", re.I),
|
||||
re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I),
|
||||
re.compile(
|
||||
r"\b(?:does not|doesn't|cannot|can't)\s+(?:itself\s+)?(?:prove|establish|support|show)|"
|
||||
r"\b(?:does not|doesn't|cannot|can't)\s+rule out\b|"
|
||||
r"\bunsupported leap\b",
|
||||
r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+(?:itself\s+)?"
|
||||
r"(?:prove|establish|support|show)|"
|
||||
r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+rule out\b|"
|
||||
r"\b(?:not paid for|outruns?).{0,40}\b(?:the )?evidence\b|\bunsupported leap\b",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
"narrower_claim_revision": (
|
||||
re.compile(
|
||||
r"\b(?:narrower\s+(?:revision|claim|formulation)|revision\s*:|"
|
||||
r"\b(?:narrower\s+(?:revision|claim|formulation|candidate(?: claim)?)|revision\s*:|"
|
||||
r"should be narrowed to|defensible formulation|revise(?:d)?\b.{0,40}\bto\b)",
|
||||
re.I,
|
||||
),
|
||||
re.compile(
|
||||
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|"
|
||||
r"unproven|open constraint)\b",
|
||||
r"undemonstrated|unproven|not established|not demonstrated|remains unknown|open constraint)\b",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
|
|
@ -567,7 +599,7 @@ COUNT_INVARIANT_REJECTION_RE = re.compile(
|
|||
)
|
||||
|
||||
SCHEMA_GAP_QUALIFIER_RE = re.compile(
|
||||
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|"
|
||||
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
|
||||
r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|"
|
||||
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
|
||||
r"schema extension|extension proposal)\b|"
|
||||
|
|
@ -656,6 +688,11 @@ DB_ONLY_CAUSALITY_RE = re.compile(
|
|||
RESTART_ERASES_ALL_RE = re.compile(
|
||||
r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S
|
||||
)
|
||||
DURABLE_SESSION_EPHEMERAL_RE = re.compile(
|
||||
r"session JSONL.{0,100}(?:is|are|gets?|becomes?).{0,30}(?:gone|lost|deleted|discarded)|"
|
||||
r"session JSONL.{0,100}(?:gone|lost|deleted|discarded).{0,50}(?:session closes?|restart)",
|
||||
re.I | re.S,
|
||||
)
|
||||
REASONING_TOOL_CLAIM_EDGE_RE = re.compile(
|
||||
r"claims?.{0,120}(?:get|have|create|use).{0,50}(?:claim_)?edges?.{0,100}"
|
||||
r"(?:reasoning_tool|tool(?:'s)? ID)|"
|
||||
|
|
@ -718,7 +755,7 @@ CLAIM_EVIDENCE_CONFLATION_RE = re.compile(
|
|||
REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P<value>[^\n]+)", re.I)
|
||||
REVISION_BOUNDARY_RE = re.compile(
|
||||
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|"
|
||||
r"open constraint)\b",
|
||||
r"undemonstrated|not established|not demonstrated|remains unknown|open constraint)\b",
|
||||
re.I,
|
||||
)
|
||||
REVISION_REPETITION_RE = re.compile(
|
||||
|
|
@ -849,6 +886,8 @@ def broad_semantic_issues(reply: str) -> list[str]:
|
|||
re.I | re.S,
|
||||
):
|
||||
findings.add("restart_called_total_memory_erasure")
|
||||
if DURABLE_SESSION_EPHEMERAL_RE.search(reply):
|
||||
findings.add("durable_session_called_ephemeral")
|
||||
if HANDLER_TELEGRAM_OVERCLAIM_RE.search(reply):
|
||||
findings.add("handler_proof_called_telegram_live")
|
||||
if FORECAST_HISTORY_REWRITE_RE.search(reply):
|
||||
|
|
|
|||
|
|
@ -866,6 +866,7 @@ def _executed_behavior_ablation(grounded_report: dict[str, Any], baseline_report
|
|||
stable_keys = {
|
||||
"schema",
|
||||
"model_runtime",
|
||||
"python_runtime",
|
||||
"hermes_runtime",
|
||||
"teleo_infrastructure_runtime",
|
||||
"components",
|
||||
|
|
@ -1061,11 +1062,11 @@ def _receipt_score(
|
|||
}
|
||||
reply = str(result.get("reply") or "")
|
||||
reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(reply)}
|
||||
reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = (
|
||||
_resolve_supported_citations(reply, CLAIM_ID_CITATION_RE, receipt_claim_ids)
|
||||
reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = _resolve_supported_citations(
|
||||
reply, CLAIM_ID_CITATION_RE, receipt_claim_ids
|
||||
)
|
||||
reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = (
|
||||
_resolve_supported_citations(reply, SOURCE_ID_CITATION_RE, receipt_source_ids)
|
||||
reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = _resolve_supported_citations(
|
||||
reply, SOURCE_ID_CITATION_RE, receipt_source_ids
|
||||
)
|
||||
unsupported_identifiers = sorted(reply_identifiers - supported_identifiers)
|
||||
receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records]
|
||||
|
|
@ -1169,9 +1170,7 @@ def _benchmark_execution_chain(
|
|||
and local_state.get("status_sha256") == hashlib.sha256(b"?? goal.md\n").hexdigest()
|
||||
)
|
||||
supported_worktree_mode = clean_worktree_exact or control_goal_only_exact
|
||||
worktree_allowed_missing = (
|
||||
CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset()
|
||||
)
|
||||
worktree_allowed_missing = CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset()
|
||||
mode = report.get("grounding_mode")
|
||||
if mode == "grounded":
|
||||
allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing,)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,53 @@ def test_plugin_registers_generation_and_delivery_hooks() -> None:
|
|||
assert [name for name, _callback in registered] == ["pre_llm_call", "post_llm_call"]
|
||||
|
||||
|
||||
def test_mixed_packet_contract_rejects_framework_mapped_to_claim() -> None:
|
||||
module = load_plugin()
|
||||
contracts = [{"id": "mixed_packet_composition"}]
|
||||
|
||||
issues = module.response_contract_issues(
|
||||
"The evidence-backed fact maps to a claim, and the reasoning framework maps to a normative claim.",
|
||||
contracts,
|
||||
)
|
||||
|
||||
assert "framework_mapped_to_claim" in issues
|
||||
|
||||
|
||||
def test_runtime_persistence_contract_rejects_lost_session_jsonl() -> None:
|
||||
module = load_plugin()
|
||||
contracts = [{"id": "runtime_persistence"}]
|
||||
|
||||
bad = module.response_contract_issues(
|
||||
"state.db persists, but the session JSONL is lost when the session closes.",
|
||||
contracts,
|
||||
)
|
||||
good = module.response_contract_issues(
|
||||
"state.db and the session JSONL persist on disk and survive a service restart.",
|
||||
contracts,
|
||||
)
|
||||
|
||||
assert "runtime_persistence_contradiction" in bad
|
||||
assert "runtime_persistence_contradiction" not in good
|
||||
|
||||
|
||||
def test_forecast_contract_requires_reviewed_apply_boundary() -> None:
|
||||
module = load_plugin()
|
||||
contracts = [{"id": "forecast_resolution_schema"}]
|
||||
|
||||
incomplete = module.response_contract_issues(
|
||||
"Preserve the original forecast and stage a reviewed schema proposal.",
|
||||
contracts,
|
||||
)
|
||||
complete = module.response_contract_issues(
|
||||
"Preserve the original forecast because no criteria existed. Stage a reviewed proposal, but do not apply "
|
||||
"or invent the new fields in the current schema.",
|
||||
contracts,
|
||||
)
|
||||
|
||||
assert "forecast_review_apply_incomplete" in incomplete
|
||||
assert "forecast_review_apply_incomplete" not in complete
|
||||
|
||||
|
||||
def test_plugin_injects_bounded_retrieval_rows_and_writes_body_redacted_trace(tmp_path: Path, monkeypatch) -> None:
|
||||
module = load_plugin()
|
||||
home = tmp_path / "profile"
|
||||
|
|
@ -591,7 +638,9 @@ def test_plugin_rejects_concrete_runtime_schema_contradictions_without_requiring
|
|||
|
||||
assert (
|
||||
module.response_contract_issues(
|
||||
"There is no resolves edge or resolved_at field; stage a reviewed schema proposal.", forecast
|
||||
"There is no resolves edge or resolved_at field; stage a reviewed schema proposal and do not apply "
|
||||
"it until approved.",
|
||||
forecast,
|
||||
)
|
||||
== []
|
||||
)
|
||||
|
|
@ -609,7 +658,7 @@ def test_plugin_replaces_schema_contradiction_with_current_compiled_contract() -
|
|||
contracts = [{"id": "forecast_resolution_schema"}]
|
||||
compiled = (
|
||||
"Preserve the original forecast. There is no resolves edge or resolved_at field in the current schema; "
|
||||
"stage a reviewed schema proposal for explicit resolution semantics."
|
||||
"stage a reviewed schema proposal for explicit resolution semantics and do not apply it until approved."
|
||||
)
|
||||
module._store_snapshot(
|
||||
"forecast-contract",
|
||||
|
|
|
|||
|
|
@ -38,6 +38,16 @@ def test_leoclean_bridge_files_are_present_and_parseable() -> None:
|
|||
subprocess.run(["bash", "-n", str(wrapper)], check=True)
|
||||
|
||||
|
||||
def test_vps_bridge_recognizes_natural_mixed_briefing_contract() -> None:
|
||||
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
||||
contracts = module.operational_contracts(
|
||||
"Compose this briefing from evidence-backed facts, a reasoning framework, a contested position, "
|
||||
"an operating rule, and a correction."
|
||||
)
|
||||
|
||||
assert "mixed_packet_composition" in {item["id"] for item in contracts}
|
||||
|
||||
|
||||
def test_vps_bridge_global_options_do_not_accept_untraced_abbreviations(monkeypatch) -> None:
|
||||
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
||||
monkeypatch.setattr(
|
||||
|
|
|
|||
|
|
@ -339,9 +339,7 @@ def test_valid_response_transform_binds_raw_and_delivered_hashes() -> None:
|
|||
|
||||
|
||||
def test_dirty_or_untracked_harness_cannot_produce_complete_manifest() -> None:
|
||||
value = build(
|
||||
harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64}
|
||||
)
|
||||
value = build(harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64})
|
||||
|
||||
assert "harness_worktree_clean" in value["attribution"]["missing_required_bindings"]
|
||||
|
||||
|
|
@ -382,6 +380,103 @@ def test_write_capable_database_tool_trace_fails_closed() -> None:
|
|||
assert "database_tools_read_only" in value["attribution"]["missing_required_bindings"]
|
||||
|
||||
|
||||
def test_hash_bound_read_only_tool_error_remains_attributable() -> None:
|
||||
result = turn_result()
|
||||
result["database_tool_trace"] = {
|
||||
"schema": "livingip.leoKbToolTrace.v1",
|
||||
"database_tool_call_count": 2,
|
||||
"database_tool_completed_count": 1,
|
||||
"database_tool_calls_read_only": True,
|
||||
"calls": [
|
||||
{
|
||||
"tool_call_id_sha256": "1" * 64,
|
||||
"arguments_sha256": "2" * 64,
|
||||
"database_invocations": [
|
||||
{
|
||||
"access_mode": "read_only",
|
||||
"command_sha256": "3" * 64,
|
||||
"executable": "teleo-kb",
|
||||
"subcommand": "context",
|
||||
}
|
||||
],
|
||||
"result": {
|
||||
"content_sha256": "4" * 64,
|
||||
"content_bytes": 17,
|
||||
"error_detected": True,
|
||||
"nonempty": True,
|
||||
"row_ids": [],
|
||||
"sha256_values": [],
|
||||
},
|
||||
},
|
||||
{
|
||||
"tool_call_id_sha256": "5" * 64,
|
||||
"arguments_sha256": "6" * 64,
|
||||
"database_invocations": [
|
||||
{
|
||||
"access_mode": "read_only",
|
||||
"command_sha256": "7" * 64,
|
||||
"executable": "teleo-kb",
|
||||
"subcommand": "context",
|
||||
}
|
||||
],
|
||||
"result": {
|
||||
"content_sha256": "8" * 64,
|
||||
"content_bytes": 23,
|
||||
"error_detected": False,
|
||||
"nonempty": True,
|
||||
"row_ids": [],
|
||||
"sha256_values": [],
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
value = build(result)
|
||||
binding = value["canonical_database"]["database_tool_binding"]
|
||||
|
||||
assert binding["all_results_content_bound"] is True
|
||||
assert binding["all_calls_succeeded"] is False
|
||||
assert "database_tool_results" not in value["attribution"]["missing_required_bindings"]
|
||||
assert value["attribution"]["status"] == "complete"
|
||||
|
||||
|
||||
def test_unbound_database_tool_error_remains_incomplete() -> None:
|
||||
result = turn_result()
|
||||
result["database_tool_trace"] = {
|
||||
"schema": "livingip.leoKbToolTrace.v1",
|
||||
"database_tool_call_count": 1,
|
||||
"database_tool_completed_count": 0,
|
||||
"database_tool_calls_read_only": True,
|
||||
"calls": [
|
||||
{
|
||||
"tool_call_id_sha256": "1" * 64,
|
||||
"arguments_sha256": "2" * 64,
|
||||
"database_invocations": [
|
||||
{
|
||||
"access_mode": "read_only",
|
||||
"command_sha256": "3" * 64,
|
||||
"executable": "teleo-kb",
|
||||
"subcommand": "context",
|
||||
}
|
||||
],
|
||||
"result": {
|
||||
"content_sha256": None,
|
||||
"content_bytes": 0,
|
||||
"error_detected": True,
|
||||
"nonempty": False,
|
||||
"row_ids": [],
|
||||
"sha256_values": [],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
value = build(result)
|
||||
|
||||
assert value["canonical_database"]["database_tool_binding"]["all_results_content_bound"] is False
|
||||
assert "database_tool_results" in value["attribution"]["missing_required_bindings"]
|
||||
|
||||
|
||||
def test_later_turn_binds_previous_execution_and_conversation_state() -> None:
|
||||
first_result = turn_result()
|
||||
first = build(first_result)
|
||||
|
|
|
|||
|
|
@ -745,6 +745,108 @@ def test_oos_state_boundary_accepts_approved_is_not_applied() -> None:
|
|||
assert benchmark.matched_concept("Approved is not applied.", "state_boundary") is True
|
||||
|
||||
|
||||
def test_oos_state_boundary_accepts_natural_apply_language() -> None:
|
||||
reply = (
|
||||
"Reviewer approval is intent. The canonical KB changes only when an authorized apply writes the rows and "
|
||||
"applied_at is recorded in the postflight readback."
|
||||
)
|
||||
|
||||
assert benchmark.matched_concept(reply, "state_boundary") is True
|
||||
assert benchmark.matched_concept(reply, "canonical_readback") is True
|
||||
|
||||
|
||||
def test_oos_source_chain_accepts_document_pointer_audit_language() -> None:
|
||||
reply = (
|
||||
"The extracted document artifact is a pointer, not yet canonical support. public.sources has no matching "
|
||||
"source row and claim_evidence has no link, so audit the path, source row, and join before-and-after."
|
||||
)
|
||||
|
||||
assert benchmark.matched_concept(reply, "source_evidence_chain") is True
|
||||
assert benchmark.matched_concept(reply, "canonical_evidence_boundary") is True
|
||||
|
||||
|
||||
def test_oos_runtime_reasoning_accepts_recycle_and_content_hash_language() -> None:
|
||||
reply = (
|
||||
"Postgres canonical rows and artifact_state_sha256 are unchanged, but identical counts do not prove "
|
||||
"behavior or content. Deployed skills and SOUL.md affect the answer. state.db and session JSONL persist "
|
||||
"conversation continuity, so a service recycle does not erase the session."
|
||||
)
|
||||
|
||||
assert benchmark.matched_concept(reply, "runtime_inputs") is True
|
||||
assert benchmark.matched_concept(reply, "durable_session_continuity") is True
|
||||
assert benchmark.matched_concept(reply, "row_content_proof") is True
|
||||
assert benchmark.broad_semantic_issues(reply) == []
|
||||
|
||||
|
||||
def test_oos_runtime_reasoning_rejects_durable_session_as_ephemeral() -> None:
|
||||
reply = "state.db survives, but the session JSONL is lost when the session closes."
|
||||
|
||||
assert benchmark.broad_semantic_issues(reply) == ["durable_session_called_ephemeral"]
|
||||
|
||||
|
||||
def test_oos_agent_positions_accept_attribution_without_literal_column_name() -> None:
|
||||
reply = (
|
||||
"public.beliefs stores each stance attributed to that agent with confidence. There is no claim-ID foreign "
|
||||
"key, so a direct link to the shared claim remains a schema gap."
|
||||
)
|
||||
|
||||
assert benchmark.matched_concept(reply, "agent_specific_positions") is True
|
||||
|
||||
|
||||
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
|
||||
reply = (
|
||||
"Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved "
|
||||
"field, and there is no resolves edge. Stage a proposal, obtain reviewer approval, then apply it and "
|
||||
"record applied_at."
|
||||
)
|
||||
|
||||
assert benchmark.matched_concept(reply, "forecast_history") is True
|
||||
assert benchmark.matched_concept(reply, "forecast_schema_gap") is True
|
||||
assert benchmark.matched_concept(reply, "staged_review_apply") is True
|
||||
assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(reply)
|
||||
assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(
|
||||
"The schema is missing a resolves edge."
|
||||
)
|
||||
|
||||
|
||||
def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None:
|
||||
reply = (
|
||||
"Challenge: the sources and claim bodies do not establish causality; the conclusion is not paid for by "
|
||||
"the evidence. A narrower candidate claim says only that the association is observed, while the mechanism "
|
||||
"remains undemonstrated."
|
||||
)
|
||||
|
||||
assert benchmark.matched_concept(reply, "evidence_specific_challenge") is True
|
||||
assert benchmark.matched_concept(reply, "narrower_claim_revision") is True
|
||||
|
||||
|
||||
def test_oos_semantic_scorer_accepts_natural_receipt_and_policy_paraphrases() -> None:
|
||||
assert benchmark.matched_concept(
|
||||
"Identical counts prove nothing about content; compare artifact_state_sha256.",
|
||||
"row_content_proof",
|
||||
)
|
||||
assert benchmark.matched_concept(
|
||||
"approve_claim does not support behavioral_rules; the separate apply capability must be reviewed and authorized.",
|
||||
"reviewed_policy_apply",
|
||||
)
|
||||
assert benchmark.matched_concept(
|
||||
"behavioral_rules are outside approve_claim's apply capability and need a separate authorization.",
|
||||
"reviewed_policy_apply",
|
||||
)
|
||||
assert benchmark.matched_concept(
|
||||
"Leave the original claim untouched with confidence 0.6 because no resolution criteria existed.",
|
||||
"forecast_history",
|
||||
)
|
||||
assert benchmark.matched_concept(
|
||||
"Challenged inference: the claim body and evidence do not establish causality.",
|
||||
"evidence_specific_challenge",
|
||||
)
|
||||
assert benchmark.matched_concept(
|
||||
"Revision: adoption is documented, while lock-in remains undemonstrated.",
|
||||
"narrower_claim_revision",
|
||||
)
|
||||
|
||||
|
||||
def test_oos_schema_gap_accepts_explicitly_absent_edge_type() -> None:
|
||||
reply = "Current claim_edges has edge_type; there is no `resolves` edge type."
|
||||
assert benchmark.current_schema_overclaims(reply) == []
|
||||
|
|
|
|||
|
|
@ -253,6 +253,7 @@ def fake_behavior_manifest(*, grounded: bool) -> dict:
|
|||
stable = {
|
||||
"schema": "livingip.leoBehaviorManifest.v1",
|
||||
"model_runtime": {"model": "test-model"},
|
||||
"python_runtime": {"python_version": "3.11.15", "implementation": "CPython"},
|
||||
"hermes_runtime": {
|
||||
"git_head": "e" * 40,
|
||||
"source_tree": {"sha256": "f" * 64, "file_count": 1},
|
||||
|
|
@ -738,8 +739,7 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
|
|||
assert validation["local_state_checks"]["control_goal_only_exact"] is True
|
||||
assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing)
|
||||
assert all(
|
||||
result["execution_manifest"]["attribution"]["missing_required_bindings"]
|
||||
== sorted(expected_missing)
|
||||
result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing)
|
||||
for result in report["results"]
|
||||
)
|
||||
|
||||
|
|
@ -804,9 +804,7 @@ def test_execution_chain_accepts_clean_harness_without_cleanliness_omission() ->
|
|||
result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing)
|
||||
for result in report["results"]
|
||||
)
|
||||
assert all(
|
||||
checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values()
|
||||
)
|
||||
assert all(checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values())
|
||||
|
||||
manifest = report["results"][-1]["execution_manifest"]
|
||||
manifest["attribution"]["missing_required_bindings"].append("harness_worktree_clean")
|
||||
|
|
@ -855,6 +853,7 @@ def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal
|
|||
for key in (
|
||||
"schema",
|
||||
"model_runtime",
|
||||
"python_runtime",
|
||||
"hermes_runtime",
|
||||
"teleo_infrastructure_runtime",
|
||||
"components",
|
||||
|
|
|
|||
Loading…
Reference in a new issue