teleo/teleo-infrastructure
8
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Harden blinded Leo reasoning repair gate

Browse source
This commit is contained in:
twentyOne2x 2026-07-16 05:19:50 +02:00
parent 4daa7853fe
commit 813d4cd91d
10 changed files with 370 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
hermes-agent/leoclean-bin/kb_tool.py
View file

@ -855,11 +855,17 @@ def operational_contracts(
mixed_markers = (
"packet",
"briefing",
"factual observation",
"evidence-backed fact",
"strategic framework",
"reasoning framework",
"disputed interpretation",
"contested position",
"governance rule",
"operating rule",
"old belief",
"correction",
"compose the database",
)
if sum(marker in lowered for marker in mixed_markers) >= 2:

15
hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py
View file

@ -568,6 +568,13 @@ def _mixed_packet_issues(response: str) -> list[str]:
re.I | re.S,
):
issues.append("unsupported_collection_listed_in_apply")
if re.search(
r"(?:reasoning|strategic) framework.{0,80}"
r"(?:is|as|into|maps? to|mapped to|classif(?:y|ied) as) (?:an? )?(?:normative )?claim",
response,
re.I | re.S,
):
issues.append("framework_mapped_to_claim")
for segment in re.split(r"(?<=[.!?])\s+|\n+", response):
negative = re.search(r"\b(?:no|not|never|without|must not|do not|does not|cannot)\b", segment, re.I)
@ -807,7 +814,7 @@ def _schema_contract_issues(response: str, contracts: list[dict[str, Any]]) -> l
if "runtime_persistence" in contract_ids and affirmative(
r"(?:state\.db|session\s+jsonl).{0,100}"
r"(?:in[- ]memory|ephemeral|disappears?|vanishes?|erased|discarded)"
r"(?:in[- ]memory|ephemeral|disappears?|vanishes?|erased|discarded|gone|lost|deleted)"
):
issues.append("runtime_persistence_contradiction")
@ -833,6 +840,12 @@ def _schema_contract_issues(response: str, contracts: list[dict[str, Any]]) -> l
issues.append("forecast_resolution_edge_contradiction")
if affirmative(r"public\.claims.{0,100}(?:has|stores?|contains?).{0,80}(?:forecast_resolution|resolved_at)"):
issues.append("forecast_resolution_field_contradiction")
if not (
re.search(r"\b(?:stage|proposal)\b", response, re.I)
and re.search(r"\breview(?:ed|er|ing)?\b", response, re.I)
and re.search(r"\bappl(?:y|ied|ication)\b", response, re.I)
):
issues.append("forecast_review_apply_incomplete")
return sorted(set(issues))

14
scripts/leo_turn_execution_manifest.py
View file

@ -345,21 +345,26 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
)
tool_call_count = trace.get("database_tool_call_count", 0)
completed_count = trace.get("database_tool_completed_count", 0)
all_results_bound = bool(
all_results_content_bound = bool(
_non_negative_int(tool_call_count)
and _non_negative_int(completed_count)
and completed_count == tool_call_count
and completed_count <= tool_call_count
and len(calls) == tool_call_count
and all(
_is_sha256(call.get("tool_call_id_sha256"))
and _is_sha256(call.get("arguments_sha256"))
and _is_sha256(call.get("result_content_sha256"))
and _non_negative_int(call.get("result_content_bytes"))
and call.get("result_error_detected") is False
and isinstance(call.get("result_error_detected"), bool)
and call.get("result_nonempty") is True
for call in calls
)
)
all_calls_succeeded = bool(
all_results_content_bound
and completed_count == tool_call_count
and all(call.get("result_error_detected") is False for call in calls)
)
all_calls_read_only = bool(
trace.get("database_tool_calls_read_only") is True
and all(
@ -376,7 +381,8 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
"tool_call_count": tool_call_count,
"completed_count": completed_count,
"all_calls_read_only": all_calls_read_only if tool_call_count else True,
"all_results_content_bound": all_results_bound,
"all_results_content_bound": all_results_content_bound,
"all_calls_succeeded": all_calls_succeeded,
"calls": calls,
"trace_sha256": canonical_sha256(trace),
}

99
scripts/working_leo_m3taversal_oos_benchmark.py
View file

@ -254,7 +254,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
"canonical_readback": (
re.compile(
r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)|"
r"canonical (?:DB|database)|applied_at.{0,100}readiness",
r"canonical (?:DB|database|KB)|(?:KB|knowledge database).{0,50}(?:changed|updated)|"
r"applied_at.{0,100}(?:readiness|postflight|canonical)",
re.I | re.S,
),
),
@ -263,7 +264,12 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
r"approved is not (?:the same as )?applied|applied_at:\s*(?:none|null)|not canonical|"
r"proposal does not commit|no receipt,? no durable knowledge|"
r"approval.{0,60}not (?:a )?(?:DB|database|canonical) write|"
r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied",
r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied|"
r"(?:reviewer )?approval is intent.{0,100}canonical (?:update|changes?).{0,80}(?:apply|applied_at)|"
r"(?:reviewer )?approval is intent.{0,100}(?:canonical )?KB changes? only when.{0,60}"
r"(?:apply|applied_at)|"
r"(?:reviewer )?approval.{0,100}(?:canonical|KB).{0,60}(?:requires|happens only when).{0,50}"
r"(?:apply|applied_at)",
re.I,
),
),
@ -280,11 +286,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(r"provenance|stable (?:reference|ref)|source row|author/channel|(?:file|content) hash", re.I),
),
"deduplication": (re.compile(r"deduplic|duplicate|existing claim|overlap", re.I),),
"contradiction": (re.compile(r"contradict|divergence|competing interpretation|disagree", re.I),),
"contradiction": (
re.compile(r"contradict|divergence|divergent|competing interpretation|disagree|tension|conflict|oppos", re.I),
),
"staged_review_apply": (
re.compile(r"staging|stage|proposal", re.I),
re.compile(r"review|approve|reviewer", re.I),
re.compile(r"apply|canonical", re.I),
re.compile(r"appl(?:y|ied|ication)|canonical", re.I),
),
"receipt": (
re.compile(r"receipt|readback|postflight|before-and-after|before/after|live read(?:ing)?|live proof", re.I),
@ -310,13 +318,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
),
),
"source_evidence_chain": (
re.compile(r"file|attachment|source_ref", re.I),
re.compile(r"file|attachment|source_ref|document|artifact|extracted (?:text|markdown)|path|pointer", re.I),
re.compile(r"public\.sources", re.I),
re.compile(r"claim_evidence", re.I),
re.compile(r"audit|link|join|row chain|before-and-after|before/after", re.I),
),
"canonical_evidence_boundary": (
re.compile(r"canonical evidence", re.I),
re.compile(r"canonical (?:evidence|support|linkage)|not yet canonical|not canonical", re.I),
re.compile(r"claim_evidence", re.I),
re.compile(r"public\.sources|source row", re.I),
re.compile(
@ -325,13 +333,19 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
r"(?:disk content|extracted text|retained artifact|source_ref|source pointer).{0,140}"
r"(?:is not|isn't|does not|doesn't|alone|until|unless).{0,80}(?:canonical evidence|finished provenance)|"
r"(?:canonical evidence|canonical link).{0,100}(?:requires|exists only when|is complete only when).{0,100}"
r"(?:public\.sources|source row|claim_evidence)",
r"(?:public\.sources|source row|claim_evidence)|"
r"(?:not yet canonical|staging-only).{0,300}(?:public\.sources|source row).{0,200}claim_evidence|"
r"(?:public\.sources|source row).{0,120}(?:absent|missing|requires?).{0,200}claim_evidence",
re.I | re.S,
),
),
"evidence_provenance_quality": (
re.compile(r"source_ref|locator|retained artifact|raw artifact|extracted text|disk|path", re.I),
re.compile(r"weak|unresolved|not traceable|verified|hash_matches_db|provenance", re.I),
re.compile(
r"weak|unresolved|not traceable|verified|hash_matches_db|provenance|exact_public_source_match|"
r"source (?:match|audit)|link audit",
re.I,
),
re.compile(r"canonical evidence|canonical link|public\.sources|claim_evidence", re.I),
),
"heterogeneous_types": (
@ -352,19 +366,28 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(
r"does not (?:accept|insert|support|cover|write)|supports neither|neither.{0,80}nor|"
r"(?:behavioral_rules|governance_gates).{0,120}(?:cannot be applied by|sits? outside) approve_claim|"
r"outside approve_claim(?:'s)? apply capability|approve_claim applies only|"
r"approve_claim.{0,120}(?:cannot apply|cannot write|does not write)",
re.I | re.S,
),
re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply (?:capability|path)", re.I | re.S),
re.compile(
r"separate.{0,60}(?:reviewed|authorized|authorization).{0,60}(?:apply )?(?:capability|path|write)|"
r"separate (?:reviewed )?(?:apply )?(?:capability|authorization)|"
r"separate.{0,50}apply capability.{0,60}(?:reviewed|authorized)|"
r"reviewed.{0,50}apply (?:capability|path)",
re.I | re.S,
),
),
"runtime_inputs": (
re.compile(r"Postgres|canonical (?:DB|database|counts?)|(?:DB|database) totals?", re.I),
re.compile(r"Postgres|canonical (?:DB|database|counts?|rows?)|(?:DB|database) totals?", re.I),
re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I),
re.compile(r"session|conversation context", re.I),
re.compile(
r"unchanged.{0,120}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)|"
r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)|"
r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)",
r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)|"
r"(?:identical|unchanged).{0,40}(?:counts|totals).{0,60}(?:prove nothing about content|"
r"not proof of content)",
re.I | re.S,
),
),
@ -372,23 +395,28 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(r"state\.db|session JSONL", re.I),
re.compile(r"persist|durable|continuity", re.I),
re.compile(
r"restart.{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|"
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}restart|"
r"(?:facts?|context).{0,80}not erased by restart",
r"(?:restart|recycle).{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|"
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}(?:restart|recycle)|"
r"(?:facts?|context).{0,80}not erased by (?:restart|recycle)",
re.I | re.S,
),
),
"row_content_proof": (
re.compile(r"unchanged (?:database )?(?:counts?|totals?)", re.I),
re.compile(r"does not prove|doesn't prove|do not prove|say nothing", re.I),
re.compile(r"(?:unchanged|identical) (?:database )?(?:counts?|totals?)", re.I),
re.compile(r"does not prove|doesn't prove|do not prove|prove(?:s)? nothing|say nothing", re.I),
re.compile(
r"row (?:IDs?|hashes?)|fingerprints?|timestamps?|balanced (?:insert|write|change)|"
r"individual rows?.{0,60}(?:mutated|changed|updated)",
r"individual rows?.{0,60}(?:mutated|changed|updated)|artifact_state_sha256|content hashes?|"
r"row[- ]level (?:hash|readback)",
re.I | re.S,
),
),
"proof_tiers": (
re.compile(r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3", re.I | re.S),
re.compile(
r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3|"
r"canonical rows.{0,600}deployed runtime inputs.{0,600}durable session state",
re.I | re.S,
),
re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I),
re.compile(r"runtime|skills?|session|SOUL\.md", re.I),
),
@ -407,26 +435,29 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
),
"agent_specific_positions": (
re.compile(r"public\.beliefs", re.I),
re.compile(r"agent_id", re.I),
re.compile(r"agent_id|agent attribution|attributed to (?:that|each|an?) agent", re.I),
re.compile(r"belief|position|stance|confidence", re.I),
re.compile(r"no.{0,40}claim(?:-ID|_id).{0,30}(?:foreign key|link)|schema gap", re.I | re.S),
),
"forecast_history": (
re.compile(r"original (?:probability|confidence)|60%|history", re.I),
re.compile(r"original (?:probability|confidence)|original.{0,120}(?:probability|confidence)|60%|history", re.I),
re.compile(
r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|original.{0,60}survives",
r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|"
r"leave.{0,60}(?:untouched|unmodified)|original.{0,60}survives",
re.I | re.S,
),
re.compile(
r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria|"
r"lack(?:ed|s|ing).{0,30}(?:success|resolution) criteria|"
r"criteria.{0,40}(?:never existed|were never defined|did not exist)|without.{0,30}criteria",
re.I | re.S,
),
),
"forecast_schema_gap": (
re.compile(r"current (?:v1|schema)|public\.claims", re.I),
re.compile(r"current (?:v1|schema)|public\.claims|the schema|schema has|claims table", re.I),
re.compile(
r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|"
r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
r"(?:does not exist|is absent|is not present|isn't present)|"
r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present",
@ -531,24 +562,25 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
),
),
"evidence_specific_challenge": (
re.compile(r"\b(?:challenge|limitation|however|unsupported leap|caveat)\b", re.I),
re.compile(r"\b(?:challeng(?:e|ed|ing)|limitation|however|unsupported leap|caveat)\b", re.I),
re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I),
re.compile(
r"\b(?:does not|doesn't|cannot|can't)\s+(?:itself\s+)?(?:prove|establish|support|show)|"
r"\b(?:does not|doesn't|cannot|can't)\s+rule out\b|"
r"\bunsupported leap\b",
r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+(?:itself\s+)?"
r"(?:prove|establish|support|show)|"
r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+rule out\b|"
r"\b(?:not paid for|outruns?).{0,40}\b(?:the )?evidence\b|\bunsupported leap\b",
re.I,
),
),
"narrower_claim_revision": (
re.compile(
r"\b(?:narrower\s+(?:revision|claim|formulation)|revision\s*:|"
r"\b(?:narrower\s+(?:revision|claim|formulation|candidate(?: claim)?)|revision\s*:|"
r"should be narrowed to|defensible formulation|revise(?:d)?\b.{0,40}\bto\b)",
re.I,
),
re.compile(
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|"
r"unproven|open constraint)\b",
r"undemonstrated|unproven|not established|not demonstrated|remains unknown|open constraint)\b",
re.I,
),
),
@ -567,7 +599,7 @@ COUNT_INVARIANT_REJECTION_RE = re.compile(
)
SCHEMA_GAP_QUALIFIER_RE = re.compile(
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|"
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|"
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
r"schema extension|extension proposal)\b|"
@ -656,6 +688,11 @@ DB_ONLY_CAUSALITY_RE = re.compile(
RESTART_ERASES_ALL_RE = re.compile(
r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S
)
DURABLE_SESSION_EPHEMERAL_RE = re.compile(
r"session JSONL.{0,100}(?:is|are|gets?|becomes?).{0,30}(?:gone|lost|deleted|discarded)|"
r"session JSONL.{0,100}(?:gone|lost|deleted|discarded).{0,50}(?:session closes?|restart)",
re.I | re.S,
)
REASONING_TOOL_CLAIM_EDGE_RE = re.compile(
r"claims?.{0,120}(?:get|have|create|use).{0,50}(?:claim_)?edges?.{0,100}"
r"(?:reasoning_tool|tool(?:'s)? ID)|"
@ -718,7 +755,7 @@ CLAIM_EVIDENCE_CONFLATION_RE = re.compile(
REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P<value>[^\n]+)", re.I)
REVISION_BOUNDARY_RE = re.compile(
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|"
r"open constraint)\b",
r"undemonstrated|not established|not demonstrated|remains unknown|open constraint)\b",
re.I,
)
REVISION_REPETITION_RE = re.compile(
@ -849,6 +886,8 @@ def broad_semantic_issues(reply: str) -> list[str]:
re.I | re.S,
):
findings.add("restart_called_total_memory_erasure")
if DURABLE_SESSION_EPHEMERAL_RE.search(reply):
findings.add("durable_session_called_ephemeral")
if HANDLER_TELEGRAM_OVERCLAIM_RE.search(reply):
findings.add("handler_proof_called_telegram_live")
if FORECAST_HISTORY_REWRITE_RE.search(reply):

13
scripts/working_leo_m3taversal_oos_protocol.py
View file

@ -866,6 +866,7 @@ def _executed_behavior_ablation(grounded_report: dict[str, Any], baseline_report
stable_keys = {
"schema",
"model_runtime",
"python_runtime",
"hermes_runtime",
"teleo_infrastructure_runtime",
"components",
@ -1061,11 +1062,11 @@ def _receipt_score(
}
reply = str(result.get("reply") or "")
reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(reply)}
reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = (
_resolve_supported_citations(reply, CLAIM_ID_CITATION_RE, receipt_claim_ids)
reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = _resolve_supported_citations(
reply, CLAIM_ID_CITATION_RE, receipt_claim_ids
)
reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = (
_resolve_supported_citations(reply, SOURCE_ID_CITATION_RE, receipt_source_ids)
reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = _resolve_supported_citations(
reply, SOURCE_ID_CITATION_RE, receipt_source_ids
)
unsupported_identifiers = sorted(reply_identifiers - supported_identifiers)
receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records]
@ -1169,9 +1170,7 @@ def _benchmark_execution_chain(
and local_state.get("status_sha256") == hashlib.sha256(b"?? goal.md\n").hexdigest()
)
supported_worktree_mode = clean_worktree_exact or control_goal_only_exact
worktree_allowed_missing = (
CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset()
)
worktree_allowed_missing = CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset()
mode = report.get("grounding_mode")
if mode == "grounded":
allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing,)

53
tests/test_hermes_leoclean_db_context_plugin.py
View file

@ -28,6 +28,53 @@ def test_plugin_registers_generation_and_delivery_hooks() -> None:
assert [name for name, _callback in registered] == ["pre_llm_call", "post_llm_call"]
def test_mixed_packet_contract_rejects_framework_mapped_to_claim() -> None:
module = load_plugin()
contracts = [{"id": "mixed_packet_composition"}]
issues = module.response_contract_issues(
"The evidence-backed fact maps to a claim, and the reasoning framework maps to a normative claim.",
contracts,
)
assert "framework_mapped_to_claim" in issues
def test_runtime_persistence_contract_rejects_lost_session_jsonl() -> None:
module = load_plugin()
contracts = [{"id": "runtime_persistence"}]
bad = module.response_contract_issues(
"state.db persists, but the session JSONL is lost when the session closes.",
contracts,
)
good = module.response_contract_issues(
"state.db and the session JSONL persist on disk and survive a service restart.",
contracts,
)
assert "runtime_persistence_contradiction" in bad
assert "runtime_persistence_contradiction" not in good
def test_forecast_contract_requires_reviewed_apply_boundary() -> None:
module = load_plugin()
contracts = [{"id": "forecast_resolution_schema"}]
incomplete = module.response_contract_issues(
"Preserve the original forecast and stage a reviewed schema proposal.",
contracts,
)
complete = module.response_contract_issues(
"Preserve the original forecast because no criteria existed. Stage a reviewed proposal, but do not apply "
"or invent the new fields in the current schema.",
contracts,
)
assert "forecast_review_apply_incomplete" in incomplete
assert "forecast_review_apply_incomplete" not in complete
def test_plugin_injects_bounded_retrieval_rows_and_writes_body_redacted_trace(tmp_path: Path, monkeypatch) -> None:
module = load_plugin()
home = tmp_path / "profile"
@ -591,7 +638,9 @@ def test_plugin_rejects_concrete_runtime_schema_contradictions_without_requiring
assert (
module.response_contract_issues(
"There is no resolves edge or resolved_at field; stage a reviewed schema proposal.", forecast
"There is no resolves edge or resolved_at field; stage a reviewed schema proposal and do not apply "
"it until approved.",
forecast,
)
== []
)
@ -609,7 +658,7 @@ def test_plugin_replaces_schema_contradiction_with_current_compiled_contract() -
contracts = [{"id": "forecast_resolution_schema"}]
compiled = (
"Preserve the original forecast. There is no resolves edge or resolved_at field in the current schema; "
"stage a reviewed schema proposal for explicit resolution semantics."
"stage a reviewed schema proposal for explicit resolution semantics and do not apply it until approved."
)
module._store_snapshot(
"forecast-contract",

10
tests/test_hermes_leoclean_kb_bridge_source.py
View file

@ -38,6 +38,16 @@ def test_leoclean_bridge_files_are_present_and_parseable() -> None:
subprocess.run(["bash", "-n", str(wrapper)], check=True)
def test_vps_bridge_recognizes_natural_mixed_briefing_contract() -> None:
module = _load_module(BRIDGE_DIR / "kb_tool.py")
contracts = module.operational_contracts(
"Compose this briefing from evidence-backed facts, a reasoning framework, a contested position, "
"an operating rule, and a correction."
)
assert "mixed_packet_composition" in {item["id"] for item in contracts}
def test_vps_bridge_global_options_do_not_accept_untraced_abbreviations(monkeypatch) -> None:
module = _load_module(BRIDGE_DIR / "kb_tool.py")
monkeypatch.setattr(

101
tests/test_leo_turn_execution_manifest.py
View file

@ -339,9 +339,7 @@ def test_valid_response_transform_binds_raw_and_delivered_hashes() -> None:
def test_dirty_or_untracked_harness_cannot_produce_complete_manifest() -> None:
value = build(
harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64}
)
value = build(harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64})
assert "harness_worktree_clean" in value["attribution"]["missing_required_bindings"]
@ -382,6 +380,103 @@ def test_write_capable_database_tool_trace_fails_closed() -> None:
assert "database_tools_read_only" in value["attribution"]["missing_required_bindings"]
def test_hash_bound_read_only_tool_error_remains_attributable() -> None:
result = turn_result()
result["database_tool_trace"] = {
"schema": "livingip.leoKbToolTrace.v1",
"database_tool_call_count": 2,
"database_tool_completed_count": 1,
"database_tool_calls_read_only": True,
"calls": [
{
"tool_call_id_sha256": "1" * 64,
"arguments_sha256": "2" * 64,
"database_invocations": [
{
"access_mode": "read_only",
"command_sha256": "3" * 64,
"executable": "teleo-kb",
"subcommand": "context",
}
],
"result": {
"content_sha256": "4" * 64,
"content_bytes": 17,
"error_detected": True,
"nonempty": True,
"row_ids": [],
"sha256_values": [],
},
},
{
"tool_call_id_sha256": "5" * 64,
"arguments_sha256": "6" * 64,
"database_invocations": [
{
"access_mode": "read_only",
"command_sha256": "7" * 64,
"executable": "teleo-kb",
"subcommand": "context",
}
],
"result": {
"content_sha256": "8" * 64,
"content_bytes": 23,
"error_detected": False,
"nonempty": True,
"row_ids": [],
"sha256_values": [],
},
},
],
}
value = build(result)
binding = value["canonical_database"]["database_tool_binding"]
assert binding["all_results_content_bound"] is True
assert binding["all_calls_succeeded"] is False
assert "database_tool_results" not in value["attribution"]["missing_required_bindings"]
assert value["attribution"]["status"] == "complete"
def test_unbound_database_tool_error_remains_incomplete() -> None:
result = turn_result()
result["database_tool_trace"] = {
"schema": "livingip.leoKbToolTrace.v1",
"database_tool_call_count": 1,
"database_tool_completed_count": 0,
"database_tool_calls_read_only": True,
"calls": [
{
"tool_call_id_sha256": "1" * 64,
"arguments_sha256": "2" * 64,
"database_invocations": [
{
"access_mode": "read_only",
"command_sha256": "3" * 64,
"executable": "teleo-kb",
"subcommand": "context",
}
],
"result": {
"content_sha256": None,
"content_bytes": 0,
"error_detected": True,
"nonempty": False,
"row_ids": [],
"sha256_values": [],
},
}
],
}
value = build(result)
assert value["canonical_database"]["database_tool_binding"]["all_results_content_bound"] is False
assert "database_tool_results" in value["attribution"]["missing_required_bindings"]
def test_later_turn_binds_previous_execution_and_conversation_state() -> None:
first_result = turn_result()
first = build(first_result)

102
tests/test_working_leo_m3taversal_oos_benchmark.py
View file

@ -745,6 +745,108 @@ def test_oos_state_boundary_accepts_approved_is_not_applied() -> None:
assert benchmark.matched_concept("Approved is not applied.", "state_boundary") is True
def test_oos_state_boundary_accepts_natural_apply_language() -> None:
reply = (
"Reviewer approval is intent. The canonical KB changes only when an authorized apply writes the rows and "
"applied_at is recorded in the postflight readback."
)
assert benchmark.matched_concept(reply, "state_boundary") is True
assert benchmark.matched_concept(reply, "canonical_readback") is True
def test_oos_source_chain_accepts_document_pointer_audit_language() -> None:
reply = (
"The extracted document artifact is a pointer, not yet canonical support. public.sources has no matching "
"source row and claim_evidence has no link, so audit the path, source row, and join before-and-after."
)
assert benchmark.matched_concept(reply, "source_evidence_chain") is True
assert benchmark.matched_concept(reply, "canonical_evidence_boundary") is True
def test_oos_runtime_reasoning_accepts_recycle_and_content_hash_language() -> None:
reply = (
"Postgres canonical rows and artifact_state_sha256 are unchanged, but identical counts do not prove "
"behavior or content. Deployed skills and SOUL.md affect the answer. state.db and session JSONL persist "
"conversation continuity, so a service recycle does not erase the session."
)
assert benchmark.matched_concept(reply, "runtime_inputs") is True
assert benchmark.matched_concept(reply, "durable_session_continuity") is True
assert benchmark.matched_concept(reply, "row_content_proof") is True
assert benchmark.broad_semantic_issues(reply) == []
def test_oos_runtime_reasoning_rejects_durable_session_as_ephemeral() -> None:
reply = "state.db survives, but the session JSONL is lost when the session closes."
assert benchmark.broad_semantic_issues(reply) == ["durable_session_called_ephemeral"]
def test_oos_agent_positions_accept_attribution_without_literal_column_name() -> None:
reply = (
"public.beliefs stores each stance attributed to that agent with confidence. There is no claim-ID foreign "
"key, so a direct link to the shared claim remains a schema gap."
)
assert benchmark.matched_concept(reply, "agent_specific_positions") is True
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
reply = (
"Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved "
"field, and there is no resolves edge. Stage a proposal, obtain reviewer approval, then apply it and "
"record applied_at."
)
assert benchmark.matched_concept(reply, "forecast_history") is True
assert benchmark.matched_concept(reply, "forecast_schema_gap") is True
assert benchmark.matched_concept(reply, "staged_review_apply") is True
assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(reply)
assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(
"The schema is missing a resolves edge."
)
def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None:
reply = (
"Challenge: the sources and claim bodies do not establish causality; the conclusion is not paid for by "
"the evidence. A narrower candidate claim says only that the association is observed, while the mechanism "
"remains undemonstrated."
)
assert benchmark.matched_concept(reply, "evidence_specific_challenge") is True
assert benchmark.matched_concept(reply, "narrower_claim_revision") is True
def test_oos_semantic_scorer_accepts_natural_receipt_and_policy_paraphrases() -> None:
assert benchmark.matched_concept(
"Identical counts prove nothing about content; compare artifact_state_sha256.",
"row_content_proof",
)
assert benchmark.matched_concept(
"approve_claim does not support behavioral_rules; the separate apply capability must be reviewed and authorized.",
"reviewed_policy_apply",
)
assert benchmark.matched_concept(
"behavioral_rules are outside approve_claim's apply capability and need a separate authorization.",
"reviewed_policy_apply",
)
assert benchmark.matched_concept(
"Leave the original claim untouched with confidence 0.6 because no resolution criteria existed.",
"forecast_history",
)
assert benchmark.matched_concept(
"Challenged inference: the claim body and evidence do not establish causality.",
"evidence_specific_challenge",
)
assert benchmark.matched_concept(
"Revision: adoption is documented, while lock-in remains undemonstrated.",
"narrower_claim_revision",
)
def test_oos_schema_gap_accepts_explicitly_absent_edge_type() -> None:
reply = "Current claim_edges has edge_type; there is no `resolves` edge type."
assert benchmark.current_schema_overclaims(reply) == []

9
tests/test_working_leo_m3taversal_oos_protocol.py
View file

@ -253,6 +253,7 @@ def fake_behavior_manifest(*, grounded: bool) -> dict:
stable = {
"schema": "livingip.leoBehaviorManifest.v1",
"model_runtime": {"model": "test-model"},
"python_runtime": {"python_version": "3.11.15", "implementation": "CPython"},
"hermes_runtime": {
"git_head": "e" * 40,
"source_tree": {"sha256": "f" * 64, "file_count": 1},
@ -738,8 +739,7 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non
assert validation["local_state_checks"]["control_goal_only_exact"] is True
assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing)
assert all(
result["execution_manifest"]["attribution"]["missing_required_bindings"]
== sorted(expected_missing)
result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing)
for result in report["results"]
)
@ -804,9 +804,7 @@ def test_execution_chain_accepts_clean_harness_without_cleanliness_omission() ->
result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing)
for result in report["results"]
)
assert all(
checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values()
)
assert all(checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values())
manifest = report["results"][-1]["execution_manifest"]
manifest["attribution"]["missing_required_bindings"].append("harness_worktree_clean")
@ -855,6 +853,7 @@ def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal
for key in (
"schema",
"model_runtime",
"python_runtime",
"hermes_runtime",
"teleo_infrastructure_runtime",
"components",