diff --git a/hermes-agent/leoclean-bin/kb_tool.py b/hermes-agent/leoclean-bin/kb_tool.py index f7a25a2..478f827 100755 --- a/hermes-agent/leoclean-bin/kb_tool.py +++ b/hermes-agent/leoclean-bin/kb_tool.py @@ -855,11 +855,17 @@ def operational_contracts( mixed_markers = ( "packet", + "briefing", "factual observation", + "evidence-backed fact", "strategic framework", + "reasoning framework", "disputed interpretation", + "contested position", "governance rule", + "operating rule", "old belief", + "correction", "compose the database", ) if sum(marker in lowered for marker in mixed_markers) >= 2: diff --git a/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py b/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py index 262f22a..9794b7d 100644 --- a/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py +++ b/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py @@ -568,6 +568,13 @@ def _mixed_packet_issues(response: str) -> list[str]: re.I | re.S, ): issues.append("unsupported_collection_listed_in_apply") + if re.search( + r"(?:reasoning|strategic) framework.{0,80}" + r"(?:is|as|into|maps? to|mapped to|classif(?:y|ied) as) (?:an? )?(?:normative )?claim", + response, + re.I | re.S, + ): + issues.append("framework_mapped_to_claim") for segment in re.split(r"(?<=[.!?])\s+|\n+", response): negative = re.search(r"\b(?:no|not|never|without|must not|do not|does not|cannot)\b", segment, re.I) @@ -807,7 +814,7 @@ def _schema_contract_issues(response: str, contracts: list[dict[str, Any]]) -> l if "runtime_persistence" in contract_ids and affirmative( r"(?:state\.db|session\s+jsonl).{0,100}" - r"(?:in[- ]memory|ephemeral|disappears?|vanishes?|erased|discarded)" + r"(?:in[- ]memory|ephemeral|disappears?|vanishes?|erased|discarded|gone|lost|deleted)" ): issues.append("runtime_persistence_contradiction") @@ -833,6 +840,12 @@ def _schema_contract_issues(response: str, contracts: list[dict[str, Any]]) -> l issues.append("forecast_resolution_edge_contradiction") if affirmative(r"public\.claims.{0,100}(?:has|stores?|contains?).{0,80}(?:forecast_resolution|resolved_at)"): issues.append("forecast_resolution_field_contradiction") + if not ( + re.search(r"\b(?:stage|proposal)\b", response, re.I) + and re.search(r"\breview(?:ed|er|ing)?\b", response, re.I) + and re.search(r"\bappl(?:y|ied|ication)\b", response, re.I) + ): + issues.append("forecast_review_apply_incomplete") return sorted(set(issues)) diff --git a/scripts/leo_turn_execution_manifest.py b/scripts/leo_turn_execution_manifest.py index 0fcdb14..4843fc3 100644 --- a/scripts/leo_turn_execution_manifest.py +++ b/scripts/leo_turn_execution_manifest.py @@ -345,21 +345,26 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]: ) tool_call_count = trace.get("database_tool_call_count", 0) completed_count = trace.get("database_tool_completed_count", 0) - all_results_bound = bool( + all_results_content_bound = bool( _non_negative_int(tool_call_count) and _non_negative_int(completed_count) - and completed_count == tool_call_count + and completed_count <= tool_call_count and len(calls) == tool_call_count and all( _is_sha256(call.get("tool_call_id_sha256")) and _is_sha256(call.get("arguments_sha256")) and _is_sha256(call.get("result_content_sha256")) and _non_negative_int(call.get("result_content_bytes")) - and call.get("result_error_detected") is False + and isinstance(call.get("result_error_detected"), bool) and call.get("result_nonempty") is True for call in calls ) ) + all_calls_succeeded = bool( + all_results_content_bound + and completed_count == tool_call_count + and all(call.get("result_error_detected") is False for call in calls) + ) all_calls_read_only = bool( trace.get("database_tool_calls_read_only") is True and all( @@ -376,7 +381,8 @@ def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]: "tool_call_count": tool_call_count, "completed_count": completed_count, "all_calls_read_only": all_calls_read_only if tool_call_count else True, - "all_results_content_bound": all_results_bound, + "all_results_content_bound": all_results_content_bound, + "all_calls_succeeded": all_calls_succeeded, "calls": calls, "trace_sha256": canonical_sha256(trace), } diff --git a/scripts/working_leo_m3taversal_oos_benchmark.py b/scripts/working_leo_m3taversal_oos_benchmark.py index cf851b4..c7be6a0 100755 --- a/scripts/working_leo_m3taversal_oos_benchmark.py +++ b/scripts/working_leo_m3taversal_oos_benchmark.py @@ -254,7 +254,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { "canonical_readback": ( re.compile( r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)|" - r"canonical (?:DB|database)|applied_at.{0,100}readiness", + r"canonical (?:DB|database|KB)|(?:KB|knowledge database).{0,50}(?:changed|updated)|" + r"applied_at.{0,100}(?:readiness|postflight|canonical)", re.I | re.S, ), ), @@ -263,7 +264,12 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { r"approved is not (?:the same as )?applied|applied_at:\s*(?:none|null)|not canonical|" r"proposal does not commit|no receipt,? no durable knowledge|" r"approval.{0,60}not (?:a )?(?:DB|database|canonical) write|" - r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied", + r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied|" + r"(?:reviewer )?approval is intent.{0,100}canonical (?:update|changes?).{0,80}(?:apply|applied_at)|" + r"(?:reviewer )?approval is intent.{0,100}(?:canonical )?KB changes? only when.{0,60}" + r"(?:apply|applied_at)|" + r"(?:reviewer )?approval.{0,100}(?:canonical|KB).{0,60}(?:requires|happens only when).{0,50}" + r"(?:apply|applied_at)", re.I, ), ), @@ -280,11 +286,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"provenance|stable (?:reference|ref)|source row|author/channel|(?:file|content) hash", re.I), ), "deduplication": (re.compile(r"deduplic|duplicate|existing claim|overlap", re.I),), - "contradiction": (re.compile(r"contradict|divergence|competing interpretation|disagree", re.I),), + "contradiction": ( + re.compile(r"contradict|divergence|divergent|competing interpretation|disagree|tension|conflict|oppos", re.I), + ), "staged_review_apply": ( re.compile(r"staging|stage|proposal", re.I), re.compile(r"review|approve|reviewer", re.I), - re.compile(r"apply|canonical", re.I), + re.compile(r"appl(?:y|ied|ication)|canonical", re.I), ), "receipt": ( re.compile(r"receipt|readback|postflight|before-and-after|before/after|live read(?:ing)?|live proof", re.I), @@ -310,13 +318,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { ), ), "source_evidence_chain": ( - re.compile(r"file|attachment|source_ref", re.I), + re.compile(r"file|attachment|source_ref|document|artifact|extracted (?:text|markdown)|path|pointer", re.I), re.compile(r"public\.sources", re.I), re.compile(r"claim_evidence", re.I), re.compile(r"audit|link|join|row chain|before-and-after|before/after", re.I), ), "canonical_evidence_boundary": ( - re.compile(r"canonical evidence", re.I), + re.compile(r"canonical (?:evidence|support|linkage)|not yet canonical|not canonical", re.I), re.compile(r"claim_evidence", re.I), re.compile(r"public\.sources|source row", re.I), re.compile( @@ -325,13 +333,19 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { r"(?:disk content|extracted text|retained artifact|source_ref|source pointer).{0,140}" r"(?:is not|isn't|does not|doesn't|alone|until|unless).{0,80}(?:canonical evidence|finished provenance)|" r"(?:canonical evidence|canonical link).{0,100}(?:requires|exists only when|is complete only when).{0,100}" - r"(?:public\.sources|source row|claim_evidence)", + r"(?:public\.sources|source row|claim_evidence)|" + r"(?:not yet canonical|staging-only).{0,300}(?:public\.sources|source row).{0,200}claim_evidence|" + r"(?:public\.sources|source row).{0,120}(?:absent|missing|requires?).{0,200}claim_evidence", re.I | re.S, ), ), "evidence_provenance_quality": ( re.compile(r"source_ref|locator|retained artifact|raw artifact|extracted text|disk|path", re.I), - re.compile(r"weak|unresolved|not traceable|verified|hash_matches_db|provenance", re.I), + re.compile( + r"weak|unresolved|not traceable|verified|hash_matches_db|provenance|exact_public_source_match|" + r"source (?:match|audit)|link audit", + re.I, + ), re.compile(r"canonical evidence|canonical link|public\.sources|claim_evidence", re.I), ), "heterogeneous_types": ( @@ -352,19 +366,28 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile( r"does not (?:accept|insert|support|cover|write)|supports neither|neither.{0,80}nor|" r"(?:behavioral_rules|governance_gates).{0,120}(?:cannot be applied by|sits? outside) approve_claim|" + r"outside approve_claim(?:'s)? apply capability|approve_claim applies only|" r"approve_claim.{0,120}(?:cannot apply|cannot write|does not write)", re.I | re.S, ), - re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply (?:capability|path)", re.I | re.S), + re.compile( + r"separate.{0,60}(?:reviewed|authorized|authorization).{0,60}(?:apply )?(?:capability|path|write)|" + r"separate (?:reviewed )?(?:apply )?(?:capability|authorization)|" + r"separate.{0,50}apply capability.{0,60}(?:reviewed|authorized)|" + r"reviewed.{0,50}apply (?:capability|path)", + re.I | re.S, + ), ), "runtime_inputs": ( - re.compile(r"Postgres|canonical (?:DB|database|counts?)|(?:DB|database) totals?", re.I), + re.compile(r"Postgres|canonical (?:DB|database|counts?|rows?)|(?:DB|database) totals?", re.I), re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I), re.compile(r"session|conversation context", re.I), re.compile( r"unchanged.{0,120}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)|" r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)|" - r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)", + r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)|" + r"(?:identical|unchanged).{0,40}(?:counts|totals).{0,60}(?:prove nothing about content|" + r"not proof of content)", re.I | re.S, ), ), @@ -372,23 +395,28 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"state\.db|session JSONL", re.I), re.compile(r"persist|durable|continuity", re.I), re.compile( - r"restart.{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|" - r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}restart|" - r"(?:facts?|context).{0,80}not erased by restart", + r"(?:restart|recycle).{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|" + r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}(?:restart|recycle)|" + r"(?:facts?|context).{0,80}not erased by (?:restart|recycle)", re.I | re.S, ), ), "row_content_proof": ( - re.compile(r"unchanged (?:database )?(?:counts?|totals?)", re.I), - re.compile(r"does not prove|doesn't prove|do not prove|say nothing", re.I), + re.compile(r"(?:unchanged|identical) (?:database )?(?:counts?|totals?)", re.I), + re.compile(r"does not prove|doesn't prove|do not prove|prove(?:s)? nothing|say nothing", re.I), re.compile( r"row (?:IDs?|hashes?)|fingerprints?|timestamps?|balanced (?:insert|write|change)|" - r"individual rows?.{0,60}(?:mutated|changed|updated)", + r"individual rows?.{0,60}(?:mutated|changed|updated)|artifact_state_sha256|content hashes?|" + r"row[- ]level (?:hash|readback)", re.I | re.S, ), ), "proof_tiers": ( - re.compile(r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3", re.I | re.S), + re.compile( + r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3|" + r"canonical rows.{0,600}deployed runtime inputs.{0,600}durable session state", + re.I | re.S, + ), re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I), re.compile(r"runtime|skills?|session|SOUL\.md", re.I), ), @@ -407,26 +435,29 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { ), "agent_specific_positions": ( re.compile(r"public\.beliefs", re.I), - re.compile(r"agent_id", re.I), + re.compile(r"agent_id|agent attribution|attributed to (?:that|each|an?) agent", re.I), re.compile(r"belief|position|stance|confidence", re.I), re.compile(r"no.{0,40}claim(?:-ID|_id).{0,30}(?:foreign key|link)|schema gap", re.I | re.S), ), "forecast_history": ( - re.compile(r"original (?:probability|confidence)|60%|history", re.I), + re.compile(r"original (?:probability|confidence)|original.{0,120}(?:probability|confidence)|60%|history", re.I), re.compile( - r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|original.{0,60}survives", + r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|" + r"leave.{0,60}(?:untouched|unmodified)|original.{0,60}survives", re.I | re.S, ), re.compile( r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria|" + r"lack(?:ed|s|ing).{0,30}(?:success|resolution) criteria|" r"criteria.{0,40}(?:never existed|were never defined|did not exist)|without.{0,30}criteria", re.I | re.S, ), ), "forecast_schema_gap": ( - re.compile(r"current (?:v1|schema)|public\.claims", re.I), + re.compile(r"current (?:v1|schema)|public\.claims|the schema|schema has|claims table", re.I), re.compile( r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution|" + r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|" r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}" r"(?:does not exist|is absent|is not present|isn't present)|" r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present", @@ -531,24 +562,25 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { ), ), "evidence_specific_challenge": ( - re.compile(r"\b(?:challenge|limitation|however|unsupported leap|caveat)\b", re.I), + re.compile(r"\b(?:challeng(?:e|ed|ing)|limitation|however|unsupported leap|caveat)\b", re.I), re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I), re.compile( - r"\b(?:does not|doesn't|cannot|can't)\s+(?:itself\s+)?(?:prove|establish|support|show)|" - r"\b(?:does not|doesn't|cannot|can't)\s+rule out\b|" - r"\bunsupported leap\b", + r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+(?:itself\s+)?" + r"(?:prove|establish|support|show)|" + r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+rule out\b|" + r"\b(?:not paid for|outruns?).{0,40}\b(?:the )?evidence\b|\bunsupported leap\b", re.I, ), ), "narrower_claim_revision": ( re.compile( - r"\b(?:narrower\s+(?:revision|claim|formulation)|revision\s*:|" + r"\b(?:narrower\s+(?:revision|claim|formulation|candidate(?: claim)?)|revision\s*:|" r"should be narrowed to|defensible formulation|revise(?:d)?\b.{0,40}\bto\b)", re.I, ), re.compile( r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|" - r"unproven|open constraint)\b", + r"undemonstrated|unproven|not established|not demonstrated|remains unknown|open constraint)\b", re.I, ), ), @@ -567,7 +599,7 @@ COUNT_INVARIANT_REJECTION_RE = re.compile( ) SCHEMA_GAP_QUALIFIER_RE = re.compile( - r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|" + r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|" r"has no|have no|there is no|no column|not an edge|would require|schema gap|must be added|" r"does not support|doesn't support|supports neither|not supported|must not|do not invent|" r"schema extension|extension proposal)\b|" @@ -656,6 +688,11 @@ DB_ONLY_CAUSALITY_RE = re.compile( RESTART_ERASES_ALL_RE = re.compile( r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S ) +DURABLE_SESSION_EPHEMERAL_RE = re.compile( + r"session JSONL.{0,100}(?:is|are|gets?|becomes?).{0,30}(?:gone|lost|deleted|discarded)|" + r"session JSONL.{0,100}(?:gone|lost|deleted|discarded).{0,50}(?:session closes?|restart)", + re.I | re.S, +) REASONING_TOOL_CLAIM_EDGE_RE = re.compile( r"claims?.{0,120}(?:get|have|create|use).{0,50}(?:claim_)?edges?.{0,100}" r"(?:reasoning_tool|tool(?:'s)? ID)|" @@ -718,7 +755,7 @@ CLAIM_EVIDENCE_CONFLATION_RE = re.compile( REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P[^\n]+)", re.I) REVISION_BOUNDARY_RE = re.compile( r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|" - r"open constraint)\b", + r"undemonstrated|not established|not demonstrated|remains unknown|open constraint)\b", re.I, ) REVISION_REPETITION_RE = re.compile( @@ -849,6 +886,8 @@ def broad_semantic_issues(reply: str) -> list[str]: re.I | re.S, ): findings.add("restart_called_total_memory_erasure") + if DURABLE_SESSION_EPHEMERAL_RE.search(reply): + findings.add("durable_session_called_ephemeral") if HANDLER_TELEGRAM_OVERCLAIM_RE.search(reply): findings.add("handler_proof_called_telegram_live") if FORECAST_HISTORY_REWRITE_RE.search(reply): diff --git a/scripts/working_leo_m3taversal_oos_protocol.py b/scripts/working_leo_m3taversal_oos_protocol.py index ebbefae..3d2cf03 100644 --- a/scripts/working_leo_m3taversal_oos_protocol.py +++ b/scripts/working_leo_m3taversal_oos_protocol.py @@ -866,6 +866,7 @@ def _executed_behavior_ablation(grounded_report: dict[str, Any], baseline_report stable_keys = { "schema", "model_runtime", + "python_runtime", "hermes_runtime", "teleo_infrastructure_runtime", "components", @@ -1061,11 +1062,11 @@ def _receipt_score( } reply = str(result.get("reply") or "") reply_identifiers = {match.group(0).lower() for match in UUID_RE.finditer(reply)} - reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = ( - _resolve_supported_citations(reply, CLAIM_ID_CITATION_RE, receipt_claim_ids) + reply_claim_citation_tokens, reply_claim_citations, unresolved_claim_citations = _resolve_supported_citations( + reply, CLAIM_ID_CITATION_RE, receipt_claim_ids ) - reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = ( - _resolve_supported_citations(reply, SOURCE_ID_CITATION_RE, receipt_source_ids) + reply_source_citation_tokens, reply_source_citations, unresolved_source_citations = _resolve_supported_citations( + reply, SOURCE_ID_CITATION_RE, receipt_source_ids ) unsupported_identifiers = sorted(reply_identifiers - supported_identifiers) receipt_counts = [(item.get("retrieval_receipt") or {}).get("counts") or {} for item in retrieval_records] @@ -1169,9 +1170,7 @@ def _benchmark_execution_chain( and local_state.get("status_sha256") == hashlib.sha256(b"?? goal.md\n").hexdigest() ) supported_worktree_mode = clean_worktree_exact or control_goal_only_exact - worktree_allowed_missing = ( - CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset() - ) + worktree_allowed_missing = CONTROL_GOAL_EXECUTION_ALLOWED_MISSING if control_goal_only_exact else frozenset() mode = report.get("grounding_mode") if mode == "grounded": allowed_missing_sets = (GROUNDED_EXECUTION_ALLOWED_MISSING | worktree_allowed_missing,) diff --git a/tests/test_hermes_leoclean_db_context_plugin.py b/tests/test_hermes_leoclean_db_context_plugin.py index 22024f6..594c46f 100644 --- a/tests/test_hermes_leoclean_db_context_plugin.py +++ b/tests/test_hermes_leoclean_db_context_plugin.py @@ -28,6 +28,53 @@ def test_plugin_registers_generation_and_delivery_hooks() -> None: assert [name for name, _callback in registered] == ["pre_llm_call", "post_llm_call"] +def test_mixed_packet_contract_rejects_framework_mapped_to_claim() -> None: + module = load_plugin() + contracts = [{"id": "mixed_packet_composition"}] + + issues = module.response_contract_issues( + "The evidence-backed fact maps to a claim, and the reasoning framework maps to a normative claim.", + contracts, + ) + + assert "framework_mapped_to_claim" in issues + + +def test_runtime_persistence_contract_rejects_lost_session_jsonl() -> None: + module = load_plugin() + contracts = [{"id": "runtime_persistence"}] + + bad = module.response_contract_issues( + "state.db persists, but the session JSONL is lost when the session closes.", + contracts, + ) + good = module.response_contract_issues( + "state.db and the session JSONL persist on disk and survive a service restart.", + contracts, + ) + + assert "runtime_persistence_contradiction" in bad + assert "runtime_persistence_contradiction" not in good + + +def test_forecast_contract_requires_reviewed_apply_boundary() -> None: + module = load_plugin() + contracts = [{"id": "forecast_resolution_schema"}] + + incomplete = module.response_contract_issues( + "Preserve the original forecast and stage a reviewed schema proposal.", + contracts, + ) + complete = module.response_contract_issues( + "Preserve the original forecast because no criteria existed. Stage a reviewed proposal, but do not apply " + "or invent the new fields in the current schema.", + contracts, + ) + + assert "forecast_review_apply_incomplete" in incomplete + assert "forecast_review_apply_incomplete" not in complete + + def test_plugin_injects_bounded_retrieval_rows_and_writes_body_redacted_trace(tmp_path: Path, monkeypatch) -> None: module = load_plugin() home = tmp_path / "profile" @@ -591,7 +638,9 @@ def test_plugin_rejects_concrete_runtime_schema_contradictions_without_requiring assert ( module.response_contract_issues( - "There is no resolves edge or resolved_at field; stage a reviewed schema proposal.", forecast + "There is no resolves edge or resolved_at field; stage a reviewed schema proposal and do not apply " + "it until approved.", + forecast, ) == [] ) @@ -609,7 +658,7 @@ def test_plugin_replaces_schema_contradiction_with_current_compiled_contract() - contracts = [{"id": "forecast_resolution_schema"}] compiled = ( "Preserve the original forecast. There is no resolves edge or resolved_at field in the current schema; " - "stage a reviewed schema proposal for explicit resolution semantics." + "stage a reviewed schema proposal for explicit resolution semantics and do not apply it until approved." ) module._store_snapshot( "forecast-contract", diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index a73b42c..3a325ac 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -38,6 +38,16 @@ def test_leoclean_bridge_files_are_present_and_parseable() -> None: subprocess.run(["bash", "-n", str(wrapper)], check=True) +def test_vps_bridge_recognizes_natural_mixed_briefing_contract() -> None: + module = _load_module(BRIDGE_DIR / "kb_tool.py") + contracts = module.operational_contracts( + "Compose this briefing from evidence-backed facts, a reasoning framework, a contested position, " + "an operating rule, and a correction." + ) + + assert "mixed_packet_composition" in {item["id"] for item in contracts} + + def test_vps_bridge_global_options_do_not_accept_untraced_abbreviations(monkeypatch) -> None: module = _load_module(BRIDGE_DIR / "kb_tool.py") monkeypatch.setattr( diff --git a/tests/test_leo_turn_execution_manifest.py b/tests/test_leo_turn_execution_manifest.py index d4045e8..c7b51e4 100644 --- a/tests/test_leo_turn_execution_manifest.py +++ b/tests/test_leo_turn_execution_manifest.py @@ -339,9 +339,7 @@ def test_valid_response_transform_binds_raw_and_delivered_hashes() -> None: def test_dirty_or_untracked_harness_cannot_produce_complete_manifest() -> None: - value = build( - harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64} - ) + value = build(harness_source={"git_head": "8" * 40, "worktree_clean": False, "status_sha256": "1" * 64}) assert "harness_worktree_clean" in value["attribution"]["missing_required_bindings"] @@ -382,6 +380,103 @@ def test_write_capable_database_tool_trace_fails_closed() -> None: assert "database_tools_read_only" in value["attribution"]["missing_required_bindings"] +def test_hash_bound_read_only_tool_error_remains_attributable() -> None: + result = turn_result() + result["database_tool_trace"] = { + "schema": "livingip.leoKbToolTrace.v1", + "database_tool_call_count": 2, + "database_tool_completed_count": 1, + "database_tool_calls_read_only": True, + "calls": [ + { + "tool_call_id_sha256": "1" * 64, + "arguments_sha256": "2" * 64, + "database_invocations": [ + { + "access_mode": "read_only", + "command_sha256": "3" * 64, + "executable": "teleo-kb", + "subcommand": "context", + } + ], + "result": { + "content_sha256": "4" * 64, + "content_bytes": 17, + "error_detected": True, + "nonempty": True, + "row_ids": [], + "sha256_values": [], + }, + }, + { + "tool_call_id_sha256": "5" * 64, + "arguments_sha256": "6" * 64, + "database_invocations": [ + { + "access_mode": "read_only", + "command_sha256": "7" * 64, + "executable": "teleo-kb", + "subcommand": "context", + } + ], + "result": { + "content_sha256": "8" * 64, + "content_bytes": 23, + "error_detected": False, + "nonempty": True, + "row_ids": [], + "sha256_values": [], + }, + }, + ], + } + + value = build(result) + binding = value["canonical_database"]["database_tool_binding"] + + assert binding["all_results_content_bound"] is True + assert binding["all_calls_succeeded"] is False + assert "database_tool_results" not in value["attribution"]["missing_required_bindings"] + assert value["attribution"]["status"] == "complete" + + +def test_unbound_database_tool_error_remains_incomplete() -> None: + result = turn_result() + result["database_tool_trace"] = { + "schema": "livingip.leoKbToolTrace.v1", + "database_tool_call_count": 1, + "database_tool_completed_count": 0, + "database_tool_calls_read_only": True, + "calls": [ + { + "tool_call_id_sha256": "1" * 64, + "arguments_sha256": "2" * 64, + "database_invocations": [ + { + "access_mode": "read_only", + "command_sha256": "3" * 64, + "executable": "teleo-kb", + "subcommand": "context", + } + ], + "result": { + "content_sha256": None, + "content_bytes": 0, + "error_detected": True, + "nonempty": False, + "row_ids": [], + "sha256_values": [], + }, + } + ], + } + + value = build(result) + + assert value["canonical_database"]["database_tool_binding"]["all_results_content_bound"] is False + assert "database_tool_results" in value["attribution"]["missing_required_bindings"] + + def test_later_turn_binds_previous_execution_and_conversation_state() -> None: first_result = turn_result() first = build(first_result) diff --git a/tests/test_working_leo_m3taversal_oos_benchmark.py b/tests/test_working_leo_m3taversal_oos_benchmark.py index d97e893..70b48ac 100644 --- a/tests/test_working_leo_m3taversal_oos_benchmark.py +++ b/tests/test_working_leo_m3taversal_oos_benchmark.py @@ -745,6 +745,108 @@ def test_oos_state_boundary_accepts_approved_is_not_applied() -> None: assert benchmark.matched_concept("Approved is not applied.", "state_boundary") is True +def test_oos_state_boundary_accepts_natural_apply_language() -> None: + reply = ( + "Reviewer approval is intent. The canonical KB changes only when an authorized apply writes the rows and " + "applied_at is recorded in the postflight readback." + ) + + assert benchmark.matched_concept(reply, "state_boundary") is True + assert benchmark.matched_concept(reply, "canonical_readback") is True + + +def test_oos_source_chain_accepts_document_pointer_audit_language() -> None: + reply = ( + "The extracted document artifact is a pointer, not yet canonical support. public.sources has no matching " + "source row and claim_evidence has no link, so audit the path, source row, and join before-and-after." + ) + + assert benchmark.matched_concept(reply, "source_evidence_chain") is True + assert benchmark.matched_concept(reply, "canonical_evidence_boundary") is True + + +def test_oos_runtime_reasoning_accepts_recycle_and_content_hash_language() -> None: + reply = ( + "Postgres canonical rows and artifact_state_sha256 are unchanged, but identical counts do not prove " + "behavior or content. Deployed skills and SOUL.md affect the answer. state.db and session JSONL persist " + "conversation continuity, so a service recycle does not erase the session." + ) + + assert benchmark.matched_concept(reply, "runtime_inputs") is True + assert benchmark.matched_concept(reply, "durable_session_continuity") is True + assert benchmark.matched_concept(reply, "row_content_proof") is True + assert benchmark.broad_semantic_issues(reply) == [] + + +def test_oos_runtime_reasoning_rejects_durable_session_as_ephemeral() -> None: + reply = "state.db survives, but the session JSONL is lost when the session closes." + + assert benchmark.broad_semantic_issues(reply) == ["durable_session_called_ephemeral"] + + +def test_oos_agent_positions_accept_attribution_without_literal_column_name() -> None: + reply = ( + "public.beliefs stores each stance attributed to that agent with confidence. There is no claim-ID foreign " + "key, so a direct link to the shared claim remains a schema gap." + ) + + assert benchmark.matched_concept(reply, "agent_specific_positions") is True + + +def test_oos_forecast_accepts_natural_missing_resolution_language() -> None: + reply = ( + "Preserve the original 60% forecast; it lacked success criteria. The schema has no outcome or resolved " + "field, and there is no resolves edge. Stage a proposal, obtain reviewer approval, then apply it and " + "record applied_at." + ) + + assert benchmark.matched_concept(reply, "forecast_history") is True + assert benchmark.matched_concept(reply, "forecast_schema_gap") is True + assert benchmark.matched_concept(reply, "staged_review_apply") is True + assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues(reply) + assert "current_schema_column_hallucination" not in benchmark.broad_semantic_issues( + "The schema is missing a resolves edge." + ) + + +def test_oos_challenge_accepts_plural_and_evidence_bounded_language() -> None: + reply = ( + "Challenge: the sources and claim bodies do not establish causality; the conclusion is not paid for by " + "the evidence. A narrower candidate claim says only that the association is observed, while the mechanism " + "remains undemonstrated." + ) + + assert benchmark.matched_concept(reply, "evidence_specific_challenge") is True + assert benchmark.matched_concept(reply, "narrower_claim_revision") is True + + +def test_oos_semantic_scorer_accepts_natural_receipt_and_policy_paraphrases() -> None: + assert benchmark.matched_concept( + "Identical counts prove nothing about content; compare artifact_state_sha256.", + "row_content_proof", + ) + assert benchmark.matched_concept( + "approve_claim does not support behavioral_rules; the separate apply capability must be reviewed and authorized.", + "reviewed_policy_apply", + ) + assert benchmark.matched_concept( + "behavioral_rules are outside approve_claim's apply capability and need a separate authorization.", + "reviewed_policy_apply", + ) + assert benchmark.matched_concept( + "Leave the original claim untouched with confidence 0.6 because no resolution criteria existed.", + "forecast_history", + ) + assert benchmark.matched_concept( + "Challenged inference: the claim body and evidence do not establish causality.", + "evidence_specific_challenge", + ) + assert benchmark.matched_concept( + "Revision: adoption is documented, while lock-in remains undemonstrated.", + "narrower_claim_revision", + ) + + def test_oos_schema_gap_accepts_explicitly_absent_edge_type() -> None: reply = "Current claim_edges has edge_type; there is no `resolves` edge type." assert benchmark.current_schema_overclaims(reply) == [] diff --git a/tests/test_working_leo_m3taversal_oos_protocol.py b/tests/test_working_leo_m3taversal_oos_protocol.py index 3107007..561887e 100644 --- a/tests/test_working_leo_m3taversal_oos_protocol.py +++ b/tests/test_working_leo_m3taversal_oos_protocol.py @@ -253,6 +253,7 @@ def fake_behavior_manifest(*, grounded: bool) -> dict: stable = { "schema": "livingip.leoBehaviorManifest.v1", "model_runtime": {"model": "test-model"}, + "python_runtime": {"python_version": "3.11.15", "implementation": "CPython"}, "hermes_runtime": { "git_head": "e" * 40, "source_tree": {"sha256": "f" * 64, "file_count": 1}, @@ -738,8 +739,7 @@ def test_execution_chain_allows_only_declared_ablation_and_control_goal() -> Non assert validation["local_state_checks"]["control_goal_only_exact"] is True assert validation["allowed_missing_binding_sets"][0] == sorted(expected_missing) assert all( - result["execution_manifest"]["attribution"]["missing_required_bindings"] - == sorted(expected_missing) + result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing) for result in report["results"] ) @@ -804,9 +804,7 @@ def test_execution_chain_accepts_clean_harness_without_cleanliness_omission() -> result["execution_manifest"]["attribution"]["missing_required_bindings"] == sorted(expected_missing) for result in report["results"] ) - assert all( - checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values() - ) + assert all(checks["declared_missing_exact"] is True for checks in validation["turn_checks"].values()) manifest = report["results"][-1]["execution_manifest"] manifest["attribution"]["missing_required_bindings"].append("harness_worktree_clean") @@ -855,6 +853,7 @@ def test_comparison_rejects_any_executed_profile_delta_beyond_db_context_removal for key in ( "schema", "model_runtime", + "python_runtime", "hermes_runtime", "teleo_infrastructure_runtime", "components",