From 8cbda554353a326a1a0b796e3adca8cceeac5371 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Thu, 16 Jul 2026 05:59:09 +0200 Subject: [PATCH] Preserve subject and memory boundaries in Leo replies --- hermes-agent/leoclean-bin/kb_tool.py | 57 ++++++++++++------- .../vps/leo-db-context/__init__.py | 18 ++++++ .../working_leo_m3taversal_oos_benchmark.py | 21 +++++-- .../test_hermes_leoclean_db_context_plugin.py | 15 +++++ .../test_hermes_leoclean_kb_bridge_source.py | 9 +++ ...st_working_leo_m3taversal_oos_benchmark.py | 17 ++++++ 6 files changed, 113 insertions(+), 24 deletions(-) diff --git a/hermes-agent/leoclean-bin/kb_tool.py b/hermes-agent/leoclean-bin/kb_tool.py index 00a19e1..bfbf6c7 100755 --- a/hermes-agent/leoclean-bin/kb_tool.py +++ b/hermes-agent/leoclean-bin/kb_tool.py @@ -663,6 +663,20 @@ def operational_contracts( for term in ("challenge", "unsupported", "narrower", "assumption", "what supports", "falsif") ) ) + conversation_memory_question = any( + term in lowered + for term in ( + "chat memory only", + "chat-only label", + "temporary label", + "temporary conversation mnemonic", + "recall the temporary label", + "retrieve the mnemonic", + "bind it to", + "until my next question", + "for the next turn only", + ) + ) if claim_evidence_challenge_question: contracts.append( { @@ -683,6 +697,7 @@ def operational_contracts( broad_kb_change_question = _has_unnegated_action(query, ("change", "changed", "update", "updated", "landed")) proposal_state_question = ( kb_scope + and not conversation_memory_question and not forecast_resolution_question and not claim_reasoning_question and _has_unnegated_database_state_request(query) @@ -694,26 +709,30 @@ def operational_contracts( named_packet_question = "helmer" in lowered and any( term in lowered for term in ("7 powers", "seven powers", "powers framework", "powers packet") ) - demo_question = "demo" in lowered and ( - any( - term in lowered - for term in ( - "change the kb", - "changes the kb", - "changed the kb", - "change the knowledge base", - "changes the knowledge base", - "changed the knowledge base", - "change the database", - "changes the database", - "changed the database", + demo_question = ( + not conversation_memory_question + and "demo" in lowered + and ( + any( + term in lowered + for term in ( + "change the kb", + "changes the kb", + "changed the kb", + "change the knowledge base", + "changes the knowledge base", + "changed the knowledge base", + "change the database", + "changes the database", + "changed the database", + ) ) - ) - or ( - kb_scope - and ( - any(term in lowered for term in ("learned", "updated", "update", "live", "current", "working")) - or _has_unnegated_action(query, ("write", "apply", "mutate", "stage")) + or ( + kb_scope + and ( + any(term in lowered for term in ("learned", "updated", "update", "live", "current", "working")) + or _has_unnegated_action(query, ("write", "apply", "mutate", "stage")) + ) ) ) ) diff --git a/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py b/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py index cd7a691..e28e1f3 100644 --- a/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py +++ b/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py @@ -27,6 +27,10 @@ DATABASE_REASONING_PROTOCOL_VERSION = "leo-db-reasoning-v2" WORD_RE = re.compile(r"\b\w+(?:[-']\w+)*\b") LIST_PREFIX_RE = re.compile(r"^(\s*(?:[-*]|\d+[.)])\s+)(.*)$") SENTENCE_BREAK_RE = re.compile(r"(?<=[.!?])\s+") +REQUESTED_SUBJECT_RE = re.compile( + r"name the subject exactly once as\s+`(?P[^`\r\n]{1,120})`\s+in your answer", + re.I, +) COMPILED_CONTRACT_IDS = frozenset( { "mixed_packet_composition", @@ -527,6 +531,18 @@ def _reply_hard_max(contracts: list[dict[str, Any]]) -> int | None: return None +def _ensure_requested_subject(response: str, user_message: str) -> tuple[str, bool]: + """Preserve an explicit short subject label when a compiled fallback is delivered.""" + + match = REQUESTED_SUBJECT_RE.search(user_message) + if not match: + return response, False + subject = " ".join(match.group("subject").split()) + if not subject or re.search(re.escape(subject), response, re.I): + return response, False + return f"Subject: {subject}\n\n{response}", True + + def _mixed_packet_issues(response: str) -> list[str]: lowered = response.lower() issues: list[str] = [] @@ -981,6 +997,7 @@ def _post_llm_call(**kwargs: Any) -> dict[str, str] | None: delivered = str(compiled_response) else: delivered = response + delivered, requested_subject_injected = _ensure_requested_subject(delivered, user_message) hard_max = _reply_hard_max(contracts) budget_compacted = bool(not fail_closed and hard_max is not None and _word_count(delivered) > hard_max) if budget_compacted: @@ -999,6 +1016,7 @@ def _post_llm_call(**kwargs: Any) -> dict[str, str] | None: "transformed": transformed, "budget_compacted": budget_compacted, "compiled_response_enforced": compiled_fallback_required, + "requested_subject_injected": requested_subject_injected, "fail_closed": fail_closed, "delivered_response_sha256": hashlib.sha256(delivered.encode("utf-8")).hexdigest(), } diff --git a/scripts/working_leo_m3taversal_oos_benchmark.py b/scripts/working_leo_m3taversal_oos_benchmark.py index eb14737..fe11f99 100755 --- a/scripts/working_leo_m3taversal_oos_benchmark.py +++ b/scripts/working_leo_m3taversal_oos_benchmark.py @@ -426,12 +426,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile( r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|" r"keep the factual claim once|store it once|one public\.claims row|" - r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)", + r"one shared public\.claims row|fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)", re.I | re.S, ), re.compile(r"source|evidence", re.I), re.compile( - r"do not duplicate|don'?t duplicate|one shared claim|single shared claim|duplicating (?:the )?claim", + r"do not duplicate|don'?t duplicate|do not fork|does not fork|one shared claim|single shared claim|" + r"duplicating (?:the )?claim", re.I, ), ), @@ -440,7 +441,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile( r"agent_id|agent attribution|attributed to (?:that|each|an?) agent|" r"one (?:public\.beliefs |beliefs )?row per agent(?: position)?|" - r"each agent(?:'s)? (?:belief|position|stance)", + r"each agent(?:'s)? (?:belief|position|stance)|" + r"each agent.{0,60}(?:public\.)?beliefs row", re.I, ), re.compile(r"belief|position|stance|confidence", re.I), @@ -748,7 +750,12 @@ UNSUPPORTED_COMPOSITION_APPLY_RE = re.compile( re.I | re.S, ) APPROVED_APPLY_ACTION_RE = re.compile( - r"approved.{0,240}(?:guarded )?apply|(?:guarded )?apply.{0,240}approved", + r"approved.{0,200}(?:authorize|run|execute|perform|proceed).{0,60}(?:guarded )?apply|" + r"(?:authorize(?:d)?(?: and)?|run|execute|perform|proceed with).{0,60}(?:guarded )?apply.{0,200}approved", + re.I | re.S, +) +APPLY_ACTION_DISCLAIMER_RE = re.compile( + r"proof condition.{0,100}not authorization|not authorization.{0,100}(?:run|execute|perform|apply)", re.I | re.S, ) CANONICAL_SOURCE_BEFORE_REVIEW_RE = re.compile( @@ -934,7 +941,11 @@ def broad_semantic_issues(reply: str) -> list[str]: def proposal_readiness_issues(prompt_id: str, reply: str) -> list[str]: if prompt_id not in {"OOS-01", "OOS-04", "OOS-07", "OOS-08"}: return [] - if APPROVED_APPLY_ACTION_RE.search(reply) and not APPLYABILITY_GAP_RE.search(reply): + if ( + APPROVED_APPLY_ACTION_RE.search(reply) + and not APPLYABILITY_GAP_RE.search(reply) + and not APPLY_ACTION_DISCLAIMER_RE.search(reply) + ): return ["approved_proposal_applyability_overclaim"] return [] diff --git a/tests/test_hermes_leoclean_db_context_plugin.py b/tests/test_hermes_leoclean_db_context_plugin.py index cf26aae..c0acf3c 100644 --- a/tests/test_hermes_leoclean_db_context_plugin.py +++ b/tests/test_hermes_leoclean_db_context_plugin.py @@ -28,6 +28,21 @@ def test_plugin_registers_generation_and_delivery_hooks() -> None: assert [name for name, _callback in registered] == ["pre_llm_call", "post_llm_call"] +def test_compiled_fallback_preserves_explicit_subject_label_once() -> None: + module = load_plugin() + prompt = "Audit the database. Name the subject exactly once as `partner readiness` in your answer." + + injected, changed = module._ensure_requested_subject("No. Approved is not applied.", prompt) + preserved, preserved_changed = module._ensure_requested_subject( + "Subject: partner readiness\n\nNo. Approved is not applied.", prompt + ) + + assert injected.count("partner readiness") == 1 + assert changed is True + assert preserved.count("partner readiness") == 1 + assert preserved_changed is False + + def test_mixed_packet_contract_rejects_framework_mapped_to_claim() -> None: module = load_plugin() contracts = [{"id": "mixed_packet_composition"}] diff --git a/tests/test_hermes_leoclean_kb_bridge_source.py b/tests/test_hermes_leoclean_kb_bridge_source.py index cbb9001..2ec4969 100644 --- a/tests/test_hermes_leoclean_kb_bridge_source.py +++ b/tests/test_hermes_leoclean_kb_bridge_source.py @@ -933,6 +933,15 @@ def test_vps_bridge_direct_intents_reject_unrelated_and_resolve_overlaps() -> No } assert reasoning_followup_ids & direct_ids == set() + chat_memory_ids = { + item["id"] + for item in module.operational_contracts( + "Use chat memory only: bind the approved-versus-applied canonical blocker to TEMP until my next " + "question. Do not stage or apply anything." + ) + } + assert chat_memory_ids == {"reply_budget"} + def test_vps_bridge_named_document_question_matches_exact_proposal_and_compiles_receipt(monkeypatch) -> None: module = _load_module(BRIDGE_DIR / "kb_tool.py") diff --git a/tests/test_working_leo_m3taversal_oos_benchmark.py b/tests/test_working_leo_m3taversal_oos_benchmark.py index 4e821e4..25cb687 100644 --- a/tests/test_working_leo_m3taversal_oos_benchmark.py +++ b/tests/test_working_leo_m3taversal_oos_benchmark.py @@ -508,6 +508,15 @@ def test_oos_direct_apply_case_rejects_approved_rows_without_readiness_gap() -> assert benchmark.proposal_readiness_issues(prompt["id"], repaired) == [] +def test_oos_closure_receipt_does_not_imply_approved_row_is_directly_applyable() -> None: + reply = ( + "The approved-versus-applied blocker closes only when a future apply run returns applied_at plus exact " + "canonical row IDs and hashes. This is a proof condition, not authorization to run apply." + ) + + assert benchmark.proposal_readiness_issues("OOS-08", reply) == [] + + def test_oos_direct_apply_case_accepts_natural_live_readback_wording() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[3] @@ -822,6 +831,14 @@ def test_oos_agent_positions_accept_attribution_without_literal_column_name() -> assert benchmark.matched_concept(reply, "agent_specific_positions") is True + natural = ( + "The fact and its source evidence live in one shared public.claims row, and both analysts do not fork the " + "claim. Each agent files one public.beliefs row for its position. There is no claim-ID foreign key, so that " + "link is a schema gap." + ) + assert benchmark.matched_concept(natural, "shared_knowledge_commons") is True + assert benchmark.matched_concept(natural, "agent_specific_positions") is True + def test_oos_forecast_accepts_natural_missing_resolution_language() -> None: reply = (