Preserve subject and memory boundaries in Leo replies
This commit is contained in:
parent
3cbbbfaa18
commit
8cbda55435
6 changed files with 113 additions and 24 deletions
|
|
@ -663,6 +663,20 @@ def operational_contracts(
|
||||||
for term in ("challenge", "unsupported", "narrower", "assumption", "what supports", "falsif")
|
for term in ("challenge", "unsupported", "narrower", "assumption", "what supports", "falsif")
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
conversation_memory_question = any(
|
||||||
|
term in lowered
|
||||||
|
for term in (
|
||||||
|
"chat memory only",
|
||||||
|
"chat-only label",
|
||||||
|
"temporary label",
|
||||||
|
"temporary conversation mnemonic",
|
||||||
|
"recall the temporary label",
|
||||||
|
"retrieve the mnemonic",
|
||||||
|
"bind it to",
|
||||||
|
"until my next question",
|
||||||
|
"for the next turn only",
|
||||||
|
)
|
||||||
|
)
|
||||||
if claim_evidence_challenge_question:
|
if claim_evidence_challenge_question:
|
||||||
contracts.append(
|
contracts.append(
|
||||||
{
|
{
|
||||||
|
|
@ -683,6 +697,7 @@ def operational_contracts(
|
||||||
broad_kb_change_question = _has_unnegated_action(query, ("change", "changed", "update", "updated", "landed"))
|
broad_kb_change_question = _has_unnegated_action(query, ("change", "changed", "update", "updated", "landed"))
|
||||||
proposal_state_question = (
|
proposal_state_question = (
|
||||||
kb_scope
|
kb_scope
|
||||||
|
and not conversation_memory_question
|
||||||
and not forecast_resolution_question
|
and not forecast_resolution_question
|
||||||
and not claim_reasoning_question
|
and not claim_reasoning_question
|
||||||
and _has_unnegated_database_state_request(query)
|
and _has_unnegated_database_state_request(query)
|
||||||
|
|
@ -694,7 +709,10 @@ def operational_contracts(
|
||||||
named_packet_question = "helmer" in lowered and any(
|
named_packet_question = "helmer" in lowered and any(
|
||||||
term in lowered for term in ("7 powers", "seven powers", "powers framework", "powers packet")
|
term in lowered for term in ("7 powers", "seven powers", "powers framework", "powers packet")
|
||||||
)
|
)
|
||||||
demo_question = "demo" in lowered and (
|
demo_question = (
|
||||||
|
not conversation_memory_question
|
||||||
|
and "demo" in lowered
|
||||||
|
and (
|
||||||
any(
|
any(
|
||||||
term in lowered
|
term in lowered
|
||||||
for term in (
|
for term in (
|
||||||
|
|
@ -717,6 +735,7 @@ def operational_contracts(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
)
|
||||||
identity_question = (
|
identity_question = (
|
||||||
any(term in lowered for term in ("soul.md", "soul file"))
|
any(term in lowered for term in ("soul.md", "soul file"))
|
||||||
and any(term in lowered for term in ("canonical identity", "identity", "source of truth"))
|
and any(term in lowered for term in ("canonical identity", "identity", "source of truth"))
|
||||||
|
|
|
||||||
|
|
@ -27,6 +27,10 @@ DATABASE_REASONING_PROTOCOL_VERSION = "leo-db-reasoning-v2"
|
||||||
WORD_RE = re.compile(r"\b\w+(?:[-']\w+)*\b")
|
WORD_RE = re.compile(r"\b\w+(?:[-']\w+)*\b")
|
||||||
LIST_PREFIX_RE = re.compile(r"^(\s*(?:[-*]|\d+[.)])\s+)(.*)$")
|
LIST_PREFIX_RE = re.compile(r"^(\s*(?:[-*]|\d+[.)])\s+)(.*)$")
|
||||||
SENTENCE_BREAK_RE = re.compile(r"(?<=[.!?])\s+")
|
SENTENCE_BREAK_RE = re.compile(r"(?<=[.!?])\s+")
|
||||||
|
REQUESTED_SUBJECT_RE = re.compile(
|
||||||
|
r"name the subject exactly once as\s+`(?P<subject>[^`\r\n]{1,120})`\s+in your answer",
|
||||||
|
re.I,
|
||||||
|
)
|
||||||
COMPILED_CONTRACT_IDS = frozenset(
|
COMPILED_CONTRACT_IDS = frozenset(
|
||||||
{
|
{
|
||||||
"mixed_packet_composition",
|
"mixed_packet_composition",
|
||||||
|
|
@ -527,6 +531,18 @@ def _reply_hard_max(contracts: list[dict[str, Any]]) -> int | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_requested_subject(response: str, user_message: str) -> tuple[str, bool]:
|
||||||
|
"""Preserve an explicit short subject label when a compiled fallback is delivered."""
|
||||||
|
|
||||||
|
match = REQUESTED_SUBJECT_RE.search(user_message)
|
||||||
|
if not match:
|
||||||
|
return response, False
|
||||||
|
subject = " ".join(match.group("subject").split())
|
||||||
|
if not subject or re.search(re.escape(subject), response, re.I):
|
||||||
|
return response, False
|
||||||
|
return f"Subject: {subject}\n\n{response}", True
|
||||||
|
|
||||||
|
|
||||||
def _mixed_packet_issues(response: str) -> list[str]:
|
def _mixed_packet_issues(response: str) -> list[str]:
|
||||||
lowered = response.lower()
|
lowered = response.lower()
|
||||||
issues: list[str] = []
|
issues: list[str] = []
|
||||||
|
|
@ -981,6 +997,7 @@ def _post_llm_call(**kwargs: Any) -> dict[str, str] | None:
|
||||||
delivered = str(compiled_response)
|
delivered = str(compiled_response)
|
||||||
else:
|
else:
|
||||||
delivered = response
|
delivered = response
|
||||||
|
delivered, requested_subject_injected = _ensure_requested_subject(delivered, user_message)
|
||||||
hard_max = _reply_hard_max(contracts)
|
hard_max = _reply_hard_max(contracts)
|
||||||
budget_compacted = bool(not fail_closed and hard_max is not None and _word_count(delivered) > hard_max)
|
budget_compacted = bool(not fail_closed and hard_max is not None and _word_count(delivered) > hard_max)
|
||||||
if budget_compacted:
|
if budget_compacted:
|
||||||
|
|
@ -999,6 +1016,7 @@ def _post_llm_call(**kwargs: Any) -> dict[str, str] | None:
|
||||||
"transformed": transformed,
|
"transformed": transformed,
|
||||||
"budget_compacted": budget_compacted,
|
"budget_compacted": budget_compacted,
|
||||||
"compiled_response_enforced": compiled_fallback_required,
|
"compiled_response_enforced": compiled_fallback_required,
|
||||||
|
"requested_subject_injected": requested_subject_injected,
|
||||||
"fail_closed": fail_closed,
|
"fail_closed": fail_closed,
|
||||||
"delivered_response_sha256": hashlib.sha256(delivered.encode("utf-8")).hexdigest(),
|
"delivered_response_sha256": hashlib.sha256(delivered.encode("utf-8")).hexdigest(),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -426,12 +426,13 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||||
re.compile(
|
re.compile(
|
||||||
r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|"
|
r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|"
|
||||||
r"keep the factual claim once|store it once|one public\.claims row|"
|
r"keep the factual claim once|store it once|one public\.claims row|"
|
||||||
r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
|
r"one shared public\.claims row|fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
|
||||||
re.I | re.S,
|
re.I | re.S,
|
||||||
),
|
),
|
||||||
re.compile(r"source|evidence", re.I),
|
re.compile(r"source|evidence", re.I),
|
||||||
re.compile(
|
re.compile(
|
||||||
r"do not duplicate|don'?t duplicate|one shared claim|single shared claim|duplicating (?:the )?claim",
|
r"do not duplicate|don'?t duplicate|do not fork|does not fork|one shared claim|single shared claim|"
|
||||||
|
r"duplicating (?:the )?claim",
|
||||||
re.I,
|
re.I,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
|
@ -440,7 +441,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
||||||
re.compile(
|
re.compile(
|
||||||
r"agent_id|agent attribution|attributed to (?:that|each|an?) agent|"
|
r"agent_id|agent attribution|attributed to (?:that|each|an?) agent|"
|
||||||
r"one (?:public\.beliefs |beliefs )?row per agent(?: position)?|"
|
r"one (?:public\.beliefs |beliefs )?row per agent(?: position)?|"
|
||||||
r"each agent(?:'s)? (?:belief|position|stance)",
|
r"each agent(?:'s)? (?:belief|position|stance)|"
|
||||||
|
r"each agent.{0,60}(?:public\.)?beliefs row",
|
||||||
re.I,
|
re.I,
|
||||||
),
|
),
|
||||||
re.compile(r"belief|position|stance|confidence", re.I),
|
re.compile(r"belief|position|stance|confidence", re.I),
|
||||||
|
|
@ -748,7 +750,12 @@ UNSUPPORTED_COMPOSITION_APPLY_RE = re.compile(
|
||||||
re.I | re.S,
|
re.I | re.S,
|
||||||
)
|
)
|
||||||
APPROVED_APPLY_ACTION_RE = re.compile(
|
APPROVED_APPLY_ACTION_RE = re.compile(
|
||||||
r"approved.{0,240}(?:guarded )?apply|(?:guarded )?apply.{0,240}approved",
|
r"approved.{0,200}(?:authorize|run|execute|perform|proceed).{0,60}(?:guarded )?apply|"
|
||||||
|
r"(?:authorize(?:d)?(?: and)?|run|execute|perform|proceed with).{0,60}(?:guarded )?apply.{0,200}approved",
|
||||||
|
re.I | re.S,
|
||||||
|
)
|
||||||
|
APPLY_ACTION_DISCLAIMER_RE = re.compile(
|
||||||
|
r"proof condition.{0,100}not authorization|not authorization.{0,100}(?:run|execute|perform|apply)",
|
||||||
re.I | re.S,
|
re.I | re.S,
|
||||||
)
|
)
|
||||||
CANONICAL_SOURCE_BEFORE_REVIEW_RE = re.compile(
|
CANONICAL_SOURCE_BEFORE_REVIEW_RE = re.compile(
|
||||||
|
|
@ -934,7 +941,11 @@ def broad_semantic_issues(reply: str) -> list[str]:
|
||||||
def proposal_readiness_issues(prompt_id: str, reply: str) -> list[str]:
|
def proposal_readiness_issues(prompt_id: str, reply: str) -> list[str]:
|
||||||
if prompt_id not in {"OOS-01", "OOS-04", "OOS-07", "OOS-08"}:
|
if prompt_id not in {"OOS-01", "OOS-04", "OOS-07", "OOS-08"}:
|
||||||
return []
|
return []
|
||||||
if APPROVED_APPLY_ACTION_RE.search(reply) and not APPLYABILITY_GAP_RE.search(reply):
|
if (
|
||||||
|
APPROVED_APPLY_ACTION_RE.search(reply)
|
||||||
|
and not APPLYABILITY_GAP_RE.search(reply)
|
||||||
|
and not APPLY_ACTION_DISCLAIMER_RE.search(reply)
|
||||||
|
):
|
||||||
return ["approved_proposal_applyability_overclaim"]
|
return ["approved_proposal_applyability_overclaim"]
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,21 @@ def test_plugin_registers_generation_and_delivery_hooks() -> None:
|
||||||
assert [name for name, _callback in registered] == ["pre_llm_call", "post_llm_call"]
|
assert [name for name, _callback in registered] == ["pre_llm_call", "post_llm_call"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_compiled_fallback_preserves_explicit_subject_label_once() -> None:
|
||||||
|
module = load_plugin()
|
||||||
|
prompt = "Audit the database. Name the subject exactly once as `partner readiness` in your answer."
|
||||||
|
|
||||||
|
injected, changed = module._ensure_requested_subject("No. Approved is not applied.", prompt)
|
||||||
|
preserved, preserved_changed = module._ensure_requested_subject(
|
||||||
|
"Subject: partner readiness\n\nNo. Approved is not applied.", prompt
|
||||||
|
)
|
||||||
|
|
||||||
|
assert injected.count("partner readiness") == 1
|
||||||
|
assert changed is True
|
||||||
|
assert preserved.count("partner readiness") == 1
|
||||||
|
assert preserved_changed is False
|
||||||
|
|
||||||
|
|
||||||
def test_mixed_packet_contract_rejects_framework_mapped_to_claim() -> None:
|
def test_mixed_packet_contract_rejects_framework_mapped_to_claim() -> None:
|
||||||
module = load_plugin()
|
module = load_plugin()
|
||||||
contracts = [{"id": "mixed_packet_composition"}]
|
contracts = [{"id": "mixed_packet_composition"}]
|
||||||
|
|
|
||||||
|
|
@ -933,6 +933,15 @@ def test_vps_bridge_direct_intents_reject_unrelated_and_resolve_overlaps() -> No
|
||||||
}
|
}
|
||||||
assert reasoning_followup_ids & direct_ids == set()
|
assert reasoning_followup_ids & direct_ids == set()
|
||||||
|
|
||||||
|
chat_memory_ids = {
|
||||||
|
item["id"]
|
||||||
|
for item in module.operational_contracts(
|
||||||
|
"Use chat memory only: bind the approved-versus-applied canonical blocker to TEMP until my next "
|
||||||
|
"question. Do not stage or apply anything."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
assert chat_memory_ids == {"reply_budget"}
|
||||||
|
|
||||||
|
|
||||||
def test_vps_bridge_named_document_question_matches_exact_proposal_and_compiles_receipt(monkeypatch) -> None:
|
def test_vps_bridge_named_document_question_matches_exact_proposal_and_compiles_receipt(monkeypatch) -> None:
|
||||||
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
||||||
|
|
|
||||||
|
|
@ -508,6 +508,15 @@ def test_oos_direct_apply_case_rejects_approved_rows_without_readiness_gap() ->
|
||||||
assert benchmark.proposal_readiness_issues(prompt["id"], repaired) == []
|
assert benchmark.proposal_readiness_issues(prompt["id"], repaired) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_oos_closure_receipt_does_not_imply_approved_row_is_directly_applyable() -> None:
|
||||||
|
reply = (
|
||||||
|
"The approved-versus-applied blocker closes only when a future apply run returns applied_at plus exact "
|
||||||
|
"canonical row IDs and hashes. This is a proof condition, not authorization to run apply."
|
||||||
|
)
|
||||||
|
|
||||||
|
assert benchmark.proposal_readiness_issues("OOS-08", reply) == []
|
||||||
|
|
||||||
|
|
||||||
def test_oos_direct_apply_case_accepts_natural_live_readback_wording() -> None:
|
def test_oos_direct_apply_case_accepts_natural_live_readback_wording() -> None:
|
||||||
token = "demo-ledger-deadbeef"
|
token = "demo-ledger-deadbeef"
|
||||||
prompt = benchmark.prompt_catalog(token)[3]
|
prompt = benchmark.prompt_catalog(token)[3]
|
||||||
|
|
@ -822,6 +831,14 @@ def test_oos_agent_positions_accept_attribution_without_literal_column_name() ->
|
||||||
|
|
||||||
assert benchmark.matched_concept(reply, "agent_specific_positions") is True
|
assert benchmark.matched_concept(reply, "agent_specific_positions") is True
|
||||||
|
|
||||||
|
natural = (
|
||||||
|
"The fact and its source evidence live in one shared public.claims row, and both analysts do not fork the "
|
||||||
|
"claim. Each agent files one public.beliefs row for its position. There is no claim-ID foreign key, so that "
|
||||||
|
"link is a schema gap."
|
||||||
|
)
|
||||||
|
assert benchmark.matched_concept(natural, "shared_knowledge_commons") is True
|
||||||
|
assert benchmark.matched_concept(natural, "agent_specific_positions") is True
|
||||||
|
|
||||||
|
|
||||||
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
|
def test_oos_forecast_accepts_natural_missing_resolution_language() -> None:
|
||||||
reply = (
|
reply = (
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue