teleo/teleo-infrastructure
8
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Bind Leo responses without masking semantic failures

Browse source
This commit is contained in:
twentyOne2x 2026-07-15 05:54:45 +02:00
parent da65e05316
commit 8506dd9359
10 changed files with 143 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py
View file

@ -870,7 +870,8 @@ def _post_llm_call(**kwargs: Any) -> dict[str, str] | None:
_trace(
base_record
| {
"status": "error" if fail_closed or delivered_issues else "ok",
"status": "error" if fail_closed else "ok",
"contract_satisfied": not delivered_issues,
"contract_ids": sorted(contract_ids),
"issues": issues,
"delivered_issues": delivered_issues,

1
scripts/leo_turn_execution_manifest.py
View file

@ -293,6 +293,7 @@ def _database_context_binding(
"pre_injected": pre.get("injected"),
"post_status": post.get("status"),
"post_validated": post.get("validated"),
"post_contract_satisfied": post.get("contract_satisfied"),
"post_transformed": post.get("transformed"),
"raw_response_sha256": raw_response_sha256,
"delivered_response_sha256": delivered_response_sha256,

58
scripts/run_leo_direct_claim_handler_suite.py
View file

@ -61,6 +61,41 @@ DB_CONTEXT_PLUGIN = (
KB_TOOL = ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py"
BEHAVIOR_MANIFEST = ROOT / "scripts" / "leo_behavior_manifest.py"
RESPONSE_BINDING_HELPER_SOURCE = r'''
def response_bindings_are_hash_bound(results, records):
if not isinstance(results, list) or not isinstance(records, list):
return False
result_rows = [item for item in results if isinstance(item, dict)]
record_rows = [item for item in records if isinstance(item, dict)]
if len(result_rows) != len(results) or len(record_rows) != len(records):
return False
def text_sha256(value):
return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest()
def valid_sha256(value):
return (
isinstance(value, str)
and len(value) == 64
and all(character in "0123456789abcdef" for character in value)
)
expected = sorted(
(text_sha256(item.get("prompt")), text_sha256(item.get("reply")))
for item in result_rows
)
observed = sorted(
(str(item.get("query_sha256") or ""), str(item.get("delivered_response_sha256") or ""))
for item in record_rows
)
return bool(expected) and len(record_rows) == len(result_rows) and observed == expected and all(
item.get("status") == "ok"
and item.get("validated") is True
and valid_sha256(item.get("response_sha256"))
for item in record_rows
)
'''
REMOTE_SCRIPT = r'''
import asyncio
@ -77,6 +112,8 @@ import types
from datetime import datetime, timezone
from pathlib import Path
__RESPONSE_BINDING_HELPER_SOURCE__
LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent")
DEPLOY_ROOT = Path("/opt/teleo-eval/workspaces/deploy-infra")
@ -742,7 +779,7 @@ async def run_suite():
context_injections = [
item for item in database_context_trace if item.get("event", "pre_llm_call") == "pre_llm_call"
]
response_validations = [
response_bindings = [
item for item in database_context_trace if item.get("event") == "post_llm_call"
]
report["database_context_injection_count"] = len(context_injections)
@ -750,13 +787,21 @@ async def run_suite():
item.get("status") == "ok" and item.get("injected") is True
for item in context_injections
)
report["database_response_validation_count"] = len(response_validations)
report["database_response_validation_all_ok"] = bool(response_validations) and all(
item.get("status") == "ok" and item.get("validated") is True
for item in response_validations
report["database_response_binding_count"] = len(response_bindings)
report["database_response_binding_all_ok"] = response_bindings_are_hash_bound(
report.get("results") or [], response_bindings
)
report["database_response_contract_reported_count"] = sum(
isinstance(item.get("contract_satisfied"), bool) for item in response_bindings
)
report["database_response_contract_satisfied_count"] = sum(
item.get("contract_satisfied") is True for item in response_bindings
)
report["database_response_contract_all_satisfied"] = bool(response_bindings) and all(
item.get("contract_satisfied") is True for item in response_bindings
)
report["database_response_transform_count"] = sum(
item.get("transformed") is True for item in response_validations
item.get("transformed") is True for item in response_bindings
)
tool_traces = [
result.get("database_tool_trace") or {}
@ -816,6 +861,7 @@ def build_remote_script(
.replace("__SUITE_MODE_JSON__", json.dumps(suite_mode))
.replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix))
.replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note))
.replace("__RESPONSE_BINDING_HELPER_SOURCE__", RESPONSE_BINDING_HELPER_SOURCE)
.replace("__DB_CONTEXT_PLUGIN_SOURCE_JSON__", json.dumps(DB_CONTEXT_PLUGIN.read_text(encoding="utf-8")))
.replace("__KB_TOOL_SOURCE_JSON__", json.dumps(KB_TOOL.read_text(encoding="utf-8")))
.replace("__BEHAVIOR_MANIFEST_SOURCE_JSON__", json.dumps(BEHAVIOR_MANIFEST.read_text(encoding="utf-8")))

19
scripts/run_leo_m3taversal_oos_handler_suite.py
View file

@ -464,8 +464,10 @@ def write_score_markdown(path: Path, report: dict[str, Any]) -> None:
f"Temporary profile removed: `{report['temp_profile_removed']}`",
f"Database context injections: `{report['database_context_injection_count']}`",
f"Database context all OK: `{report['database_context_all_ok']}`",
f"Database response validations: `{report['database_response_validation_count']}`",
f"Database response validation all OK: `{report['database_response_validation_all_ok']}`",
f"Database response bindings: `{report['database_response_binding_count']}`",
f"Database response binding all OK: `{report['database_response_binding_all_ok']}`",
f"Database response contracts satisfied: `{report['database_response_contract_satisfied_count']}`",
f"Database response contracts all satisfied: `{report['database_response_contract_all_satisfied']}`",
f"Database-composed replacements: `{report['database_response_transform_count']}`",
f"Posted to Telegram: `{report['posted_to_telegram']}`",
"",
@ -504,8 +506,11 @@ def build_score_report(report: dict[str, Any], *, memory_token: str) -> dict[str
"temp_profile_removed": report.get("temp_profile_removed"),
"database_context_injection_count": report.get("database_context_injection_count"),
"database_context_all_ok": report.get("database_context_all_ok"),
"database_response_validation_count": report.get("database_response_validation_count"),
"database_response_validation_all_ok": report.get("database_response_validation_all_ok"),
"database_response_binding_count": report.get("database_response_binding_count"),
"database_response_binding_all_ok": report.get("database_response_binding_all_ok"),
"database_response_contract_reported_count": report.get("database_response_contract_reported_count"),
"database_response_contract_satisfied_count": report.get("database_response_contract_satisfied_count"),
"database_response_contract_all_satisfied": report.get("database_response_contract_all_satisfied"),
"database_response_transform_count": report.get("database_response_transform_count"),
"posted_to_telegram": report.get("posted_to_telegram"),
"production_db_apply_ran": False,
@ -523,8 +528,10 @@ def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) ->
and report.get("temp_profile_removed") is True
and report.get("database_context_all_ok") is True
and int(report.get("database_context_injection_count") or 0) >= len(report.get("results") or [])
and report.get("database_response_validation_all_ok") is True
and int(report.get("database_response_validation_count") or 0) >= len(report.get("results") or [])
and report.get("database_response_binding_all_ok") is True
and int(report.get("database_response_binding_count") or 0) >= len(report.get("results") or [])
and report.get("database_response_contract_all_satisfied") is True
and int(report.get("database_response_contract_reported_count") or 0) >= len(report.get("results") or [])
and report.get("posted_to_telegram") is False
and score_report["score"]["pass"]
)

4
scripts/verify_leo_db_first_oos_canary.py
View file

@ -125,8 +125,10 @@ def verify_report(
),
"delivered_reply_within_budget": (
word_count <= 220
and report.get("database_response_validation_all_ok") is True
and report.get("database_response_binding_all_ok") is True
and report.get("database_response_contract_all_satisfied") is True
and post_record.get("delivered_issues") == []
and post_record.get("contract_satisfied") is True
and post_record.get("validated") is True
),
"canonical_counts_unchanged": (

10
scripts/working_leo_m3taversal_oos_protocol.py
View file

@ -939,6 +939,9 @@ def _receipt_score(
for item in traces
if item.get("event") == "post_llm_call" and item.get("status") == "ok" and item.get("validated") is True
]
post_contract_satisfaction_reported = bool(post) and all(
isinstance(item.get("contract_satisfied"), bool) for item in post
)
prompt_sha256 = hashlib.sha256(str(result.get("prompt") or "").encode()).hexdigest()
pre_hashes = {item.get("query_sha256") for item in pre if item.get("query_sha256")}
post_hashes = {item.get("query_sha256") for item in post if item.get("query_sha256")}
@ -1059,7 +1062,11 @@ def _receipt_score(
and len(traces) == len(raw_traces) == 2
and len(pre) == len(post) == 1,
"context_injected": len(pre) == 1,
"response_validated": len(post) == 1,
"response_trace_bound": len(post) == 1,
# Contract satisfaction is semantic evidence, not an execution-binding gate.
# The answer scorer independently decides whether the delivered response is
# correct; a false value must remain visible without invalidating the receipt.
"post_contract_satisfaction_reported": post_contract_satisfaction_reported,
"contract_ids_are_typed_lists": contract_ids_are_lists,
"context_response_query_hash_bound": pre_hashes == post_hashes == {prompt_sha256},
"delivered_response_hash_bound": len(post) == 1
@ -1101,6 +1108,7 @@ def _receipt_score(
"supported_contract_row_ids": sorted(receipt_contract_row_ids),
"supported_identifiers": sorted(supported_identifiers),
"unsupported_identifiers": unsupported_identifiers,
"post_contract_satisfied": post[0].get("contract_satisfied") if len(post) == 1 else None,
"pass": all(checks.values()),
}

18
tests/test_hermes_leoclean_db_context_plugin.py
View file

@ -376,6 +376,8 @@ def test_plugin_preserves_noncontradictory_status_draft_but_replaces_contradicti
kb_tool.parent.mkdir(parents=True)
kb_tool.write_text("# fixture\n", encoding="utf-8")
monkeypatch.setenv("HERMES_HOME", str(home))
trace = tmp_path / "status-trace.jsonl"
monkeypatch.setenv("LEO_DB_CONTEXT_TRACE_PATH", str(trace))
database_status = {
"high_signal_rows": {
"claims": 1837,
@ -431,6 +433,15 @@ def test_plugin_preserves_noncontradictory_status_draft_but_replaces_contradicti
session_id="session-contradictory", user_message=query, assistant_response=contradictory
) == {"assistant_response": compiled}
post_records = [
json.loads(line)
for line in trace.read_text(encoding="utf-8").splitlines()
if json.loads(line).get("event") == "post_llm_call"
]
assert [record["status"] for record in post_records] == ["ok", "ok", "ok"]
assert [record["contract_satisfied"] for record in post_records] == [False, True, True]
assert post_records[0]["delivered_response_sha256"] == module.hashlib.sha256(invalid.encode()).hexdigest()
def test_plugin_requires_named_proposal_receipt_when_live_match_exists() -> None:
module = load_plugin()
@ -595,8 +606,11 @@ def test_handler_harness_retains_database_context_proof_fields() -> None:
assert '"database_context_trace"' in source
assert '"database_context_injection_count"' in source
assert '"database_context_all_ok"' in source
assert '"database_response_validation_count"' in source
assert '"database_response_validation_all_ok"' in source
assert '"database_response_binding_count"' in source
assert '"database_response_binding_all_ok"' in source
assert '"database_response_contract_reported_count"' in source
assert '"database_response_contract_satisfied_count"' in source
assert '"database_response_contract_all_satisfied"' in source
assert '"database_response_transform_count"' in source
assert '"database_tool_trace"' in source
assert '"database_tool_call_proven"' in source

31
tests/test_run_leo_direct_claim_handler_suite.py
View file

@ -1,11 +1,18 @@
from __future__ import annotations
import hashlib
import json
import subprocess
from scripts import run_leo_direct_claim_handler_suite as suite
def response_binding_helper():
namespace = {"hashlib": hashlib}
exec(suite.RESPONSE_BINDING_HELPER_SOURCE, namespace)
return namespace["response_bindings_are_hash_bound"]
def safe_report() -> dict:
return {
"remote_returncode": 0,
@ -112,3 +119,27 @@ def test_safety_gate_rejects_missing_no_send_declaration() -> None:
assert gate["status"] == "fail"
assert "no_telegram_post" in gate["failed_checks"]
def test_response_binding_helper_requires_exact_prompt_and_delivered_reply_hashes() -> None:
prompt = "Broad ID-free read-only question"
reply = "A bounded grounded answer"
result = {"prompt": prompt, "reply": reply}
record = {
"status": "ok",
"validated": True,
"query_sha256": hashlib.sha256(prompt.encode()).hexdigest(),
"response_sha256": "a" * 64,
"delivered_response_sha256": hashlib.sha256(reply.encode()).hexdigest(),
}
helper = response_binding_helper()
assert helper([result], [record]) is True
for field in ("query_sha256", "response_sha256", "delivered_response_sha256"):
missing = dict(record)
missing.pop(field)
assert helper([result], [missing]) is False
mismatched = dict(record, delivered_response_sha256="b" * 64)
assert helper([result], [mismatched]) is False

4
tests/test_verify_leo_db_first_oos_canary.py
View file

@ -63,7 +63,8 @@ def passing_report() -> dict:
"live_behavior_manifest_unchanged": True,
"live_behavior_manifest_before": behavior,
"live_behavior_manifest_after": behavior,
"database_response_validation_all_ok": True,
"database_response_binding_all_ok": True,
"database_response_contract_all_satisfied": True,
"execution_manifest_summary": {"all_turns_attribution_complete": True},
"safety_gate": {"status": "pass"},
"temp_profile_removed": True,
@ -81,6 +82,7 @@ def passing_report() -> dict:
{
"event": "post_llm_call",
"delivered_issues": [],
"contract_satisfied": True,
"validated": True,
}
],

13
tests/test_working_leo_m3taversal_oos_protocol.py
View file

@ -124,6 +124,7 @@ def result_for(prompt: dict, token: str, turn: int, *, grounded: bool) -> dict:
"event": "post_llm_call",
"status": "ok",
"validated": True,
"contract_satisfied": True,
"query_sha256": query_sha256,
"contract_ids": ["reply_budget", "demo_capability_readback"],
"delivered_response_sha256": hashlib.sha256(reply.encode()).hexdigest(),
@ -895,6 +896,18 @@ def test_autonomous_retrieval_requires_nonempty_rows_and_receipted_reply_ids() -
assert valid["pass"] is True
assert valid["supported_claim_ids"] == [CLAIM_UUID]
assert valid["supported_source_ids"] == [SOURCE_UUID]
assert valid["post_contract_satisfied"] is True
semantically_incomplete = copy.deepcopy(result)
semantically_incomplete["database_context_trace"][-1]["contract_satisfied"] = False
incomplete_score = protocol_lib._receipt_score(
semantically_incomplete,
require_database_contract=False,
require_database_receipt=True,
require_grounded_rows=True,
)
assert incomplete_score["pass"] is True
assert incomplete_score["post_contract_satisfied"] is False
paraphrased = copy.deepcopy(result)
paraphrased["reply"] = (