diff --git a/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py b/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py index 0ae4ca8..262f8be 100644 --- a/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py +++ b/hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py @@ -870,7 +870,8 @@ def _post_llm_call(**kwargs: Any) -> dict[str, str] | None: _trace( base_record | { - "status": "error" if fail_closed or delivered_issues else "ok", + "status": "error" if fail_closed else "ok", + "contract_satisfied": not delivered_issues, "contract_ids": sorted(contract_ids), "issues": issues, "delivered_issues": delivered_issues, diff --git a/scripts/leo_turn_execution_manifest.py b/scripts/leo_turn_execution_manifest.py index 964548c..3c8b911 100644 --- a/scripts/leo_turn_execution_manifest.py +++ b/scripts/leo_turn_execution_manifest.py @@ -293,6 +293,7 @@ def _database_context_binding( "pre_injected": pre.get("injected"), "post_status": post.get("status"), "post_validated": post.get("validated"), + "post_contract_satisfied": post.get("contract_satisfied"), "post_transformed": post.get("transformed"), "raw_response_sha256": raw_response_sha256, "delivered_response_sha256": delivered_response_sha256, diff --git a/scripts/run_leo_direct_claim_handler_suite.py b/scripts/run_leo_direct_claim_handler_suite.py index 9272964..9cd30d7 100755 --- a/scripts/run_leo_direct_claim_handler_suite.py +++ b/scripts/run_leo_direct_claim_handler_suite.py @@ -61,6 +61,41 @@ DB_CONTEXT_PLUGIN = ( KB_TOOL = ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py" BEHAVIOR_MANIFEST = ROOT / "scripts" / "leo_behavior_manifest.py" +RESPONSE_BINDING_HELPER_SOURCE = r''' +def response_bindings_are_hash_bound(results, records): + if not isinstance(results, list) or not isinstance(records, list): + return False + result_rows = [item for item in results if isinstance(item, dict)] + record_rows = [item for item in records if isinstance(item, dict)] + if len(result_rows) != len(results) or len(record_rows) != len(records): + return False + + def text_sha256(value): + return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest() + + def valid_sha256(value): + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + expected = sorted( + (text_sha256(item.get("prompt")), text_sha256(item.get("reply"))) + for item in result_rows + ) + observed = sorted( + (str(item.get("query_sha256") or ""), str(item.get("delivered_response_sha256") or "")) + for item in record_rows + ) + return bool(expected) and len(record_rows) == len(result_rows) and observed == expected and all( + item.get("status") == "ok" + and item.get("validated") is True + and valid_sha256(item.get("response_sha256")) + for item in record_rows + ) +''' + REMOTE_SCRIPT = r''' import asyncio @@ -77,6 +112,8 @@ import types from datetime import datetime, timezone from pathlib import Path +__RESPONSE_BINDING_HELPER_SOURCE__ + LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean") AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent") DEPLOY_ROOT = Path("/opt/teleo-eval/workspaces/deploy-infra") @@ -742,7 +779,7 @@ async def run_suite(): context_injections = [ item for item in database_context_trace if item.get("event", "pre_llm_call") == "pre_llm_call" ] - response_validations = [ + response_bindings = [ item for item in database_context_trace if item.get("event") == "post_llm_call" ] report["database_context_injection_count"] = len(context_injections) @@ -750,13 +787,21 @@ async def run_suite(): item.get("status") == "ok" and item.get("injected") is True for item in context_injections ) - report["database_response_validation_count"] = len(response_validations) - report["database_response_validation_all_ok"] = bool(response_validations) and all( - item.get("status") == "ok" and item.get("validated") is True - for item in response_validations + report["database_response_binding_count"] = len(response_bindings) + report["database_response_binding_all_ok"] = response_bindings_are_hash_bound( + report.get("results") or [], response_bindings + ) + report["database_response_contract_reported_count"] = sum( + isinstance(item.get("contract_satisfied"), bool) for item in response_bindings + ) + report["database_response_contract_satisfied_count"] = sum( + item.get("contract_satisfied") is True for item in response_bindings + ) + report["database_response_contract_all_satisfied"] = bool(response_bindings) and all( + item.get("contract_satisfied") is True for item in response_bindings ) report["database_response_transform_count"] = sum( - item.get("transformed") is True for item in response_validations + item.get("transformed") is True for item in response_bindings ) tool_traces = [ result.get("database_tool_trace") or {} @@ -816,6 +861,7 @@ def build_remote_script( .replace("__SUITE_MODE_JSON__", json.dumps(suite_mode)) .replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix)) .replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note)) + .replace("__RESPONSE_BINDING_HELPER_SOURCE__", RESPONSE_BINDING_HELPER_SOURCE) .replace("__DB_CONTEXT_PLUGIN_SOURCE_JSON__", json.dumps(DB_CONTEXT_PLUGIN.read_text(encoding="utf-8"))) .replace("__KB_TOOL_SOURCE_JSON__", json.dumps(KB_TOOL.read_text(encoding="utf-8"))) .replace("__BEHAVIOR_MANIFEST_SOURCE_JSON__", json.dumps(BEHAVIOR_MANIFEST.read_text(encoding="utf-8"))) diff --git a/scripts/run_leo_m3taversal_oos_handler_suite.py b/scripts/run_leo_m3taversal_oos_handler_suite.py index 46fe739..d401936 100755 --- a/scripts/run_leo_m3taversal_oos_handler_suite.py +++ b/scripts/run_leo_m3taversal_oos_handler_suite.py @@ -34,8 +34,10 @@ def write_score_markdown(path: Path, report: dict[str, Any]) -> None: f"Temporary profile removed: `{report['temp_profile_removed']}`", f"Database context injections: `{report['database_context_injection_count']}`", f"Database context all OK: `{report['database_context_all_ok']}`", - f"Database response validations: `{report['database_response_validation_count']}`", - f"Database response validation all OK: `{report['database_response_validation_all_ok']}`", + f"Database response bindings: `{report['database_response_binding_count']}`", + f"Database response binding all OK: `{report['database_response_binding_all_ok']}`", + f"Database response contracts satisfied: `{report['database_response_contract_satisfied_count']}`", + f"Database response contracts all satisfied: `{report['database_response_contract_all_satisfied']}`", f"Database-composed replacements: `{report['database_response_transform_count']}`", f"Posted to Telegram: `{report['posted_to_telegram']}`", "", @@ -74,8 +76,11 @@ def build_score_report(report: dict[str, Any], *, memory_token: str) -> dict[str "temp_profile_removed": report.get("temp_profile_removed"), "database_context_injection_count": report.get("database_context_injection_count"), "database_context_all_ok": report.get("database_context_all_ok"), - "database_response_validation_count": report.get("database_response_validation_count"), - "database_response_validation_all_ok": report.get("database_response_validation_all_ok"), + "database_response_binding_count": report.get("database_response_binding_count"), + "database_response_binding_all_ok": report.get("database_response_binding_all_ok"), + "database_response_contract_reported_count": report.get("database_response_contract_reported_count"), + "database_response_contract_satisfied_count": report.get("database_response_contract_satisfied_count"), + "database_response_contract_all_satisfied": report.get("database_response_contract_all_satisfied"), "database_response_transform_count": report.get("database_response_transform_count"), "posted_to_telegram": report.get("posted_to_telegram"), "production_db_apply_ran": False, @@ -93,8 +98,10 @@ def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) -> and report.get("temp_profile_removed") is True and report.get("database_context_all_ok") is True and int(report.get("database_context_injection_count") or 0) >= len(report.get("results") or []) - and report.get("database_response_validation_all_ok") is True - and int(report.get("database_response_validation_count") or 0) >= len(report.get("results") or []) + and report.get("database_response_binding_all_ok") is True + and int(report.get("database_response_binding_count") or 0) >= len(report.get("results") or []) + and report.get("database_response_contract_all_satisfied") is True + and int(report.get("database_response_contract_reported_count") or 0) >= len(report.get("results") or []) and report.get("posted_to_telegram") is False and score_report["score"]["pass"] ) diff --git a/scripts/verify_leo_db_first_oos_canary.py b/scripts/verify_leo_db_first_oos_canary.py index 9676fa2..8353904 100644 --- a/scripts/verify_leo_db_first_oos_canary.py +++ b/scripts/verify_leo_db_first_oos_canary.py @@ -125,8 +125,10 @@ def verify_report( ), "delivered_reply_within_budget": ( word_count <= 220 - and report.get("database_response_validation_all_ok") is True + and report.get("database_response_binding_all_ok") is True + and report.get("database_response_contract_all_satisfied") is True and post_record.get("delivered_issues") == [] + and post_record.get("contract_satisfied") is True and post_record.get("validated") is True ), "canonical_counts_unchanged": ( diff --git a/tests/test_hermes_leoclean_db_context_plugin.py b/tests/test_hermes_leoclean_db_context_plugin.py index a7afbb5..9b625fe 100644 --- a/tests/test_hermes_leoclean_db_context_plugin.py +++ b/tests/test_hermes_leoclean_db_context_plugin.py @@ -376,6 +376,8 @@ def test_plugin_preserves_noncontradictory_status_draft_but_replaces_contradicti kb_tool.parent.mkdir(parents=True) kb_tool.write_text("# fixture\n", encoding="utf-8") monkeypatch.setenv("HERMES_HOME", str(home)) + trace = tmp_path / "status-trace.jsonl" + monkeypatch.setenv("LEO_DB_CONTEXT_TRACE_PATH", str(trace)) database_status = { "high_signal_rows": { "claims": 1837, @@ -431,6 +433,15 @@ def test_plugin_preserves_noncontradictory_status_draft_but_replaces_contradicti session_id="session-contradictory", user_message=query, assistant_response=contradictory ) == {"assistant_response": compiled} + post_records = [ + json.loads(line) + for line in trace.read_text(encoding="utf-8").splitlines() + if json.loads(line).get("event") == "post_llm_call" + ] + assert [record["status"] for record in post_records] == ["ok", "ok", "ok"] + assert [record["contract_satisfied"] for record in post_records] == [False, True, True] + assert post_records[0]["delivered_response_sha256"] == module.hashlib.sha256(invalid.encode()).hexdigest() + def test_plugin_requires_named_proposal_receipt_when_live_match_exists() -> None: module = load_plugin() @@ -595,8 +606,11 @@ def test_handler_harness_retains_database_context_proof_fields() -> None: assert '"database_context_trace"' in source assert '"database_context_injection_count"' in source assert '"database_context_all_ok"' in source - assert '"database_response_validation_count"' in source - assert '"database_response_validation_all_ok"' in source + assert '"database_response_binding_count"' in source + assert '"database_response_binding_all_ok"' in source + assert '"database_response_contract_reported_count"' in source + assert '"database_response_contract_satisfied_count"' in source + assert '"database_response_contract_all_satisfied"' in source assert '"database_response_transform_count"' in source assert '"database_tool_trace"' in source assert '"database_tool_call_proven"' in source diff --git a/tests/test_run_leo_direct_claim_handler_suite.py b/tests/test_run_leo_direct_claim_handler_suite.py index 3732ac2..b22eb0c 100644 --- a/tests/test_run_leo_direct_claim_handler_suite.py +++ b/tests/test_run_leo_direct_claim_handler_suite.py @@ -1,11 +1,18 @@ from __future__ import annotations +import hashlib import json import subprocess from scripts import run_leo_direct_claim_handler_suite as suite +def response_binding_helper(): + namespace = {"hashlib": hashlib} + exec(suite.RESPONSE_BINDING_HELPER_SOURCE, namespace) + return namespace["response_bindings_are_hash_bound"] + + def safe_report() -> dict: return { "remote_returncode": 0, @@ -112,3 +119,27 @@ def test_safety_gate_rejects_missing_no_send_declaration() -> None: assert gate["status"] == "fail" assert "no_telegram_post" in gate["failed_checks"] + + +def test_response_binding_helper_requires_exact_prompt_and_delivered_reply_hashes() -> None: + prompt = "Broad ID-free read-only question" + reply = "A bounded grounded answer" + result = {"prompt": prompt, "reply": reply} + record = { + "status": "ok", + "validated": True, + "query_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "response_sha256": "a" * 64, + "delivered_response_sha256": hashlib.sha256(reply.encode()).hexdigest(), + } + + helper = response_binding_helper() + assert helper([result], [record]) is True + + for field in ("query_sha256", "response_sha256", "delivered_response_sha256"): + missing = dict(record) + missing.pop(field) + assert helper([result], [missing]) is False + + mismatched = dict(record, delivered_response_sha256="b" * 64) + assert helper([result], [mismatched]) is False diff --git a/tests/test_verify_leo_db_first_oos_canary.py b/tests/test_verify_leo_db_first_oos_canary.py index 7d89fb6..2a28d12 100644 --- a/tests/test_verify_leo_db_first_oos_canary.py +++ b/tests/test_verify_leo_db_first_oos_canary.py @@ -63,7 +63,8 @@ def passing_report() -> dict: "live_behavior_manifest_unchanged": True, "live_behavior_manifest_before": behavior, "live_behavior_manifest_after": behavior, - "database_response_validation_all_ok": True, + "database_response_binding_all_ok": True, + "database_response_contract_all_satisfied": True, "execution_manifest_summary": {"all_turns_attribution_complete": True}, "safety_gate": {"status": "pass"}, "temp_profile_removed": True, @@ -81,6 +82,7 @@ def passing_report() -> dict: { "event": "post_llm_call", "delivered_issues": [], + "contract_satisfied": True, "validated": True, } ],