feat: bind tested Leo turns to reproducible inputs

This commit is contained in:
twentyOne2x 2026-07-14 23:45:22 +02:00
parent 794635d17f
commit cc46d66713
9 changed files with 1042 additions and 24 deletions

View file

@ -103,7 +103,7 @@ fi
# Syntax check all Python files before copying # Syntax check all Python files before copying
ERRORS=0 ERRORS=0
for f in lib/*.py *.py diagnostics/*.py telegram/*.py tests/*.py hermes-agent/leoclean-bin/*.py hermes-agent/leoclean-plugins/vps/*/*.py hermes-agent/patches/*.py scripts/compile_kb_source_packet.py scripts/kb_proposal_normalize.py scripts/prepare_kb_source_manifest.py scripts/leo_behavior_manifest.py; do for f in lib/*.py *.py diagnostics/*.py telegram/*.py tests/*.py hermes-agent/leoclean-bin/*.py hermes-agent/leoclean-plugins/vps/*/*.py hermes-agent/patches/*.py scripts/compile_kb_source_packet.py scripts/kb_proposal_normalize.py scripts/prepare_kb_source_manifest.py scripts/leo_behavior_manifest.py scripts/leo_turn_execution_manifest.py; do
[ -f "$f" ] || continue [ -f "$f" ] || continue
if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>&1; then if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>&1; then
log "SYNTAX ERROR: $f" log "SYNTAX ERROR: $f"

View file

@ -48,7 +48,7 @@ echo ""
# Syntax check all Python files before deploying # Syntax check all Python files before deploying
echo "=== Pre-deploy syntax check ===" echo "=== Pre-deploy syntax check ==="
ERRORS=0 ERRORS=0
for f in "$REPO_ROOT/lib/"*.py "$REPO_ROOT/"*.py "$REPO_ROOT/diagnostics/"*.py "$REPO_ROOT/telegram/"*.py "$REPO_ROOT/hermes-agent/leoclean-bin/"*.py "$REPO_ROOT/hermes-agent/leoclean-plugins/vps/"*/*.py "$REPO_ROOT/hermes-agent/patches/"*.py "$REPO_ROOT/scripts/compile_kb_source_packet.py" "$REPO_ROOT/scripts/kb_proposal_normalize.py" "$REPO_ROOT/scripts/prepare_kb_source_manifest.py" "$REPO_ROOT/scripts/leo_behavior_manifest.py"; do for f in "$REPO_ROOT/lib/"*.py "$REPO_ROOT/"*.py "$REPO_ROOT/diagnostics/"*.py "$REPO_ROOT/telegram/"*.py "$REPO_ROOT/hermes-agent/leoclean-bin/"*.py "$REPO_ROOT/hermes-agent/leoclean-plugins/vps/"*/*.py "$REPO_ROOT/hermes-agent/patches/"*.py "$REPO_ROOT/scripts/compile_kb_source_packet.py" "$REPO_ROOT/scripts/kb_proposal_normalize.py" "$REPO_ROOT/scripts/prepare_kb_source_manifest.py" "$REPO_ROOT/scripts/leo_behavior_manifest.py" "$REPO_ROOT/scripts/leo_turn_execution_manifest.py"; do
[ -f "$f" ] || continue [ -f "$f" ] || continue
if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>/dev/null; then if ! python3 -c "import ast, sys; ast.parse(open(sys.argv[1]).read())" "$f" 2>/dev/null; then
echo "SYNTAX ERROR: $f" echo "SYNTAX ERROR: $f"

View file

@ -71,6 +71,41 @@ def _trace(record: dict[str, Any]) -> None:
pass pass
def _retrieval_receipt_trace(value: Any) -> dict[str, Any] | None:
"""Retain the exact read receipt without retaining claim or source bodies."""
if not isinstance(value, dict) or value.get("schema") != "livingip.teleoKbRetrievalReceipt.v1":
return None
consistency = value.get("read_consistency") if isinstance(value.get("read_consistency"), dict) else {}
counts = value.get("counts") if isinstance(value.get("counts"), dict) else {}
safe = {
"schema": value.get("schema"),
"query_sha256": value.get("query_sha256"),
"semantic_context_sha256": value.get("semantic_context_sha256"),
"artifact_state_sha256": value.get("artifact_state_sha256"),
"claim_ids": sorted(str(item) for item in value.get("claim_ids") or []),
"source_ids": sorted(str(item) for item in value.get("source_ids") or []),
"counts": {
key: item
for key, item in sorted(counts.items())
if isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool)
},
"read_consistency": {
"status": consistency.get("status"),
"attempts": consistency.get("attempts"),
"database": consistency.get("database"),
"database_user": consistency.get("database_user"),
"system_identifier": consistency.get("system_identifier"),
"wal_lsn_before": consistency.get("wal_lsn_before"),
"wal_lsn_after": consistency.get("wal_lsn_after"),
},
}
safe["receipt_sha256"] = hashlib.sha256(
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
return safe
def _failure_context(reason: str) -> str: def _failure_context(reason: str) -> str:
return ( return (
'<leo_current_runtime_contracts source="teleo-kb" status="unavailable">\n' '<leo_current_runtime_contracts source="teleo-kb" status="unavailable">\n'
@ -246,6 +281,7 @@ def _load_database_snapshot(
compiled_response = payload.get("compiled_response") compiled_response = payload.get("compiled_response")
if compiled_response is not None and not isinstance(compiled_response, str): if compiled_response is not None and not isinstance(compiled_response, str):
raise ValueError("compiled_response_invalid") raise ValueError("compiled_response_invalid")
retrieval_receipt = _retrieval_receipt_trace(payload.get("retrieval_receipt"))
except (json.JSONDecodeError, TypeError, ValueError) as exc: except (json.JSONDecodeError, TypeError, ValueError) as exc:
record = base_record | {"status": "error", "error": str(exc), "injected": True} record = base_record | {"status": "error", "error": str(exc), "injected": True}
_trace(record) _trace(record)
@ -259,6 +295,8 @@ def _load_database_snapshot(
"contract_sha256": hashlib.sha256(contract_json.encode("utf-8")).hexdigest(), "contract_sha256": hashlib.sha256(contract_json.encode("utf-8")).hexdigest(),
"compiled_response_available": bool(compiled_response), "compiled_response_available": bool(compiled_response),
} }
if retrieval_receipt is not None:
record["retrieval_receipt"] = retrieval_receipt
_trace(record) _trace(record)
return { return {
"status": "ok", "status": "ok",

View file

@ -23,6 +23,7 @@ import yaml
SCHEMA = "livingip.leoBehaviorManifest.v1" SCHEMA = "livingip.leoBehaviorManifest.v1"
DEFAULT_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean") DEFAULT_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean")
DEFAULT_HERMES_ROOT = Path("/home/teleo/.hermes/hermes-agent") DEFAULT_HERMES_ROOT = Path("/home/teleo/.hermes/hermes-agent")
DEFAULT_DEPLOYMENT_ROOT = Path("/opt/teleo-eval/workspaces/deploy-infra")
def sha256_bytes(value: bytes) -> str: def sha256_bytes(value: bytes) -> str:
@ -190,9 +191,14 @@ def component(
} }
def build_manifest(profile: Path = DEFAULT_PROFILE, hermes_root: Path = DEFAULT_HERMES_ROOT) -> dict[str, Any]: def build_manifest(
profile: Path = DEFAULT_PROFILE,
hermes_root: Path = DEFAULT_HERMES_ROOT,
deployment_root: Path = DEFAULT_DEPLOYMENT_ROOT,
) -> dict[str, Any]:
profile = profile.resolve() profile = profile.resolve()
hermes_root = hermes_root.resolve() hermes_root = hermes_root.resolve()
deployment_root = deployment_root.resolve()
config = safe_model_config(profile / "config.yaml") config = safe_model_config(profile / "config.yaml")
components = { components = {
"profile_config": component( "profile_config": component(
@ -271,6 +277,16 @@ def build_manifest(profile: Path = DEFAULT_PROFILE, hermes_root: Path = DEFAULT_
profile, (hermes_root / "run_agent.py", hermes_root / "agent" / "prompt_builder.py") profile, (hermes_root / "run_agent.py", hermes_root / "agent" / "prompt_builder.py")
), ),
}, },
"teleo_infrastructure_runtime": {
"git_head": git_head(deployment_root),
"source_tree": path_manifest(
profile,
(
deployment_root / "scripts" / "leo_behavior_manifest.py",
deployment_root / "scripts" / "leo_tool_trace.py",
),
),
},
"components": components, "components": components,
"canonical_database": { "canonical_database": {
"included": False, "included": False,
@ -282,6 +298,7 @@ def build_manifest(profile: Path = DEFAULT_PROFILE, hermes_root: Path = DEFAULT_
"generated_at_utc": datetime.now(timezone.utc).isoformat(), "generated_at_utc": datetime.now(timezone.utc).isoformat(),
"profile_root": str(profile), "profile_root": str(profile),
"hermes_root": str(hermes_root), "hermes_root": str(hermes_root),
"deployment_root": str(deployment_root),
"behavior_sha256": canonical_sha256(stable), "behavior_sha256": canonical_sha256(stable),
} }
@ -290,9 +307,10 @@ def main() -> int:
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--profile", type=Path, default=DEFAULT_PROFILE) parser.add_argument("--profile", type=Path, default=DEFAULT_PROFILE)
parser.add_argument("--hermes-root", type=Path, default=DEFAULT_HERMES_ROOT) parser.add_argument("--hermes-root", type=Path, default=DEFAULT_HERMES_ROOT)
parser.add_argument("--deployment-root", type=Path, default=DEFAULT_DEPLOYMENT_ROOT)
parser.add_argument("--output", type=Path) parser.add_argument("--output", type=Path)
args = parser.parse_args() args = parser.parse_args()
manifest = build_manifest(args.profile, args.hermes_root) manifest = build_manifest(args.profile, args.hermes_root, args.deployment_root)
encoded = json.dumps(manifest, indent=2, sort_keys=True) + "\n" encoded = json.dumps(manifest, indent=2, sort_keys=True) + "\n"
if args.output: if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True) args.output.parent.mkdir(parents=True, exist_ok=True)

View file

@ -0,0 +1,476 @@
#!/usr/bin/env python3
"""Bind one tested Leo turn to its runtime, model call, session, and KB read.
The manifest is deliberately content-minimizing: prompts, replies, raw database
rows, tool arguments, provider URLs, and session identifiers are represented by
hashes. It proves attribution of a tested turn, not deterministic regeneration
of externally hosted model weights.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
SCHEMA = "livingip.leoTurnExecutionManifest.v1"
RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1"
def canonical_sha256(value: Any) -> str:
encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def text_sha256(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def git_state(repo: Path) -> dict[str, Any]:
try:
head = subprocess.run(
["git", "-C", str(repo), "rev-parse", "HEAD"],
check=False,
capture_output=True,
text=True,
timeout=10,
)
status = subprocess.run(
["git", "-C", str(repo), "status", "--porcelain", "--untracked-files=no"],
check=False,
capture_output=True,
text=True,
timeout=10,
)
except (OSError, subprocess.TimeoutExpired):
return {"git_head": None, "tracked_tree_clean": None}
git_head = head.stdout.strip() if head.returncode == 0 and len(head.stdout.strip()) == 40 else None
return {
"git_head": git_head,
"tracked_tree_clean": status.returncode == 0 and not status.stdout.strip(),
}
def _safe_usage(value: Any) -> dict[str, int | float | None]:
if not isinstance(value, dict):
return {}
safe: dict[str, int | float | None] = {}
for key in (
"input_tokens",
"output_tokens",
"prompt_tokens",
"completion_tokens",
"total_tokens",
"cache_read_tokens",
"cache_write_tokens",
"reasoning_tokens",
):
item = value.get(key)
if isinstance(item, (int, float)) and not isinstance(item, bool):
safe[key] = item
return safe
def _model_calls(result: dict[str, Any]) -> list[dict[str, Any]]:
calls: list[dict[str, Any]] = []
for raw in result.get("model_call_trace") or []:
if not isinstance(raw, dict) or raw.get("event") != "post_api_request":
continue
calls.append(
{
"api_call_count": raw.get("api_call_count"),
"api_mode": raw.get("api_mode"),
"model": raw.get("model"),
"provider": raw.get("provider"),
"response_model": raw.get("response_model"),
"finish_reason": raw.get("finish_reason"),
"message_count": raw.get("message_count"),
"assistant_content_chars": raw.get("assistant_content_chars"),
"assistant_tool_call_count": raw.get("assistant_tool_call_count"),
"task_id_sha256": raw.get("task_id_sha256"),
"session_id_sha256": raw.get("session_id_sha256"),
"base_url_sha256": raw.get("base_url_sha256"),
"usage": _safe_usage(raw.get("usage")),
"provider_response_id": raw.get("provider_response_id"),
"provider_response_id_status": raw.get("provider_response_id_status"),
}
)
return calls
def _safe_retrieval_receipt(value: Any) -> dict[str, Any] | None:
if not isinstance(value, dict) or value.get("schema") != RETRIEVAL_RECEIPT_SCHEMA:
return None
consistency = value.get("read_consistency") if isinstance(value.get("read_consistency"), dict) else {}
counts = value.get("counts") if isinstance(value.get("counts"), dict) else {}
return {
"schema": value.get("schema"),
"query_sha256": value.get("query_sha256"),
"semantic_context_sha256": value.get("semantic_context_sha256"),
"artifact_state_sha256": value.get("artifact_state_sha256"),
"claim_ids_sha256": canonical_sha256(sorted(str(item) for item in value.get("claim_ids") or [])),
"source_ids_sha256": canonical_sha256(sorted(str(item) for item in value.get("source_ids") or [])),
"counts": {
key: item
for key, item in sorted(counts.items())
if isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool)
},
"read_consistency": {
"status": consistency.get("status"),
"attempts": consistency.get("attempts"),
"database": consistency.get("database"),
"database_user": consistency.get("database_user"),
"system_identifier": consistency.get("system_identifier"),
"wal_lsn_before": consistency.get("wal_lsn_before"),
"wal_lsn_after": consistency.get("wal_lsn_after"),
},
"receipt_sha256": canonical_sha256(value),
}
def _context_receipts(result: dict[str, Any]) -> list[dict[str, Any]]:
receipts = []
for record in result.get("database_context_trace") or []:
if not isinstance(record, dict) or record.get("event") != "pre_llm_call":
continue
receipt = _safe_retrieval_receipt(record.get("retrieval_receipt"))
if receipt is not None:
receipts.append(receipt)
return receipts
def _tool_receipts(result: dict[str, Any]) -> list[dict[str, Any]]:
receipts = []
trace = result.get("database_tool_trace") or {}
for call in trace.get("calls") or []:
if not isinstance(call, dict):
continue
raw = ((call.get("result") or {}).get("retrieval_receipt") or {})
if not isinstance(raw, dict) or not raw.get("semantic_context_sha256"):
continue
receipts.append(
{
"schema": raw.get("schema"),
"semantic_context_sha256": raw.get("semantic_context_sha256"),
"artifact_state_sha256": raw.get("artifact_state_sha256"),
"read_consistency": {"status": raw.get("read_consistency_status")},
"receipt_sha256": canonical_sha256(raw),
}
)
return receipts
def _database_required(result: dict[str, Any]) -> bool:
for record in result.get("database_context_trace") or []:
if not isinstance(record, dict) or record.get("event") != "pre_llm_call":
continue
contract_ids = {str(item) for item in record.get("contract_ids") or []}
if contract_ids - {"reply_budget"}:
return True
trace = result.get("database_tool_trace") or {}
return bool(trace.get("database_tool_call_count"))
def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]:
trace = result.get("database_tool_trace") or {}
calls = []
for call in trace.get("calls") or []:
if not isinstance(call, dict):
continue
call_result = call.get("result") if isinstance(call.get("result"), dict) else {}
calls.append(
{
"tool_call_id_sha256": call.get("tool_call_id_sha256"),
"arguments_sha256": call.get("arguments_sha256"),
"database_invocations": call.get("database_invocations") or [],
"result_content_sha256": call_result.get("content_sha256"),
"result_content_bytes": call_result.get("content_bytes"),
"result_error_detected": call_result.get("error_detected"),
"result_row_ids_sha256": canonical_sha256(sorted(call_result.get("row_ids") or [])),
"result_sha256_values_sha256": canonical_sha256(sorted(call_result.get("sha256_values") or [])),
}
)
all_results_bound = bool(calls) and all(
call.get("result_content_sha256") and call.get("result_error_detected") is False for call in calls
)
return {
"schema": trace.get("schema"),
"tool_call_count": trace.get("database_tool_call_count", 0),
"completed_count": trace.get("database_tool_completed_count", 0),
"all_calls_read_only": trace.get("database_tool_calls_read_only"),
"all_results_content_bound": all_results_bound,
"calls": calls,
"trace_sha256": canonical_sha256(trace),
}
def _database_fingerprint_complete(value: dict[str, Any]) -> bool:
consistency = value.get("read_consistency") if isinstance(value.get("read_consistency"), dict) else {}
return bool(
value.get("status") == "ok"
and value.get("fingerprint_sha256")
and value.get("table_count")
and value.get("total_rows")
and consistency.get("status") == "stable_wal_marker"
)
def _required_missing(
*,
behavior: dict[str, Any],
calls: list[dict[str, Any]],
receipts: list[dict[str, Any]],
database_required: bool,
session_key: str,
prompt: str,
reply: str,
harness_source: dict[str, Any],
database_tool_binding: dict[str, Any],
db_fingerprint_before: dict[str, Any],
db_fingerprint_after: dict[str, Any],
) -> list[str]:
missing = []
checks = {
"prompt": bool(prompt),
"reply": bool(reply),
"session_boundary": bool(session_key),
"behavior_sha256": bool(behavior.get("behavior_sha256")),
"hermes_git_head": bool((behavior.get("hermes_runtime") or {}).get("git_head")),
"teleo_infrastructure_git_head": bool(
(behavior.get("teleo_infrastructure_runtime") or {}).get("git_head")
),
"harness_git_head": bool(harness_source.get("git_head")),
"actual_model_call": bool(
calls
and all(
call.get("model") and call.get("provider") and call.get("response_model")
for call in calls
)
),
"database_retrieval_receipt": not database_required or bool(receipts),
"database_tool_results": not database_tool_binding.get("tool_call_count")
or bool(database_tool_binding.get("all_results_content_bound")),
"database_fingerprint_before": not database_required
or _database_fingerprint_complete(db_fingerprint_before),
"database_fingerprint_after": not database_required
or _database_fingerprint_complete(db_fingerprint_after),
"database_fingerprint_unchanged": not database_required
or db_fingerprint_before.get("fingerprint_sha256") == db_fingerprint_after.get("fingerprint_sha256"),
}
for label, present in checks.items():
if not present:
missing.append(label)
return missing
def build_turn_manifest(
*,
result: dict[str, Any],
behavior_manifest: dict[str, Any],
session_key: str,
source: dict[str, Any],
suite_mode: str,
remote_run_id: str | None,
db_counts_before: dict[str, Any] | None,
db_counts_after: dict[str, Any] | None,
db_counts_changed: bool | None,
db_fingerprint_before: dict[str, Any] | None,
db_fingerprint_after: dict[str, Any] | None,
posted_to_telegram: bool,
mutates_kb_by_harness: bool,
harness_source: dict[str, Any],
) -> dict[str, Any]:
prompt = str(result.get("prompt") or "")
reply = str(result.get("reply") or "")
calls = _model_calls(result)
context_receipts = _context_receipts(result)
tool_receipts = _tool_receipts(result)
receipts = context_receipts + tool_receipts
database_required = _database_required(result)
database_tool_binding = _database_tool_binding(result)
fingerprint_before = db_fingerprint_before or {}
fingerprint_after = db_fingerprint_after or {}
missing = _required_missing(
behavior=behavior_manifest,
calls=calls,
receipts=receipts,
database_required=database_required,
session_key=session_key,
prompt=prompt,
reply=reply,
harness_source=harness_source,
database_tool_binding=database_tool_binding,
db_fingerprint_before=fingerprint_before,
db_fingerprint_after=fingerprint_after,
)
stable = {
"schema": SCHEMA,
"turn": {
"prompt_id": result.get("prompt_id"),
"turn_index": result.get("turn"),
"remote_run_id": remote_run_id,
"started_at_utc": result.get("started_at_utc"),
"ended_at_utc": result.get("ended_at_utc"),
"prompt_sha256": text_sha256(prompt),
"reply_sha256": text_sha256(reply),
"prompt_bytes": len(prompt.encode("utf-8")),
"reply_bytes": len(reply.encode("utf-8")),
},
"session_boundary": {
"session_key_sha256": text_sha256(session_key) if session_key else None,
"source_platform": source.get("platform"),
"chat_type": source.get("chat_type"),
"fresh_temp_profile": True,
"prior_dynamic_state_excluded": True,
},
"runtime": {
"behavior_sha256": behavior_manifest.get("behavior_sha256"),
"hermes_runtime": behavior_manifest.get("hermes_runtime"),
"teleo_infrastructure_runtime": behavior_manifest.get("teleo_infrastructure_runtime"),
"profile_component_hashes": {
name: (component.get("content") or {}).get("sha256")
for name, component in sorted((behavior_manifest.get("components") or {}).items())
if isinstance(component, dict)
},
"harness_source": harness_source,
},
"model_execution": {
"call_count": len(calls),
"calls": calls,
"externally_hosted_weights_hashable": False,
"provider_response_id_required_for_attribution": False,
},
"canonical_database": {
"required_for_turn": database_required,
"binding_status": "retrieval_receipt_bound"
if receipts
else "not_required"
if not database_required
else "missing",
"context_retrieval_receipts": context_receipts,
"explicit_tool_retrieval_receipts": tool_receipts,
"database_tool_binding": database_tool_binding,
"fingerprint_before": fingerprint_before,
"fingerprint_after": fingerprint_after,
"fingerprint_unchanged": fingerprint_before.get("fingerprint_sha256")
== fingerprint_after.get("fingerprint_sha256"),
"suite_counts_before_sha256": canonical_sha256(db_counts_before or {}),
"suite_counts_after_sha256": canonical_sha256(db_counts_after or {}),
"suite_counts_changed": db_counts_changed,
"full_database_content_fingerprint_included": bool(
fingerprint_before.get("fingerprint_sha256") and fingerprint_after.get("fingerprint_sha256")
),
"full_database_snapshot_artifact_included": False,
},
"delivery_and_safety": {
"suite_mode": suite_mode,
"posted_to_telegram": posted_to_telegram,
"kb_mutation_by_harness": mutates_kb_by_harness,
"raw_prompt_retained_in_manifest": False,
"raw_reply_retained_in_manifest": False,
"raw_database_rows_retained_in_manifest": False,
"raw_session_identifier_retained_in_manifest": False,
},
"attribution": {
"status": "complete" if not missing else "incomplete",
"missing_required_bindings": missing,
},
"replay_claim": {
"status": "bounded_recorded_inputs_pinned" if not missing else "incomplete_inputs",
"local_runtime_inputs_reconstructible": not missing,
"database_state_identified_by_content_fingerprint": bool(
not missing and fingerprint_before.get("fingerprint_sha256")
),
"database_snapshot_reconstructible_from_this_manifest_alone": False,
"external_model_state_reconstructible": False,
"bitwise_identical_model_output_guaranteed": False,
"reason": (
"The tested turn is attributable to exact local/runtime inputs and DB reads, but hosted model "
"weights and sampling are not locally hashable or deterministic."
),
},
}
return {
**stable,
"generated_at_utc": datetime.now(UTC).isoformat(),
"execution_sha256": canonical_sha256(stable),
}
def validate_turn_manifest(manifest: dict[str, Any]) -> list[str]:
problems = []
if manifest.get("schema") != SCHEMA:
problems.append("schema_mismatch")
stable = {
key: value
for key, value in manifest.items()
if key not in {"generated_at_utc", "execution_sha256"}
}
if manifest.get("execution_sha256") != canonical_sha256(stable):
problems.append("execution_sha256_mismatch")
missing = (manifest.get("attribution") or {}).get("missing_required_bindings")
expected_status = "complete" if not missing else "incomplete"
if (manifest.get("attribution") or {}).get("status") != expected_status:
problems.append("attribution_status_mismatch")
return problems
def attach_execution_manifests(report: dict[str, Any], *, repo_root: Path) -> dict[str, Any]:
behavior = report.get("executed_behavior_manifest") or report.get("live_behavior_manifest_before") or {}
handler = report.get("handler") or {}
harness_source = git_state(repo_root)
results = report.get("results") or []
complete = 0
for result in results:
if not isinstance(result, dict):
continue
manifest = build_turn_manifest(
result=result,
behavior_manifest=behavior,
session_key=str(handler.get("session_key") or ""),
source=report.get("source") or {},
suite_mode=str(report.get("mode") or ""),
remote_run_id=report.get("remote_run_id"),
db_counts_before=report.get("db_counts_before"),
db_counts_after=report.get("db_counts_after"),
db_counts_changed=report.get("db_counts_changed"),
db_fingerprint_before=report.get("db_fingerprint_before"),
db_fingerprint_after=report.get("db_fingerprint_after"),
posted_to_telegram=bool(report.get("posted_to_telegram")),
mutates_kb_by_harness=bool(report.get("mutates_kb_by_harness")),
harness_source=harness_source,
)
result["execution_manifest"] = manifest
if manifest["attribution"]["status"] == "complete" and not validate_turn_manifest(manifest):
complete += 1
report["execution_manifest_summary"] = {
"schema": SCHEMA,
"turn_count": len(results),
"attribution_complete_count": complete,
"all_turns_attribution_complete": bool(results) and complete == len(results),
"harness_source": harness_source,
}
return report
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--suite-report", required=True, type=Path)
parser.add_argument("--repo-root", default=Path(__file__).resolve().parents[1], type=Path)
parser.add_argument("--output", required=True, type=Path)
args = parser.parse_args()
report = json.loads(args.suite_report.read_text(encoding="utf-8"))
attach_execution_manifests(report, repo_root=args.repo_root)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
summary = report["execution_manifest_summary"]
print(json.dumps(summary, indent=2, sort_keys=True))
return 0 if summary["all_turns_attribution_complete"] else 1
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -20,6 +20,11 @@ from typing import Any
import working_leo_open_ended_benchmark as benchmark import working_leo_open_ended_benchmark as benchmark
try:
from leo_turn_execution_manifest import attach_execution_manifests
except ImportError: # pragma: no cover - imported as scripts.* in tests
from scripts.leo_turn_execution_manifest import attach_execution_manifests
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709" REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
OUTPUT_JSON = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json" OUTPUT_JSON = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json"
@ -40,9 +45,14 @@ SECRET_PATTERNS = [
re.compile(r"((?:api[_-]?key|token|secret|password)[=:]\s*)([A-Za-z0-9._-]+)", re.IGNORECASE), re.compile(r"((?:api[_-]?key|token|secret|password)[=:]\s*)([A-Za-z0-9._-]+)", re.IGNORECASE),
] ]
DB_CONTEXT_PLUGIN = (
ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py"
)
REMOTE_SCRIPT = r''' REMOTE_SCRIPT = r'''
import asyncio import asyncio
import hashlib
import json import json
import os import os
import shutil import shutil
@ -63,6 +73,83 @@ RUN_ID = "__RUN_ID__"
PROMPTS = __PROMPTS_JSON__ PROMPTS = __PROMPTS_JSON__
REPORT_PREFIX = __REPORT_PREFIX_JSON__ REPORT_PREFIX = __REPORT_PREFIX_JSON__
REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json") REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json")
DB_CONTEXT_PLUGIN_SOURCE = __DB_CONTEXT_PLUGIN_SOURCE_JSON__
TURN_TRACE_PLUGIN_SOURCE = """\
import hashlib
import json
import os
from datetime import datetime, timezone
from pathlib import Path
def _safe_usage(value):
if not isinstance(value, dict):
return {}
safe = {}
for key in (
"input_tokens",
"output_tokens",
"prompt_tokens",
"completion_tokens",
"total_tokens",
"cache_read_tokens",
"cache_write_tokens",
"reasoning_tokens",
):
item = value.get(key)
if isinstance(item, (int, float)) and not isinstance(item, bool):
safe[key] = item
return safe
def _sha(value):
return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest()
def _post_api_request(**kwargs):
raw_path = os.getenv("LEO_TURN_API_TRACE_PATH", "").strip()
if not raw_path:
return None
path = Path(raw_path)
path.parent.mkdir(parents=True, exist_ok=True)
record = {
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
"event": "post_api_request",
"task_id_sha256": _sha(kwargs.get("task_id")),
"session_id_sha256": _sha(kwargs.get("session_id")),
"model": kwargs.get("model"),
"provider": kwargs.get("provider"),
"base_url_sha256": _sha(kwargs.get("base_url")),
"api_mode": kwargs.get("api_mode"),
"api_call_count": kwargs.get("api_call_count"),
"finish_reason": kwargs.get("finish_reason"),
"message_count": kwargs.get("message_count"),
"response_model": kwargs.get("response_model"),
"usage": _safe_usage(kwargs.get("usage")),
"assistant_content_chars": kwargs.get("assistant_content_chars"),
"assistant_tool_call_count": kwargs.get("assistant_tool_call_count"),
"provider_response_id": None,
"provider_response_id_status": "not_exposed_by_hermes_post_api_request_hook",
}
with path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(record, sort_keys=True) + "\\n")
try:
path.chmod(0o600)
except OSError:
pass
return None
def register(ctx):
ctx.register_hook("post_api_request", _post_api_request)
"""
TURN_TRACE_PLUGIN_MANIFEST = """\
name: leo-turn-execution-trace
version: "1.0.0"
description: Secret-safe model-call trace for no-send Leo harnesses.
provides_hooks:
- post_api_request
"""
def service_state(): def service_state():
@ -114,6 +201,127 @@ select jsonb_build_object(
return {"error": str(exc)} return {"error": str(exc)}
def db_read_marker():
sql = """
select jsonb_build_object(
'database', current_database(),
'database_user', current_user,
'system_identifier', (select system_identifier::text from pg_control_system()),
'wal_lsn', pg_current_wal_lsn()::text
)::text;
"""
raw = subprocess.check_output(
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
input=sql,
text=True,
stderr=subprocess.STDOUT,
)
return json.loads(raw.strip())
def db_fingerprint():
sql_path = DEPLOY_ROOT / "ops" / "postgres_parity_manifest.sql"
before = db_read_marker()
raw = subprocess.check_output(
["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"],
input=sql_path.read_text(encoding="utf-8"),
text=True,
stderr=subprocess.STDOUT,
timeout=180,
)
after = db_read_marker()
rows = []
for line in raw.splitlines():
candidate = line.strip()
if not candidate or candidate in {"BEGIN", "SET", "ROLLBACK"}:
continue
value = json.loads(candidate)
if isinstance(value, dict):
rows.append(value)
tables = {
f"{row['schema']}.{row['table']}": {
"row_count": int(row["row_count"]),
"rowset_md5": row["rowset_md5"],
}
for row in rows
if row.get("kind") == "table"
}
singleton = {
str(row["kind"]): row.get("items", [])
for row in rows
if row.get("kind")
in {
"schemas",
"extensions",
"application_roles",
"columns",
"constraints",
"indexes",
"sequences",
"views",
"functions",
"triggers",
"types",
"policies",
}
}
identity_rows = [row for row in rows if row.get("kind") == "identity"]
if len(identity_rows) != 1 or not tables:
raise RuntimeError("canonical database fingerprint manifest is incomplete")
identity = identity_rows[0]
stable = {
"identity": {
key: identity.get(key)
for key in (
"database",
"server_version_num",
"server_encoding",
"database_collation",
"database_ctype",
"transaction_read_only",
)
},
"singleton_sha256": {
key: hashlib.sha256(
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
for key, value in sorted(singleton.items())
},
"tables": tables,
}
return {
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
"status": "ok",
"fingerprint_sha256": hashlib.sha256(
json.dumps(stable, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest(),
"table_count": len(tables),
"total_rows": sum(item["row_count"] for item in tables.values()),
"table_rows_sha256": hashlib.sha256(
json.dumps(tables, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest(),
"structure_sha256": hashlib.sha256(
json.dumps(stable["singleton_sha256"], sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest(),
"read_consistency": {
"status": "stable_wal_marker" if before == after else "wal_changed_during_fingerprint",
"before": before,
"after": after,
},
}
def safe_db_fingerprint():
try:
return db_fingerprint()
except Exception as exc:
return {
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
"status": "error",
"error": type(exc).__name__,
}
def copy_profile() -> Path: def copy_profile() -> Path:
tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-")) tmp_root = Path(tempfile.mkdtemp(prefix="leo-direct-claim-handler-"))
target = tmp_root / "profile" target = tmp_root / "profile"
@ -155,6 +363,31 @@ def copy_profile() -> Path:
return target return target
def install_harness_plugins(profile):
db_context = profile / "plugins" / "leo-db-context" / "__init__.py"
db_context.parent.mkdir(parents=True, exist_ok=True)
db_context.write_text(DB_CONTEXT_PLUGIN_SOURCE, encoding="utf-8")
turn_trace = profile / "plugins" / "leo-turn-execution-trace"
turn_trace.mkdir(parents=True, exist_ok=True)
(turn_trace / "__init__.py").write_text(TURN_TRACE_PLUGIN_SOURCE, encoding="utf-8")
(turn_trace / "plugin.yaml").write_text(TURN_TRACE_PLUGIN_MANIFEST, encoding="utf-8")
def read_jsonl(path):
if not path or not path.exists():
return []
records = []
for line in path.read_text(encoding="utf-8").splitlines():
try:
value = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(value, dict):
records.append(value)
return records
def remove_tree(path): def remove_tree(path):
if not path or not path.exists(): if not path or not path.exists():
return True return True
@ -192,17 +425,7 @@ def write_report_snapshot(report):
def read_database_context_trace(path): def read_database_context_trace(path):
if not path or not path.exists(): return read_jsonl(path)
return []
records = []
for line in path.read_text(encoding="utf-8").splitlines():
try:
value = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(value, dict):
records.append(value)
return records
def current_agent_messages(runner, session_key): def current_agent_messages(runner, session_key):
@ -222,14 +445,47 @@ def current_agent_messages(runner, session_key):
async def run_suite(): async def run_suite():
sys.path.insert(0, str(DEPLOY_ROOT / "scripts")) sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
from leo_behavior_manifest import build_manifest as build_behavior_manifest import leo_behavior_manifest as behavior_manifest_module
from leo_tool_trace import extract_kb_tool_trace from leo_tool_trace import extract_kb_tool_trace
def build_behavior_manifest(profile):
try:
return behavior_manifest_module.build_manifest(profile, AGENT_ROOT, DEPLOY_ROOT)
except TypeError:
manifest = behavior_manifest_module.build_manifest(profile, AGENT_ROOT)
manifest["teleo_infrastructure_runtime"] = {
"git_head": behavior_manifest_module.git_head(DEPLOY_ROOT),
"source_tree": behavior_manifest_module.path_manifest(
profile,
(
DEPLOY_ROOT / "scripts" / "leo_behavior_manifest.py",
DEPLOY_ROOT / "scripts" / "leo_tool_trace.py",
),
),
}
stable = {
key: value
for key, value in manifest.items()
if key
not in {
"generated_at_utc",
"profile_root",
"hermes_root",
"deployment_root",
"behavior_sha256",
}
}
manifest["deployment_root"] = str(DEPLOY_ROOT)
manifest["behavior_sha256"] = behavior_manifest_module.canonical_sha256(stable)
return manifest
before_service = service_state() before_service = service_state()
before_counts = db_counts() before_counts = db_counts()
behavior_before = build_behavior_manifest(LIVE_PROFILE, AGENT_ROOT) before_fingerprint = safe_db_fingerprint()
behavior_before = build_behavior_manifest(LIVE_PROFILE)
temp_profile = None temp_profile = None
context_trace_path = None context_trace_path = None
model_trace_path = None
temp_removed = False temp_removed = False
report = { report = {
"generated_at_utc": datetime.now(timezone.utc).isoformat(), "generated_at_utc": datetime.now(timezone.utc).isoformat(),
@ -264,16 +520,21 @@ async def run_suite():
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.", "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery.",
], ],
"db_counts_before": before_counts, "db_counts_before": before_counts,
"db_fingerprint_before": before_fingerprint,
"before_service": before_service, "before_service": before_service,
"remote_report_path": str(REPORT_PATH), "remote_report_path": str(REPORT_PATH),
} }
write_report_snapshot(report) write_report_snapshot(report)
try: try:
temp_profile = copy_profile() temp_profile = copy_profile()
install_harness_plugins(temp_profile)
os.environ["HERMES_HOME"] = str(temp_profile) os.environ["HERMES_HOME"] = str(temp_profile)
os.environ["HOME"] = "/home/teleo" os.environ["HOME"] = "/home/teleo"
context_trace_path = temp_profile / "state" / "leo-db-context-trace.jsonl" context_trace_path = temp_profile / "state" / "leo-db-context-trace.jsonl"
model_trace_path = temp_profile / "state" / "leo-turn-api-trace.jsonl"
os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path) os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path)
os.environ["LEO_TURN_API_TRACE_PATH"] = str(model_trace_path)
report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile)
sys.path.insert(0, str(AGENT_ROOT)) sys.path.insert(0, str(AGENT_ROOT))
from gateway.config import Platform from gateway.config import Platform
@ -308,6 +569,7 @@ async def run_suite():
results = [] results = []
for index, prompt in enumerate(PROMPTS, start=1): for index, prompt in enumerate(PROMPTS, start=1):
trace_start = len(read_database_context_trace(context_trace_path)) trace_start = len(read_database_context_trace(context_trace_path))
model_trace_start = len(read_jsonl(model_trace_path))
event = MessageEvent( event = MessageEvent(
text=prompt["message"], text=prompt["message"],
message_type=MessageType.TEXT, message_type=MessageType.TEXT,
@ -333,6 +595,7 @@ async def run_suite():
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
"database_context_trace": read_database_context_trace(context_trace_path)[trace_start:], "database_context_trace": read_database_context_trace(context_trace_path)[trace_start:],
"database_tool_trace": database_tool_trace, "database_tool_trace": database_tool_trace,
"model_call_trace": read_jsonl(model_trace_path)[model_trace_start:],
} }
) )
report["results"] = results report["results"] = results
@ -348,10 +611,15 @@ async def run_suite():
return report return report
finally: finally:
after_counts = db_counts() after_counts = db_counts()
after_fingerprint = safe_db_fingerprint()
after_service = service_state() after_service = service_state()
behavior_after = build_behavior_manifest(LIVE_PROFILE, AGENT_ROOT) behavior_after = build_behavior_manifest(LIVE_PROFILE)
report["db_counts_after"] = after_counts report["db_counts_after"] = after_counts
report["db_counts_changed"] = after_counts != before_counts report["db_counts_changed"] = after_counts != before_counts
report["db_fingerprint_after"] = after_fingerprint
report["db_fingerprint_unchanged"] = (
before_fingerprint.get("fingerprint_sha256") == after_fingerprint.get("fingerprint_sha256")
)
report["live_behavior_manifest_after"] = behavior_after report["live_behavior_manifest_after"] = behavior_after
report["changed_live_profile"] = behavior_before["behavior_sha256"] != behavior_after["behavior_sha256"] report["changed_live_profile"] = behavior_before["behavior_sha256"] != behavior_after["behavior_sha256"]
report["live_behavior_manifest_unchanged"] = not report["changed_live_profile"] report["live_behavior_manifest_unchanged"] = not report["changed_live_profile"]
@ -361,11 +629,20 @@ async def run_suite():
"unchanged_from_preexisting_live_readback": before_service == after_service, "unchanged_from_preexisting_live_readback": before_service == after_service,
} }
database_context_trace = read_database_context_trace(context_trace_path) database_context_trace = read_database_context_trace(context_trace_path)
model_call_trace = read_jsonl(model_trace_path)
if temp_profile is not None: if temp_profile is not None:
tmp_root = temp_profile.parent tmp_root = temp_profile.parent
temp_removed = remove_tree(tmp_root) temp_removed = remove_tree(tmp_root)
report["temp_profile_removed"] = temp_removed report["temp_profile_removed"] = temp_removed
report["database_context_trace"] = database_context_trace report["database_context_trace"] = database_context_trace
report["model_call_trace_count"] = len(model_call_trace)
report["model_call_trace_all_bound"] = bool(model_call_trace) and all(
item.get("event") == "post_api_request"
and item.get("model")
and item.get("provider")
and item.get("response_model")
for item in model_call_trace
)
context_injections = [ context_injections = [
item for item in database_context_trace if item.get("event", "pre_llm_call") == "pre_llm_call" item for item in database_context_trace if item.get("event", "pre_llm_call") == "pre_llm_call"
] ]
@ -443,6 +720,7 @@ def build_remote_script(
.replace("__SUITE_MODE_JSON__", json.dumps(suite_mode)) .replace("__SUITE_MODE_JSON__", json.dumps(suite_mode))
.replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix)) .replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix))
.replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note)) .replace("__PROMPT_NOTE_JSON__", json.dumps(prompt_note))
.replace("__DB_CONTEXT_PLUGIN_SOURCE_JSON__", json.dumps(DB_CONTEXT_PLUGIN.read_text(encoding="utf-8")))
) )
@ -561,6 +839,7 @@ def write_output(remote: dict[str, Any], *, output_json: Path = OUTPUT_JSON) ->
report["remote_run_id"] = remote.get("run_id") report["remote_run_id"] = remote.get("run_id")
if remote.get("stderr"): if remote.get("stderr"):
report["remote_stderr"] = remote["stderr"] report["remote_stderr"] = remote["stderr"]
attach_execution_manifests(report, repo_root=ROOT)
output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") output_json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
return report return report

View file

@ -43,6 +43,24 @@ def test_plugin_injects_only_operational_contracts_and_writes_redacted_trace(tmp
{"id": "source_intake", "capability_tier": "build-local compiler only"}, {"id": "source_intake", "capability_tier": "build-local compiler only"},
], ],
"compiled_response": "must not be injected either", "compiled_response": "must not be injected either",
"retrieval_receipt": {
"schema": "livingip.teleoKbRetrievalReceipt.v1",
"query_sha256": "1" * 64,
"semantic_context_sha256": "2" * 64,
"artifact_state_sha256": "3" * 64,
"claim_ids": ["claim-1"],
"source_ids": ["source-1"],
"counts": {"claims": 1, "sources": 1},
"read_consistency": {
"status": "stable_wal_marker",
"attempts": 1,
"database": "teleo",
"database_user": "runtime",
"system_identifier": "system-1",
"wal_lsn_before": "0/123",
"wal_lsn_after": "0/123",
},
},
} }
calls = [] calls = []
@ -63,6 +81,9 @@ def test_plugin_injects_only_operational_contracts_and_writes_redacted_trace(tmp
assert record["status"] == "ok" assert record["status"] == "ok"
assert record["injected"] is True assert record["injected"] is True
assert record["contract_ids"] == ["reply_budget", "source_intake"] assert record["contract_ids"] == ["reply_budget", "source_intake"]
assert record["retrieval_receipt"]["semantic_context_sha256"] == "2" * 64
assert record["retrieval_receipt"]["read_consistency"]["system_identifier"] == "system-1"
assert len(record["retrieval_receipt"]["receipt_sha256"]) == 64
command, kwargs = calls[0] command, kwargs = calls[0]
assert command[2:5] == ["--local", "context", "operator secret-shaped but nonsecret message"] assert command[2:5] == ["--local", "context", "operator secret-shaped but nonsecret message"]
assert command[-6:] == ["--limit", "0", "--context-limit", "0", "--format", "json"] assert command[-6:] == ["--limit", "0", "--context-limit", "0", "--format", "json"]
@ -475,3 +496,9 @@ def test_handler_harness_retains_database_context_proof_fields() -> None:
assert '"database_tool_call_proven"' in source assert '"database_tool_call_proven"' in source
assert '"database_retrieval_receipt_proven"' in source assert '"database_retrieval_receipt_proven"' in source
assert '"database_tool_calls_read_only"' in source assert '"database_tool_calls_read_only"' in source
assert "LEO_TURN_API_TRACE_PATH" in source
assert '"model_call_trace"' in source
assert "attach_execution_manifests" in source
assert '"db_fingerprint_before"' in source
assert '"db_fingerprint_after"' in source
assert '"db_fingerprint_unchanged"' in source

View file

@ -16,6 +16,7 @@ def _write(path: Path, value: str) -> None:
def test_behavior_manifest_is_deterministic_and_never_emits_config_secrets(tmp_path: Path) -> None: def test_behavior_manifest_is_deterministic_and_never_emits_config_secrets(tmp_path: Path) -> None:
profile = tmp_path / "profile" profile = tmp_path / "profile"
hermes = tmp_path / "hermes" hermes = tmp_path / "hermes"
deployment = tmp_path / "deployment"
_write( _write(
profile / "config.yaml", profile / "config.yaml",
"""model: """model:
@ -40,12 +41,15 @@ gateway:
_write(profile / "state.db", "runtime-state") _write(profile / "state.db", "runtime-state")
_write(hermes / "run_agent.py", "# runtime\n") _write(hermes / "run_agent.py", "# runtime\n")
_write(hermes / "agent" / "prompt_builder.py", "# prompts\n") _write(hermes / "agent" / "prompt_builder.py", "# prompts\n")
_write(deployment / "scripts" / "leo_behavior_manifest.py", "# manifest\n")
_write(deployment / "scripts" / "leo_tool_trace.py", "# tool trace\n")
first = manifest.build_manifest(profile, hermes) first = manifest.build_manifest(profile, hermes, deployment)
second = manifest.build_manifest(profile, hermes) second = manifest.build_manifest(profile, hermes, deployment)
assert first["behavior_sha256"] == second["behavior_sha256"] assert first["behavior_sha256"] == second["behavior_sha256"]
assert first["model_runtime"]["default_model"] == "anthropic/claude-sonnet-4-6" assert first["model_runtime"]["default_model"] == "anthropic/claude-sonnet-4-6"
assert first["teleo_infrastructure_runtime"]["source_tree"]["file_count"] == 2
assert first["components"]["persistent_memory"]["content"]["file_count"] == 1 assert first["components"]["persistent_memory"]["content"]["file_count"] == 1
encoded = json.dumps(first) encoded = json.dumps(first)
assert "super-secret-token" not in encoded assert "super-secret-token" not in encoded
@ -56,14 +60,17 @@ gateway:
def test_behavior_manifest_changes_when_a_behavior_layer_changes(tmp_path: Path) -> None: def test_behavior_manifest_changes_when_a_behavior_layer_changes(tmp_path: Path) -> None:
profile = tmp_path / "profile" profile = tmp_path / "profile"
hermes = tmp_path / "hermes" hermes = tmp_path / "hermes"
deployment = tmp_path / "deployment"
_write(profile / "config.yaml", "model:\n provider: openrouter\n default: model-a\n") _write(profile / "config.yaml", "model:\n provider: openrouter\n default: model-a\n")
_write(profile / "SOUL.md", "identity-a\n") _write(profile / "SOUL.md", "identity-a\n")
_write(hermes / "run_agent.py", "runtime\n") _write(hermes / "run_agent.py", "runtime\n")
_write(hermes / "agent" / "prompt_builder.py", "prompts\n") _write(hermes / "agent" / "prompt_builder.py", "prompts\n")
_write(deployment / "scripts" / "leo_behavior_manifest.py", "manifest\n")
_write(deployment / "scripts" / "leo_tool_trace.py", "tool trace\n")
before = manifest.build_manifest(profile, hermes) before = manifest.build_manifest(profile, hermes, deployment)
_write(profile / "SOUL.md", "identity-b\n") _write(profile / "SOUL.md", "identity-b\n")
after = manifest.build_manifest(profile, hermes) after = manifest.build_manifest(profile, hermes, deployment)
assert before["behavior_sha256"] != after["behavior_sha256"] assert before["behavior_sha256"] != after["behavior_sha256"]
assert ( assert (
@ -75,6 +82,7 @@ def test_behavior_manifest_changes_when_a_behavior_layer_changes(tmp_path: Path)
def test_behavior_manifest_hashes_content_behind_symlinks(tmp_path: Path) -> None: def test_behavior_manifest_hashes_content_behind_symlinks(tmp_path: Path) -> None:
profile = tmp_path / "profile" profile = tmp_path / "profile"
hermes = tmp_path / "hermes" hermes = tmp_path / "hermes"
deployment = tmp_path / "deployment"
target = tmp_path / "deployed-skill.md" target = tmp_path / "deployed-skill.md"
_write(profile / "config.yaml", "model:\n provider: test\n") _write(profile / "config.yaml", "model:\n provider: test\n")
_write(profile / "SOUL.md", "identity\n") _write(profile / "SOUL.md", "identity\n")
@ -83,10 +91,12 @@ def test_behavior_manifest_hashes_content_behind_symlinks(tmp_path: Path) -> Non
(profile / "skills" / "SKILL.md").symlink_to(target) (profile / "skills" / "SKILL.md").symlink_to(target)
_write(hermes / "run_agent.py", "runtime\n") _write(hermes / "run_agent.py", "runtime\n")
_write(hermes / "agent" / "prompt_builder.py", "prompts\n") _write(hermes / "agent" / "prompt_builder.py", "prompts\n")
_write(deployment / "scripts" / "leo_behavior_manifest.py", "manifest\n")
_write(deployment / "scripts" / "leo_tool_trace.py", "tool trace\n")
before = manifest.build_manifest(profile, hermes) before = manifest.build_manifest(profile, hermes, deployment)
_write(target, "version-b\n") _write(target, "version-b\n")
after = manifest.build_manifest(profile, hermes) after = manifest.build_manifest(profile, hermes, deployment)
assert before["behavior_sha256"] != after["behavior_sha256"] assert before["behavior_sha256"] != after["behavior_sha256"]
symlinks = after["components"]["procedural_skills"]["content"]["symlinks"] symlinks = after["components"]["procedural_skills"]["content"]["symlinks"]

View file

@ -0,0 +1,170 @@
from __future__ import annotations
import copy
from scripts import leo_turn_execution_manifest as manifest
def behavior_manifest() -> dict:
return {
"behavior_sha256": "a" * 64,
"hermes_runtime": {"git_head": "b" * 40, "source_tree": {"sha256": "c" * 64}},
"teleo_infrastructure_runtime": {
"git_head": "d" * 40,
"source_tree": {"sha256": "e" * 64},
},
"components": {
"runtime_identity": {"content": {"sha256": "f" * 64}},
"procedural_skills": {"content": {"sha256": "1" * 64}},
},
}
def retrieval_receipt() -> dict:
return {
"schema": "livingip.teleoKbRetrievalReceipt.v1",
"query_sha256": "2" * 64,
"semantic_context_sha256": "3" * 64,
"artifact_state_sha256": "4" * 64,
"claim_ids": ["claim-b", "claim-a"],
"source_ids": ["source-a"],
"counts": {"claims": 2, "context_rows": 1},
"read_consistency": {
"status": "stable_wal_marker",
"attempts": 1,
"database": "teleo",
"database_user": "runtime",
"system_identifier": "system-1",
"wal_lsn_before": "0/123",
"wal_lsn_after": "0/123",
},
}
def turn_result() -> dict:
return {
"turn": 1,
"prompt_id": "OOS-01",
"prompt": "Challenge the weak acquisition claim.",
"reply": "The current evidence is shallow; here is a narrower candidate claim.",
"started_at_utc": "2026-07-14T12:00:00+00:00",
"ended_at_utc": "2026-07-14T12:00:02+00:00",
"database_context_trace": [
{
"event": "pre_llm_call",
"contract_ids": ["reply_budget", "source_link_audit"],
"retrieval_receipt": retrieval_receipt(),
}
],
"database_tool_trace": {"database_tool_call_count": 0, "calls": []},
"model_call_trace": [
{
"event": "post_api_request",
"api_call_count": 1,
"api_mode": "openai_chat_completions",
"model": "anthropic/claude-sonnet-4-6",
"provider": "openrouter",
"response_model": "anthropic/claude-sonnet-4-6",
"finish_reason": "stop",
"message_count": 4,
"assistant_content_chars": 120,
"assistant_tool_call_count": 0,
"task_id_sha256": "5" * 64,
"session_id_sha256": "6" * 64,
"base_url_sha256": "7" * 64,
"usage": {"input_tokens": 100, "output_tokens": 20, "ignored": "secret-shaped"},
"provider_response_id": None,
"provider_response_id_status": "not_exposed_by_hermes_post_api_request_hook",
}
],
}
def database_fingerprint() -> dict:
marker = {
"database": "teleo",
"database_user": "runtime",
"system_identifier": "system-1",
"wal_lsn": "0/123",
}
return {
"schema": "livingip.teleoCanonicalDatabaseFingerprint.v1",
"status": "ok",
"fingerprint_sha256": "9" * 64,
"table_count": 39,
"total_rows": 52167,
"table_rows_sha256": "a" * 64,
"structure_sha256": "b" * 64,
"read_consistency": {"status": "stable_wal_marker", "before": marker, "after": marker},
}
def build(result: dict | None = None, *, fingerprint: dict | None = None) -> dict:
selected_fingerprint = database_fingerprint() if fingerprint is None else fingerprint
return manifest.build_turn_manifest(
result=result or turn_result(),
behavior_manifest=behavior_manifest(),
session_key="agent:main:telegram:group:private",
source={"platform": "telegram", "chat_type": "group"},
suite_mode="live_vps_gatewayrunner_temp_profile",
remote_run_id="run-1",
db_counts_before={"public.claims": 1837},
db_counts_after={"public.claims": 1837},
db_counts_changed=False,
db_fingerprint_before=selected_fingerprint,
db_fingerprint_after=selected_fingerprint,
posted_to_telegram=False,
mutates_kb_by_harness=False,
harness_source={"git_head": "8" * 40, "tracked_tree_clean": True},
)
def test_turn_manifest_binds_model_runtime_session_and_exact_database_read() -> None:
value = build()
assert value["attribution"]["status"] == "complete"
assert value["replay_claim"]["status"] == "bounded_recorded_inputs_pinned"
assert value["model_execution"]["calls"][0]["response_model"] == "anthropic/claude-sonnet-4-6"
assert value["model_execution"]["calls"][0]["usage"] == {"input_tokens": 100, "output_tokens": 20}
assert value["canonical_database"]["binding_status"] == "retrieval_receipt_bound"
assert value["canonical_database"]["fingerprint_before"]["table_count"] == 39
assert value["canonical_database"]["fingerprint_unchanged"] is True
receipt = value["canonical_database"]["context_retrieval_receipts"][0]
assert receipt["read_consistency"]["system_identifier"] == "system-1"
assert receipt["read_consistency"]["wal_lsn_before"] == receipt["read_consistency"]["wal_lsn_after"]
assert value["delivery_and_safety"]["posted_to_telegram"] is False
assert value["delivery_and_safety"]["kb_mutation_by_harness"] is False
assert manifest.validate_turn_manifest(value) == []
encoded = str(value)
assert "Challenge the weak acquisition claim" not in encoded
assert "narrower candidate claim" not in encoded
assert "agent:main:telegram" not in encoded
assert "secret-shaped" not in encoded
def test_turn_manifest_detects_missing_actual_model_call() -> None:
result = turn_result()
result["model_call_trace"] = []
value = build(result)
assert value["attribution"]["status"] == "incomplete"
assert "actual_model_call" in value["attribution"]["missing_required_bindings"]
assert value["replay_claim"]["local_runtime_inputs_reconstructible"] is False
def test_turn_manifest_hash_detects_tampering() -> None:
value = build()
tampered = copy.deepcopy(value)
tampered["turn"]["reply_sha256"] = "0" * 64
assert manifest.validate_turn_manifest(tampered) == ["execution_sha256_mismatch"]
def test_database_question_requires_a_stable_full_database_fingerprint() -> None:
value = build(fingerprint={"status": "error"})
assert value["attribution"]["status"] == "incomplete"
assert "database_fingerprint_before" in value["attribution"]["missing_required_bindings"]
assert "database_fingerprint_after" in value["attribution"]["missing_required_bindings"]