#!/usr/bin/env python3 """Bind one tested Leo turn to its runtime, model call, session, and KB read. The manifest is deliberately content-minimizing: prompts, replies, raw database rows, tool arguments, provider URLs, and session identifiers are represented by hashes. It proves attribution of a tested turn, not deterministic regeneration of externally hosted model weights. """ from __future__ import annotations import argparse import hashlib import json import subprocess from datetime import UTC, datetime from pathlib import Path from typing import Any SCHEMA = "livingip.leoTurnExecutionManifest.v1" RETRIEVAL_RECEIPT_SCHEMA = "livingip.teleoKbRetrievalReceipt.v1" def canonical_sha256(value: Any) -> str: encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") return hashlib.sha256(encoded).hexdigest() def text_sha256(value: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest() def git_state(repo: Path) -> dict[str, Any]: try: head = subprocess.run( ["git", "-C", str(repo), "rev-parse", "HEAD"], check=False, capture_output=True, text=True, timeout=10, ) status = subprocess.run( ["git", "-C", str(repo), "status", "--porcelain", "--untracked-files=no"], check=False, capture_output=True, text=True, timeout=10, ) except (OSError, subprocess.TimeoutExpired): return {"git_head": None, "tracked_tree_clean": None} git_head = head.stdout.strip() if head.returncode == 0 and len(head.stdout.strip()) == 40 else None return { "git_head": git_head, "tracked_tree_clean": status.returncode == 0 and not status.stdout.strip(), } def _safe_usage(value: Any) -> dict[str, int | float | None]: if not isinstance(value, dict): return {} safe: dict[str, int | float | None] = {} for key in ( "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens", "total_tokens", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens", ): item = value.get(key) if isinstance(item, (int, float)) and not isinstance(item, bool): safe[key] = item return safe def _model_calls(result: dict[str, Any]) -> list[dict[str, Any]]: calls: list[dict[str, Any]] = [] for raw in result.get("model_call_trace") or []: if not isinstance(raw, dict) or raw.get("event") != "post_api_request": continue calls.append( { "api_call_count": raw.get("api_call_count"), "api_mode": raw.get("api_mode"), "model": raw.get("model"), "provider": raw.get("provider"), "response_model": raw.get("response_model"), "finish_reason": raw.get("finish_reason"), "message_count": raw.get("message_count"), "assistant_content_chars": raw.get("assistant_content_chars"), "assistant_tool_call_count": raw.get("assistant_tool_call_count"), "task_id_sha256": raw.get("task_id_sha256"), "session_id_sha256": raw.get("session_id_sha256"), "base_url_sha256": raw.get("base_url_sha256"), "usage": _safe_usage(raw.get("usage")), "provider_response_id": raw.get("provider_response_id"), "provider_response_id_status": raw.get("provider_response_id_status"), } ) return calls def _safe_retrieval_receipt(value: Any) -> dict[str, Any] | None: if not isinstance(value, dict) or value.get("schema") != RETRIEVAL_RECEIPT_SCHEMA: return None consistency = value.get("read_consistency") if isinstance(value.get("read_consistency"), dict) else {} counts = value.get("counts") if isinstance(value.get("counts"), dict) else {} return { "schema": value.get("schema"), "query_sha256": value.get("query_sha256"), "semantic_context_sha256": value.get("semantic_context_sha256"), "artifact_state_sha256": value.get("artifact_state_sha256"), "claim_ids_sha256": canonical_sha256(sorted(str(item) for item in value.get("claim_ids") or [])), "source_ids_sha256": canonical_sha256(sorted(str(item) for item in value.get("source_ids") or [])), "counts": { key: item for key, item in sorted(counts.items()) if isinstance(key, str) and isinstance(item, int) and not isinstance(item, bool) }, "read_consistency": { "status": consistency.get("status"), "attempts": consistency.get("attempts"), "database": consistency.get("database"), "database_user": consistency.get("database_user"), "system_identifier": consistency.get("system_identifier"), "wal_lsn_before": consistency.get("wal_lsn_before"), "wal_lsn_after": consistency.get("wal_lsn_after"), }, "receipt_sha256": canonical_sha256(value), } def _context_receipts(result: dict[str, Any]) -> list[dict[str, Any]]: receipts = [] for record in result.get("database_context_trace") or []: if not isinstance(record, dict) or record.get("event") != "pre_llm_call": continue receipt = _safe_retrieval_receipt(record.get("retrieval_receipt")) if receipt is not None: receipts.append(receipt) return receipts def _tool_receipts(result: dict[str, Any]) -> list[dict[str, Any]]: receipts = [] trace = result.get("database_tool_trace") or {} for call in trace.get("calls") or []: if not isinstance(call, dict): continue raw = ((call.get("result") or {}).get("retrieval_receipt") or {}) if not isinstance(raw, dict) or not raw.get("semantic_context_sha256"): continue receipts.append( { "schema": raw.get("schema"), "semantic_context_sha256": raw.get("semantic_context_sha256"), "artifact_state_sha256": raw.get("artifact_state_sha256"), "read_consistency": {"status": raw.get("read_consistency_status")}, "receipt_sha256": canonical_sha256(raw), } ) return receipts def _database_required(result: dict[str, Any]) -> bool: for record in result.get("database_context_trace") or []: if not isinstance(record, dict) or record.get("event") != "pre_llm_call": continue contract_ids = {str(item) for item in record.get("contract_ids") or []} if contract_ids - {"reply_budget"}: return True trace = result.get("database_tool_trace") or {} return bool(trace.get("database_tool_call_count")) def _database_tool_binding(result: dict[str, Any]) -> dict[str, Any]: trace = result.get("database_tool_trace") or {} calls = [] for call in trace.get("calls") or []: if not isinstance(call, dict): continue call_result = call.get("result") if isinstance(call.get("result"), dict) else {} calls.append( { "tool_call_id_sha256": call.get("tool_call_id_sha256"), "arguments_sha256": call.get("arguments_sha256"), "database_invocations": call.get("database_invocations") or [], "result_content_sha256": call_result.get("content_sha256"), "result_content_bytes": call_result.get("content_bytes"), "result_error_detected": call_result.get("error_detected"), "result_row_ids_sha256": canonical_sha256(sorted(call_result.get("row_ids") or [])), "result_sha256_values_sha256": canonical_sha256(sorted(call_result.get("sha256_values") or [])), } ) all_results_bound = bool(calls) and all( call.get("result_content_sha256") and call.get("result_error_detected") is False for call in calls ) return { "schema": trace.get("schema"), "tool_call_count": trace.get("database_tool_call_count", 0), "completed_count": trace.get("database_tool_completed_count", 0), "all_calls_read_only": trace.get("database_tool_calls_read_only"), "all_results_content_bound": all_results_bound, "calls": calls, "trace_sha256": canonical_sha256(trace), } def _database_fingerprint_complete(value: dict[str, Any]) -> bool: consistency = value.get("read_consistency") if isinstance(value.get("read_consistency"), dict) else {} return bool( value.get("status") == "ok" and value.get("fingerprint_sha256") and value.get("table_count") and value.get("total_rows") and consistency.get("status") == "stable_wal_marker" ) def _required_missing( *, behavior: dict[str, Any], calls: list[dict[str, Any]], receipts: list[dict[str, Any]], database_required: bool, session_key: str, prompt: str, reply: str, harness_source: dict[str, Any], database_tool_binding: dict[str, Any], db_fingerprint_before: dict[str, Any], db_fingerprint_after: dict[str, Any], ) -> list[str]: missing = [] checks = { "prompt": bool(prompt), "reply": bool(reply), "session_boundary": bool(session_key), "behavior_sha256": bool(behavior.get("behavior_sha256")), "hermes_git_head": bool((behavior.get("hermes_runtime") or {}).get("git_head")), "teleo_infrastructure_git_head": bool( (behavior.get("teleo_infrastructure_runtime") or {}).get("git_head") ), "harness_git_head": bool(harness_source.get("git_head")), "actual_model_call": bool( calls and all( call.get("model") and call.get("provider") and call.get("response_model") for call in calls ) ), "database_retrieval_receipt": not database_required or bool(receipts), "database_tool_results": not database_tool_binding.get("tool_call_count") or bool(database_tool_binding.get("all_results_content_bound")), "database_fingerprint_before": not database_required or _database_fingerprint_complete(db_fingerprint_before), "database_fingerprint_after": not database_required or _database_fingerprint_complete(db_fingerprint_after), "database_fingerprint_unchanged": not database_required or db_fingerprint_before.get("fingerprint_sha256") == db_fingerprint_after.get("fingerprint_sha256"), } for label, present in checks.items(): if not present: missing.append(label) return missing def build_turn_manifest( *, result: dict[str, Any], behavior_manifest: dict[str, Any], session_key: str, source: dict[str, Any], suite_mode: str, remote_run_id: str | None, db_counts_before: dict[str, Any] | None, db_counts_after: dict[str, Any] | None, db_counts_changed: bool | None, db_fingerprint_before: dict[str, Any] | None, db_fingerprint_after: dict[str, Any] | None, posted_to_telegram: bool, mutates_kb_by_harness: bool, harness_source: dict[str, Any], ) -> dict[str, Any]: prompt = str(result.get("prompt") or "") reply = str(result.get("reply") or "") calls = _model_calls(result) context_receipts = _context_receipts(result) tool_receipts = _tool_receipts(result) receipts = context_receipts + tool_receipts database_required = _database_required(result) database_tool_binding = _database_tool_binding(result) fingerprint_before = db_fingerprint_before or {} fingerprint_after = db_fingerprint_after or {} fingerprint_unchanged = bool( fingerprint_before.get("fingerprint_sha256") and fingerprint_after.get("fingerprint_sha256") and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256") ) missing = _required_missing( behavior=behavior_manifest, calls=calls, receipts=receipts, database_required=database_required, session_key=session_key, prompt=prompt, reply=reply, harness_source=harness_source, database_tool_binding=database_tool_binding, db_fingerprint_before=fingerprint_before, db_fingerprint_after=fingerprint_after, ) stable = { "schema": SCHEMA, "turn": { "prompt_id": result.get("prompt_id"), "turn_index": result.get("turn"), "remote_run_id": remote_run_id, "started_at_utc": result.get("started_at_utc"), "ended_at_utc": result.get("ended_at_utc"), "prompt_sha256": text_sha256(prompt), "reply_sha256": text_sha256(reply), "prompt_bytes": len(prompt.encode("utf-8")), "reply_bytes": len(reply.encode("utf-8")), }, "session_boundary": { "session_key_sha256": text_sha256(session_key) if session_key else None, "source_platform": source.get("platform"), "chat_type": source.get("chat_type"), "fresh_temp_profile": True, "prior_dynamic_state_excluded": True, }, "runtime": { "behavior_sha256": behavior_manifest.get("behavior_sha256"), "hermes_runtime": behavior_manifest.get("hermes_runtime"), "teleo_infrastructure_runtime": behavior_manifest.get("teleo_infrastructure_runtime"), "profile_component_hashes": { name: (component.get("content") or {}).get("sha256") for name, component in sorted((behavior_manifest.get("components") or {}).items()) if isinstance(component, dict) }, "harness_source": harness_source, }, "model_execution": { "call_count": len(calls), "calls": calls, "externally_hosted_weights_hashable": False, "provider_response_id_required_for_attribution": False, }, "canonical_database": { "required_for_turn": database_required, "binding_status": "retrieval_receipt_bound" if receipts else "not_required" if not database_required else "missing", "context_retrieval_receipts": context_receipts, "explicit_tool_retrieval_receipts": tool_receipts, "database_tool_binding": database_tool_binding, "fingerprint_before": fingerprint_before, "fingerprint_after": fingerprint_after, "fingerprint_unchanged": fingerprint_unchanged, "suite_counts_before_sha256": canonical_sha256(db_counts_before or {}), "suite_counts_after_sha256": canonical_sha256(db_counts_after or {}), "suite_counts_changed": db_counts_changed, "full_database_content_fingerprint_included": bool( fingerprint_before.get("fingerprint_sha256") and fingerprint_after.get("fingerprint_sha256") ), "full_database_snapshot_artifact_included": False, }, "delivery_and_safety": { "suite_mode": suite_mode, "posted_to_telegram": posted_to_telegram, "kb_mutation_by_harness": mutates_kb_by_harness, "raw_prompt_retained_in_manifest": False, "raw_reply_retained_in_manifest": False, "raw_database_rows_retained_in_manifest": False, "raw_session_identifier_retained_in_manifest": False, }, "attribution": { "status": "complete" if not missing else "incomplete", "missing_required_bindings": missing, }, "replay_claim": { "status": "bounded_recorded_inputs_pinned" if not missing else "incomplete_inputs", "local_runtime_inputs_reconstructible": not missing, "database_state_identified_by_content_fingerprint": bool( not missing and fingerprint_before.get("fingerprint_sha256") ), "database_snapshot_reconstructible_from_this_manifest_alone": False, "external_model_state_reconstructible": False, "bitwise_identical_model_output_guaranteed": False, "reason": ( "The tested turn is attributable to exact local/runtime inputs and DB reads, but hosted model " "weights and sampling are not locally hashable or deterministic." ), }, } return { **stable, "generated_at_utc": datetime.now(UTC).isoformat(), "execution_sha256": canonical_sha256(stable), } def validate_turn_manifest(manifest: dict[str, Any]) -> list[str]: problems = [] if manifest.get("schema") != SCHEMA: problems.append("schema_mismatch") stable = { key: value for key, value in manifest.items() if key not in {"generated_at_utc", "execution_sha256"} } if manifest.get("execution_sha256") != canonical_sha256(stable): problems.append("execution_sha256_mismatch") missing = (manifest.get("attribution") or {}).get("missing_required_bindings") expected_status = "complete" if not missing else "incomplete" if (manifest.get("attribution") or {}).get("status") != expected_status: problems.append("attribution_status_mismatch") return problems def attach_execution_manifests(report: dict[str, Any], *, repo_root: Path) -> dict[str, Any]: behavior = report.get("executed_behavior_manifest") or report.get("live_behavior_manifest_before") or {} handler = report.get("handler") or {} harness_source = git_state(repo_root) results = report.get("results") or [] complete = 0 for result in results: if not isinstance(result, dict): continue manifest = build_turn_manifest( result=result, behavior_manifest=behavior, session_key=str(handler.get("session_key") or ""), source=report.get("source") or {}, suite_mode=str(report.get("mode") or ""), remote_run_id=report.get("remote_run_id"), db_counts_before=report.get("db_counts_before"), db_counts_after=report.get("db_counts_after"), db_counts_changed=report.get("db_counts_changed"), db_fingerprint_before=report.get("db_fingerprint_before"), db_fingerprint_after=report.get("db_fingerprint_after"), posted_to_telegram=bool(report.get("posted_to_telegram")), mutates_kb_by_harness=bool(report.get("mutates_kb_by_harness")), harness_source=harness_source, ) result["execution_manifest"] = manifest if manifest["attribution"]["status"] == "complete" and not validate_turn_manifest(manifest): complete += 1 report["execution_manifest_summary"] = { "schema": SCHEMA, "turn_count": len(results), "attribution_complete_count": complete, "all_turns_attribution_complete": bool(results) and complete == len(results), "harness_source": harness_source, } return report def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--suite-report", required=True, type=Path) parser.add_argument("--repo-root", default=Path(__file__).resolve().parents[1], type=Path) parser.add_argument("--output", required=True, type=Path) args = parser.parse_args() report = json.loads(args.suite_report.read_text(encoding="utf-8")) attach_execution_manifests(report, repo_root=args.repo_root) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") summary = report["execution_manifest_summary"] print(json.dumps(summary, indent=2, sort_keys=True)) return 0 if summary["all_turns_attribution_complete"] else 1 if __name__ == "__main__": raise SystemExit(main())