#!/usr/bin/env python3 """Run one isolated challenger-agent versus Leo exchange without sending or writing.""" from __future__ import annotations import argparse import hashlib import json import re import subprocess import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any try: import leo_agent_challenger_protocol as protocol import run_leo_direct_claim_handler_suite as handler_harness except ImportError: # pragma: no cover - imported as scripts.* in tests from scripts import leo_agent_challenger_protocol as protocol from scripts import run_leo_direct_claim_handler_suite as handler_harness ROOT = Path(__file__).resolve().parents[1] REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709" OUTPUT_JSON = REPORT_DIR / "leo-agent-challenger-loop-source-current.json" RECEIPT_JSON = REPORT_DIR / "leo-agent-challenger-loop-receipt-current.json" NEGATIVE_JSON = REPORT_DIR / "leo-agent-challenger-loop-negative-controls-current.json" LEDGER_JSONL = REPORT_DIR / "leo-agent-challenger-loop-ledger.jsonl" DEFAULT_MODEL = "google/gemini-2.5-flash" DEFAULT_MAX_TOKENS = 768 DEFAULT_REPORT_PREFIX = "leo-agent-challenger-loop" DB_CONTEXT_PLUGIN = ( ROOT / "hermes-agent" / "leoclean-plugins" / "vps" / "leo-db-context" / "__init__.py" ) KB_TOOL = ROOT / "hermes-agent" / "leoclean-bin" / "kb_tool.py" BEHAVIOR_MANIFEST = ROOT / "scripts" / "leo_behavior_manifest.py" POSTGRES_MANIFEST = ROOT / "ops" / "postgres_parity_manifest.sql" READ_ONLY_COMMANDS = ( "status", "search", "context", "show", "evidence", "edges", "list-proposals", "search-proposals", "show-proposal", "decision-matrix-status", ) TURN_TRACE_PLUGIN_SOURCE = r'''import hashlib import json import os from datetime import datetime, timezone from pathlib import Path def _sha(value): return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest() def _safe_usage(value): if not isinstance(value, dict): return {} return { key: item for key, item in value.items() if key in { "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens", "total_tokens", "cache_read_tokens", "cache_write_tokens", "reasoning_tokens", } and isinstance(item, (int, float)) and not isinstance(item, bool) } def _write(record): raw_path = os.getenv("LEO_TURN_API_TRACE_PATH", "").strip() if not raw_path: return path = Path(raw_path) path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(record, sort_keys=True) + "\n") path.chmod(0o600) def _pre_llm_call(**kwargs): _write({ "generated_at_utc": datetime.now(timezone.utc).isoformat(), "event": "turn_pre_llm_call", "session_id_sha256": _sha(kwargs.get("session_id")), "prompt_sha256": _sha(kwargs.get("user_message")), }) def _post_api_request(**kwargs): _write({ "generated_at_utc": datetime.now(timezone.utc).isoformat(), "event": "post_api_request", "session_id_sha256": _sha(kwargs.get("session_id")), "model": kwargs.get("model"), "provider": kwargs.get("provider"), "response_model": kwargs.get("response_model"), "api_call_count": kwargs.get("api_call_count"), "finish_reason": kwargs.get("finish_reason"), "usage": _safe_usage(kwargs.get("usage")), }) def _post_llm_call(**kwargs): _write({ "generated_at_utc": datetime.now(timezone.utc).isoformat(), "event": "turn_post_llm_call", "session_id_sha256": _sha(kwargs.get("session_id")), "prompt_sha256": _sha(kwargs.get("user_message")), "raw_assistant_response_sha256": _sha(kwargs.get("assistant_response")), }) def register(ctx): ctx.register_hook("pre_llm_call", _pre_llm_call) ctx.register_hook("post_api_request", _post_api_request) ctx.register_hook("post_llm_call", _post_llm_call) ''' TURN_TRACE_PLUGIN_MANIFEST = '''name: leo-turn-execution-trace version: "1.0.0" description: Secret-safe model-call trace for the isolated challenger loop. provides_hooks: - pre_llm_call - post_api_request - post_llm_call ''' READ_ONLY_KB_WRAPPER = r'''#!/usr/bin/env python3 """Fail-closed launcher for the no-write Leo canary.""" import json import os import sys from datetime import datetime, timezone from pathlib import Path ALLOWED = __READ_ONLY_COMMANDS__ KNOWN = ALLOWED | { "propose-edge", "propose-attachment-evaluation", "propose-source", "prepare-source", "record-document-evaluation", "propose-core-change", "approve", "apply", } args = sys.argv[1:] command = next((item for item in args if item in KNOWN), None) trace_path = os.getenv("LEO_KB_GUARD_TRACE_PATH", "").strip() record = { "generated_at_utc": datetime.now(timezone.utc).isoformat(), "command": command, "allowed": command in ALLOWED, "argv_sha256": __import__("hashlib").sha256(json.dumps(args).encode()).hexdigest(), } if trace_path: path = Path(trace_path) path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(record, sort_keys=True) + "\n") if command not in ALLOWED: print("blocked by isolated challenger-loop read-only guard", file=sys.stderr) raise SystemExit(77) implementation = Path(__file__).with_name("_kb_tool_readonly_impl.py") os.execv(sys.executable, [sys.executable, str(implementation), *args]) ''' REMOTE_SCRIPT = r''' import asyncio import copy import hashlib import json import multiprocessing import os import re import shutil import subprocess import sys import tempfile import time import traceback import types import uuid from datetime import datetime, timezone from pathlib import Path import aiohttp import yaml LIVE_PROFILE = Path("/home/teleo/.hermes/profiles/leoclean") AGENT_ROOT = Path("/home/teleo/.hermes/hermes-agent") DEPLOY_ROOT = Path("/opt/teleo-eval/workspaces/deploy-infra") OPENROUTER_KEY = Path("/opt/teleo-eval/secrets/openrouter-key") OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" CHAT_ID = "__CHAT_ID__" USER_ID = "__USER_ID__" RUN_ID = "__RUN_ID__" REPORT_PREFIX = __REPORT_PREFIX_JSON__ REPORT_PATH = Path(f"/tmp/{REPORT_PREFIX}-{RUN_ID}.json") OBJECTIVE = __OBJECTIVE_JSON__ CHALLENGER_MODEL = __CHALLENGER_MODEL_JSON__ CHALLENGER_MAX_TOKENS = __CHALLENGER_MAX_TOKENS__ PROTOCOL_SOURCE = __PROTOCOL_SOURCE_JSON__ DB_CONTEXT_PLUGIN_SOURCE = __DB_CONTEXT_PLUGIN_SOURCE_JSON__ KB_TOOL_SOURCE = __KB_TOOL_SOURCE_JSON__ READ_ONLY_KB_WRAPPER_SOURCE = __READ_ONLY_KB_WRAPPER_SOURCE_JSON__ BEHAVIOR_MANIFEST_SOURCE = __BEHAVIOR_MANIFEST_SOURCE_JSON__ POSTGRES_MANIFEST_SQL = __POSTGRES_MANIFEST_SQL_JSON__ TURN_TRACE_PLUGIN_SOURCE = __TURN_TRACE_PLUGIN_SOURCE_JSON__ TURN_TRACE_PLUGIN_MANIFEST = __TURN_TRACE_PLUGIN_MANIFEST_JSON__ def canonical_sha256(value): return hashlib.sha256( json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str).encode("utf-8") ).hexdigest() def text_sha256(value): return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest() def process_identity(): pid = os.getpid() try: start_ticks = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8").split()[21] except Exception: start_ticks = "unavailable" return { "process_pid": pid, "process_start_ticks": start_ticks, "process_identity_sha256": text_sha256(f"{RUN_ID}:{pid}:{start_ticks}"), } def service_state(): raw = subprocess.check_output( [ "systemctl", "show", "leoclean-gateway.service", "-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts", "-p", "ExecMainStartTimestamp", ], text=True, ) return dict(line.split("=", 1) for line in raw.splitlines() if "=" in line) def db_read_marker(): sql = """ select jsonb_build_object( 'database', current_database(), 'database_user', current_user, 'system_identifier', (select system_identifier::text from pg_control_system()), 'wal_lsn', pg_current_wal_lsn()::text )::text; """ raw = subprocess.check_output( ["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"], input=sql, text=True, stderr=subprocess.STDOUT, ) return json.loads(raw.strip()) def db_fingerprint(): marker_before = db_read_marker() raw = subprocess.check_output( ["docker", "exec", "-i", "teleo-pg", "psql", "-U", "postgres", "-d", "teleo", "-At", "-q"], input=POSTGRES_MANIFEST_SQL, text=True, stderr=subprocess.STDOUT, timeout=180, ) marker_after = db_read_marker() rows = [] for line in raw.splitlines(): candidate = line.strip() if not candidate or candidate in {"BEGIN", "SET", "ROLLBACK"}: continue value = json.loads(candidate) if isinstance(value, dict): rows.append(value) tables = { f"{row['schema']}.{row['table']}": { "row_count": int(row["row_count"]), "rowset_md5": row["rowset_md5"], } for row in rows if row.get("kind") == "table" } singleton = { str(row["kind"]): row.get("items", []) for row in rows if row.get("kind") in { "schemas", "extensions", "application_roles", "columns", "constraints", "indexes", "sequences", "views", "functions", "triggers", "types", "policies", } } identities = [row for row in rows if row.get("kind") == "identity"] if len(identities) != 1 or not tables: raise RuntimeError("canonical database fingerprint manifest is incomplete") identity = identities[0] stable = { "identity": { key: identity.get(key) for key in ( "database", "server_version_num", "server_encoding", "database_collation", "database_ctype", "transaction_read_only", ) }, "singleton_sha256": { key: canonical_sha256(value) for key, value in sorted(singleton.items()) }, "tables": tables, } return { "schema": "livingip.teleoCanonicalDatabaseFingerprint.v1", "status": "ok", "fingerprint_sha256": canonical_sha256(stable), "table_count": len(tables), "total_rows": sum(item["row_count"] for item in tables.values()), "table_rows_sha256": canonical_sha256(tables), "structure_sha256": canonical_sha256(stable["singleton_sha256"]), "read_consistency": { "status": "stable_wal_marker" if marker_before == marker_after else "wal_changed_during_fingerprint", "before": marker_before, "after": marker_after, }, } def safe_db_fingerprint(): try: return db_fingerprint() except Exception as exc: return { "schema": "livingip.teleoCanonicalDatabaseFingerprint.v1", "status": "error", "error": type(exc).__name__, } def write_report(report): try: REPORT_PATH.write_text(json.dumps(report, sort_keys=True, default=str), encoding="utf-8") REPORT_PATH.chmod(0o600) except Exception: pass def read_jsonl(path): if not path or not path.exists(): return [] values = [] for line in path.read_text(encoding="utf-8").splitlines(): try: value = json.loads(line) except json.JSONDecodeError: continue if isinstance(value, dict): values.append(value) return values def safe_file_manifest(profile): included = [] for relative in ( "config.yaml", "SOUL.md", "bin/kb_tool.py", "bin/_kb_tool_readonly_impl.py", "plugins/leo-db-context/__init__.py", "plugins/leo-db-context/plugin.yaml", "plugins/leo-turn-execution-trace/__init__.py", "plugins/leo-turn-execution-trace/plugin.yaml", ): path = profile / relative included.append( { "path": relative, "sha256": hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None, "is_symlink": path.is_symlink(), } ) state_dirs = {} for name in ("memories", "sessions", "state", "skills", "workspace"): path = profile / name state_dirs[name] = sorted(item.name for item in path.iterdir()) if path.is_dir() else None safe = { "included_files": included, "state_directories_at_seed": state_dirs, "secret_files_copied_but_not_hashed_or_retained": [".env", "auth.json"], "symlink_count": sum(path.is_symlink() for path in profile.rglob("*")), } return {**safe, "manifest_sha256": canonical_sha256(safe)} def build_leo_profile(tmp_root): profile = tmp_root / "leo-profile" profile.mkdir(mode=0o700) for name in (".env", "auth.json", "config.yaml", "SOUL.md"): shutil.copy2(LIVE_PROFILE / name, profile / name) if (LIVE_PROFILE / "models_dev_cache.json").is_file(): shutil.copy2(LIVE_PROFILE / "models_dev_cache.json", profile / "models_dev_cache.json") config = yaml.safe_load((profile / "config.yaml").read_text(encoding="utf-8")) or {} memory = config.setdefault("memory", {}) memory["enabled"] = False memory["memory_enabled"] = False memory["user_profile_enabled"] = False config["tools"] = {key: False for key in (config.get("tools") or {})} config["toolsets"] = [] config.setdefault("skills", {})["external_dirs"] = [] config.setdefault("checkpoints", {})["enabled"] = False config.setdefault("compression", {})["enabled"] = False config.setdefault("gateway", {}).setdefault("telegram", {})["enabled"] = False (profile / "config.yaml").write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") for name in ("memories", "sessions", "state", "plugins", "skills", "workspace", "bin", "cache", "logs"): (profile / name).mkdir(mode=0o700, exist_ok=True) db_plugin = profile / "plugins" / "leo-db-context" db_plugin.mkdir(parents=True) db_plugin_source = DB_CONTEXT_PLUGIN_SOURCE.replace( '"--limit",\n "0",', '"--limit",\n "4",' ).replace('"--context-limit",\n "0",', '"--context-limit",\n "6",') (db_plugin / "__init__.py").write_text(db_plugin_source, encoding="utf-8") (db_plugin / "plugin.yaml").write_text( 'name: leo-db-context\nversion: "1.0.0"\ndescription: Read-only current database context.\n' 'provides_hooks:\n - pre_llm_call\n - post_llm_call\n', encoding="utf-8", ) trace_plugin = profile / "plugins" / "leo-turn-execution-trace" trace_plugin.mkdir(parents=True) (trace_plugin / "__init__.py").write_text(TURN_TRACE_PLUGIN_SOURCE, encoding="utf-8") (trace_plugin / "plugin.yaml").write_text(TURN_TRACE_PLUGIN_MANIFEST, encoding="utf-8") implementation = profile / "bin" / "_kb_tool_readonly_impl.py" implementation.write_text(KB_TOOL_SOURCE, encoding="utf-8") implementation.chmod(0o700) wrapper = profile / "bin" / "kb_tool.py" wrapper.write_text(READ_ONLY_KB_WRAPPER_SOURCE, encoding="utf-8") wrapper.chmod(0o700) manifest = safe_file_manifest(profile) contract = { "profile_realpath": str(profile.resolve()), "profile_realpath_sha256": text_sha256(str(profile.resolve())), "profile_manifest": manifest, "memory_seed_empty": manifest["state_directories_at_seed"]["memories"] == [], "session_seed_empty": manifest["state_directories_at_seed"]["sessions"] == [], "state_seed_empty": manifest["state_directories_at_seed"]["state"] == [], "tools_enabled": any(bool(value) for value in config.get("tools", {}).values()) or bool(config.get("toolsets")), "kb_plugin_present": (db_plugin / "__init__.py").is_file(), "soul_sha256": hashlib.sha256((profile / "SOUL.md").read_bytes()).hexdigest(), } return profile, contract def conversation_rows(messages): rows = [] for message in messages: content = message.get("content") text = content if isinstance(content, str) else json.dumps(content, sort_keys=True, default=str) tool_calls = message.get("tool_calls") or [] rows.append( { "role": message.get("role"), "content_sha256": text_sha256(text), "content_bytes": len(text.encode("utf-8")), "tool_calls_sha256": canonical_sha256(tool_calls), "tool_call_count": len(tool_calls), } ) return rows def conversation_trace(messages): rows = conversation_rows(messages) return { "message_count": len(rows), "messages_sha256": canonical_sha256(rows), "last_message_role": rows[-1]["role"] if rows else None, "last_message_content_sha256": rows[-1]["content_sha256"] if rows else None, } def current_agent_messages(runner, session_key): cache = getattr(runner, "_agent_cache", None) or {} cache_lock = getattr(runner, "_agent_cache_lock", None) if cache_lock is not None: with cache_lock: cached = cache.get(session_key) else: cached = cache.get(session_key) if not cached: return [] messages = getattr(cached[0], "_session_messages", None) or [] return [copy.deepcopy(message) for message in messages if isinstance(message, dict)] def model_response_trace(messages): return [row for message, row in zip(messages, conversation_rows(messages)) if message.get("role") == "assistant"] def tool_audit(messages): calls = [] forbidden_pattern = re.compile( r"(?:propose(?:-|_)|record-document|approve_proposal|apply_proposal|stage_normalized|" r"\b(?:insert|update|delete|alter|drop|truncate|create)\b)", re.IGNORECASE, ) for message in messages: if not isinstance(message, dict): continue for call in message.get("tool_calls") or []: function = call.get("function") if isinstance(call, dict) else {} name = str((function or {}).get("name") or (call or {}).get("name") or "") arguments = (function or {}).get("arguments") if isinstance(function, dict) else None raw = arguments if isinstance(arguments, str) else json.dumps(arguments, sort_keys=True, default=str) calls.append( { "tool_name": name, "arguments_sha256": text_sha256(raw), "forbidden_mutation": bool(forbidden_pattern.search(f"{name} {raw}")), } ) return { "tool_call_count": len(calls), "calls": calls, "forbidden_mutation_detected": any(call["forbidden_mutation"] for call in calls), "unknown_tool_call_detected": bool(calls), } def database_tool_trace(context_events): calls = [] for event in context_events: receipt = event.get("retrieval_receipt") if isinstance(event, dict) else None if event.get("event", "pre_llm_call") != "pre_llm_call" or not isinstance(receipt, dict): continue row_ids = [str(item) for item in receipt.get("claim_ids") or []] source_ids = [str(item) for item in receipt.get("source_ids") or []] calls.append( { "database_invocations": [ {"executable": "kb_tool.py", "subcommand": "context", "access_mode": "read_only"} ], "result": { "row_ids": row_ids, "source_ids": source_ids, "content_sha256": receipt.get("receipt_sha256"), "nonempty": bool(row_ids or source_ids), "error_detected": False, }, } ) return { "database_tool_call_count": len(calls), "database_tool_completed_count": len(calls), "database_tool_call_proven": bool(calls), "database_retrieval_receipt_proven": bool(calls), "database_tool_calls_read_only": bool(calls), "calls": calls, } def leo_worker(profile, connection): identity = process_identity() try: os.environ["HERMES_HOME"] = str(profile) os.environ["HOME"] = "/home/teleo" context_trace_path = profile / "state" / "leo-db-context-trace.jsonl" model_trace_path = profile / "state" / "leo-turn-api-trace.jsonl" guard_trace_path = profile / "state" / "leo-kb-guard-trace.jsonl" os.environ["LEO_DB_CONTEXT_TRACE_PATH"] = str(context_trace_path) os.environ["LEO_TURN_API_TRACE_PATH"] = str(model_trace_path) os.environ["LEO_KB_GUARD_TRACE_PATH"] = str(guard_trace_path) sys.path.insert(0, str(AGENT_ROOT)) from gateway.config import Platform from gateway.platforms.base import MessageEvent, MessageType from gateway.run import GatewayRunner from gateway.session import SessionSource source = SessionSource( platform=Platform.TELEGRAM, chat_id=CHAT_ID, chat_name="Leo isolated no-send canary", chat_type="group", user_id=USER_ID, user_name="isolated challenger-loop operator", ) runner = GatewayRunner() session_key = runner._session_key_for_source(source) ready = { "event": "ready", **identity, "authorized": runner._is_user_authorized(source), "session_key_sha256": text_sha256(session_key), } connection.send(ready) if not ready["authorized"]: return async def loop(): while True: command = await asyncio.to_thread(connection.recv) if command.get("action") == "stop": connection.send({"event": "stopped", **identity}) return prompt = str(command.get("prompt") or "") turn_id = str(command.get("turn_id") or "") context_start = len(read_jsonl(context_trace_path)) model_start = len(read_jsonl(model_trace_path)) messages_before = current_agent_messages(runner, session_key) event = MessageEvent( text=prompt, message_type=MessageType.TEXT, source=source, message_id=f"agent-challenger-{RUN_ID}-{turn_id}", ) started = datetime.now(timezone.utc) reply = await asyncio.wait_for(runner._handle_message(event), timeout=360) ended = datetime.now(timezone.utc) messages_after = current_agent_messages(runner, session_key) before_rows = conversation_rows(messages_before) after_rows = conversation_rows(messages_after) prefix = after_rows[: len(before_rows)] == before_rows new_messages = messages_after[len(messages_before):] if prefix else messages_after context_events = read_jsonl(context_trace_path)[context_start:] connection.send( { "event": "turn", "turn_id": turn_id, "actor": "leo", "input_prompt": prompt, "message": reply or "", "started_at_utc": started.isoformat(), "ended_at_utc": ended.isoformat(), "process_identity_sha256": identity["process_identity_sha256"], "handler_session_key_sha256": text_sha256(session_key), "model_call_trace": read_jsonl(model_trace_path)[model_start:], "model_response_trace": model_response_trace(new_messages), "database_context_trace": context_events, "database_tool_trace": database_tool_trace(context_events), "tool_audit": tool_audit(new_messages), "conversation_before": conversation_trace(messages_before), "conversation_after": conversation_trace(messages_after), "conversation_history_prefix_preserved": prefix, } ) asyncio.run(loop()) except EOFError: return except Exception as exc: try: connection.send( { "event": "error", "error": type(exc).__name__, "message": str(exc), "traceback_tail": traceback.format_exc().splitlines()[-12:], **identity, } ) except Exception: pass finally: connection.close() def recv_with_timeout(connection, timeout): if not connection.poll(timeout): raise TimeoutError(f"Leo worker did not reply within {timeout}s") value = connection.recv() if not isinstance(value, dict): raise RuntimeError("Leo worker returned a non-object") if value.get("event") == "error": raise RuntimeError(f"Leo worker error: {value.get('error')}: {value.get('message')}") return value async def challenger_turn(prompt, session_id_sha256, process_row): if not OPENROUTER_KEY.is_file(): raise RuntimeError("OpenRouter key file unavailable") key = OPENROUTER_KEY.read_text(encoding="utf-8").strip() started = datetime.now(timezone.utc) payload = { "model": CHALLENGER_MODEL, "messages": [ {"role": "system", "content": protocol.CHALLENGER_SOUL}, {"role": "user", "content": prompt}, ], "max_tokens": CHALLENGER_MAX_TOKENS, "temperature": 0.2, } async with aiohttp.ClientSession() as session: async with session.post( OPENROUTER_URL, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, json=payload, timeout=aiohttp.ClientTimeout(total=180), ) as response: status = response.status data = await response.json() if status >= 400: raise RuntimeError(f"challenger provider returned HTTP {status}") message = str((data.get("choices") or [{}])[0].get("message", {}).get("content") or "").strip() if not message: raise RuntimeError("challenger provider returned an empty response") ended = datetime.now(timezone.utc) prompt_sha = text_sha256(prompt) reply_sha = text_sha256(message) provider_id_sha = text_sha256(str(data.get("id") or "")) usage = { key: value for key, value in (data.get("usage") or {}).items() if key in {"prompt_tokens", "completion_tokens", "total_tokens"} and isinstance(value, (int, float)) } trace = [ { "event": "turn_pre_llm_call", "session_id_sha256": session_id_sha256, "prompt_sha256": prompt_sha, }, { "event": "post_api_request", "session_id_sha256": session_id_sha256, "prompt_sha256": prompt_sha, "provider": "openrouter", "model": CHALLENGER_MODEL, "response_model": data.get("model") or CHALLENGER_MODEL, "provider_response_id_sha256": provider_id_sha, "usage": usage, }, { "event": "turn_post_llm_call", "session_id_sha256": session_id_sha256, "prompt_sha256": prompt_sha, "raw_assistant_response_sha256": reply_sha, }, ] rows = [ { "role": "user", "content_sha256": prompt_sha, "content_bytes": len(prompt.encode("utf-8")), "tool_calls_sha256": canonical_sha256([]), "tool_call_count": 0, }, { "role": "assistant", "content_sha256": reply_sha, "content_bytes": len(message.encode("utf-8")), "tool_calls_sha256": canonical_sha256([]), "tool_call_count": 0, }, ] return { "actor": "challenger", "input_prompt": prompt, "message": message, "started_at_utc": started.isoformat(), "ended_at_utc": ended.isoformat(), "process_identity_sha256": process_row["process_identity_sha256"], "handler_session_key_sha256": session_id_sha256, "model_call_trace": trace, "model_response_trace": [rows[-1]], "database_context_trace": [], "database_tool_trace": { "database_tool_call_count": 0, "database_tool_completed_count": 0, "database_tool_call_proven": False, "database_retrieval_receipt_proven": False, "database_tool_calls_read_only": True, "calls": [], }, "tool_audit": { "tool_call_count": 0, "calls": [], "forbidden_mutation_detected": False, "unknown_tool_call_detected": False, }, "conversation_before": { "message_count": 0, "messages_sha256": canonical_sha256([]), "last_message_role": None, "last_message_content_sha256": None, }, "conversation_after": { "message_count": 2, "messages_sha256": canonical_sha256(rows), "last_message_role": "assistant", "last_message_content_sha256": reply_sha, }, "conversation_history_prefix_preserved": True, "stateless_input_rebuilt": True, "usage": usage, } def build_safety_gate(report): turns = report.get("turns") or [] before = report.get("db_fingerprint_before") or {} after = report.get("db_fingerprint_after") or {} cleanup = report.get("cleanup") or {} service = report.get("service_before_after") or {} checks = { "six_runtime_turns_completed": len(turns) == 6 and all(str(item.get("message") or "").strip() for item in turns), "leo_worker_authorized": (report.get("handler") or {}).get("authorized") is True, "leo_model_tools_disabled": (report.get("isolation") or {}).get("roles", {}).get("leo", {}).get("tools_enabled") is False, "read_only_guard_blocked_mutation_preflight": (report.get("read_only_guard_preflight") or {}).get("blocked") is True, "no_turn_used_model_tools": all((item.get("tool_audit") or {}).get("tool_call_count") == 0 for item in turns), "no_telegram_post": report.get("posted_to_telegram") is False, "harness_declared_no_kb_mutation": report.get("mutates_kb_by_harness") is False, "database_fingerprint_unchanged": report.get("db_fingerprint_unchanged") is True and before.get("fingerprint_sha256") == after.get("fingerprint_sha256") and before.get("table_rows_sha256") == after.get("table_rows_sha256") and before.get("structure_sha256") == after.get("structure_sha256"), "live_behavior_unchanged": report.get("live_behavior_manifest_unchanged") is True, "gateway_service_unchanged": service.get("unchanged_from_preexisting_live_readback") is True, "workers_and_profiles_cleaned": cleanup.get("all_workers_stopped") is True and cleanup.get("temp_root_removed") is True and cleanup.get("orphan_worker_count") == 0, "no_runtime_error": not report.get("error"), } failed = [name for name, value in checks.items() if value is not True] return {"status": "pass" if not failed else "fail", "checks": checks, "failed_checks": failed} def run_suite(): protocol_module = types.ModuleType("leo_agent_challenger_protocol_embedded") protocol_module.__file__ = "" exec(compile(PROTOCOL_SOURCE, protocol_module.__file__, "exec"), protocol_module.__dict__) globals()["protocol"] = protocol_module behavior_module = types.ModuleType("leo_behavior_manifest_embedded") behavior_module.__file__ = "" exec(compile(BEHAVIOR_MANIFEST_SOURCE, behavior_module.__file__, "exec"), behavior_module.__dict__) parent_identity = process_identity() before_service = service_state() before_fingerprint = safe_db_fingerprint() behavior_before = behavior_module.build_manifest(LIVE_PROFILE, AGENT_ROOT, DEPLOY_ROOT) tmp_root = Path(tempfile.mkdtemp(prefix="leo-agent-challenger-loop-")) profile = None worker = None parent_connection = None worker_ready = None stopped_event = None report = { "schema": protocol.REPORT_SCHEMA, "run_id": RUN_ID, "generated_at_utc": datetime.now(timezone.utc).isoformat(), "mode": "live_vps_isolated_agent_challenger_no_send_no_write", "objective": OBJECTIVE, "challenger_model": CHALLENGER_MODEL, "challenger_max_tokens_per_turn": CHALLENGER_MAX_TOKENS, "posted_to_telegram": False, "mutates_kb_by_harness": False, "db_fingerprint_before": before_fingerprint, "before_service": before_service, "live_behavior_manifest_before": behavior_before, "turns": [], "remote_report_path": str(REPORT_PATH), } write_report(report) try: profile, profile_contract = build_leo_profile(tmp_root) guard_env = os.environ.copy() guard_env["LEO_KB_GUARD_TRACE_PATH"] = str(profile / "state" / "guard-preflight.jsonl") guard = subprocess.run( [sys.executable, str(profile / "bin" / "kb_tool.py"), "--local", "propose-edge", "a", "supports", "b", "--rationale", "must block"], text=True, capture_output=True, env=guard_env, timeout=15, ) report["read_only_guard_preflight"] = { "blocked": guard.returncode == 77, "returncode": guard.returncode, "stderr_sha256": text_sha256(guard.stderr), } if guard.returncode != 77: raise RuntimeError("read-only KB guard did not block mutation preflight") ctx = multiprocessing.get_context("fork") parent_connection, child_connection = ctx.Pipe() worker = ctx.Process(target=leo_worker, args=(profile, child_connection), name=f"leo-challenger-{RUN_ID}") worker.start() child_connection.close() worker_ready = recv_with_timeout(parent_connection, 90) if worker_ready.get("event") != "ready" or worker_ready.get("authorized") is not True: raise RuntimeError("Leo worker failed authorization preflight") challenger_session_id = f"challenger:{RUN_ID}:{uuid.uuid4().hex}" challenger_session_sha = text_sha256(challenger_session_id) challenger_role = { "role": "challenger", "runtime_kind": "stateless_openrouter_agent", **parent_identity, "session_key_sha256": challenger_session_sha, "profile_realpath": None, "profile_realpath_sha256": text_sha256("no-profile:stateless-openrouter-agent"), "profile_manifest": { "kind": "no_writable_profile", "system_prompt_sha256": text_sha256(protocol.CHALLENGER_SOUL), "manifest_sha256": canonical_sha256( {"kind": "no_writable_profile", "system_prompt_sha256": text_sha256(protocol.CHALLENGER_SOUL)} ), }, "memory_seed_empty": True, "session_seed_empty": True, "state_seed_empty": True, "tools_enabled": False, "kb_plugin_present": False, "soul_sha256": text_sha256(protocol.CHALLENGER_SOUL), "input_boundary": "fixed role contract plus user objective plus transcript-visible messages only", } challenger_role["contract_sha256"] = protocol.compute_runtime_role_contract_sha256(challenger_role) leo_role = { "role": "leo", "runtime_kind": "gatewayrunner_temp_profile_no_model_tools", **{key: worker_ready.get(key) for key in ("process_pid", "process_start_ticks", "process_identity_sha256")}, "session_key_sha256": worker_ready.get("session_key_sha256"), **profile_contract, "input_boundary": "challenger message plus Leo-local prior conversation and read-only current DB context hook", } leo_role["contract_sha256"] = protocol.compute_runtime_role_contract_sha256(leo_role) report["handler"] = { "authorized": worker_ready.get("authorized"), "session_key_sha256": worker_ready.get("session_key_sha256"), } report["isolation"] = { "distinct_processes": challenger_role["process_identity_sha256"] != leo_role["process_identity_sha256"], "writable_roots_disjoint": True, "distinct_profile_roots": True, "transcript_only_bridge": True, "roles": {"challenger": challenger_role, "leo": leo_role}, } write_report(report) async def exchange(): for step in (1, 2, 3): prompt = protocol.build_challenger_prompt(step, OBJECTIVE, report["turns"]) challenger = await challenger_turn(prompt, challenger_session_sha, parent_identity) challenger["turn_id"] = f"C{step}" challenger["visible_turn_ids"] = protocol.expected_visible_turn_ids(step) challenger["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(challenger) report["turns"].append(challenger) write_report(report) parent_connection.send( {"action": "turn", "turn_id": f"L{step}", "prompt": challenger["message"]} ) leo = await asyncio.to_thread(recv_with_timeout, parent_connection, 420) if leo.get("event") != "turn": raise RuntimeError("unexpected Leo worker response") leo.pop("event", None) leo["visible_turn_ids"] = [f"C{step}"] leo["turn_execution_sha256"] = protocol.compute_turn_execution_sha256(leo) report["turns"].append(leo) write_report(report) asyncio.run(exchange()) report["proposal_packet"] = protocol.build_proposal_packet(report) except Exception as exc: report["error"] = f"{type(exc).__name__}: {exc}" report["traceback_tail"] = traceback.format_exc().splitlines()[-16:] finally: if parent_connection is not None: try: parent_connection.send({"action": "stop"}) stopped_event = recv_with_timeout(parent_connection, 30) except Exception: stopped_event = None if worker is not None: worker.join(timeout=30) if worker.is_alive(): worker.terminate() worker.join(timeout=10) worker_exitcode = worker.exitcode if worker is not None else None worker_alive = worker.is_alive() if worker is not None else False after_fingerprint = safe_db_fingerprint() after_service = service_state() behavior_after = behavior_module.build_manifest(LIVE_PROFILE, AGENT_ROOT, DEPLOY_ROOT) report["db_fingerprint_after"] = after_fingerprint report["db_fingerprint_unchanged"] = bool( before_fingerprint.get("status") == "ok" and after_fingerprint.get("status") == "ok" and before_fingerprint.get("fingerprint_sha256") == after_fingerprint.get("fingerprint_sha256") and before_fingerprint.get("table_rows_sha256") == after_fingerprint.get("table_rows_sha256") and before_fingerprint.get("structure_sha256") == after_fingerprint.get("structure_sha256") ) report["live_behavior_manifest_after"] = behavior_after report["live_behavior_manifest_unchanged"] = behavior_before.get("behavior_sha256") == behavior_after.get("behavior_sha256") report["service_before_after"] = { "before": before_service, "after": after_service, "unchanged_from_preexisting_live_readback": before_service == after_service, } leo_profile_removed = True if profile is not None and profile.exists(): shutil.rmtree(profile, ignore_errors=True) leo_profile_removed = not profile.exists() if tmp_root.exists(): shutil.rmtree(tmp_root, ignore_errors=True) report["cleanup"] = { "challenger_profile_removed": True, "leo_profile_removed": leo_profile_removed, "temp_root_removed": not tmp_root.exists(), "all_workers_stopped": not worker_alive and worker_exitcode == 0 and bool(stopped_event), "orphan_worker_count": 0 if not worker_alive else 1, "leo_worker_exitcode": worker_exitcode, "graceful_stop_receipt": bool(stopped_event and stopped_event.get("event") == "stopped"), } report["safety_gate"] = build_safety_gate(report) write_report(report) return report print(json.dumps(run_suite(), sort_keys=True, default=str)) ''' def _patched_db_context_source() -> str: source = DB_CONTEXT_PLUGIN.read_text(encoding="utf-8") patched = source.replace('"--limit",\n "0",', '"--limit",\n "4",').replace( '"--context-limit",\n "0",', '"--context-limit",\n "6",' ) if patched == source: raise RuntimeError("database context source no longer matches the bounded harness patch") return patched def build_remote_script( run_id: str, *, objective: str = protocol.DEFAULT_OBJECTIVE, challenger_model: str = DEFAULT_MODEL, challenger_max_tokens: int = DEFAULT_MAX_TOKENS, report_prefix: str = DEFAULT_REPORT_PREFIX, ) -> str: if not re.fullmatch(r"[A-Za-z0-9._-]+", report_prefix): raise ValueError("report_prefix must contain only letters, numbers, dot, underscore, or hyphen") if not 128 <= challenger_max_tokens <= 2048: raise ValueError("challenger_max_tokens must be between 128 and 2048") wrapper = READ_ONLY_KB_WRAPPER.replace("__READ_ONLY_COMMANDS__", repr(set(READ_ONLY_COMMANDS))) return ( REMOTE_SCRIPT.replace("__CHAT_ID__", handler_harness.CHAT_ID) .replace("__USER_ID__", handler_harness.USER_ID) .replace("__RUN_ID__", run_id) .replace("__REPORT_PREFIX_JSON__", json.dumps(report_prefix)) .replace("__OBJECTIVE_JSON__", json.dumps(objective)) .replace("__CHALLENGER_MODEL_JSON__", json.dumps(challenger_model)) .replace("__CHALLENGER_MAX_TOKENS__", str(challenger_max_tokens)) .replace("__PROTOCOL_SOURCE_JSON__", json.dumps((ROOT / "scripts" / "leo_agent_challenger_protocol.py").read_text(encoding="utf-8"))) .replace("__DB_CONTEXT_PLUGIN_SOURCE_JSON__", json.dumps(_patched_db_context_source())) .replace("__KB_TOOL_SOURCE_JSON__", json.dumps(KB_TOOL.read_text(encoding="utf-8"))) .replace("__READ_ONLY_KB_WRAPPER_SOURCE_JSON__", json.dumps(wrapper)) .replace("__BEHAVIOR_MANIFEST_SOURCE_JSON__", json.dumps(BEHAVIOR_MANIFEST.read_text(encoding="utf-8"))) .replace("__POSTGRES_MANIFEST_SQL_JSON__", json.dumps(POSTGRES_MANIFEST.read_text(encoding="utf-8"))) .replace("__TURN_TRACE_PLUGIN_SOURCE_JSON__", json.dumps(TURN_TRACE_PLUGIN_SOURCE)) .replace("__TURN_TRACE_PLUGIN_MANIFEST_JSON__", json.dumps(TURN_TRACE_PLUGIN_MANIFEST)) ) def fetch_remote_report(run_id: str, *, report_prefix: str = DEFAULT_REPORT_PREFIX) -> dict[str, Any] | None: report_path = f"/tmp/{report_prefix}-{run_id}.json" proc = subprocess.run( [ "ssh", "-i", str(handler_harness.SSH_KEY), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", handler_harness.VPS, "sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -", ], input=( "from pathlib import Path\n" f"p=Path({report_path!r})\n" "if p.exists():\n" " print(p.read_text(encoding='utf-8'))\n" " p.unlink()\n" ), text=True, capture_output=True, timeout=60, ) if proc.returncode != 0 or not proc.stdout.strip(): return None return json.loads(handler_harness.redact(proc.stdout.strip())) def run_remote( *, objective: str = protocol.DEFAULT_OBJECTIVE, challenger_model: str = DEFAULT_MODEL, challenger_max_tokens: int = DEFAULT_MAX_TOKENS, report_prefix: str = DEFAULT_REPORT_PREFIX, ) -> dict[str, Any]: run_id = uuid.uuid4().hex[:12] proc = subprocess.run( [ "ssh", "-i", str(handler_harness.SSH_KEY), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", handler_harness.VPS, "cd /home/teleo && sudo -u teleo -H /home/teleo/.hermes/hermes-agent/venv/bin/python -", ], input=build_remote_script( run_id, objective=objective, challenger_model=challenger_model, challenger_max_tokens=challenger_max_tokens, report_prefix=report_prefix, ), text=True, capture_output=True, timeout=3600, ) result: dict[str, Any] = { "returncode": proc.returncode, "stdout": handler_harness.redact(proc.stdout), "stderr": handler_harness.redact(proc.stderr), "run_id": run_id, } for line in reversed(proc.stdout.splitlines()): candidate = line.strip() if not candidate.startswith("{"): continue try: result["parsed"] = json.loads(handler_harness.redact(candidate)) break except json.JSONDecodeError: continue fetched = fetch_remote_report(run_id, report_prefix=report_prefix) if "parsed" not in result and fetched is not None: result["parsed"] = fetched result["parsed_from_report_file"] = True return result def write_outputs(remote: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: REPORT_DIR.mkdir(parents=True, exist_ok=True) report = remote.get("parsed") if isinstance(remote.get("parsed"), dict) else {} report["remote_returncode"] = remote.get("returncode") report["remote_run_id"] = remote.get("run_id") if remote.get("stderr"): report["remote_stderr"] = remote["stderr"] OUTPUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") source_sha = hashlib.sha256(OUTPUT_JSON.read_bytes()).hexdigest() receipt = protocol.verify_report(report, source_report_sha256=source_sha) RECEIPT_JSON.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") negative = { "schema": "livingip.leoAgentChallengerNegativeControlReceipt.v1", "benchmark_id": protocol.BENCHMARK_ID, "source_report_sha256": source_sha, "status": ( "pass" if receipt.get("negative_controls") and all(item.get("status") == "caught" for item in receipt["negative_controls"]) else "fail" ), "controls": receipt.get("negative_controls") or [], } NEGATIVE_JSON.write_text(json.dumps(negative, indent=2, sort_keys=True) + "\n", encoding="utf-8") ledger_event = { "generated_at_utc": datetime.now(timezone.utc).isoformat(), "benchmark_id": protocol.BENCHMARK_ID, "remote_run_id": remote.get("run_id"), "status": receipt.get("status"), "required_tier": receipt.get("required_tier"), "current_tier": receipt.get("current_tier"), "source_report_sha256": source_sha, "proposal_packet_sha256": receipt.get("proposal_packet_sha256"), "negative_controls": [item.get("status") for item in receipt.get("negative_controls") or []], "decision": "keep" if receipt.get("status") == "pass" else "repair", } with LEDGER_JSONL.open("a", encoding="utf-8") as handle: handle.write(json.dumps(ledger_event, sort_keys=True) + "\n") return report, receipt def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--challenger-model", default=DEFAULT_MODEL) parser.add_argument("--challenger-max-tokens", type=int, default=DEFAULT_MAX_TOKENS) parser.add_argument("--objective", default=protocol.DEFAULT_OBJECTIVE) args = parser.parse_args() remote = run_remote( objective=args.objective, challenger_model=args.challenger_model, challenger_max_tokens=args.challenger_max_tokens, ) report, receipt = write_outputs(remote) print( json.dumps( { "source": str(OUTPUT_JSON), "receipt": str(RECEIPT_JSON), "negative_controls": str(NEGATIVE_JSON), "runtime_safety": (report.get("safety_gate") or {}).get("status"), "verification": receipt.get("status"), }, indent=2, sort_keys=True, ) ) return 0 if receipt.get("status") == "pass" else 1 if __name__ == "__main__": raise SystemExit(main())