#!/usr/bin/env python3 """Run and score the read-only m3taversal out-of-sample suite on the live VPS handler.""" from __future__ import annotations import argparse import hashlib import json import re import subprocess from datetime import datetime, timezone from pathlib import Path from typing import Any import leo_oos_readonly_guard as readonly_guard import run_leo_direct_claim_handler_suite as handler import working_leo_m3taversal_oos_benchmark as benchmark import working_leo_m3taversal_oos_protocol as protocol_lib BASE_HANDLER_SCRIPT_BUILDER = handler.build_remote_script ROOT = Path(__file__).resolve().parents[1] REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709" RESULTS_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-current.json" SCORE_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.json" SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.md" PROTOCOL_REPORT_DIR = ROOT / "docs" / "reports" / "leo-oos-reasoning-benchmark-20260715" GROUNDING_MODES = ("grounded", "db_tool_ablated") RUNTIME_PROMPT_SCAN_ROOTS = ( ROOT / "hermes-agent" / "leoclean-skills", ROOT / "hermes-agent" / "leoclean-plugins", ROOT / "hermes-agent" / "leoclean-bin", ) def prompt_leakage_scan(protocol: dict[str, Any]) -> dict[str, Any]: """Fail if a frozen prompt was copied into a runtime skill/plugin/tool.""" prompt_markers = [ (str(prompt["id"]), str(marker)) for trial in protocol.get("trials") or [] for prompt in trial.get("prompts") or [] for marker in prompt.get("leakage_markers") or [] ] matches: list[dict[str, str]] = [] scanned_files = 0 scanned_bytes = 0 scan_errors: list[dict[str, str]] = [] symlink_entries: list[dict[str, str]] = [] expected_roots = [ { "path": str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path), "exists": path.exists(), "is_symlink": path.is_symlink(), } for path in RUNTIME_PROMPT_SCAN_ROOTS ] for root in RUNTIME_PROMPT_SCAN_ROOTS: if not root.exists() or root.is_symlink(): continue try: entries = sorted(root.rglob("*")) except OSError as exc: scan_errors.append( { "path_sha256": hashlib.sha256(str(root).encode()).hexdigest(), "error_type": type(exc).__name__, } ) continue for path in entries: if path.is_symlink(): symlink_entries.append({"path_sha256": hashlib.sha256(str(path).encode()).hexdigest()}) paths = [item for item in entries if item.is_file() and not item.is_symlink()] for path in paths: try: text = path.read_text(encoding="utf-8") except UnicodeDecodeError: continue except OSError as exc: scan_errors.append( { "path_sha256": hashlib.sha256(str(path).encode()).hexdigest(), "error_type": type(exc).__name__, } ) continue scanned_files += 1 scanned_bytes += len(text.encode()) normalized = " ".join(re.findall(r"[a-z0-9_]+", text.lower())) for prompt_id, marker in prompt_markers: if marker in normalized: display_path = str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path) matches.append( { "prompt_id": prompt_id, "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(), "path": display_path, } ) return { "expected_roots": expected_roots, "scanned_roots": [item["path"] for item in expected_roots], "scanned_files": scanned_files, "scanned_bytes": scanned_bytes, "errors": scan_errors, "symlink_entries": symlink_entries, "exact_prompt_matches": matches, "pass": bool( all(item["exists"] and not item["is_symlink"] for item in expected_roots) and scanned_files > 0 and scanned_bytes > 0 and not scan_errors and not symlink_entries and not matches ), } def build_guarded_remote_script( run_id: str, *, prompts: list[dict[str, Any]], suite_mode: str, report_prefix: str, prompt_note: str, grounding_mode: str, leakage_scan: dict[str, Any], ) -> str: """Inject the reviewed fail-closed tool surface before GatewayRunner starts.""" if grounding_mode not in GROUNDING_MODES: raise ValueError(f"unsupported grounding mode: {grounding_mode}") script = BASE_HANDLER_SCRIPT_BUILDER( run_id, prompts=prompts, suite_mode=suite_mode, report_prefix=report_prefix, prompt_note=prompt_note, ) original_plugin_source = handler.DB_CONTEXT_PLUGIN.read_text(encoding="utf-8") instrumented_plugin_source = protocol_lib.instrument_db_context_plugin_source(original_plugin_source) encoded_original_plugin_source = json.dumps(original_plugin_source) if script.count(encoded_original_plugin_source) != 1: raise RuntimeError("embedded DB context plugin source marker changed") script = script.replace( encoded_original_plugin_source, json.dumps(instrumented_plugin_source), 1, ) tool_trace_source = (ROOT / "scripts" / "leo_tool_trace.py").read_text(encoding="utf-8") remote_tool_trace_import = """ sys.path.insert(0, str(DEPLOY_ROOT / "scripts")) from leo_tool_trace import extract_kb_tool_trace """ embedded_tool_trace_import = """ tool_trace_module = types.ModuleType("leo_tool_trace_harness") tool_trace_module.__file__ = "" exec(compile(LEO_TOOL_TRACE_SOURCE, tool_trace_module.__file__, "exec"), tool_trace_module.__dict__) extract_kb_tool_trace = tool_trace_module.extract_kb_tool_trace """ if script.count(remote_tool_trace_import) != 1: raise RuntimeError("remote leo_tool_trace import marker changed") script = script.replace(remote_tool_trace_import, embedded_tool_trace_import, 1) guard_source = Path(readonly_guard.__file__).resolve().read_text(encoding="utf-8") function_marker = "async def run_suite():" if script.count(function_marker) != 1: raise RuntimeError("remote harness run_suite marker changed") script = script.replace( function_marker, "OOS_READONLY_GUARD_SOURCE = " + json.dumps(guard_source) + "\nLEO_TOOL_TRACE_SOURCE = " + json.dumps(tool_trace_source) + "\nLEO_TOOL_TRACE_SOURCE_SHA256 = " + json.dumps(hashlib.sha256(tool_trace_source.encode()).hexdigest()) + "\nOOS_GROUNDING_MODE = " + json.dumps(grounding_mode) + "\nOOS_GUARD_SOURCE_SHA256 = " + json.dumps(hashlib.sha256(guard_source.encode()).hexdigest()) + "\nOOS_PROMPT_LEAKAGE_SCAN = " + repr(leakage_scan) + "\n\n" + function_marker, ) report_marker = ' "mutates_kb_by_harness": False,\n' if script.count(report_marker) != 1: raise RuntimeError("remote report tool-trace marker changed") script = script.replace( report_marker, report_marker + ' "tool_trace_source_sha256": LEO_TOOL_TRACE_SOURCE_SHA256,\n', 1, ) gateway_marker = " from gateway.session import SessionSource\n\n source = SessionSource(" if script.count(gateway_marker) != 1: raise RuntimeError("remote harness GatewayRunner marker changed") injection = """ import re from gateway.session import SessionSource from gateway.platforms.telegram import TelegramAdapter db_context_dir = temp_profile / "plugins" / "leo-db-context" if OOS_GROUNDING_MODE == "db_tool_ablated": if db_context_dir.is_symlink(): db_context_dir.unlink() elif db_context_dir.exists(): shutil.rmtree(db_context_dir) guard_module = types.ModuleType("leo_oos_readonly_guard_embedded") guard_module.__file__ = "" exec(compile(OOS_READONLY_GUARD_SOURCE, guard_module.__file__, "exec"), guard_module.__dict__) tool_surface = guard_module.install_read_only_tool_surface( temp_profile, allow_kb_reads=OOS_GROUNDING_MODE == "grounded", ) report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile) telegram_transport_attempts = [] telegram_outbound_methods = ( "_send_with_retry", "edit_message", "play_tts", "send", "send_animation", "send_document", "send_image", "send_image_file", "send_model_picker", "send_typing", "send_update_prompt", "send_video", "send_voice", ) patched_telegram_methods = [] def make_transport_deny(method_name): async def deny_transport(*args, **kwargs): telegram_transport_attempts.append({ "method": method_name, "positional_argument_count": len(args), "keyword_names": sorted(str(key) for key in kwargs), }) report["telegram_transport_deny"]["attempt_count"] = len(telegram_transport_attempts) report["telegram_transport_deny"]["attempts"] = list(telegram_transport_attempts) write_report_snapshot(report) raise RuntimeError("Telegram transport is denied in the OOS no-post harness") return deny_transport for method_name in telegram_outbound_methods: if hasattr(TelegramAdapter, method_name): setattr(TelegramAdapter, method_name, make_transport_deny(method_name)) patched_telegram_methods.append(method_name) remote_leakage_matches = [] remote_scanned_files = 0 remote_scanned_bytes = 0 remote_scan_errors = [] remote_symlink_targets = 0 prompt_markers = [] for prompt in PROMPTS: words = re.findall(r"[a-z0-9_]+", str(prompt.get("message") or "").lower()) width = 16 starts = (0,) if len(words) <= width else (0, max(0, (len(words) - width) // 2), len(words) - width) for start in starts: marker = " ".join(words[start:start + width]) if marker: prompt_markers.append((str(prompt.get("id") or ""), marker)) remote_scan_roots = { "profile": temp_profile, "skills": temp_profile / "skills", "plugins": temp_profile / "plugins", "bin": temp_profile / "bin", } excluded_profile_parts = { ".git", "__pycache__", "memories", "sessions", "state", "venv", } expected_roots = [ {"name": name, "exists": path.exists(), "is_symlink": path.is_symlink()} for name, path in remote_scan_roots.items() ] scanned_paths = set() scanned_directories = set() pending_paths = list(remote_scan_roots.values()) candidate_files = [] while pending_paths: scan_path = pending_paths.pop() if set(scan_path.parts) & excluded_profile_parts: continue try: if scan_path.is_symlink(): remote_symlink_targets += 1 resolved = scan_path.resolve(strict=True) if resolved.is_file(): candidate_files.append(resolved) continue if not resolved.is_dir(): continue directory_identity = str(resolved) if directory_identity in scanned_directories: continue scanned_directories.add(directory_identity) pending_paths.extend(sorted(resolved.iterdir(), reverse=True)) except OSError as exc: remote_scan_errors.append({ "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(), "error_type": type(exc).__name__, }) for scan_path in sorted(candidate_files): if set(scan_path.parts) & excluded_profile_parts: continue try: path_identity = str(scan_path.resolve(strict=True)) if path_identity in scanned_paths: continue scanned_paths.add(path_identity) scan_bytes = scan_path.read_bytes() except OSError as exc: remote_scan_errors.append({ "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(), "error_type": type(exc).__name__, }) continue remote_scanned_files += 1 remote_scanned_bytes += len(scan_bytes) normalized = b" ".join(re.findall(rb"[a-z0-9_]+", scan_bytes.lower())).decode("ascii") for prompt_id, marker in prompt_markers: if marker in normalized: remote_leakage_matches.append({ "prompt_id": prompt_id, "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(), "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(), }) report["grounding_mode"] = OOS_GROUNDING_MODE report["db_context_plugin_enabled"] = db_context_dir.exists() report["read_only_tool_surface"] = tool_surface report["readonly_guard_source_sha256"] = OOS_GUARD_SOURCE_SHA256 report["prompt_leakage_scan"] = OOS_PROMPT_LEAKAGE_SCAN report["remote_temp_profile_prompt_leakage_scan"] = { "scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv", "expected_roots": expected_roots, "scanned_files": remote_scanned_files, "scanned_bytes": remote_scanned_bytes, "symlink_targets_encountered": remote_symlink_targets, "errors": remote_scan_errors, "matches": remote_leakage_matches, "pass": bool( all(item["exists"] for item in expected_roots) and remote_scanned_files > 0 and remote_scanned_bytes > 0 and not remote_scan_errors and not remote_leakage_matches ), } report["telegram_transport_deny"] = { "enabled": set(patched_telegram_methods) == set(telegram_outbound_methods), "expected_methods": list(telegram_outbound_methods), "patched_methods": patched_telegram_methods, "attempt_count": 0, "attempts": [], "runner_adapters_required_empty": True, } report["oos_max_turn_seconds"] = 180 report["preexecution_safety_gate"] = { "status": "pass" if ( tool_surface.get("send_message_tool_enabled") is False and tool_surface.get("mutating_bridge_commands_exposed") is False and tool_surface.get("terminal_restricted_to_exact_wrapper") is True and OOS_PROMPT_LEAKAGE_SCAN.get("pass") is True and report["remote_temp_profile_prompt_leakage_scan"]["pass"] is True and report["telegram_transport_deny"]["enabled"] is True and report["telegram_transport_deny"]["attempt_count"] == 0 ) else "fail", "grounding_mode": OOS_GROUNDING_MODE, } write_report_snapshot(report) if report["preexecution_safety_gate"]["status"] != "pass": raise RuntimeError("OOS pre-execution safety gate failed") source = SessionSource(""" script = script.replace(gateway_marker, injection) turn_timeout_marker = "reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)" if script.count(turn_timeout_marker) != 1: raise RuntimeError("remote harness per-turn timeout marker changed") script = script.replace( turn_timeout_marker, "reply = await asyncio.wait_for(runner._handle_message(event), timeout=180)", ) runner_marker = " runner = GatewayRunner()\n" if script.count(runner_marker) != 1: raise RuntimeError("remote harness runner marker changed") script = script.replace( runner_marker, runner_marker + " runner.adapters.clear()\n" + " runner.delivery_router.adapters = runner.adapters\n" + " report['telegram_transport_deny']['runner_adapters_empty'] = not runner.adapters\n" + " write_report_snapshot(report)\n", ) return script def _remote_orphan_readback(parsed: dict[str, Any]) -> dict[str, Any]: temp_profile = str((parsed.get("handler") or {}).get("temp_profile") or "") if not temp_profile: return {"status": "missing_temp_profile_identity", "no_matching_processes": False, "matches": []} proc = subprocess.run( [ "ssh", "-i", str(handler.SSH_KEY), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", handler.VPS, "ps -eo pid=,args=", ], text=True, capture_output=True, timeout=60, ) matches = [handler.redact(line.strip()) for line in proc.stdout.splitlines() if temp_profile in line] return { "status": "ok" if proc.returncode == 0 else "ssh_error", "returncode": proc.returncode, "temp_profile_sha256": hashlib.sha256(temp_profile.encode()).hexdigest(), "matches": matches, "no_matching_processes": proc.returncode == 0 and not matches, } def _local_harness_git_state() -> dict[str, Any]: head = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, capture_output=True, timeout=30, check=False, ) status = subprocess.run( ["git", "status", "--porcelain", "--untracked-files=all"], cwd=ROOT, text=True, capture_output=True, timeout=30, check=False, ) status_text = status.stdout if status.returncode == 0 else "" status_lines = status_text.splitlines() return { "git_head": head.stdout.strip() if head.returncode == 0 else None, "worktree_clean": status.returncode == 0 and not status_text.strip(), "status_sha256": hashlib.sha256(status_text.encode()).hexdigest() if status.returncode == 0 else None, "status_lines": status_lines, "only_control_goal_untracked": status.returncode == 0 and status_lines == ["?? goal.md"], } def _portable_artifact_path(path: Path) -> str: resolved = path.resolve() try: return str(resolved.relative_to(ROOT)) except ValueError: return str(resolved) def run_guarded_remote( *, prompts: list[dict[str, Any]], suite_mode: str, report_prefix: str, prompt_note: str, grounding_mode: str, leakage_scan: dict[str, Any], ) -> dict[str, Any]: """Use the generic transport while substituting only the guarded builder.""" original_builder = handler.build_remote_script def guarded_builder(run_id: str, **kwargs: Any) -> str: return build_guarded_remote_script( run_id, prompts=kwargs.get("prompts") or prompts, suite_mode=str(kwargs.get("suite_mode") or suite_mode), report_prefix=str(kwargs.get("report_prefix") or report_prefix), prompt_note=str(kwargs.get("prompt_note") or prompt_note), grounding_mode=grounding_mode, leakage_scan=leakage_scan, ) handler.build_remote_script = guarded_builder try: remote = handler.run_remote( prompts=prompts, suite_mode=suite_mode, report_prefix=report_prefix, prompt_note=prompt_note, ) finally: handler.build_remote_script = original_builder parsed = remote.get("parsed") if isinstance(parsed, dict): parsed["post_run_orphan_readback"] = _remote_orphan_readback(parsed) return remote def write_score_markdown(path: Path, report: dict[str, Any]) -> None: score = report["score"] lines = [ "# Working Leo m3taversal Out-of-Sample Score", "", f"Generated UTC: `{report['generated_at_utc']}`", f"Pass: `{score['pass']}`", f"Prompts: `{score['passes']}/{score['expected_prompt_count']}`", f"DB counts changed: `{report['db_counts_changed']}`", f"Live behavior manifest unchanged: `{report['live_behavior_manifest_unchanged']}`", f"Service unchanged: `{report['service_unchanged']}`", f"Temporary profile removed: `{report['temp_profile_removed']}`", f"Database context injections: `{report['database_context_injection_count']}`", f"Database context all OK: `{report['database_context_all_ok']}`", f"Database response bindings: `{report['database_response_binding_count']}`", f"Database response binding all OK: `{report['database_response_binding_all_ok']}`", f"Database response contracts satisfied: `{report['database_response_contract_satisfied_count']}`", f"Database response contracts all satisfied: `{report['database_response_contract_all_satisfied']}`", f"Database-composed replacements: `{report['database_response_transform_count']}`", f"Posted to Telegram: `{report['posted_to_telegram']}`", "", "## Prompt Scores", "", ] for item in score["scores"]: lines.append(f"- `{item['prompt_id']}` / `{item['dimension']}`: `pass={item['pass']}`") lines.extend( [ "", "## Claim Ceiling", "", "A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner " "using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible " "Telegram delivery, production DB apply, or GCP parity.", "", ] ) path.write_text("\n".join(lines), encoding="utf-8") def build_score_report(report: dict[str, Any], *, memory_token: str) -> dict[str, Any]: score = benchmark.score_results(report.get("results") or [], memory_token=memory_token) service_unchanged = bool((report.get("service_before_after") or {}).get("unchanged_from_preexisting_live_readback")) return { "generated_at_utc": datetime.now(timezone.utc).isoformat(), "mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score", "source_results_json": str(RESULTS_JSON), "memory_token": memory_token, "score": score, "db_counts_changed": report.get("db_counts_changed"), "changed_live_profile": report.get("changed_live_profile"), "live_behavior_manifest_unchanged": report.get("live_behavior_manifest_unchanged"), "service_unchanged": service_unchanged, "temp_profile_removed": report.get("temp_profile_removed"), "database_context_injection_count": report.get("database_context_injection_count"), "database_context_all_ok": report.get("database_context_all_ok"), "database_response_binding_count": report.get("database_response_binding_count"), "database_response_binding_all_ok": report.get("database_response_binding_all_ok"), "database_response_contract_reported_count": report.get("database_response_contract_reported_count"), "database_response_contract_satisfied_count": report.get("database_response_contract_satisfied_count"), "database_response_contract_all_satisfied": report.get("database_response_contract_all_satisfied"), "database_response_transform_count": report.get("database_response_transform_count"), "posted_to_telegram": report.get("posted_to_telegram"), "production_db_apply_ran": False, } def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) -> bool: return bool( report.get("remote_returncode") == 0 and report.get("pass_runtime") is True and report.get("db_counts_changed") is False and report.get("changed_live_profile") is False and report.get("live_behavior_manifest_unchanged") is True and score_report["service_unchanged"] and report.get("temp_profile_removed") is True and report.get("database_context_all_ok") is True and int(report.get("database_context_injection_count") or 0) >= len(report.get("results") or []) and report.get("database_response_binding_all_ok") is True and int(report.get("database_response_binding_count") or 0) >= len(report.get("results") or []) and report.get("database_response_contract_all_satisfied") is True and int(report.get("database_response_contract_reported_count") or 0) >= len(report.get("results") or []) and report.get("posted_to_telegram") is False and score_report["score"]["pass"] ) def write_protocol_score_markdown(path: Path, score: dict[str, Any]) -> None: lines = [ "# Leo Blinded OOS Reasoning Trial", "", f"Generated UTC: `{score['generated_at_utc']}`", f"Protocol: `{score['protocol_id']}` / `{score['protocol_hash_sha256']}`", f"Trial: `{score['trial_id']}` / `{score['session_mode']}`", f"Pass: `{score['pass']}`", f"Grounded prompts: `{score['grounded_passes']}/{score['prompt_count']}`", f"Grounded rate: `{score['grounded_pass_rate']:.3f}`", f"No-DB ablation grounded rate: `{score['receipt_ablation']['grounded_pass_rate']:.3f}`", "", "## Prompt scores", "", ] for item in score["prompt_scores"]: lines.append( f"- `{item['prompt_id']}` / `{item['family_id']}`: semantic=`{item['semantic_pass']}`, " f"subject=`{item['subject_alignment']}`, receipts=`{item['receipt_pass']}`, " f"grounded=`{item['grounded_pass']}`" ) lines.extend( [ "", "## Safety", "", f"- Grounded live safety: `{score['top_level_safety']['pass']}`", f"- Ablation live safety: `{score['baseline_top_level_safety']['pass']}`", f"- Restart receipt: `{score['restart_receipt_validation']['pass']}`", f"- Exact prompt binding: `{score['prompt_binding']['pass']}`", "", "## Claim ceiling", "", "This trial proves a direct live VPS GatewayRunner reply path with a fail-closed read-only tool surface, " "full unchanged DB fingerprints, and no Telegram post. It does not prove Telegram-visible delivery or " "any production database apply.", "", ] ) path.write_text("\n".join(lines), encoding="utf-8") def _run_protocol_mode( protocol: dict[str, Any], trial: dict[str, Any], *, grounding_mode: str, output_dir: Path, leakage_scan: dict[str, Any], ) -> tuple[dict[str, Any], Path]: prompts = [ {"id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"]} for prompt in trial["prompts"] ] safe_protocol_id = re.sub(r"[^A-Za-z0-9._-]", "-", str(protocol["protocol_id"])) report_prefix = f"leo-oos-{safe_protocol_id}-{trial['trial_id']}-{grounding_mode}" output_path = output_dir / f"{trial['trial_id']}-{grounding_mode}-handler.json" remote = run_guarded_remote( prompts=prompts, suite_mode=f"live_vps_gatewayrunner_blinded_oos_{grounding_mode}", report_prefix=report_prefix, prompt_note=( f"Frozen blinded protocol {protocol['protocol_hash_sha256']}; trial {trial['trial_id']}; " f"grounding mode {grounding_mode}; direct handler only, no Telegram post, no database mutation." ), grounding_mode=grounding_mode, leakage_scan=leakage_scan, ) report = handler.write_output(remote, output_json=output_path) report["oos_harness_git_state"] = _local_harness_git_state() report["protocol_id"] = protocol["protocol_id"] report["protocol_hash_sha256"] = protocol["protocol_hash_sha256"] report["trial_id"] = trial["trial_id"] report["trial_prompt_set_sha256"] = trial["prompt_set_sha256"] report["scorer_version"] = protocol["scorer_version"] report["source_hashes"] = protocol["source_hashes"] output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") return report, output_path def _load_restart_receipt(path: Path | None, trial: dict[str, Any]) -> dict[str, Any] | None: if trial["session_mode"] != "post_restart_clean_session": return None if path is None: raise SystemExit("--restart-receipt is required for a post-restart trial") return json.loads(path.read_text(encoding="utf-8")) def _ssh_readback(command: str, *, timeout: int = 120) -> subprocess.CompletedProcess[str]: return subprocess.run( [ "ssh", "-i", str(handler.SSH_KEY), "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", handler.VPS, command, ], text=True, capture_output=True, timeout=timeout, ) def _deploy_identity() -> dict[str, Any]: proc = _ssh_readback( "cd /opt/teleo-eval/workspaces/deploy-infra && " "printf 'head=' && git rev-parse HEAD && " "printf 'stamp=' && cat /opt/teleo-eval/.last-deploy-sha", timeout=60, ) values: dict[str, str] = {} for line in proc.stdout.splitlines(): if "=" in line: key, value = line.split("=", 1) values[key] = value.strip() return { "returncode": proc.returncode, "head": values.get("head"), "stamp": values.get("stamp"), "stderr": handler.redact(proc.stderr), } def _restart_state_probe( protocol: dict[str, Any], *, label: str, output_dir: Path, leakage_scan: dict[str, Any], ) -> tuple[dict[str, Any], Path]: output_path = output_dir / f"restart-{label}-state.json" remote = run_guarded_remote( prompts=[], suite_mode=f"live_vps_gateway_restart_{label}_readonly_state_probe", report_prefix=f"leo-oos-restart-{label}-{protocol['protocol_id']}", prompt_note=( f"Read-only restart {label} state probe for frozen protocol {protocol['protocol_hash_sha256']}; " "zero prompts, no Telegram post, no database mutation." ), grounding_mode="grounded", leakage_scan=leakage_scan, ) report = handler.write_output(remote, output_json=output_path) report["oos_harness_git_state"] = _local_harness_git_state() report["protocol_id"] = protocol["protocol_id"] report["protocol_hash_sha256"] = protocol["protocol_hash_sha256"] report["source_hashes"] = protocol["source_hashes"] output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") return report, output_path def _restart_probe_passes(report: dict[str, Any]) -> bool: before = report.get("db_fingerprint_before") or {} after = report.get("db_fingerprint_after") or {} counts_before = report.get("db_counts_before") counts_after = report.get("db_counts_after") transport = report.get("telegram_transport_deny") or {} return bool( report.get("remote_returncode") == 0 and report.get("pass_runtime") is True and report.get("posted_to_telegram") is False and report.get("db_counts_changed") is False and isinstance(counts_before, dict) and bool(counts_before) and counts_before == counts_after and all(isinstance(value, int) and not isinstance(value, bool) for value in counts_before.values()) and report.get("db_fingerprint_unchanged") is True and before.get("status") == "ok" and after.get("status") == "ok" and bool(re.fullmatch(r"[0-9a-f]{64}", str(before.get("fingerprint_sha256") or ""))) and before.get("fingerprint_sha256") == after.get("fingerprint_sha256") and (report.get("preexecution_safety_gate") or {}).get("status") == "pass" and transport.get("enabled") is True and isinstance(transport.get("attempt_count"), int) and not isinstance(transport.get("attempt_count"), bool) and transport.get("attempt_count") == 0 and transport.get("runner_adapters_empty") is True and report.get("temp_profile_removed") is True and (report.get("post_run_orphan_readback") or {}).get("no_matching_processes") is True ) def collect_restart_receipt(args: argparse.Namespace) -> int: protocol = json.loads(args.protocol.read_text(encoding="utf-8")) validation = protocol_lib.validate_protocol(protocol, verify_source_hashes=True) if not validation["pass"]: raise SystemExit(f"frozen protocol validation failed: {validation['issues']}") restart_trials = [ item for item in protocol.get("trials") or [] if item.get("session_mode") == "post_restart_clean_session" ] if len(restart_trials) != 1: raise SystemExit("frozen protocol must identify exactly one post-restart trial") next_trial = restart_trials[0] leakage_scan = prompt_leakage_scan(protocol) if not leakage_scan["pass"]: raise SystemExit(f"runtime prompt leakage detected: {leakage_scan['exact_prompt_matches']}") args.output_dir.mkdir(parents=True, exist_ok=True) before_deploy = _deploy_identity() before_report, before_path = _restart_state_probe( protocol, label="before", output_dir=args.output_dir, leakage_scan=leakage_scan, ) if ( before_deploy.get("returncode") != 0 or not re.fullmatch(r"[0-9a-f]{40}", str(before_deploy.get("head") or "")) or before_deploy.get("head") != before_deploy.get("stamp") ): raise SystemExit(f"deploy identity preflight failed: {before_deploy}") if not _restart_probe_passes(before_report): raise SystemExit(f"read-only pre-restart probe failed: {before_path}") restart_started_at_utc = datetime.now(timezone.utc).isoformat() restart = _ssh_readback( "systemctl restart leoclean-gateway.service && " "for i in $(seq 1 60); do " 'if [ "$(systemctl is-active leoclean-gateway.service)" = active ]; then break; fi; sleep 1; done; ' "systemctl show leoclean-gateway.service " "-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp", timeout=120, ) restart_ended_at_utc = datetime.now(timezone.utc).isoformat() after_deploy = _deploy_identity() after_report, after_path = _restart_state_probe( protocol, label="after", output_dir=args.output_dir, leakage_scan=leakage_scan, ) service_before = (before_report.get("service_before_after") or {}).get("after") or {} service_after = after_report.get("before_service") or {} fingerprint_before = before_report.get("db_fingerprint_after") or {} fingerprint_after = after_report.get("db_fingerprint_before") or {} counts_before = before_report.get("db_counts_after") or {} counts_after = after_report.get("db_counts_before") or {} receipt = { "schema": "livingip.leoGatewayRestartReceipt.v1", "generated_at_utc": datetime.now(timezone.utc).isoformat(), "protocol_id": protocol["protocol_id"], "protocol_hash_sha256": protocol["protocol_hash_sha256"], "next_trial_id": next_trial["trial_id"], "next_trial_prompt_set_sha256": next_trial["prompt_set_sha256"], "authorization_basis": "goal.md requires restart trials; service restart only, no Telegram post and no DB mutation", "restart_command": "systemctl restart leoclean-gateway.service", "restart_started_at_utc": restart_started_at_utc, "restart_ended_at_utc": restart_ended_at_utc, "restart_returncode": restart.returncode, "restart_stdout": handler.redact(restart.stdout), "restart_stderr": handler.redact(restart.stderr), "service_before": service_before, "service_after": service_after, "deploy_before": before_deploy, "deploy_after": after_deploy, "db_counts_before": counts_before, "db_counts_after": counts_after, "db_counts_changed": counts_before != counts_after, "db_fingerprint_before": fingerprint_before, "db_fingerprint_after": fingerprint_after, "db_fingerprint_unchanged": bool( fingerprint_before.get("status") == "ok" and fingerprint_after.get("status") == "ok" and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256") ), "posted_to_telegram": False, "before_probe": { "path": _portable_artifact_path(before_path), "sha256": hashlib.sha256(before_path.read_bytes()).hexdigest(), "pass": _restart_probe_passes(before_report), }, "after_probe": { "path": _portable_artifact_path(after_path), "sha256": hashlib.sha256(after_path.read_bytes()).hexdigest(), "pass": _restart_probe_passes(after_report), }, } receipt["checks"] = { "restart_command_succeeded": receipt["restart_returncode"] == 0, "service_active_after": service_after.get("ActiveState") == "active" and service_after.get("SubState") == "running", "service_pid_changed": bool( service_before.get("MainPID") and service_before.get("MainPID") != service_after.get("MainPID") ), "deploy_identity_unchanged": before_deploy.get("returncode") == 0 and after_deploy.get("returncode") == 0 and bool(re.fullmatch(r"[0-9a-f]{40}", str(before_deploy.get("head") or ""))) and before_deploy.get("head") == before_deploy.get("stamp") == after_deploy.get("head") == after_deploy.get("stamp"), "db_counts_unchanged": receipt["db_counts_changed"] is False, "db_counts_complete": isinstance(counts_before, dict) and bool(counts_before) and counts_before == counts_after and all(isinstance(value, int) and not isinstance(value, bool) for value in counts_before.values()), "db_fingerprint_unchanged": receipt["db_fingerprint_unchanged"] is True, "db_fingerprint_complete": bool( re.fullmatch(r"[0-9a-f]{64}", str(fingerprint_before.get("fingerprint_sha256") or "")) and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256") ), "before_probe_passed": receipt["before_probe"]["pass"] is True, "after_probe_passed": receipt["after_probe"]["pass"] is True, } receipt["pass"] = all(receipt["checks"].values()) args.collect_restart_receipt.parent.mkdir(parents=True, exist_ok=True) args.collect_restart_receipt.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8") print( json.dumps( { "restart_receipt": str(args.collect_restart_receipt), "pass": receipt["pass"], "checks": receipt["checks"], }, indent=2, sort_keys=True, ) ) return 0 if receipt["pass"] else 1 def run_or_score_protocol_trial(args: argparse.Namespace) -> int: protocol = json.loads(args.protocol.read_text(encoding="utf-8")) validation = protocol_lib.validate_protocol(protocol, verify_source_hashes=True) if not validation["pass"]: raise SystemExit(f"frozen protocol validation failed: {validation['issues']}") trial = next((item for item in protocol["trials"] if item["trial_id"] == args.trial_id), None) if trial is None: raise SystemExit(f"unknown trial id: {args.trial_id}") leakage_scan = prompt_leakage_scan(protocol) if not leakage_scan["pass"]: raise SystemExit(f"runtime prompt leakage detected: {leakage_scan['exact_prompt_matches']}") args.output_dir.mkdir(parents=True, exist_ok=True) restart_receipt = _load_restart_receipt(args.restart_receipt, trial) if bool(args.grounded_report) != bool(args.baseline_report): raise SystemExit("--grounded-report and --baseline-report must be supplied together") if args.grounded_report: grounded_report = json.loads(args.grounded_report.read_text(encoding="utf-8")) baseline_report = json.loads(args.baseline_report.read_text(encoding="utf-8")) grounded_path = args.grounded_report baseline_path = args.baseline_report elif args.grounding_mode: report, output_path = _run_protocol_mode( protocol, trial, grounding_mode=args.grounding_mode, output_dir=args.output_dir, leakage_scan=leakage_scan, ) mode_safety = protocol_lib._top_level_safety( report, require_handler_safety_gate=args.grounding_mode == "grounded", expected_harness_git_head=protocol["harness_git_head"], ) mode_checks = { "expected_grounding_mode": report.get("grounding_mode") == args.grounding_mode, "db_context_state": report.get("db_context_plugin_enabled") is (args.grounding_mode == "grounded"), "tool_surface_mode": (report.get("read_only_tool_surface") or {}).get("mode") == ("read_only_kb" if args.grounding_mode == "grounded" else "no_db_ablation"), "zero_context_in_ablation": args.grounding_mode != "db_tool_ablated" or all(not (item.get("database_context_trace") or []) for item in report.get("results") or []), } mode_pass = mode_safety["pass"] and all(mode_checks.values()) print( json.dumps( { "report": str(output_path), "grounding_mode": args.grounding_mode, "handler_safety_gate": report.get("safety_gate"), "post_run_orphan_readback": report.get("post_run_orphan_readback"), "mode_safety": mode_safety, "mode_checks": mode_checks, "pass": mode_pass, }, indent=2, sort_keys=True, ) ) return 0 if mode_pass else 1 else: grounded_report, grounded_path = _run_protocol_mode( protocol, trial, grounding_mode="grounded", output_dir=args.output_dir, leakage_scan=leakage_scan, ) baseline_report, baseline_path = _run_protocol_mode( protocol, trial, grounding_mode="db_tool_ablated", output_dir=args.output_dir, leakage_scan=leakage_scan, ) score = protocol_lib.score_live_trial( protocol, trial["trial_id"], grounded_report, baseline_report=baseline_report, restart_receipt=restart_receipt, ) score["grounded_report_path"] = _portable_artifact_path(grounded_path) score["baseline_report_path"] = _portable_artifact_path(baseline_path) score["grounded_report_sha256"] = hashlib.sha256(grounded_path.read_bytes()).hexdigest() score["baseline_report_sha256"] = hashlib.sha256(baseline_path.read_bytes()).hexdigest() if restart_receipt is not None and args.restart_receipt is not None: score["restart_receipt_path"] = _portable_artifact_path(args.restart_receipt) score["restart_receipt_sha256"] = hashlib.sha256(args.restart_receipt.read_bytes()).hexdigest() score["restart_receipt_payload_sha256"] = protocol_lib.canonical_sha256(restart_receipt) score_path = args.output_dir / f"{trial['trial_id']}-score.json" markdown_path = args.output_dir / f"{trial['trial_id']}-score.md" score_path.write_text(json.dumps(score, indent=2, sort_keys=True) + "\n", encoding="utf-8") write_protocol_score_markdown(markdown_path, score) print( json.dumps( { "grounded_report": str(grounded_path), "baseline_report": str(baseline_path), "score_json": str(score_path), "score_markdown": str(markdown_path), "pass": score["pass"], "grounded_rate": score["grounded_pass_rate"], "baseline_grounded_rate": score["receipt_ablation"]["grounded_pass_rate"], }, indent=2, sort_keys=True, ) ) return 0 if score["pass"] else 1 def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--score-existing", action="store_true", help="Rescore the retained live transcript without making another remote model call.", ) parser.add_argument("--protocol", type=Path) parser.add_argument("--trial-id") parser.add_argument("--output-dir", type=Path, default=PROTOCOL_REPORT_DIR) parser.add_argument("--grounding-mode", choices=GROUNDING_MODES) parser.add_argument("--grounded-report", type=Path) parser.add_argument("--baseline-report", type=Path) parser.add_argument("--restart-receipt", type=Path) parser.add_argument("--collect-restart-receipt", type=Path) args = parser.parse_args() if args.collect_restart_receipt: if not args.protocol or args.trial_id: raise SystemExit("--collect-restart-receipt requires --protocol and does not accept --trial-id") return collect_restart_receipt(args) if args.protocol or args.trial_id: if not args.protocol or not args.trial_id: raise SystemExit("--protocol and --trial-id are required together") return run_or_score_protocol_trial(args) if not args.score_existing: raise SystemExit( "refusing the legacy fixed live suite; use --protocol and --trial-id for a frozen guarded trial" ) if args.score_existing: report = json.loads(RESULTS_JSON.read_text(encoding="utf-8")) previous_score = json.loads(SCORE_JSON.read_text(encoding="utf-8")) memory_token = str(previous_score["memory_token"]) score_report = build_score_report(report, memory_token=memory_token) SCORE_JSON.write_text(json.dumps(score_report, indent=2, sort_keys=True) + "\n", encoding="utf-8") write_score_markdown(SCORE_MARKDOWN, score_report) safe_pass = score_report_passes(report, score_report) print( json.dumps( { "results_json": str(RESULTS_JSON), "score_json": str(SCORE_JSON), "score_markdown": str(SCORE_MARKDOWN), "pass": safe_pass, "prompts": f"{score_report['score']['passes']}/{score_report['score']['expected_prompt_count']}", "rescored_existing": args.score_existing, }, indent=2, sort_keys=True, ) ) return 0 if safe_pass else 1 if __name__ == "__main__": raise SystemExit(main())