#!/usr/bin/env python3
"""Run and score the read-only m3taversal out-of-sample suite on the live VPS handler."""

from __future__ import annotations

import argparse
import hashlib
import json
import re
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import leo_oos_readonly_guard as readonly_guard
import run_leo_direct_claim_handler_suite as handler
import working_leo_m3taversal_oos_benchmark as benchmark
import working_leo_m3taversal_oos_protocol as protocol_lib

BASE_HANDLER_SCRIPT_BUILDER = handler.build_remote_script

ROOT = Path(__file__).resolve().parents[1]
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
RESULTS_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-current.json"
SCORE_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.json"
SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.md"
PROTOCOL_REPORT_DIR = ROOT / "docs" / "reports" / "leo-oos-reasoning-benchmark-20260715"
GROUNDING_MODES = ("grounded", "db_tool_ablated")
RUNTIME_PROMPT_SCAN_ROOTS = (
    ROOT / "hermes-agent" / "leoclean-skills",
    ROOT / "hermes-agent" / "leoclean-plugins",
    ROOT / "hermes-agent" / "leoclean-bin",
)


def prompt_leakage_scan(protocol: dict[str, Any]) -> dict[str, Any]:
    """Fail if a frozen prompt was copied into a runtime skill/plugin/tool."""

    prompt_markers = [
        (str(prompt["id"]), str(marker))
        for trial in protocol.get("trials") or []
        for prompt in trial.get("prompts") or []
        for marker in prompt.get("leakage_markers") or []
    ]
    matches: list[dict[str, str]] = []
    scanned_files = 0
    scanned_bytes = 0
    scan_errors: list[dict[str, str]] = []
    symlink_entries: list[dict[str, str]] = []
    expected_roots = [
        {
            "path": str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path),
            "exists": path.exists(),
            "is_symlink": path.is_symlink(),
        }
        for path in RUNTIME_PROMPT_SCAN_ROOTS
    ]
    for root in RUNTIME_PROMPT_SCAN_ROOTS:
        if not root.exists() or root.is_symlink():
            continue
        try:
            entries = sorted(root.rglob("*"))
        except OSError as exc:
            scan_errors.append(
                {
                    "path_sha256": hashlib.sha256(str(root).encode()).hexdigest(),
                    "error_type": type(exc).__name__,
                }
            )
            continue
        for path in entries:
            if path.is_symlink():
                symlink_entries.append({"path_sha256": hashlib.sha256(str(path).encode()).hexdigest()})
        paths = [item for item in entries if item.is_file() and not item.is_symlink()]
        for path in paths:
            try:
                text = path.read_text(encoding="utf-8")
            except UnicodeDecodeError:
                continue
            except OSError as exc:
                scan_errors.append(
                    {
                        "path_sha256": hashlib.sha256(str(path).encode()).hexdigest(),
                        "error_type": type(exc).__name__,
                    }
                )
                continue
            scanned_files += 1
            scanned_bytes += len(text.encode())
            normalized = " ".join(re.findall(r"[a-z0-9_]+", text.lower()))
            for prompt_id, marker in prompt_markers:
                if marker in normalized:
                    display_path = str(path.relative_to(ROOT)) if path.is_relative_to(ROOT) else str(path)
                    matches.append(
                        {
                            "prompt_id": prompt_id,
                            "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(),
                            "path": display_path,
                        }
                    )
    return {
        "expected_roots": expected_roots,
        "scanned_roots": [item["path"] for item in expected_roots],
        "scanned_files": scanned_files,
        "scanned_bytes": scanned_bytes,
        "errors": scan_errors,
        "symlink_entries": symlink_entries,
        "exact_prompt_matches": matches,
        "pass": bool(
            all(item["exists"] and not item["is_symlink"] for item in expected_roots)
            and scanned_files > 0
            and scanned_bytes > 0
            and not scan_errors
            and not symlink_entries
            and not matches
        ),
    }


def build_guarded_remote_script(
    run_id: str,
    *,
    prompts: list[dict[str, Any]],
    suite_mode: str,
    report_prefix: str,
    prompt_note: str,
    grounding_mode: str,
    leakage_scan: dict[str, Any],
) -> str:
    """Inject the reviewed fail-closed tool surface before GatewayRunner starts."""

    if grounding_mode not in GROUNDING_MODES:
        raise ValueError(f"unsupported grounding mode: {grounding_mode}")
    script = BASE_HANDLER_SCRIPT_BUILDER(
        run_id,
        prompts=prompts,
        suite_mode=suite_mode,
        report_prefix=report_prefix,
        prompt_note=prompt_note,
    )
    original_plugin_source = handler.DB_CONTEXT_PLUGIN.read_text(encoding="utf-8")
    instrumented_plugin_source = protocol_lib.instrument_db_context_plugin_source(original_plugin_source)
    encoded_original_plugin_source = json.dumps(original_plugin_source)
    if script.count(encoded_original_plugin_source) != 1:
        raise RuntimeError("embedded DB context plugin source marker changed")
    script = script.replace(
        encoded_original_plugin_source,
        json.dumps(instrumented_plugin_source),
        1,
    )
    tool_trace_source = (ROOT / "scripts" / "leo_tool_trace.py").read_text(encoding="utf-8")
    remote_tool_trace_import = """    sys.path.insert(0, str(DEPLOY_ROOT / "scripts"))
    from leo_tool_trace import extract_kb_tool_trace
"""
    embedded_tool_trace_import = """    tool_trace_module = types.ModuleType("leo_tool_trace_harness")
    tool_trace_module.__file__ = "<embedded leo_tool_trace.py>"
    exec(compile(LEO_TOOL_TRACE_SOURCE, tool_trace_module.__file__, "exec"), tool_trace_module.__dict__)
    extract_kb_tool_trace = tool_trace_module.extract_kb_tool_trace
"""
    if script.count(remote_tool_trace_import) != 1:
        raise RuntimeError("remote leo_tool_trace import marker changed")
    script = script.replace(remote_tool_trace_import, embedded_tool_trace_import, 1)
    guard_source = Path(readonly_guard.__file__).resolve().read_text(encoding="utf-8")
    function_marker = "async def run_suite():"
    if script.count(function_marker) != 1:
        raise RuntimeError("remote harness run_suite marker changed")
    script = script.replace(
        function_marker,
        "OOS_READONLY_GUARD_SOURCE = "
        + json.dumps(guard_source)
        + "\nLEO_TOOL_TRACE_SOURCE = "
        + json.dumps(tool_trace_source)
        + "\nLEO_TOOL_TRACE_SOURCE_SHA256 = "
        + json.dumps(hashlib.sha256(tool_trace_source.encode()).hexdigest())
        + "\nOOS_GROUNDING_MODE = "
        + json.dumps(grounding_mode)
        + "\nOOS_GUARD_SOURCE_SHA256 = "
        + json.dumps(hashlib.sha256(guard_source.encode()).hexdigest())
        + "\nOOS_KB_TOOL_SOURCE_SHA256 = hashlib.sha256(KB_TOOL_SOURCE.encode()).hexdigest()"
        + "\nOOS_PROMPT_LEAKAGE_SCAN = "
        + repr(leakage_scan)
        + "\n\n"
        + function_marker,
    )
    report_marker = '        "mutates_kb_by_harness": False,\n'
    if script.count(report_marker) != 1:
        raise RuntimeError("remote report tool-trace marker changed")
    script = script.replace(
        report_marker,
        report_marker + '        "tool_trace_source_sha256": LEO_TOOL_TRACE_SOURCE_SHA256,\n',
        1,
    )
    gateway_marker = "        from gateway.session import SessionSource\n\n        source = SessionSource("
    if script.count(gateway_marker) != 1:
        raise RuntimeError("remote harness GatewayRunner marker changed")
    injection = """        import re
        from gateway.session import SessionSource
        from gateway.platforms.telegram import TelegramAdapter

        db_context_dir = temp_profile / "plugins" / "leo-db-context"
        if OOS_GROUNDING_MODE == "db_tool_ablated":
            if db_context_dir.is_symlink():
                db_context_dir.unlink()
            elif db_context_dir.exists():
                shutil.rmtree(db_context_dir)
        guard_module = types.ModuleType("leo_oos_readonly_guard_embedded")
        guard_module.__file__ = "<embedded leo_oos_readonly_guard.py>"
        exec(compile(OOS_READONLY_GUARD_SOURCE, guard_module.__file__, "exec"), guard_module.__dict__)
        tool_surface = guard_module.install_read_only_tool_surface(
            temp_profile,
            allow_kb_reads=OOS_GROUNDING_MODE == "grounded",
            expected_kb_tool_sha256=OOS_KB_TOOL_SOURCE_SHA256,
        )
        report["executed_behavior_manifest"] = build_behavior_manifest(temp_profile)
        telegram_transport_attempts = []
        telegram_outbound_methods = (
            "_send_with_retry",
            "edit_message",
            "play_tts",
            "send",
            "send_animation",
            "send_document",
            "send_image",
            "send_image_file",
            "send_model_picker",
            "send_typing",
            "send_update_prompt",
            "send_video",
            "send_voice",
        )
        patched_telegram_methods = []
        def make_transport_deny(method_name):
            async def deny_transport(*args, **kwargs):
                telegram_transport_attempts.append({
                    "method": method_name,
                    "positional_argument_count": len(args),
                    "keyword_names": sorted(str(key) for key in kwargs),
                })
                report["telegram_transport_deny"]["attempt_count"] = len(telegram_transport_attempts)
                report["telegram_transport_deny"]["attempts"] = list(telegram_transport_attempts)
                write_report_snapshot(report)
                raise RuntimeError("Telegram transport is denied in the OOS no-post harness")
            return deny_transport
        for method_name in telegram_outbound_methods:
            if hasattr(TelegramAdapter, method_name):
                setattr(TelegramAdapter, method_name, make_transport_deny(method_name))
                patched_telegram_methods.append(method_name)

        remote_leakage_matches = []
        remote_scanned_files = 0
        remote_scanned_bytes = 0
        remote_scan_errors = []
        remote_symlink_targets = 0
        prompt_markers = []
        for prompt in PROMPTS:
            words = re.findall(r"[a-z0-9_]+", str(prompt.get("message") or "").lower())
            width = 16
            starts = (0,) if len(words) <= width else (0, max(0, (len(words) - width) // 2), len(words) - width)
            for start in starts:
                marker = " ".join(words[start:start + width])
                if marker:
                    prompt_markers.append((str(prompt.get("id") or ""), marker))
        remote_scan_roots = {
            "profile": temp_profile,
            "skills": temp_profile / "skills",
            "plugins": temp_profile / "plugins",
            "bin": temp_profile / "bin",
        }
        excluded_profile_parts = {
            ".git", "__pycache__", "memories", "sessions", "state", "venv",
        }
        expected_roots = [
            {"name": name, "exists": path.exists(), "is_symlink": path.is_symlink()}
            for name, path in remote_scan_roots.items()
        ]
        scanned_paths = set()
        scanned_directories = set()
        pending_paths = list(remote_scan_roots.values())
        candidate_files = []
        while pending_paths:
            scan_path = pending_paths.pop()
            if set(scan_path.parts) & excluded_profile_parts:
                continue
            try:
                if scan_path.is_symlink():
                    remote_symlink_targets += 1
                resolved = scan_path.resolve(strict=True)
                if resolved.is_file():
                    candidate_files.append(resolved)
                    continue
                if not resolved.is_dir():
                    continue
                directory_identity = str(resolved)
                if directory_identity in scanned_directories:
                    continue
                scanned_directories.add(directory_identity)
                pending_paths.extend(sorted(resolved.iterdir(), reverse=True))
            except OSError as exc:
                remote_scan_errors.append({
                    "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(),
                    "error_type": type(exc).__name__,
                })
        for scan_path in sorted(candidate_files):
            if set(scan_path.parts) & excluded_profile_parts:
                continue
            try:
                path_identity = str(scan_path.resolve(strict=True))
                if path_identity in scanned_paths:
                    continue
                scanned_paths.add(path_identity)
                scan_bytes = scan_path.read_bytes()
            except OSError as exc:
                remote_scan_errors.append({
                    "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(),
                    "error_type": type(exc).__name__,
                })
                continue
            remote_scanned_files += 1
            remote_scanned_bytes += len(scan_bytes)
            normalized = b" ".join(re.findall(rb"[a-z0-9_]+", scan_bytes.lower())).decode("ascii")
            for prompt_id, marker in prompt_markers:
                if marker in normalized:
                    remote_leakage_matches.append({
                        "prompt_id": prompt_id,
                        "marker_sha256": hashlib.sha256(marker.encode()).hexdigest(),
                        "path_sha256": hashlib.sha256(str(scan_path).encode()).hexdigest(),
                    })
        report["grounding_mode"] = OOS_GROUNDING_MODE
        report["db_context_plugin_enabled"] = db_context_dir.exists()
        report["read_only_tool_surface"] = tool_surface
        report["readonly_guard_source_sha256"] = OOS_GUARD_SOURCE_SHA256
        report["prompt_leakage_scan"] = OOS_PROMPT_LEAKAGE_SCAN
        report["remote_temp_profile_prompt_leakage_scan"] = {
            "scope": "full_model_visible_temp_profile_excluding_sessions_state_memories_and_venv",
            "expected_roots": expected_roots,
            "scanned_files": remote_scanned_files,
            "scanned_bytes": remote_scanned_bytes,
            "symlink_targets_encountered": remote_symlink_targets,
            "errors": remote_scan_errors,
            "matches": remote_leakage_matches,
            "pass": bool(
                all(item["exists"] for item in expected_roots)
                and remote_scanned_files > 0
                and remote_scanned_bytes > 0
                and not remote_scan_errors
                and not remote_leakage_matches
            ),
        }
        report["telegram_transport_deny"] = {
            "enabled": set(patched_telegram_methods) == set(telegram_outbound_methods),
            "expected_methods": list(telegram_outbound_methods),
            "patched_methods": patched_telegram_methods,
            "attempt_count": 0,
            "attempts": [],
            "runner_adapters_required_empty": True,
        }
        report["oos_max_turn_seconds"] = 180
        report["preexecution_safety_gate"] = {
            "status": "pass" if (
                tool_surface.get("actual_registry_tools") == ["skill_view", "skills_list", "teleo-kb"]
                and tool_surface.get("allowed_tools") == ["skills_list", "skill_view", "teleo-kb"]
                and tool_surface.get("send_message_tool_enabled") is False
                and tool_surface.get("shell_tool_enabled") is False
                and tool_surface.get("mutating_bridge_commands_exposed") is False
                and tool_surface.get("structured_read_tool_name") == "teleo-kb"
                and tool_surface.get("structured_read_tool_restricted_to_frozen_kb_tool") is True
                and re.fullmatch(
                    r"[0-9a-f]{64}", str(tool_surface.get("structured_read_tool_definition_sha256") or "")
                ) is not None
                and tool_surface.get("kb_tool_sha256") == OOS_KB_TOOL_SOURCE_SHA256
                and isinstance(tool_surface.get("bridge_timeout_seconds"), int)
                and not isinstance(tool_surface.get("bridge_timeout_seconds"), bool)
                and 0 < tool_surface.get("bridge_timeout_seconds") <= 60
                and OOS_PROMPT_LEAKAGE_SCAN.get("pass") is True
                and report["remote_temp_profile_prompt_leakage_scan"]["pass"] is True
                and report["telegram_transport_deny"]["enabled"] is True
                and report["telegram_transport_deny"]["attempt_count"] == 0
            ) else "fail",
            "grounding_mode": OOS_GROUNDING_MODE,
        }
        write_report_snapshot(report)
        if report["preexecution_safety_gate"]["status"] != "pass":
            raise RuntimeError("OOS pre-execution safety gate failed")

        source = SessionSource("""
    script = script.replace(gateway_marker, injection)
    turn_timeout_marker = "reply = await asyncio.wait_for(runner._handle_message(event), timeout=300)"
    if script.count(turn_timeout_marker) != 1:
        raise RuntimeError("remote harness per-turn timeout marker changed")
    script = script.replace(
        turn_timeout_marker,
        "reply = await asyncio.wait_for(runner._handle_message(event), timeout=180)",
    )
    runner_marker = "        runner = GatewayRunner()\n"
    if script.count(runner_marker) != 1:
        raise RuntimeError("remote harness runner marker changed")
    script = script.replace(
        runner_marker,
        runner_marker
        + "        post_runner_tool_surface = guard_module.reassert_read_only_tool_surface(\n"
        + "            temp_profile,\n"
        + "            expected_kb_tool_sha256=OOS_KB_TOOL_SOURCE_SHA256,\n"
        + "            expected_definition_sha256=(\n"
        + "                tool_surface['structured_read_tool_definition_sha256']\n"
        + "            ),\n"
        + "        )\n"
        + "        report['post_runner_tool_surface'] = post_runner_tool_surface\n"
        + "        write_report_snapshot(report)\n"
        + "        if not post_runner_tool_surface.get('final_surface_matches_preexecution'):\n"
        + "            raise RuntimeError('OOS post-runner tool surface changed')\n"
        + "        runner.adapters.clear()\n"
        + "        runner.delivery_router.adapters = runner.adapters\n"
        + "        report['telegram_transport_deny']['runner_adapters_empty'] = not runner.adapters\n"
        + "        write_report_snapshot(report)\n",
    )
    return script


def _remote_orphan_readback(parsed: dict[str, Any]) -> dict[str, Any]:
    temp_profile = str((parsed.get("handler") or {}).get("temp_profile") or "")
    if not temp_profile:
        return {"status": "missing_temp_profile_identity", "no_matching_processes": False, "matches": []}
    proc = subprocess.run(
        [
            "ssh",
            "-i",
            str(handler.SSH_KEY),
            "-o",
            "BatchMode=yes",
            "-o",
            "StrictHostKeyChecking=accept-new",
            handler.VPS,
            "ps -eo pid=,args=",
        ],
        text=True,
        capture_output=True,
        timeout=60,
    )
    matches = [handler.redact(line.strip()) for line in proc.stdout.splitlines() if temp_profile in line]
    return {
        "status": "ok" if proc.returncode == 0 else "ssh_error",
        "returncode": proc.returncode,
        "temp_profile_sha256": hashlib.sha256(temp_profile.encode()).hexdigest(),
        "matches": matches,
        "no_matching_processes": proc.returncode == 0 and not matches,
    }


def _local_harness_git_state() -> dict[str, Any]:
    head = subprocess.run(
        ["git", "rev-parse", "HEAD"],
        cwd=ROOT,
        text=True,
        capture_output=True,
        timeout=30,
        check=False,
    )
    status = subprocess.run(
        ["git", "status", "--porcelain", "--untracked-files=all"],
        cwd=ROOT,
        text=True,
        capture_output=True,
        timeout=30,
        check=False,
    )
    status_text = status.stdout if status.returncode == 0 else ""
    status_lines = status_text.splitlines()
    return {
        "git_head": head.stdout.strip() if head.returncode == 0 else None,
        "worktree_clean": status.returncode == 0 and not status_text.strip(),
        "status_sha256": hashlib.sha256(status_text.encode()).hexdigest() if status.returncode == 0 else None,
        "status_lines": status_lines,
        "only_control_goal_untracked": status.returncode == 0 and status_lines == ["?? goal.md"],
    }


def _portable_artifact_path(path: Path) -> str:
    resolved = path.resolve()
    try:
        return str(resolved.relative_to(ROOT))
    except ValueError:
        return str(resolved)


def run_guarded_remote(
    *,
    prompts: list[dict[str, Any]],
    suite_mode: str,
    report_prefix: str,
    prompt_note: str,
    grounding_mode: str,
    leakage_scan: dict[str, Any],
) -> dict[str, Any]:
    """Use the generic transport while substituting only the guarded builder."""

    original_builder = handler.build_remote_script

    def guarded_builder(run_id: str, **kwargs: Any) -> str:
        return build_guarded_remote_script(
            run_id,
            prompts=kwargs.get("prompts") or prompts,
            suite_mode=str(kwargs.get("suite_mode") or suite_mode),
            report_prefix=str(kwargs.get("report_prefix") or report_prefix),
            prompt_note=str(kwargs.get("prompt_note") or prompt_note),
            grounding_mode=grounding_mode,
            leakage_scan=leakage_scan,
        )

    handler.build_remote_script = guarded_builder
    try:
        remote = handler.run_remote(
            prompts=prompts,
            suite_mode=suite_mode,
            report_prefix=report_prefix,
            prompt_note=prompt_note,
        )
    finally:
        handler.build_remote_script = original_builder
    parsed = remote.get("parsed")
    if isinstance(parsed, dict):
        parsed["post_run_orphan_readback"] = _remote_orphan_readback(parsed)
    return remote


def write_score_markdown(path: Path, report: dict[str, Any]) -> None:
    score = report["score"]
    lines = [
        "# Working Leo m3taversal Out-of-Sample Score",
        "",
        f"Generated UTC: `{report['generated_at_utc']}`",
        f"Pass: `{score['pass']}`",
        f"Prompts: `{score['passes']}/{score['expected_prompt_count']}`",
        f"DB counts changed: `{report['db_counts_changed']}`",
        f"Live behavior manifest unchanged: `{report['live_behavior_manifest_unchanged']}`",
        f"Service unchanged: `{report['service_unchanged']}`",
        f"Temporary profile removed: `{report['temp_profile_removed']}`",
        f"Database context injections: `{report['database_context_injection_count']}`",
        f"Database context all OK: `{report['database_context_all_ok']}`",
        f"Database response bindings: `{report['database_response_binding_count']}`",
        f"Database response binding all OK: `{report['database_response_binding_all_ok']}`",
        f"Database response contracts satisfied: `{report['database_response_contract_satisfied_count']}`",
        f"Database response contracts all satisfied: `{report['database_response_contract_all_satisfied']}`",
        f"Database-composed replacements: `{report['database_response_transform_count']}`",
        f"Posted to Telegram: `{report['posted_to_telegram']}`",
        "",
        "## Prompt Scores",
        "",
    ]
    for item in score["scores"]:
        lines.append(f"- `{item['prompt_id']}` / `{item['dimension']}`: `pass={item['pass']}`")
    lines.extend(
        [
            "",
            "## Claim Ceiling",
            "",
            "A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner "
            "using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible "
            "Telegram delivery, production DB apply, or GCP parity.",
            "",
        ]
    )
    path.write_text("\n".join(lines), encoding="utf-8")


def build_score_report(report: dict[str, Any], *, memory_token: str) -> dict[str, Any]:
    score = benchmark.score_results(report.get("results") or [], memory_token=memory_token)
    service_unchanged = bool((report.get("service_before_after") or {}).get("unchanged_from_preexisting_live_readback"))
    return {
        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
        "mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score",
        "source_results_json": str(RESULTS_JSON),
        "memory_token": memory_token,
        "score": score,
        "db_counts_changed": report.get("db_counts_changed"),
        "changed_live_profile": report.get("changed_live_profile"),
        "live_behavior_manifest_unchanged": report.get("live_behavior_manifest_unchanged"),
        "service_unchanged": service_unchanged,
        "temp_profile_removed": report.get("temp_profile_removed"),
        "database_context_injection_count": report.get("database_context_injection_count"),
        "database_context_all_ok": report.get("database_context_all_ok"),
        "database_response_binding_count": report.get("database_response_binding_count"),
        "database_response_binding_all_ok": report.get("database_response_binding_all_ok"),
        "database_response_contract_reported_count": report.get("database_response_contract_reported_count"),
        "database_response_contract_satisfied_count": report.get("database_response_contract_satisfied_count"),
        "database_response_contract_all_satisfied": report.get("database_response_contract_all_satisfied"),
        "database_response_transform_count": report.get("database_response_transform_count"),
        "posted_to_telegram": report.get("posted_to_telegram"),
        "production_db_apply_ran": False,
    }


def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) -> bool:
    return bool(
        report.get("remote_returncode") == 0
        and report.get("pass_runtime") is True
        and report.get("db_counts_changed") is False
        and report.get("changed_live_profile") is False
        and report.get("live_behavior_manifest_unchanged") is True
        and score_report["service_unchanged"]
        and report.get("temp_profile_removed") is True
        and report.get("database_context_all_ok") is True
        and int(report.get("database_context_injection_count") or 0) >= len(report.get("results") or [])
        and report.get("database_response_binding_all_ok") is True
        and int(report.get("database_response_binding_count") or 0) >= len(report.get("results") or [])
        and report.get("database_response_contract_all_satisfied") is True
        and int(report.get("database_response_contract_reported_count") or 0) >= len(report.get("results") or [])
        and report.get("posted_to_telegram") is False
        and score_report["score"]["pass"]
    )


def write_protocol_score_markdown(path: Path, score: dict[str, Any]) -> None:
    lines = [
        "# Leo Blinded OOS Reasoning Trial",
        "",
        f"Generated UTC: `{score['generated_at_utc']}`",
        f"Protocol: `{score['protocol_id']}` / `{score['protocol_hash_sha256']}`",
        f"Trial: `{score['trial_id']}` / `{score['session_mode']}`",
        f"Pass: `{score['pass']}`",
        f"Grounded prompts: `{score['grounded_passes']}/{score['prompt_count']}`",
        f"Grounded rate: `{score['grounded_pass_rate']:.3f}`",
        f"No-DB ablation grounded rate: `{score['receipt_ablation']['grounded_pass_rate']:.3f}`",
        "",
        "## Prompt scores",
        "",
    ]
    for item in score["prompt_scores"]:
        lines.append(
            f"- `{item['prompt_id']}` / `{item['family_id']}`: semantic=`{item['semantic_pass']}`, "
            f"subject=`{item['subject_alignment']}`, receipts=`{item['receipt_pass']}`, "
            f"grounded=`{item['grounded_pass']}`"
        )
    lines.extend(
        [
            "",
            "## Safety",
            "",
            f"- Grounded live safety: `{score['top_level_safety']['pass']}`",
            f"- Ablation live safety: `{score['baseline_top_level_safety']['pass']}`",
            f"- Restart receipt: `{score['restart_receipt_validation']['pass']}`",
            f"- Exact prompt binding: `{score['prompt_binding']['pass']}`",
            "",
            "## Claim ceiling",
            "",
            "This trial proves a direct live VPS GatewayRunner reply path with a fail-closed read-only tool surface, "
            "full unchanged DB fingerprints, and no Telegram post. It does not prove Telegram-visible delivery or "
            "any production database apply.",
            "",
        ]
    )
    path.write_text("\n".join(lines), encoding="utf-8")


def _run_protocol_mode(
    protocol: dict[str, Any],
    trial: dict[str, Any],
    *,
    grounding_mode: str,
    output_dir: Path,
    leakage_scan: dict[str, Any],
) -> tuple[dict[str, Any], Path]:
    prompts = [
        {"id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"]}
        for prompt in trial["prompts"]
    ]
    safe_protocol_id = re.sub(r"[^A-Za-z0-9._-]", "-", str(protocol["protocol_id"]))
    report_prefix = f"leo-oos-{safe_protocol_id}-{trial['trial_id']}-{grounding_mode}"
    output_path = output_dir / f"{trial['trial_id']}-{grounding_mode}-handler.json"
    remote = run_guarded_remote(
        prompts=prompts,
        suite_mode=f"live_vps_gatewayrunner_blinded_oos_{grounding_mode}",
        report_prefix=report_prefix,
        prompt_note=(
            f"Frozen blinded protocol {protocol['protocol_hash_sha256']}; trial {trial['trial_id']}; "
            f"grounding mode {grounding_mode}; direct handler only, no Telegram post, no database mutation."
        ),
        grounding_mode=grounding_mode,
        leakage_scan=leakage_scan,
    )
    report = handler.write_output(remote, output_json=output_path)
    report["oos_harness_git_state"] = _local_harness_git_state()
    report["protocol_id"] = protocol["protocol_id"]
    report["protocol_hash_sha256"] = protocol["protocol_hash_sha256"]
    report["trial_id"] = trial["trial_id"]
    report["trial_prompt_set_sha256"] = trial["prompt_set_sha256"]
    report["scorer_version"] = protocol["scorer_version"]
    report["source_hashes"] = protocol["source_hashes"]
    output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    return report, output_path


def _load_restart_receipt(path: Path | None, trial: dict[str, Any]) -> dict[str, Any] | None:
    if trial["session_mode"] != "post_restart_clean_session":
        return None
    if path is None:
        raise SystemExit("--restart-receipt is required for a post-restart trial")
    return json.loads(path.read_text(encoding="utf-8"))


def _ssh_readback(command: str, *, timeout: int = 120) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        [
            "ssh",
            "-i",
            str(handler.SSH_KEY),
            "-o",
            "BatchMode=yes",
            "-o",
            "StrictHostKeyChecking=accept-new",
            handler.VPS,
            command,
        ],
        text=True,
        capture_output=True,
        timeout=timeout,
    )


def _deploy_identity() -> dict[str, Any]:
    proc = _ssh_readback(
        "cd /opt/teleo-eval/workspaces/deploy-infra && "
        "printf 'head=' && git rev-parse HEAD && "
        "printf 'stamp=' && cat /opt/teleo-eval/.last-deploy-sha",
        timeout=60,
    )
    values: dict[str, str] = {}
    for line in proc.stdout.splitlines():
        if "=" in line:
            key, value = line.split("=", 1)
            values[key] = value.strip()
    return {
        "returncode": proc.returncode,
        "head": values.get("head"),
        "stamp": values.get("stamp"),
        "stderr": handler.redact(proc.stderr),
    }


def _restart_state_probe(
    protocol: dict[str, Any],
    *,
    label: str,
    output_dir: Path,
    leakage_scan: dict[str, Any],
) -> tuple[dict[str, Any], Path]:
    output_path = output_dir / f"restart-{label}-state.json"
    remote = run_guarded_remote(
        prompts=[],
        suite_mode=f"live_vps_gateway_restart_{label}_readonly_state_probe",
        report_prefix=f"leo-oos-restart-{label}-{protocol['protocol_id']}",
        prompt_note=(
            f"Read-only restart {label} state probe for frozen protocol {protocol['protocol_hash_sha256']}; "
            "zero prompts, no Telegram post, no database mutation."
        ),
        grounding_mode="grounded",
        leakage_scan=leakage_scan,
    )
    report = handler.write_output(remote, output_json=output_path)
    report["oos_harness_git_state"] = _local_harness_git_state()
    report["protocol_id"] = protocol["protocol_id"]
    report["protocol_hash_sha256"] = protocol["protocol_hash_sha256"]
    report["source_hashes"] = protocol["source_hashes"]
    output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    return report, output_path


def _restart_probe_passes(report: dict[str, Any]) -> bool:
    before = report.get("db_fingerprint_before") or {}
    after = report.get("db_fingerprint_after") or {}
    counts_before = report.get("db_counts_before")
    counts_after = report.get("db_counts_after")
    transport = report.get("telegram_transport_deny") or {}
    return bool(
        report.get("remote_returncode") == 0
        and report.get("pass_runtime") is True
        and report.get("posted_to_telegram") is False
        and report.get("db_counts_changed") is False
        and isinstance(counts_before, dict)
        and bool(counts_before)
        and counts_before == counts_after
        and all(isinstance(value, int) and not isinstance(value, bool) for value in counts_before.values())
        and report.get("db_fingerprint_unchanged") is True
        and before.get("status") == "ok"
        and after.get("status") == "ok"
        and bool(re.fullmatch(r"[0-9a-f]{64}", str(before.get("fingerprint_sha256") or "")))
        and before.get("fingerprint_sha256") == after.get("fingerprint_sha256")
        and (report.get("preexecution_safety_gate") or {}).get("status") == "pass"
        and transport.get("enabled") is True
        and isinstance(transport.get("attempt_count"), int)
        and not isinstance(transport.get("attempt_count"), bool)
        and transport.get("attempt_count") == 0
        and transport.get("runner_adapters_empty") is True
        and report.get("temp_profile_removed") is True
        and (report.get("post_run_orphan_readback") or {}).get("no_matching_processes") is True
    )


def collect_restart_receipt(args: argparse.Namespace) -> int:
    protocol = json.loads(args.protocol.read_text(encoding="utf-8"))
    validation = protocol_lib.validate_protocol(protocol, verify_source_hashes=True)
    if not validation["pass"]:
        raise SystemExit(f"frozen protocol validation failed: {validation['issues']}")
    restart_trials = [
        item for item in protocol.get("trials") or [] if item.get("session_mode") == "post_restart_clean_session"
    ]
    if len(restart_trials) != 1:
        raise SystemExit("frozen protocol must identify exactly one post-restart trial")
    next_trial = restart_trials[0]
    leakage_scan = prompt_leakage_scan(protocol)
    if not leakage_scan["pass"]:
        raise SystemExit(f"runtime prompt leakage detected: {leakage_scan['exact_prompt_matches']}")
    args.output_dir.mkdir(parents=True, exist_ok=True)
    before_deploy = _deploy_identity()
    before_report, before_path = _restart_state_probe(
        protocol,
        label="before",
        output_dir=args.output_dir,
        leakage_scan=leakage_scan,
    )
    if (
        before_deploy.get("returncode") != 0
        or not re.fullmatch(r"[0-9a-f]{40}", str(before_deploy.get("head") or ""))
        or before_deploy.get("head") != before_deploy.get("stamp")
    ):
        raise SystemExit(f"deploy identity preflight failed: {before_deploy}")
    if not _restart_probe_passes(before_report):
        raise SystemExit(f"read-only pre-restart probe failed: {before_path}")

    restart_started_at_utc = datetime.now(timezone.utc).isoformat()
    restart = _ssh_readback(
        "systemctl restart leoclean-gateway.service && "
        "for i in $(seq 1 60); do "
        'if [ "$(systemctl is-active leoclean-gateway.service)" = active ]; then break; fi; sleep 1; done; '
        "systemctl show leoclean-gateway.service "
        "-p ActiveState -p SubState -p MainPID -p NRestarts -p ExecMainStartTimestamp",
        timeout=120,
    )
    restart_ended_at_utc = datetime.now(timezone.utc).isoformat()
    after_deploy = _deploy_identity()
    after_report, after_path = _restart_state_probe(
        protocol,
        label="after",
        output_dir=args.output_dir,
        leakage_scan=leakage_scan,
    )
    service_before = (before_report.get("service_before_after") or {}).get("after") or {}
    service_after = after_report.get("before_service") or {}
    fingerprint_before = before_report.get("db_fingerprint_after") or {}
    fingerprint_after = after_report.get("db_fingerprint_before") or {}
    counts_before = before_report.get("db_counts_after") or {}
    counts_after = after_report.get("db_counts_before") or {}
    receipt = {
        "schema": "livingip.leoGatewayRestartReceipt.v1",
        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
        "protocol_id": protocol["protocol_id"],
        "protocol_hash_sha256": protocol["protocol_hash_sha256"],
        "next_trial_id": next_trial["trial_id"],
        "next_trial_prompt_set_sha256": next_trial["prompt_set_sha256"],
        "authorization_basis": "goal.md requires restart trials; service restart only, no Telegram post and no DB mutation",
        "restart_command": "systemctl restart leoclean-gateway.service",
        "restart_started_at_utc": restart_started_at_utc,
        "restart_ended_at_utc": restart_ended_at_utc,
        "restart_returncode": restart.returncode,
        "restart_stdout": handler.redact(restart.stdout),
        "restart_stderr": handler.redact(restart.stderr),
        "service_before": service_before,
        "service_after": service_after,
        "deploy_before": before_deploy,
        "deploy_after": after_deploy,
        "db_counts_before": counts_before,
        "db_counts_after": counts_after,
        "db_counts_changed": counts_before != counts_after,
        "db_fingerprint_before": fingerprint_before,
        "db_fingerprint_after": fingerprint_after,
        "db_fingerprint_unchanged": bool(
            fingerprint_before.get("status") == "ok"
            and fingerprint_after.get("status") == "ok"
            and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256")
        ),
        "posted_to_telegram": False,
        "before_probe": {
            "path": _portable_artifact_path(before_path),
            "sha256": hashlib.sha256(before_path.read_bytes()).hexdigest(),
            "pass": _restart_probe_passes(before_report),
        },
        "after_probe": {
            "path": _portable_artifact_path(after_path),
            "sha256": hashlib.sha256(after_path.read_bytes()).hexdigest(),
            "pass": _restart_probe_passes(after_report),
        },
    }
    receipt["checks"] = {
        "restart_command_succeeded": receipt["restart_returncode"] == 0,
        "service_active_after": service_after.get("ActiveState") == "active"
        and service_after.get("SubState") == "running",
        "service_pid_changed": bool(
            service_before.get("MainPID") and service_before.get("MainPID") != service_after.get("MainPID")
        ),
        "deploy_identity_unchanged": before_deploy.get("returncode") == 0
        and after_deploy.get("returncode") == 0
        and bool(re.fullmatch(r"[0-9a-f]{40}", str(before_deploy.get("head") or "")))
        and before_deploy.get("head")
        == before_deploy.get("stamp")
        == after_deploy.get("head")
        == after_deploy.get("stamp"),
        "db_counts_unchanged": receipt["db_counts_changed"] is False,
        "db_counts_complete": isinstance(counts_before, dict)
        and bool(counts_before)
        and counts_before == counts_after
        and all(isinstance(value, int) and not isinstance(value, bool) for value in counts_before.values()),
        "db_fingerprint_unchanged": receipt["db_fingerprint_unchanged"] is True,
        "db_fingerprint_complete": bool(
            re.fullmatch(r"[0-9a-f]{64}", str(fingerprint_before.get("fingerprint_sha256") or ""))
            and fingerprint_before.get("fingerprint_sha256") == fingerprint_after.get("fingerprint_sha256")
        ),
        "before_probe_passed": receipt["before_probe"]["pass"] is True,
        "after_probe_passed": receipt["after_probe"]["pass"] is True,
    }
    receipt["pass"] = all(receipt["checks"].values())
    args.collect_restart_receipt.parent.mkdir(parents=True, exist_ok=True)
    args.collect_restart_receipt.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    print(
        json.dumps(
            {
                "restart_receipt": str(args.collect_restart_receipt),
                "pass": receipt["pass"],
                "checks": receipt["checks"],
            },
            indent=2,
            sort_keys=True,
        )
    )
    return 0 if receipt["pass"] else 1


def run_or_score_protocol_trial(args: argparse.Namespace) -> int:
    protocol = json.loads(args.protocol.read_text(encoding="utf-8"))
    validation = protocol_lib.validate_protocol(protocol, verify_source_hashes=True)
    if not validation["pass"]:
        raise SystemExit(f"frozen protocol validation failed: {validation['issues']}")
    trial = next((item for item in protocol["trials"] if item["trial_id"] == args.trial_id), None)
    if trial is None:
        raise SystemExit(f"unknown trial id: {args.trial_id}")
    leakage_scan = prompt_leakage_scan(protocol)
    if not leakage_scan["pass"]:
        raise SystemExit(f"runtime prompt leakage detected: {leakage_scan['exact_prompt_matches']}")
    args.output_dir.mkdir(parents=True, exist_ok=True)
    restart_receipt = _load_restart_receipt(args.restart_receipt, trial)

    if bool(args.grounded_report) != bool(args.baseline_report):
        raise SystemExit("--grounded-report and --baseline-report must be supplied together")
    if args.grounded_report:
        grounded_report = json.loads(args.grounded_report.read_text(encoding="utf-8"))
        baseline_report = json.loads(args.baseline_report.read_text(encoding="utf-8"))
        grounded_path = args.grounded_report
        baseline_path = args.baseline_report
    elif args.grounding_mode:
        report, output_path = _run_protocol_mode(
            protocol,
            trial,
            grounding_mode=args.grounding_mode,
            output_dir=args.output_dir,
            leakage_scan=leakage_scan,
        )
        mode_safety = protocol_lib._top_level_safety(
            report,
            require_handler_safety_gate=args.grounding_mode == "grounded",
            expected_harness_git_head=protocol["harness_git_head"],
        )
        mode_checks = {
            "expected_grounding_mode": report.get("grounding_mode") == args.grounding_mode,
            "db_context_state": report.get("db_context_plugin_enabled") is (args.grounding_mode == "grounded"),
            "tool_surface_mode": (report.get("read_only_tool_surface") or {}).get("mode")
            == ("read_only_kb" if args.grounding_mode == "grounded" else "no_db_ablation"),
            "zero_context_in_ablation": args.grounding_mode != "db_tool_ablated"
            or all(not (item.get("database_context_trace") or []) for item in report.get("results") or []),
        }
        mode_pass = mode_safety["pass"] and all(mode_checks.values())
        print(
            json.dumps(
                {
                    "report": str(output_path),
                    "grounding_mode": args.grounding_mode,
                    "handler_safety_gate": report.get("safety_gate"),
                    "post_run_orphan_readback": report.get("post_run_orphan_readback"),
                    "mode_safety": mode_safety,
                    "mode_checks": mode_checks,
                    "pass": mode_pass,
                },
                indent=2,
                sort_keys=True,
            )
        )
        return 0 if mode_pass else 1
    else:
        grounded_report, grounded_path = _run_protocol_mode(
            protocol,
            trial,
            grounding_mode="grounded",
            output_dir=args.output_dir,
            leakage_scan=leakage_scan,
        )
        baseline_report, baseline_path = _run_protocol_mode(
            protocol,
            trial,
            grounding_mode="db_tool_ablated",
            output_dir=args.output_dir,
            leakage_scan=leakage_scan,
        )

    score = protocol_lib.score_live_trial(
        protocol,
        trial["trial_id"],
        grounded_report,
        baseline_report=baseline_report,
        restart_receipt=restart_receipt,
    )
    score["grounded_report_path"] = _portable_artifact_path(grounded_path)
    score["baseline_report_path"] = _portable_artifact_path(baseline_path)
    score["grounded_report_sha256"] = hashlib.sha256(grounded_path.read_bytes()).hexdigest()
    score["baseline_report_sha256"] = hashlib.sha256(baseline_path.read_bytes()).hexdigest()
    if restart_receipt is not None and args.restart_receipt is not None:
        score["restart_receipt_path"] = _portable_artifact_path(args.restart_receipt)
        score["restart_receipt_sha256"] = hashlib.sha256(args.restart_receipt.read_bytes()).hexdigest()
        score["restart_receipt_payload_sha256"] = protocol_lib.canonical_sha256(restart_receipt)
    score_path = args.output_dir / f"{trial['trial_id']}-score.json"
    markdown_path = args.output_dir / f"{trial['trial_id']}-score.md"
    score_path.write_text(json.dumps(score, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    write_protocol_score_markdown(markdown_path, score)
    print(
        json.dumps(
            {
                "grounded_report": str(grounded_path),
                "baseline_report": str(baseline_path),
                "score_json": str(score_path),
                "score_markdown": str(markdown_path),
                "pass": score["pass"],
                "grounded_rate": score["grounded_pass_rate"],
                "baseline_grounded_rate": score["receipt_ablation"]["grounded_pass_rate"],
            },
            indent=2,
            sort_keys=True,
        )
    )
    return 0 if score["pass"] else 1


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--score-existing",
        action="store_true",
        help="Rescore the retained live transcript without making another remote model call.",
    )
    parser.add_argument("--protocol", type=Path)
    parser.add_argument("--trial-id")
    parser.add_argument("--output-dir", type=Path, default=PROTOCOL_REPORT_DIR)
    parser.add_argument("--grounding-mode", choices=GROUNDING_MODES)
    parser.add_argument("--grounded-report", type=Path)
    parser.add_argument("--baseline-report", type=Path)
    parser.add_argument("--restart-receipt", type=Path)
    parser.add_argument("--collect-restart-receipt", type=Path)
    args = parser.parse_args()

    if args.collect_restart_receipt:
        if not args.protocol or args.trial_id:
            raise SystemExit("--collect-restart-receipt requires --protocol and does not accept --trial-id")
        return collect_restart_receipt(args)
    if args.protocol or args.trial_id:
        if not args.protocol or not args.trial_id:
            raise SystemExit("--protocol and --trial-id are required together")
        return run_or_score_protocol_trial(args)
    if not args.score_existing:
        raise SystemExit(
            "refusing the legacy fixed live suite; use --protocol and --trial-id for a frozen guarded trial"
        )

    if args.score_existing:
        report = json.loads(RESULTS_JSON.read_text(encoding="utf-8"))
        previous_score = json.loads(SCORE_JSON.read_text(encoding="utf-8"))
        memory_token = str(previous_score["memory_token"])

    score_report = build_score_report(report, memory_token=memory_token)
    SCORE_JSON.write_text(json.dumps(score_report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    write_score_markdown(SCORE_MARKDOWN, score_report)
    safe_pass = score_report_passes(report, score_report)
    print(
        json.dumps(
            {
                "results_json": str(RESULTS_JSON),
                "score_json": str(SCORE_JSON),
                "score_markdown": str(SCORE_MARKDOWN),
                "pass": safe_pass,
                "prompts": f"{score_report['score']['passes']}/{score_report['score']['expected_prompt_count']}",
                "rescored_existing": args.score_existing,
            },
            indent=2,
            sort_keys=True,
        )
    )
    return 0 if safe_pass else 1


if __name__ == "__main__":
    raise SystemExit(main())
