teleo-infrastructure/scripts/run_leo_m3taversal_oos_handler_suite.py

#!/usr/bin/env python3
"""Run and score the read-only m3taversal out-of-sample suite on the live VPS handler."""

from __future__ import annotations

import argparse
import json
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import run_leo_direct_claim_handler_suite as handler
import working_leo_m3taversal_oos_benchmark as benchmark

ROOT = Path(__file__).resolve().parents[1]
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
RESULTS_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-current.json"
SCORE_JSON = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.json"
SCORE_MARKDOWN = REPORT_DIR / "telegram-handler-m3taversal-oos-suite-score-current.md"


def write_score_markdown(path: Path, report: dict[str, Any]) -> None:
    score = report["score"]
    lines = [
        "# Working Leo m3taversal Out-of-Sample Score",
        "",
        f"Generated UTC: `{report['generated_at_utc']}`",
        f"Pass: `{score['pass']}`",
        f"Prompts: `{score['passes']}/{score['expected_prompt_count']}`",
        f"DB counts changed: `{report['db_counts_changed']}`",
        f"Service unchanged: `{report['service_unchanged']}`",
        f"Temporary profile removed: `{report['temp_profile_removed']}`",
        f"Database context injections: `{report['database_context_injection_count']}`",
        f"Database context all OK: `{report['database_context_all_ok']}`",
        f"Posted to Telegram: `{report['posted_to_telegram']}`",
        "",
        "## Prompt Scores",
        "",
    ]
    for item in score["scores"]:
        lines.append(f"- `{item['prompt_id']}` / `{item['dimension']}`: `pass={item['pass']}`")
    lines.extend(
        [
            "",
            "## Claim Ceiling",
            "",
            "A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner "
            "using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible "
            "Telegram delivery, production DB apply, or GCP parity.",
            "",
        ]
    )
    path.write_text("\n".join(lines), encoding="utf-8")


def build_score_report(report: dict[str, Any], *, memory_token: str) -> dict[str, Any]:
    score = benchmark.score_results(report.get("results") or [], memory_token=memory_token)
    service_unchanged = bool((report.get("service_before_after") or {}).get("unchanged_from_preexisting_live_readback"))
    return {
        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
        "mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score",
        "source_results_json": str(RESULTS_JSON),
        "memory_token": memory_token,
        "score": score,
        "db_counts_changed": report.get("db_counts_changed"),
        "service_unchanged": service_unchanged,
        "temp_profile_removed": report.get("temp_profile_removed"),
        "database_context_injection_count": report.get("database_context_injection_count"),
        "database_context_all_ok": report.get("database_context_all_ok"),
        "posted_to_telegram": report.get("posted_to_telegram"),
        "production_db_apply_ran": False,
    }


def score_report_passes(report: dict[str, Any], score_report: dict[str, Any]) -> bool:
    return bool(
        report.get("remote_returncode") == 0
        and report.get("pass_runtime") is True
        and report.get("db_counts_changed") is False
        and score_report["service_unchanged"]
        and report.get("temp_profile_removed") is True
        and report.get("database_context_all_ok") is True
        and int(report.get("database_context_injection_count") or 0) >= len(report.get("results") or [])
        and report.get("posted_to_telegram") is False
        and score_report["score"]["pass"]
    )


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--score-existing",
        action="store_true",
        help="Rescore the retained live transcript without making another remote model call.",
    )
    args = parser.parse_args()

    if args.score_existing:
        report = json.loads(RESULTS_JSON.read_text(encoding="utf-8"))
        previous_score = json.loads(SCORE_JSON.read_text(encoding="utf-8"))
        memory_token = str(previous_score["memory_token"])
    else:
        memory_token = "demo-ledger-" + uuid.uuid4().hex[:8]
        prompts = [
            {"id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"]}
            for prompt in benchmark.prompt_catalog(memory_token)
        ]
        remote = handler.run_remote(
            prompts=prompts,
            suite_mode="live_vps_gatewayrunner_temp_profile_m3taversal_out_of_sample_suite",
            report_prefix="leo-m3taversal-oos-handler-report",
            prompt_note="Broad out-of-sample m3taversal prompts plus randomized memory and participant-identity checks.",
        )
        report = handler.write_output(remote, output_json=RESULTS_JSON)

    score_report = build_score_report(report, memory_token=memory_token)
    SCORE_JSON.write_text(json.dumps(score_report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    write_score_markdown(SCORE_MARKDOWN, score_report)
    safe_pass = score_report_passes(report, score_report)
    print(
        json.dumps(
            {
                "results_json": str(RESULTS_JSON),
                "score_json": str(SCORE_JSON),
                "score_markdown": str(SCORE_MARKDOWN),
                "pass": safe_pass,
                "prompts": f"{score_report['score']['passes']}/{score_report['score']['expected_prompt_count']}",
                "rescored_existing": args.score_existing,
            },
            indent=2,
            sort_keys=True,
        )
    )
    return 0 if safe_pass else 1


if __name__ == "__main__":
    raise SystemExit(main())