teleo-infrastructure/scripts/build_telegram_visible_direct_claim_authorization_packet.py

#!/usr/bin/env python3
"""Build an exact authorization packet for a Telegram-visible DC benchmark run.

The packet is a planner, not a sender. It records the raw no-context DC-01..DC-06
messages, the target Telegram group, the expected proof artifacts, and the
required authorization sentence. It never sends Telegram messages and never
mutates the database.
"""

from __future__ import annotations

import argparse
import json
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import working_leo_open_ended_benchmark as benchmark

REPORT_DIR = Path("docs/reports/leo-working-state-20260709")
DEFAULT_OUT = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.json"
DEFAULT_MARKDOWN_OUT = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.md"
DEFAULT_LIVE_OPEN_SCORE = REPORT_DIR / "telegram-live-open-ended-suite-score-current.json"
DEFAULT_HANDLER_SUITE = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json"
DEFAULT_HANDLER_SCORE = REPORT_DIR / "telegram-handler-direct-claim-suite-score-current.json"
DEFAULT_GCP_PROBE = REPORT_DIR / "gcp-db-parity-current-probe-current.json"
DEFAULT_CAPTURE_RECEIPT = REPORT_DIR / "telegram-visible-direct-claim-capture-receipt-current.json"
DEFAULT_CAPTURE_RECEIPT_MARKDOWN = REPORT_DIR / "telegram-visible-direct-claim-capture-receipt-current.md"
DEFAULT_CAPTURE_RESULTS = REPORT_DIR / "telegram-visible-direct-claim-capture-results-current.json"
DEFAULT_CHAT_ID = "-5146042086"
DEFAULT_CHAT_LABEL = "Leo"
DEFAULT_PROOF_DIR = "outputs/telegram-visible-direct-claim-suite-20260710"


def _load_json(path: Path) -> dict[str, Any]:
    return json.loads(path.read_text(encoding="utf-8"))


def _path_status(path: Path) -> dict[str, Any]:
    exists = path.exists()
    return {"path": str(path), "exists": exists, "bytes": path.stat().st_size if exists else 0}


def current_git_sha(repo_root: Path = Path(".")) -> str:
    proc = subprocess.run(
        ["git", "rev-parse", "HEAD"],
        cwd=repo_root,
        text=True,
        capture_output=True,
        check=False,
    )
    return proc.stdout.strip() if proc.returncode == 0 else ""


def direct_claim_messages() -> list[dict[str, Any]]:
    return [
        {
            "prompt_id": prompt["id"],
            "dimension": prompt["dimension"],
            "message": prompt["message"],
            "expected_answer": prompt["expected_answer"],
            "expected_follow_up": prompt["expected_follow_up"],
            "required_signals": list(prompt["required_signals"]),
        }
        for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
    ]


def _score_passes(score: dict[str, Any], *, expected_count: int) -> bool:
    score_block = score.get("score", {})
    return (
        score_block.get("pass") is True
        and score_block.get("passes") == score_block.get("expected_prompt_count") == expected_count
        and score_block.get("failures") == []
        and score_block.get("missing_prompt_ids") == []
        and score_block.get("unexpected_prompt_ids") == []
    )


def _live_open_score_passes(score: dict[str, Any]) -> bool:
    score_block = score.get("score", {})
    return score_block.get("pass") is True and score_block.get("passes") == score_block.get(
        "expected_prompt_count"
    )


def _handler_suite_is_safe(suite: dict[str, Any], *, expected_count: int) -> bool:
    return (
        suite.get("pass_runtime") is True
        and suite.get("posted_to_telegram") is False
        and suite.get("changed_live_profile") is False
        and suite.get("mutates_kb_by_harness") is False
        and suite.get("db_counts_changed") is False
        and suite.get("temp_profile_removed") is True
        and len(suite.get("results", [])) == expected_count
    )


def build_packet(
    *,
    live_open_score_path: Path = DEFAULT_LIVE_OPEN_SCORE,
    handler_suite_path: Path = DEFAULT_HANDLER_SUITE,
    handler_score_path: Path = DEFAULT_HANDLER_SCORE,
    gcp_probe_path: Path = DEFAULT_GCP_PROBE,
    chat_id: str = DEFAULT_CHAT_ID,
    chat_label: str = DEFAULT_CHAT_LABEL,
    proof_dir: str = DEFAULT_PROOF_DIR,
    git_sha: str | None = None,
) -> dict[str, Any]:
    messages = direct_claim_messages()
    live_open_score = _load_json(live_open_score_path)
    handler_suite = _load_json(handler_suite_path)
    handler_score = _load_json(handler_score_path)
    gcp_probe = _load_json(gcp_probe_path) if gcp_probe_path.exists() else {}
    sha = git_sha if git_sha is not None else current_git_sha()
    expected_ids = [item["prompt_id"] for item in messages]

    evidence_paths = {
        "live_open_score": _path_status(live_open_score_path),
        "handler_suite": _path_status(handler_suite_path),
        "handler_score": _path_status(handler_score_path),
        "gcp_probe": _path_status(gcp_probe_path),
    }
    expected_proof_paths = {
        "results_json": f"{proof_dir}/telegram-visible-direct-claim-suite-results-current.json",
        "score_json": f"{proof_dir}/telegram-visible-direct-claim-suite-score-current.json",
        "score_markdown": f"{proof_dir}/telegram-visible-direct-claim-suite-score-current.md",
        "final_screenshot": f"{proof_dir}/telegram-visible-direct-claim-suite-final.png",
        "final_accessibility": f"{proof_dir}/telegram-visible-direct-claim-suite-final-accessibility.txt",
        "db_before_after": f"{proof_dir}/telegram-visible-direct-claim-suite-db-before-after.json",
        "service_readback": f"{proof_dir}/telegram-visible-direct-claim-suite-service-readback.txt",
        "capture_receipt_json": str(DEFAULT_CAPTURE_RECEIPT),
        "capture_receipt_markdown": str(DEFAULT_CAPTURE_RECEIPT_MARKDOWN),
        "capture_results_json": str(DEFAULT_CAPTURE_RESULTS),
    }

    checks = {
        "live_open_telegram_suite_passed": _live_open_score_passes(live_open_score),
        "handler_direct_claim_suite_safe": _handler_suite_is_safe(handler_suite, expected_count=len(messages)),
        "handler_direct_claim_score_passed": _score_passes(handler_score, expected_count=len(messages)),
        "all_evidence_paths_exist": all(item["exists"] and item["bytes"] > 0 for item in evidence_paths.values()),
        "raw_dc_messages_preserved": [item["message"] for item in messages]
        == [prompt["message"] for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS],
        "telegram_visible_messages_not_sent_by_packet": True,
        "production_apply_not_authorized_by_packet": True,
        "gcp_parity_not_required_for_vps_telegram_test": gcp_probe.get("ready_for_gcp_readonly_parity") is not True,
    }
    ready_to_request = all(checks.values())
    authorization_text = (
        "I authorize a Telegram-visible read-only direct-claim benchmark in the Leo group "
        f"`{chat_label}` (`{chat_id}`) using exactly the six raw messages DC-01 through DC-06 listed in "
        "`telegram-visible-direct-claim-authorization-packet-current.json`. Capture before/after DB counts, "
        "Telegram-visible replies, scorer output, final screenshot/accessibility proof, and service readback. "
        "Do not run production DB apply, do not edit SOUL.md, do not post any extra messages beyond those six "
        "benchmark prompts unless separately authorized."
    )

    return {
        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
        "mode": "telegram_visible_direct_claim_authorization_packet_not_sent",
        "packet_generation_git_sha": sha,
        "ready_to_request_authorization": ready_to_request,
        "authorization_gate_status": "ready_to_request_explicit_authorization" if ready_to_request else "not_ready",
        "telegram_visible_messages_sent": False,
        "telegram_visible_message_authorization_present": False,
        "production_apply_executed": False,
        "production_apply_authorization_present": False,
        "mutates_db": False,
        "target": {
            "platform": "telegram",
            "chat_label": chat_label,
            "chat_id": chat_id,
            "message_count": len(messages),
            "expected_prompt_ids": expected_ids,
        },
        "exact_messages": messages,
        "checks": checks,
        "evidence_paths": evidence_paths,
        "expected_proof_paths_after_authorized_run": expected_proof_paths,
        "explicit_operator_authorization_text": authorization_text,
        "clear_CTA": (
            "To run the Telegram-visible DC benchmark, reply with the exact authorization text above. After that, "
            "Codex should send only the six listed messages to the Leo group, capture the retained proof paths, "
            "score the retained replies with `scripts/working_leo_open_ended_benchmark.py --results-json ... "
            "--include-direct-claim-followups`, and read back DB/service stability."
        ),
        "next_non_user_action_after_authorization": [
            "Read current service state and DB counts before sending.",
            "Send exactly the six raw DC-01..DC-06 messages to Telegram group Leo.",
            "Capture each Telegram-visible Leo reply plus a final screenshot/accessibility readback.",
            "Read DB counts and service state after sending.",
            "Assemble the retained capture receipt and score it before updating the current truth index.",
        ],
        "do_not_run_without": [
            "Exact operator authorization text naming the Telegram group and six-message scope.",
            "Authenticated Telegram UI/API route that can send to the Leo group.",
            "Before-count and after-count readbacks for public.* and kb_stage.kb_proposals.",
        ],
        "claim_ceiling": (
            "This packet makes the Telegram-visible DC benchmark executable and auditable. It does not send "
            "Telegram messages, does not mutate the DB, does not authorize production apply, and does not prove the "
            "DC prompts are Telegram-visible until the authorized run is actually completed and scored."
        ),
    }


def write_json(path: Path, data: dict[str, Any]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")


def write_markdown(path: Path, data: dict[str, Any]) -> None:
    lines = [
        "# Telegram-Visible Direct-Claim Authorization Packet",
        "",
        f"Generated UTC: `{data['generated_at_utc']}`",
        f"Mode: `{data['mode']}`",
        f"Ready to request authorization: `{data['ready_to_request_authorization']}`",
        f"Authorization gate status: `{data['authorization_gate_status']}`",
        f"Telegram-visible messages sent: `{data['telegram_visible_messages_sent']}`",
        f"Telegram-visible authorization present: `{data['telegram_visible_message_authorization_present']}`",
        f"Production apply executed: `{data['production_apply_executed']}`",
        f"Mutates DB: `{data['mutates_db']}`",
        "",
        "## Target",
        "",
        f"- Chat: `{data['target']['chat_label']}` (`{data['target']['chat_id']}`)",
        f"- Message count: `{data['target']['message_count']}`",
        f"- Prompt IDs: `{', '.join(data['target']['expected_prompt_ids'])}`",
        "",
        "## Exact Messages",
        "",
    ]
    for item in data["exact_messages"]:
        lines.extend(
            [
                f"### {item['prompt_id']} / {item['dimension']}",
                "",
                item["message"],
                "",
                f"- Required signals: `{', '.join(item['required_signals'])}`",
                "",
            ]
        )
    lines.extend(["## Checks", ""])
    for key, value in sorted(data["checks"].items()):
        lines.append(f"- `{key}`: `{value}`")
    lines.extend(
        [
            "",
            "## Explicit Operator Authorization Text",
            "",
            data["explicit_operator_authorization_text"],
            "",
            "## Clear CTA",
            "",
            data["clear_CTA"],
            "",
            "## Next Non-User Action After Authorization",
            "",
        ]
    )
    lines.extend(f"- {item}" for item in data["next_non_user_action_after_authorization"])
    lines.extend(["", "## Do Not Run Without", ""])
    lines.extend(f"- {item}" for item in data["do_not_run_without"])
    lines.extend(["", "## Claim Ceiling", "", data["claim_ceiling"], ""])
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text("\n".join(lines), encoding="utf-8")


def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--live-open-score", type=Path, default=DEFAULT_LIVE_OPEN_SCORE)
    parser.add_argument("--handler-suite", type=Path, default=DEFAULT_HANDLER_SUITE)
    parser.add_argument("--handler-score", type=Path, default=DEFAULT_HANDLER_SCORE)
    parser.add_argument("--gcp-probe", type=Path, default=DEFAULT_GCP_PROBE)
    parser.add_argument("--chat-id", default=DEFAULT_CHAT_ID)
    parser.add_argument("--chat-label", default=DEFAULT_CHAT_LABEL)
    parser.add_argument("--proof-dir", default=DEFAULT_PROOF_DIR)
    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
    parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT)
    return parser.parse_args(argv)


def main(argv: list[str] | None = None) -> int:
    args = parse_args(argv)
    data = build_packet(
        live_open_score_path=args.live_open_score,
        handler_suite_path=args.handler_suite,
        handler_score_path=args.handler_score,
        gcp_probe_path=args.gcp_probe,
        chat_id=args.chat_id,
        chat_label=args.chat_label,
        proof_dir=args.proof_dir,
    )
    write_json(args.out, data)
    write_markdown(args.markdown_out, data)
    print(
        json.dumps(
            {
                "out": str(args.out),
                "markdown_out": str(args.markdown_out),
                "ready_to_request_authorization": data["ready_to_request_authorization"],
                "telegram_visible_messages_sent": data["telegram_visible_messages_sent"],
                "authorization_gate_status": data["authorization_gate_status"],
            },
            indent=2,
            sort_keys=True,
        )
    )
    return 0 if data["ready_to_request_authorization"] else 2


if __name__ == "__main__":
    raise SystemExit(main())