#!/usr/bin/env python3 """Build an exact authorization packet for a Telegram-visible DC benchmark run. The packet is a planner, not a sender. It records the raw no-context DC-01..DC-06 messages, the target Telegram group, the expected proof artifacts, and the required authorization sentence. It never sends Telegram messages and never mutates the database. """ from __future__ import annotations import argparse import json import subprocess from datetime import datetime, timezone from pathlib import Path from typing import Any import working_leo_open_ended_benchmark as benchmark REPORT_DIR = Path("docs/reports/leo-working-state-20260709") DEFAULT_OUT = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.json" DEFAULT_MARKDOWN_OUT = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.md" DEFAULT_LIVE_OPEN_SCORE = REPORT_DIR / "telegram-live-open-ended-suite-score-current.json" DEFAULT_HANDLER_SUITE = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json" DEFAULT_HANDLER_SCORE = REPORT_DIR / "telegram-handler-direct-claim-suite-score-current.json" DEFAULT_GCP_PROBE = REPORT_DIR / "gcp-db-parity-current-probe-current.json" DEFAULT_CAPTURE_RECEIPT = REPORT_DIR / "telegram-visible-direct-claim-capture-receipt-current.json" DEFAULT_CAPTURE_RECEIPT_MARKDOWN = REPORT_DIR / "telegram-visible-direct-claim-capture-receipt-current.md" DEFAULT_CAPTURE_RESULTS = REPORT_DIR / "telegram-visible-direct-claim-capture-results-current.json" DEFAULT_CHAT_ID = "-5146042086" DEFAULT_CHAT_LABEL = "Leo" DEFAULT_PROOF_DIR = "outputs/telegram-visible-direct-claim-suite-20260710" def _load_json(path: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) def _path_status(path: Path) -> dict[str, Any]: exists = path.exists() return {"path": str(path), "exists": exists, "bytes": path.stat().st_size if exists else 0} def current_git_sha(repo_root: Path = Path(".")) -> str: proc = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=repo_root, text=True, capture_output=True, check=False, ) return proc.stdout.strip() if proc.returncode == 0 else "" def direct_claim_messages() -> list[dict[str, Any]]: return [ { "prompt_id": prompt["id"], "dimension": prompt["dimension"], "message": prompt["message"], "expected_answer": prompt["expected_answer"], "expected_follow_up": prompt["expected_follow_up"], "required_signals": list(prompt["required_signals"]), } for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS ] def _score_passes(score: dict[str, Any], *, expected_count: int) -> bool: score_block = score.get("score", {}) return ( score_block.get("pass") is True and score_block.get("passes") == score_block.get("expected_prompt_count") == expected_count and score_block.get("failures") == [] and score_block.get("missing_prompt_ids") == [] and score_block.get("unexpected_prompt_ids") == [] ) def _live_open_score_passes(score: dict[str, Any]) -> bool: score_block = score.get("score", {}) return score_block.get("pass") is True and score_block.get("passes") == score_block.get( "expected_prompt_count" ) def _handler_suite_is_safe(suite: dict[str, Any], *, expected_count: int) -> bool: return ( suite.get("pass_runtime") is True and suite.get("posted_to_telegram") is False and suite.get("changed_live_profile") is False and suite.get("mutates_kb_by_harness") is False and suite.get("db_counts_changed") is False and suite.get("temp_profile_removed") is True and len(suite.get("results", [])) == expected_count ) def build_packet( *, live_open_score_path: Path = DEFAULT_LIVE_OPEN_SCORE, handler_suite_path: Path = DEFAULT_HANDLER_SUITE, handler_score_path: Path = DEFAULT_HANDLER_SCORE, gcp_probe_path: Path = DEFAULT_GCP_PROBE, chat_id: str = DEFAULT_CHAT_ID, chat_label: str = DEFAULT_CHAT_LABEL, proof_dir: str = DEFAULT_PROOF_DIR, git_sha: str | None = None, ) -> dict[str, Any]: messages = direct_claim_messages() live_open_score = _load_json(live_open_score_path) handler_suite = _load_json(handler_suite_path) handler_score = _load_json(handler_score_path) gcp_probe = _load_json(gcp_probe_path) if gcp_probe_path.exists() else {} sha = git_sha if git_sha is not None else current_git_sha() expected_ids = [item["prompt_id"] for item in messages] evidence_paths = { "live_open_score": _path_status(live_open_score_path), "handler_suite": _path_status(handler_suite_path), "handler_score": _path_status(handler_score_path), "gcp_probe": _path_status(gcp_probe_path), } expected_proof_paths = { "results_json": f"{proof_dir}/telegram-visible-direct-claim-suite-results-current.json", "score_json": f"{proof_dir}/telegram-visible-direct-claim-suite-score-current.json", "score_markdown": f"{proof_dir}/telegram-visible-direct-claim-suite-score-current.md", "final_screenshot": f"{proof_dir}/telegram-visible-direct-claim-suite-final.png", "final_accessibility": f"{proof_dir}/telegram-visible-direct-claim-suite-final-accessibility.txt", "db_before_after": f"{proof_dir}/telegram-visible-direct-claim-suite-db-before-after.json", "service_readback": f"{proof_dir}/telegram-visible-direct-claim-suite-service-readback.txt", "capture_receipt_json": str(DEFAULT_CAPTURE_RECEIPT), "capture_receipt_markdown": str(DEFAULT_CAPTURE_RECEIPT_MARKDOWN), "capture_results_json": str(DEFAULT_CAPTURE_RESULTS), } checks = { "live_open_telegram_suite_passed": _live_open_score_passes(live_open_score), "handler_direct_claim_suite_safe": _handler_suite_is_safe(handler_suite, expected_count=len(messages)), "handler_direct_claim_score_passed": _score_passes(handler_score, expected_count=len(messages)), "all_evidence_paths_exist": all(item["exists"] and item["bytes"] > 0 for item in evidence_paths.values()), "raw_dc_messages_preserved": [item["message"] for item in messages] == [prompt["message"] for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS], "telegram_visible_messages_not_sent_by_packet": True, "production_apply_not_authorized_by_packet": True, "gcp_parity_not_required_for_vps_telegram_test": gcp_probe.get("ready_for_gcp_readonly_parity") is not True, } ready_to_request = all(checks.values()) authorization_text = ( "I authorize a Telegram-visible read-only direct-claim benchmark in the Leo group " f"`{chat_label}` (`{chat_id}`) using exactly the six raw messages DC-01 through DC-06 listed in " "`telegram-visible-direct-claim-authorization-packet-current.json`. Capture before/after DB counts, " "Telegram-visible replies, scorer output, final screenshot/accessibility proof, and service readback. " "Do not run production DB apply, do not edit SOUL.md, do not post any extra messages beyond those six " "benchmark prompts unless separately authorized." ) return { "generated_at_utc": datetime.now(timezone.utc).isoformat(), "mode": "telegram_visible_direct_claim_authorization_packet_not_sent", "packet_generation_git_sha": sha, "ready_to_request_authorization": ready_to_request, "authorization_gate_status": "ready_to_request_explicit_authorization" if ready_to_request else "not_ready", "telegram_visible_messages_sent": False, "telegram_visible_message_authorization_present": False, "production_apply_executed": False, "production_apply_authorization_present": False, "mutates_db": False, "target": { "platform": "telegram", "chat_label": chat_label, "chat_id": chat_id, "message_count": len(messages), "expected_prompt_ids": expected_ids, }, "exact_messages": messages, "checks": checks, "evidence_paths": evidence_paths, "expected_proof_paths_after_authorized_run": expected_proof_paths, "explicit_operator_authorization_text": authorization_text, "clear_CTA": ( "To run the Telegram-visible DC benchmark, reply with the exact authorization text above. After that, " "Codex should send only the six listed messages to the Leo group, capture the retained proof paths, " "score the retained replies with `scripts/working_leo_open_ended_benchmark.py --results-json ... " "--include-direct-claim-followups`, and read back DB/service stability." ), "next_non_user_action_after_authorization": [ "Read current service state and DB counts before sending.", "Send exactly the six raw DC-01..DC-06 messages to Telegram group Leo.", "Capture each Telegram-visible Leo reply plus a final screenshot/accessibility readback.", "Read DB counts and service state after sending.", "Assemble the retained capture receipt and score it before updating the current truth index.", ], "do_not_run_without": [ "Exact operator authorization text naming the Telegram group and six-message scope.", "Authenticated Telegram UI/API route that can send to the Leo group.", "Before-count and after-count readbacks for public.* and kb_stage.kb_proposals.", ], "claim_ceiling": ( "This packet makes the Telegram-visible DC benchmark executable and auditable. It does not send " "Telegram messages, does not mutate the DB, does not authorize production apply, and does not prove the " "DC prompts are Telegram-visible until the authorized run is actually completed and scored." ), } def write_json(path: Path, data: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") def write_markdown(path: Path, data: dict[str, Any]) -> None: lines = [ "# Telegram-Visible Direct-Claim Authorization Packet", "", f"Generated UTC: `{data['generated_at_utc']}`", f"Mode: `{data['mode']}`", f"Ready to request authorization: `{data['ready_to_request_authorization']}`", f"Authorization gate status: `{data['authorization_gate_status']}`", f"Telegram-visible messages sent: `{data['telegram_visible_messages_sent']}`", f"Telegram-visible authorization present: `{data['telegram_visible_message_authorization_present']}`", f"Production apply executed: `{data['production_apply_executed']}`", f"Mutates DB: `{data['mutates_db']}`", "", "## Target", "", f"- Chat: `{data['target']['chat_label']}` (`{data['target']['chat_id']}`)", f"- Message count: `{data['target']['message_count']}`", f"- Prompt IDs: `{', '.join(data['target']['expected_prompt_ids'])}`", "", "## Exact Messages", "", ] for item in data["exact_messages"]: lines.extend( [ f"### {item['prompt_id']} / {item['dimension']}", "", item["message"], "", f"- Required signals: `{', '.join(item['required_signals'])}`", "", ] ) lines.extend(["## Checks", ""]) for key, value in sorted(data["checks"].items()): lines.append(f"- `{key}`: `{value}`") lines.extend( [ "", "## Explicit Operator Authorization Text", "", data["explicit_operator_authorization_text"], "", "## Clear CTA", "", data["clear_CTA"], "", "## Next Non-User Action After Authorization", "", ] ) lines.extend(f"- {item}" for item in data["next_non_user_action_after_authorization"]) lines.extend(["", "## Do Not Run Without", ""]) lines.extend(f"- {item}" for item in data["do_not_run_without"]) lines.extend(["", "## Claim Ceiling", "", data["claim_ceiling"], ""]) path.parent.mkdir(parents=True, exist_ok=True) path.write_text("\n".join(lines), encoding="utf-8") def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--live-open-score", type=Path, default=DEFAULT_LIVE_OPEN_SCORE) parser.add_argument("--handler-suite", type=Path, default=DEFAULT_HANDLER_SUITE) parser.add_argument("--handler-score", type=Path, default=DEFAULT_HANDLER_SCORE) parser.add_argument("--gcp-probe", type=Path, default=DEFAULT_GCP_PROBE) parser.add_argument("--chat-id", default=DEFAULT_CHAT_ID) parser.add_argument("--chat-label", default=DEFAULT_CHAT_LABEL) parser.add_argument("--proof-dir", default=DEFAULT_PROOF_DIR) parser.add_argument("--out", type=Path, default=DEFAULT_OUT) parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = parse_args(argv) data = build_packet( live_open_score_path=args.live_open_score, handler_suite_path=args.handler_suite, handler_score_path=args.handler_score, gcp_probe_path=args.gcp_probe, chat_id=args.chat_id, chat_label=args.chat_label, proof_dir=args.proof_dir, ) write_json(args.out, data) write_markdown(args.markdown_out, data) print( json.dumps( { "out": str(args.out), "markdown_out": str(args.markdown_out), "ready_to_request_authorization": data["ready_to_request_authorization"], "telegram_visible_messages_sent": data["telegram_visible_messages_sent"], "authorization_gate_status": data["authorization_gate_status"], }, indent=2, sort_keys=True, ) ) return 0 if data["ready_to_request_authorization"] else 2 if __name__ == "__main__": raise SystemExit(main())