310 lines
14 KiB
Python
310 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Build an exact authorization packet for a Telegram-visible DC benchmark run.
|
|
|
|
The packet is a planner, not a sender. It records the raw no-context DC-01..DC-06
|
|
messages, the target Telegram group, the expected proof artifacts, and the
|
|
required authorization sentence. It never sends Telegram messages and never
|
|
mutates the database.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import working_leo_open_ended_benchmark as benchmark
|
|
|
|
REPORT_DIR = Path("docs/reports/leo-working-state-20260709")
|
|
DEFAULT_OUT = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.json"
|
|
DEFAULT_MARKDOWN_OUT = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.md"
|
|
DEFAULT_LIVE_OPEN_SCORE = REPORT_DIR / "telegram-live-open-ended-suite-score-current.json"
|
|
DEFAULT_HANDLER_SUITE = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json"
|
|
DEFAULT_HANDLER_SCORE = REPORT_DIR / "telegram-handler-direct-claim-suite-score-current.json"
|
|
DEFAULT_GCP_PROBE = REPORT_DIR / "gcp-db-parity-current-probe-current.json"
|
|
DEFAULT_CHAT_ID = "-5146042086"
|
|
DEFAULT_CHAT_LABEL = "Leo"
|
|
DEFAULT_PROOF_DIR = "outputs/telegram-visible-direct-claim-suite-20260710"
|
|
|
|
|
|
def _load_json(path: Path) -> dict[str, Any]:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def _path_status(path: Path) -> dict[str, Any]:
|
|
exists = path.exists()
|
|
return {"path": str(path), "exists": exists, "bytes": path.stat().st_size if exists else 0}
|
|
|
|
|
|
def current_git_sha(repo_root: Path = Path(".")) -> str:
|
|
proc = subprocess.run(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=repo_root,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
return proc.stdout.strip() if proc.returncode == 0 else ""
|
|
|
|
|
|
def direct_claim_messages() -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"prompt_id": prompt["id"],
|
|
"dimension": prompt["dimension"],
|
|
"message": prompt["message"],
|
|
"expected_answer": prompt["expected_answer"],
|
|
"expected_follow_up": prompt["expected_follow_up"],
|
|
"required_signals": list(prompt["required_signals"]),
|
|
}
|
|
for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS
|
|
]
|
|
|
|
|
|
def _score_passes(score: dict[str, Any], *, expected_count: int) -> bool:
|
|
score_block = score.get("score", {})
|
|
return (
|
|
score_block.get("pass") is True
|
|
and score_block.get("passes") == score_block.get("expected_prompt_count") == expected_count
|
|
and score_block.get("failures") == []
|
|
and score_block.get("missing_prompt_ids") == []
|
|
and score_block.get("unexpected_prompt_ids") == []
|
|
)
|
|
|
|
|
|
def _live_open_score_passes(score: dict[str, Any]) -> bool:
|
|
score_block = score.get("score", {})
|
|
return score_block.get("pass") is True and score_block.get("passes") == score_block.get(
|
|
"expected_prompt_count"
|
|
)
|
|
|
|
|
|
def _handler_suite_is_safe(suite: dict[str, Any], *, expected_count: int) -> bool:
|
|
return (
|
|
suite.get("pass_runtime") is True
|
|
and suite.get("posted_to_telegram") is False
|
|
and suite.get("changed_live_profile") is False
|
|
and suite.get("mutates_kb_by_harness") is False
|
|
and suite.get("db_counts_changed") is False
|
|
and suite.get("temp_profile_removed") is True
|
|
and len(suite.get("results", [])) == expected_count
|
|
)
|
|
|
|
|
|
def build_packet(
|
|
*,
|
|
live_open_score_path: Path = DEFAULT_LIVE_OPEN_SCORE,
|
|
handler_suite_path: Path = DEFAULT_HANDLER_SUITE,
|
|
handler_score_path: Path = DEFAULT_HANDLER_SCORE,
|
|
gcp_probe_path: Path = DEFAULT_GCP_PROBE,
|
|
chat_id: str = DEFAULT_CHAT_ID,
|
|
chat_label: str = DEFAULT_CHAT_LABEL,
|
|
proof_dir: str = DEFAULT_PROOF_DIR,
|
|
git_sha: str | None = None,
|
|
) -> dict[str, Any]:
|
|
messages = direct_claim_messages()
|
|
live_open_score = _load_json(live_open_score_path)
|
|
handler_suite = _load_json(handler_suite_path)
|
|
handler_score = _load_json(handler_score_path)
|
|
gcp_probe = _load_json(gcp_probe_path) if gcp_probe_path.exists() else {}
|
|
sha = git_sha if git_sha is not None else current_git_sha()
|
|
expected_ids = [item["prompt_id"] for item in messages]
|
|
|
|
evidence_paths = {
|
|
"live_open_score": _path_status(live_open_score_path),
|
|
"handler_suite": _path_status(handler_suite_path),
|
|
"handler_score": _path_status(handler_score_path),
|
|
"gcp_probe": _path_status(gcp_probe_path),
|
|
}
|
|
expected_proof_paths = {
|
|
"results_json": f"{proof_dir}/telegram-visible-direct-claim-suite-results-current.json",
|
|
"score_json": f"{proof_dir}/telegram-visible-direct-claim-suite-score-current.json",
|
|
"score_markdown": f"{proof_dir}/telegram-visible-direct-claim-suite-score-current.md",
|
|
"final_screenshot": f"{proof_dir}/telegram-visible-direct-claim-suite-final.png",
|
|
"final_accessibility": f"{proof_dir}/telegram-visible-direct-claim-suite-final-accessibility.txt",
|
|
"db_before_after": f"{proof_dir}/telegram-visible-direct-claim-suite-db-before-after.json",
|
|
"service_readback": f"{proof_dir}/telegram-visible-direct-claim-suite-service-readback.txt",
|
|
}
|
|
|
|
checks = {
|
|
"live_open_telegram_suite_passed": _live_open_score_passes(live_open_score),
|
|
"handler_direct_claim_suite_safe": _handler_suite_is_safe(handler_suite, expected_count=len(messages)),
|
|
"handler_direct_claim_score_passed": _score_passes(handler_score, expected_count=len(messages)),
|
|
"all_evidence_paths_exist": all(item["exists"] and item["bytes"] > 0 for item in evidence_paths.values()),
|
|
"raw_dc_messages_preserved": [item["message"] for item in messages]
|
|
== [prompt["message"] for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS],
|
|
"telegram_visible_messages_not_sent_by_packet": True,
|
|
"production_apply_not_authorized_by_packet": True,
|
|
"gcp_parity_not_required_for_vps_telegram_test": gcp_probe.get("ready_for_gcp_readonly_parity") is not True,
|
|
}
|
|
ready_to_request = all(checks.values())
|
|
authorization_text = (
|
|
"I authorize a Telegram-visible read-only direct-claim benchmark in the Leo group "
|
|
f"`{chat_label}` (`{chat_id}`) using exactly the six raw messages DC-01 through DC-06 listed in "
|
|
"`telegram-visible-direct-claim-authorization-packet-current.json`. Capture before/after DB counts, "
|
|
"Telegram-visible replies, scorer output, final screenshot/accessibility proof, and service readback. "
|
|
"Do not run production DB apply, do not edit SOUL.md, do not post any extra messages beyond those six "
|
|
"benchmark prompts unless separately authorized."
|
|
)
|
|
|
|
return {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": "telegram_visible_direct_claim_authorization_packet_not_sent",
|
|
"packet_generation_git_sha": sha,
|
|
"ready_to_request_authorization": ready_to_request,
|
|
"authorization_gate_status": "ready_to_request_explicit_authorization" if ready_to_request else "not_ready",
|
|
"telegram_visible_messages_sent": False,
|
|
"telegram_visible_message_authorization_present": False,
|
|
"production_apply_executed": False,
|
|
"production_apply_authorization_present": False,
|
|
"mutates_db": False,
|
|
"target": {
|
|
"platform": "telegram",
|
|
"chat_label": chat_label,
|
|
"chat_id": chat_id,
|
|
"message_count": len(messages),
|
|
"expected_prompt_ids": expected_ids,
|
|
},
|
|
"exact_messages": messages,
|
|
"checks": checks,
|
|
"evidence_paths": evidence_paths,
|
|
"expected_proof_paths_after_authorized_run": expected_proof_paths,
|
|
"explicit_operator_authorization_text": authorization_text,
|
|
"clear_CTA": (
|
|
"To run the Telegram-visible DC benchmark, reply with the exact authorization text above. After that, "
|
|
"Codex should send only the six listed messages to the Leo group, capture the retained proof paths, "
|
|
"score the retained replies with `scripts/working_leo_open_ended_benchmark.py --results-json ... "
|
|
"--include-direct-claim-followups`, and read back DB/service stability."
|
|
),
|
|
"next_non_user_action_after_authorization": [
|
|
"Read current service state and DB counts before sending.",
|
|
"Send exactly the six raw DC-01..DC-06 messages to Telegram group Leo.",
|
|
"Capture each Telegram-visible Leo reply plus a final screenshot/accessibility readback.",
|
|
"Read DB counts and service state after sending.",
|
|
"Score the retained replies and update the current truth index.",
|
|
],
|
|
"do_not_run_without": [
|
|
"Exact operator authorization text naming the Telegram group and six-message scope.",
|
|
"Authenticated Telegram UI/API route that can send to the Leo group.",
|
|
"Before-count and after-count readbacks for public.* and kb_stage.kb_proposals.",
|
|
],
|
|
"claim_ceiling": (
|
|
"This packet makes the Telegram-visible DC benchmark executable and auditable. It does not send "
|
|
"Telegram messages, does not mutate the DB, does not authorize production apply, and does not prove the "
|
|
"DC prompts are Telegram-visible until the authorized run is actually completed and scored."
|
|
),
|
|
}
|
|
|
|
|
|
def write_json(path: Path, data: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
|
|
def write_markdown(path: Path, data: dict[str, Any]) -> None:
|
|
lines = [
|
|
"# Telegram-Visible Direct-Claim Authorization Packet",
|
|
"",
|
|
f"Generated UTC: `{data['generated_at_utc']}`",
|
|
f"Mode: `{data['mode']}`",
|
|
f"Ready to request authorization: `{data['ready_to_request_authorization']}`",
|
|
f"Authorization gate status: `{data['authorization_gate_status']}`",
|
|
f"Telegram-visible messages sent: `{data['telegram_visible_messages_sent']}`",
|
|
f"Telegram-visible authorization present: `{data['telegram_visible_message_authorization_present']}`",
|
|
f"Production apply executed: `{data['production_apply_executed']}`",
|
|
f"Mutates DB: `{data['mutates_db']}`",
|
|
"",
|
|
"## Target",
|
|
"",
|
|
f"- Chat: `{data['target']['chat_label']}` (`{data['target']['chat_id']}`)",
|
|
f"- Message count: `{data['target']['message_count']}`",
|
|
f"- Prompt IDs: `{', '.join(data['target']['expected_prompt_ids'])}`",
|
|
"",
|
|
"## Exact Messages",
|
|
"",
|
|
]
|
|
for item in data["exact_messages"]:
|
|
lines.extend(
|
|
[
|
|
f"### {item['prompt_id']} / {item['dimension']}",
|
|
"",
|
|
item["message"],
|
|
"",
|
|
f"- Required signals: `{', '.join(item['required_signals'])}`",
|
|
"",
|
|
]
|
|
)
|
|
lines.extend(["## Checks", ""])
|
|
for key, value in sorted(data["checks"].items()):
|
|
lines.append(f"- `{key}`: `{value}`")
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Explicit Operator Authorization Text",
|
|
"",
|
|
data["explicit_operator_authorization_text"],
|
|
"",
|
|
"## Clear CTA",
|
|
"",
|
|
data["clear_CTA"],
|
|
"",
|
|
"## Next Non-User Action After Authorization",
|
|
"",
|
|
]
|
|
)
|
|
lines.extend(f"- {item}" for item in data["next_non_user_action_after_authorization"])
|
|
lines.extend(["", "## Do Not Run Without", ""])
|
|
lines.extend(f"- {item}" for item in data["do_not_run_without"])
|
|
lines.extend(["", "## Claim Ceiling", "", data["claim_ceiling"], ""])
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--live-open-score", type=Path, default=DEFAULT_LIVE_OPEN_SCORE)
|
|
parser.add_argument("--handler-suite", type=Path, default=DEFAULT_HANDLER_SUITE)
|
|
parser.add_argument("--handler-score", type=Path, default=DEFAULT_HANDLER_SCORE)
|
|
parser.add_argument("--gcp-probe", type=Path, default=DEFAULT_GCP_PROBE)
|
|
parser.add_argument("--chat-id", default=DEFAULT_CHAT_ID)
|
|
parser.add_argument("--chat-label", default=DEFAULT_CHAT_LABEL)
|
|
parser.add_argument("--proof-dir", default=DEFAULT_PROOF_DIR)
|
|
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
|
parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
data = build_packet(
|
|
live_open_score_path=args.live_open_score,
|
|
handler_suite_path=args.handler_suite,
|
|
handler_score_path=args.handler_score,
|
|
gcp_probe_path=args.gcp_probe,
|
|
chat_id=args.chat_id,
|
|
chat_label=args.chat_label,
|
|
proof_dir=args.proof_dir,
|
|
)
|
|
write_json(args.out, data)
|
|
write_markdown(args.markdown_out, data)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"out": str(args.out),
|
|
"markdown_out": str(args.markdown_out),
|
|
"ready_to_request_authorization": data["ready_to_request_authorization"],
|
|
"telegram_visible_messages_sent": data["telegram_visible_messages_sent"],
|
|
"authorization_gate_status": data["authorization_gate_status"],
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if data["ready_to_request_authorization"] else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|