172 lines
9.5 KiB
JSON
172 lines
9.5 KiB
JSON
{
|
|
"authorization_gate_status": "ready_to_request_explicit_authorization",
|
|
"checks": {
|
|
"all_evidence_paths_exist": true,
|
|
"gcp_parity_not_required_for_vps_telegram_test": true,
|
|
"handler_direct_claim_score_passed": true,
|
|
"handler_direct_claim_suite_safe": true,
|
|
"live_open_telegram_suite_passed": true,
|
|
"production_apply_not_authorized_by_packet": true,
|
|
"raw_dc_messages_preserved": true,
|
|
"telegram_visible_messages_not_sent_by_packet": true
|
|
},
|
|
"claim_ceiling": "This packet makes the Telegram-visible DC benchmark executable and auditable. It does not send Telegram messages, does not mutate the DB, does not authorize production apply, and does not prove the DC prompts are Telegram-visible until the authorized run is actually completed and scored.",
|
|
"clear_CTA": "To run the Telegram-visible DC benchmark, reply with the exact authorization text above. After that, Codex should send only the six listed messages to the Leo group, capture the retained proof paths, score the retained replies with `scripts/working_leo_open_ended_benchmark.py --results-json ... --include-direct-claim-followups`, and read back DB/service stability.",
|
|
"do_not_run_without": [
|
|
"Exact operator authorization text naming the Telegram group and six-message scope.",
|
|
"Authenticated Telegram UI/API route that can send to the Leo group.",
|
|
"Before-count and after-count readbacks for public.* and kb_stage.kb_proposals."
|
|
],
|
|
"evidence_paths": {
|
|
"gcp_probe": {
|
|
"bytes": 8398,
|
|
"exists": true,
|
|
"path": "docs/reports/leo-working-state-20260709/gcp-db-parity-current-probe-current.json"
|
|
},
|
|
"handler_score": {
|
|
"bytes": 14730,
|
|
"exists": true,
|
|
"path": "docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.json"
|
|
},
|
|
"handler_suite": {
|
|
"bytes": 13329,
|
|
"exists": true,
|
|
"path": "docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-current.json"
|
|
},
|
|
"live_open_score": {
|
|
"bytes": 20925,
|
|
"exists": true,
|
|
"path": "docs/reports/leo-working-state-20260709/telegram-live-open-ended-suite-score-current.json"
|
|
}
|
|
},
|
|
"exact_messages": [
|
|
{
|
|
"dimension": "db_changed_direct_claim",
|
|
"expected_answer": "Answer with the proposed/approved/applied split. Do not claim a DB update unless canonical public.* rows or applied_at/postflight proof are present.",
|
|
"expected_follow_up": "Ask or run the row-level readback: which proposal/time window to inspect, kb_stage status, public.* before/after counts, and the next guarded apply packet if approved rows are still pending.",
|
|
"message": "Did we actually update the knowledge base, or is it still just proposals?",
|
|
"prompt_id": "DC-01",
|
|
"required_signals": [
|
|
"state_semantics",
|
|
"canonical_db",
|
|
"row_level_proof",
|
|
"cory_followup",
|
|
"no_overclaim"
|
|
]
|
|
},
|
|
{
|
|
"dimension": "helmer_live_direct_claim",
|
|
"expected_answer": "Separate usable fixture/packet readiness from production canonical state. Helmer can be packet-ready or clone-proven while still not production-applied.",
|
|
"expected_follow_up": "Offer the next proof-changing action: show the packet/readiness artifact, get explicit apply authorization, run apply in integrated order, then postflight and Telegram regression.",
|
|
"message": "Is Helmer's 7 Powers in Leo now?",
|
|
"prompt_id": "DC-02",
|
|
"required_signals": [
|
|
"canonical_db",
|
|
"row_level_proof",
|
|
"authorization",
|
|
"cory_followup",
|
|
"no_overclaim"
|
|
]
|
|
},
|
|
{
|
|
"dimension": "decision_matrix_direct_claim",
|
|
"expected_answer": "Check schema truth first. If matrix_voters/proposal_votes/proposal_decisions are absent, do not infer matrix approval from pending proposals.",
|
|
"expected_follow_up": "Follow up with the schema/readback request and then fall back to kb_stage.kb_proposals review/apply columns plus canonical row readback.",
|
|
"message": "Did the decision matrix approve this already?",
|
|
"prompt_id": "DC-03",
|
|
"required_signals": [
|
|
"decision_matrix",
|
|
"canonical_db",
|
|
"fresh_readback",
|
|
"cory_followup",
|
|
"no_overclaim"
|
|
]
|
|
},
|
|
{
|
|
"dimension": "document_source_direct_claim",
|
|
"expected_answer": "Distinguish Telegram file refs, document evaluations, proposal source_ref, raw files, and canonical public.sources. Do not reduce pending status to one document-pointer cause without row-link proof.",
|
|
"expected_follow_up": "Follow up with a row-link audit: list proposal IDs/source_ref values, match them to public.sources or missing source rows, then propose the guarded source/evidence apply contract.",
|
|
"message": "Are the pending proposals stuck because the documents are not pointed at the right source rows?",
|
|
"prompt_id": "DC-04",
|
|
"required_signals": [
|
|
"document_artifact_linking",
|
|
"staging_or_review",
|
|
"canonical_db",
|
|
"cory_followup",
|
|
"no_overclaim"
|
|
]
|
|
},
|
|
{
|
|
"dimension": "demo_changes_kb_direct_claim",
|
|
"expected_answer": "Give the operator truth ceiling: live staging and strict existing-ID apply can be shown if proven, while rich packets remain clone-proven/not production-applied unless explicitly applied.",
|
|
"expected_follow_up": "Ask what demo tier Cory wants: show existing live proof, run a safe live staging canary, or authorize the prepared production apply packet and postflight/regression sequence.",
|
|
"message": "Can I demo that Leo changes the KB?",
|
|
"prompt_id": "DC-05",
|
|
"required_signals": [
|
|
"state_semantics",
|
|
"row_level_proof",
|
|
"authorization",
|
|
"artifact",
|
|
"cory_followup",
|
|
"no_overclaim"
|
|
]
|
|
},
|
|
{
|
|
"dimension": "soul_identity_direct_claim",
|
|
"expected_answer": "Explain that direct SOUL.md edits are runtime/profile edits. Canonical identity requires DB rows plus a render/sync proof into the runtime artifact.",
|
|
"expected_follow_up": "Follow up by asking whether to audit canonical identity rows, stage the missing DB proposal, or run the renderer/sync readback if an approved identity row already exists.",
|
|
"message": "If we changed SOUL.md, did we change Leo's canonical identity?",
|
|
"prompt_id": "DC-06",
|
|
"required_signals": [
|
|
"identity_rendering",
|
|
"canonical_db",
|
|
"row_level_proof",
|
|
"cory_followup",
|
|
"no_overclaim"
|
|
]
|
|
}
|
|
],
|
|
"expected_proof_paths_after_authorized_run": {
|
|
"capture_receipt_json": "docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-capture-receipt-current.json",
|
|
"capture_receipt_markdown": "docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-capture-receipt-current.md",
|
|
"capture_results_json": "docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-capture-results-current.json",
|
|
"db_before_after": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-db-before-after.json",
|
|
"final_accessibility": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-final-accessibility.txt",
|
|
"final_screenshot": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-final.png",
|
|
"results_json": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-results-current.json",
|
|
"score_json": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-score-current.json",
|
|
"score_markdown": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-score-current.md",
|
|
"service_readback": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-service-readback.txt"
|
|
},
|
|
"explicit_operator_authorization_text": "I authorize a Telegram-visible read-only direct-claim benchmark in the Leo group `Leo` (`-5146042086`) using exactly the six raw messages DC-01 through DC-06 listed in `telegram-visible-direct-claim-authorization-packet-current.json`. Capture before/after DB counts, Telegram-visible replies, scorer output, final screenshot/accessibility proof, and service readback. Do not run production DB apply, do not edit SOUL.md, do not post any extra messages beyond those six benchmark prompts unless separately authorized.",
|
|
"generated_at_utc": "2026-07-10T03:01:26.939311+00:00",
|
|
"mode": "telegram_visible_direct_claim_authorization_packet_not_sent",
|
|
"mutates_db": false,
|
|
"next_non_user_action_after_authorization": [
|
|
"Read current service state and DB counts before sending.",
|
|
"Send exactly the six raw DC-01..DC-06 messages to Telegram group Leo.",
|
|
"Capture each Telegram-visible Leo reply plus a final screenshot/accessibility readback.",
|
|
"Read DB counts and service state after sending.",
|
|
"Assemble the retained capture receipt and score it before updating the current truth index."
|
|
],
|
|
"packet_generation_git_sha": "97755068d3fd1a97d9da11be3a64f22b751d8c41",
|
|
"production_apply_authorization_present": false,
|
|
"production_apply_executed": false,
|
|
"ready_to_request_authorization": true,
|
|
"target": {
|
|
"chat_id": "-5146042086",
|
|
"chat_label": "Leo",
|
|
"expected_prompt_ids": [
|
|
"DC-01",
|
|
"DC-02",
|
|
"DC-03",
|
|
"DC-04",
|
|
"DC-05",
|
|
"DC-06"
|
|
],
|
|
"message_count": 6,
|
|
"platform": "telegram"
|
|
},
|
|
"telegram_visible_message_authorization_present": false,
|
|
"telegram_visible_messages_sent": false
|
|
}
|