diff --git a/docs/reports/leo-working-state-20260709/current-truth-index.md b/docs/reports/leo-working-state-20260709/current-truth-index.md index 795f44c..495b0f5 100644 --- a/docs/reports/leo-working-state-20260709/current-truth-index.md +++ b/docs/reports/leo-working-state-20260709/current-truth-index.md @@ -22,6 +22,7 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c - Live VPS handler direct-claim suite report: `docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-current.json` - Live VPS handler direct-claim suite score: `docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.md` - Live VPS handler direct-claim suite review: `docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-review-current.md` +- Telegram-visible direct-claim authorization packet, ready-to-request but not sent: `docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.md` - Live VPS handler direct-claim follow-up challenge report: `docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-followup-current.json` - Fable onboarding prompt: `docs/reports/leo-working-state-20260709/fable-leo-teleo-onboarding.md` - Operator surface map: `docs/reports/leo-working-state-20260709/operator-surface-map.md` @@ -128,6 +129,7 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c - VPS Leo live-safe open-ended Telegram suite is live-proven and retained-benchmark-scored for `OE-01` through `OE-05`: full selected coverage, `5/5` scored prompts passing, no overclaim, and no marker/staging/canonical KB rows created by the suite. - The broader sandbox-first Cory-style benchmark has a full-catalog mixed-evidence score: real retained live Telegram evidence for `OE-01` through `OE-05`, sandbox fixture replies for `CS-01` through `CS-09`, and no-context direct-claim expected-answer/follow-up fixtures for `DC-01` through `DC-06`, `20/20` passing. This is not a live Telegram run for the CS/DC prompts. - The live VPS GatewayRunner direct-claim handler suite for exact `DC-01` through `DC-06` passed again after deploy SHA `7ab563e1aebe08913e72781e4e16a2b2883c5230`: `6/6` runtime replies, strict score `6/6`, no Telegram post, no live profile write, no production DB apply, unchanged DB counts (`public.claims=1837`, `public.sources=4145`, `public.claim_edges=4916`, `public.claim_evidence=4670`, `kb_stage.kb_proposals=26` before and after), temp profile removed, and `leoclean-gateway.service` stayed active/running during the harness with `MainPID=1908033`, `NRestarts=0`, `ExecMainStartTimestamp=Fri 2026-07-10 01:04:22 UTC`. +- The Telegram-visible direct-claim benchmark is now executable as an exact authorization packet, not yet run: target `Leo` group `-5146042086`, raw no-context messages `DC-01` through `DC-06`, expected proof paths, before/after DB/service readbacks, scorer output, and final screenshot/accessibility proof are retained in `telegram-visible-direct-claim-authorization-packet-current.md`; `telegram_visible_messages_sent=false`. - The highest-risk direct-claim cases are now green at handler level: `DC-02` recognizes approved Helmer staging row `a64df080-8502-42e2-98f4-9bbdecb8da73` as approved/unapplied, `DC-03` says the decision-matrix path is not shipped while matrix tables are absent, `DC-04` avoids a single-cause document pointer diagnosis, `DC-05` separates staging demo from authorized canonical apply, and `DC-06` separates `SOUL.md` runtime edits from canonical DB identity. - A strict canonical `add_edge` apply canary is live-proven. - Helmer 7 Powers is full-coverage packet and ledger clone-proven, not production-applied. diff --git a/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.json b/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.json new file mode 100644 index 0000000..066360a --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.json @@ -0,0 +1,169 @@ +{ + "authorization_gate_status": "ready_to_request_explicit_authorization", + "checks": { + "all_evidence_paths_exist": true, + "gcp_parity_not_required_for_vps_telegram_test": true, + "handler_direct_claim_score_passed": true, + "handler_direct_claim_suite_safe": true, + "live_open_telegram_suite_passed": true, + "production_apply_not_authorized_by_packet": true, + "raw_dc_messages_preserved": true, + "telegram_visible_messages_not_sent_by_packet": true + }, + "claim_ceiling": "This packet makes the Telegram-visible DC benchmark executable and auditable. It does not send Telegram messages, does not mutate the DB, does not authorize production apply, and does not prove the DC prompts are Telegram-visible until the authorized run is actually completed and scored.", + "clear_CTA": "To run the Telegram-visible DC benchmark, reply with the exact authorization text above. After that, Codex should send only the six listed messages to the Leo group, capture the retained proof paths, score the retained replies with `scripts/working_leo_open_ended_benchmark.py --results-json ... --include-direct-claim-followups`, and read back DB/service stability.", + "do_not_run_without": [ + "Exact operator authorization text naming the Telegram group and six-message scope.", + "Authenticated Telegram UI/API route that can send to the Leo group.", + "Before-count and after-count readbacks for public.* and kb_stage.kb_proposals." + ], + "evidence_paths": { + "gcp_probe": { + "bytes": 8398, + "exists": true, + "path": "docs/reports/leo-working-state-20260709/gcp-db-parity-current-probe-current.json" + }, + "handler_score": { + "bytes": 14730, + "exists": true, + "path": "docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.json" + }, + "handler_suite": { + "bytes": 13329, + "exists": true, + "path": "docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-current.json" + }, + "live_open_score": { + "bytes": 20925, + "exists": true, + "path": "docs/reports/leo-working-state-20260709/telegram-live-open-ended-suite-score-current.json" + } + }, + "exact_messages": [ + { + "dimension": "db_changed_direct_claim", + "expected_answer": "Answer with the proposed/approved/applied split. Do not claim a DB update unless canonical public.* rows or applied_at/postflight proof are present.", + "expected_follow_up": "Ask or run the row-level readback: which proposal/time window to inspect, kb_stage status, public.* before/after counts, and the next guarded apply packet if approved rows are still pending.", + "message": "Did we actually update the knowledge base, or is it still just proposals?", + "prompt_id": "DC-01", + "required_signals": [ + "state_semantics", + "canonical_db", + "row_level_proof", + "cory_followup", + "no_overclaim" + ] + }, + { + "dimension": "helmer_live_direct_claim", + "expected_answer": "Separate usable fixture/packet readiness from production canonical state. Helmer can be packet-ready or clone-proven while still not production-applied.", + "expected_follow_up": "Offer the next proof-changing action: show the packet/readiness artifact, get explicit apply authorization, run apply in integrated order, then postflight and Telegram regression.", + "message": "Is Helmer's 7 Powers in Leo now?", + "prompt_id": "DC-02", + "required_signals": [ + "canonical_db", + "row_level_proof", + "authorization", + "cory_followup", + "no_overclaim" + ] + }, + { + "dimension": "decision_matrix_direct_claim", + "expected_answer": "Check schema truth first. If matrix_voters/proposal_votes/proposal_decisions are absent, do not infer matrix approval from pending proposals.", + "expected_follow_up": "Follow up with the schema/readback request and then fall back to kb_stage.kb_proposals review/apply columns plus canonical row readback.", + "message": "Did the decision matrix approve this already?", + "prompt_id": "DC-03", + "required_signals": [ + "decision_matrix", + "canonical_db", + "fresh_readback", + "cory_followup", + "no_overclaim" + ] + }, + { + "dimension": "document_source_direct_claim", + "expected_answer": "Distinguish Telegram file refs, document evaluations, proposal source_ref, raw files, and canonical public.sources. Do not reduce pending status to one document-pointer cause without row-link proof.", + "expected_follow_up": "Follow up with a row-link audit: list proposal IDs/source_ref values, match them to public.sources or missing source rows, then propose the guarded source/evidence apply contract.", + "message": "Are the pending proposals stuck because the documents are not pointed at the right source rows?", + "prompt_id": "DC-04", + "required_signals": [ + "document_artifact_linking", + "staging_or_review", + "canonical_db", + "cory_followup", + "no_overclaim" + ] + }, + { + "dimension": "demo_changes_kb_direct_claim", + "expected_answer": "Give the operator truth ceiling: live staging and strict existing-ID apply can be shown if proven, while rich packets remain clone-proven/not production-applied unless explicitly applied.", + "expected_follow_up": "Ask what demo tier Cory wants: show existing live proof, run a safe live staging canary, or authorize the prepared production apply packet and postflight/regression sequence.", + "message": "Can I demo that Leo changes the KB?", + "prompt_id": "DC-05", + "required_signals": [ + "state_semantics", + "row_level_proof", + "authorization", + "artifact", + "cory_followup", + "no_overclaim" + ] + }, + { + "dimension": "soul_identity_direct_claim", + "expected_answer": "Explain that direct SOUL.md edits are runtime/profile edits. Canonical identity requires DB rows plus a render/sync proof into the runtime artifact.", + "expected_follow_up": "Follow up by asking whether to audit canonical identity rows, stage the missing DB proposal, or run the renderer/sync readback if an approved identity row already exists.", + "message": "If we changed SOUL.md, did we change Leo's canonical identity?", + "prompt_id": "DC-06", + "required_signals": [ + "identity_rendering", + "canonical_db", + "row_level_proof", + "cory_followup", + "no_overclaim" + ] + } + ], + "expected_proof_paths_after_authorized_run": { + "db_before_after": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-db-before-after.json", + "final_accessibility": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-final-accessibility.txt", + "final_screenshot": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-final.png", + "results_json": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-results-current.json", + "score_json": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-score-current.json", + "score_markdown": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-score-current.md", + "service_readback": "outputs/telegram-visible-direct-claim-suite-20260710/telegram-visible-direct-claim-suite-service-readback.txt" + }, + "explicit_operator_authorization_text": "I authorize a Telegram-visible read-only direct-claim benchmark in the Leo group `Leo` (`-5146042086`) using exactly the six raw messages DC-01 through DC-06 listed in `telegram-visible-direct-claim-authorization-packet-current.json`. Capture before/after DB counts, Telegram-visible replies, scorer output, final screenshot/accessibility proof, and service readback. Do not run production DB apply, do not edit SOUL.md, do not post any extra messages beyond those six benchmark prompts unless separately authorized.", + "generated_at_utc": "2026-07-10T02:29:58.153246+00:00", + "mode": "telegram_visible_direct_claim_authorization_packet_not_sent", + "mutates_db": false, + "next_non_user_action_after_authorization": [ + "Read current service state and DB counts before sending.", + "Send exactly the six raw DC-01..DC-06 messages to Telegram group Leo.", + "Capture each Telegram-visible Leo reply plus a final screenshot/accessibility readback.", + "Read DB counts and service state after sending.", + "Score the retained replies and update the current truth index." + ], + "packet_generation_git_sha": "262ddb9e0d5b3f8a00b4529dcd00f03317a8e90b", + "production_apply_authorization_present": false, + "production_apply_executed": false, + "ready_to_request_authorization": true, + "target": { + "chat_id": "-5146042086", + "chat_label": "Leo", + "expected_prompt_ids": [ + "DC-01", + "DC-02", + "DC-03", + "DC-04", + "DC-05", + "DC-06" + ], + "message_count": 6, + "platform": "telegram" + }, + "telegram_visible_message_authorization_present": false, + "telegram_visible_messages_sent": false +} diff --git a/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.md b/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.md new file mode 100644 index 0000000..997b34f --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.md @@ -0,0 +1,91 @@ +# Telegram-Visible Direct-Claim Authorization Packet + +Generated UTC: `2026-07-10T02:29:58.153246+00:00` +Mode: `telegram_visible_direct_claim_authorization_packet_not_sent` +Ready to request authorization: `True` +Authorization gate status: `ready_to_request_explicit_authorization` +Telegram-visible messages sent: `False` +Telegram-visible authorization present: `False` +Production apply executed: `False` +Mutates DB: `False` + +## Target + +- Chat: `Leo` (`-5146042086`) +- Message count: `6` +- Prompt IDs: `DC-01, DC-02, DC-03, DC-04, DC-05, DC-06` + +## Exact Messages + +### DC-01 / db_changed_direct_claim + +Did we actually update the knowledge base, or is it still just proposals? + +- Required signals: `state_semantics, canonical_db, row_level_proof, cory_followup, no_overclaim` + +### DC-02 / helmer_live_direct_claim + +Is Helmer's 7 Powers in Leo now? + +- Required signals: `canonical_db, row_level_proof, authorization, cory_followup, no_overclaim` + +### DC-03 / decision_matrix_direct_claim + +Did the decision matrix approve this already? + +- Required signals: `decision_matrix, canonical_db, fresh_readback, cory_followup, no_overclaim` + +### DC-04 / document_source_direct_claim + +Are the pending proposals stuck because the documents are not pointed at the right source rows? + +- Required signals: `document_artifact_linking, staging_or_review, canonical_db, cory_followup, no_overclaim` + +### DC-05 / demo_changes_kb_direct_claim + +Can I demo that Leo changes the KB? + +- Required signals: `state_semantics, row_level_proof, authorization, artifact, cory_followup, no_overclaim` + +### DC-06 / soul_identity_direct_claim + +If we changed SOUL.md, did we change Leo's canonical identity? + +- Required signals: `identity_rendering, canonical_db, row_level_proof, cory_followup, no_overclaim` + +## Checks + +- `all_evidence_paths_exist`: `True` +- `gcp_parity_not_required_for_vps_telegram_test`: `True` +- `handler_direct_claim_score_passed`: `True` +- `handler_direct_claim_suite_safe`: `True` +- `live_open_telegram_suite_passed`: `True` +- `production_apply_not_authorized_by_packet`: `True` +- `raw_dc_messages_preserved`: `True` +- `telegram_visible_messages_not_sent_by_packet`: `True` + +## Explicit Operator Authorization Text + +I authorize a Telegram-visible read-only direct-claim benchmark in the Leo group `Leo` (`-5146042086`) using exactly the six raw messages DC-01 through DC-06 listed in `telegram-visible-direct-claim-authorization-packet-current.json`. Capture before/after DB counts, Telegram-visible replies, scorer output, final screenshot/accessibility proof, and service readback. Do not run production DB apply, do not edit SOUL.md, do not post any extra messages beyond those six benchmark prompts unless separately authorized. + +## Clear CTA + +To run the Telegram-visible DC benchmark, reply with the exact authorization text above. After that, Codex should send only the six listed messages to the Leo group, capture the retained proof paths, score the retained replies with `scripts/working_leo_open_ended_benchmark.py --results-json ... --include-direct-claim-followups`, and read back DB/service stability. + +## Next Non-User Action After Authorization + +- Read current service state and DB counts before sending. +- Send exactly the six raw DC-01..DC-06 messages to Telegram group Leo. +- Capture each Telegram-visible Leo reply plus a final screenshot/accessibility readback. +- Read DB counts and service state after sending. +- Score the retained replies and update the current truth index. + +## Do Not Run Without + +- Exact operator authorization text naming the Telegram group and six-message scope. +- Authenticated Telegram UI/API route that can send to the Leo group. +- Before-count and after-count readbacks for public.* and kb_stage.kb_proposals. + +## Claim Ceiling + +This packet makes the Telegram-visible DC benchmark executable and auditable. It does not send Telegram messages, does not mutate the DB, does not authorize production apply, and does not prove the DC prompts are Telegram-visible until the authorized run is actually completed and scored. diff --git a/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md b/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md index 0543e50..de90ab6 100644 --- a/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md +++ b/docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md @@ -50,6 +50,12 @@ A working Leo, for Cory/m3taversal's current expectation, is a Telegram-facing a - `DC-04` correctly avoids a single-cause pointer diagnosis and separates Telegram/file refs, document evaluations, proposal source refs, and canonical `public.sources` - `DC-05` correctly separates a safe staging-write demo from an authorized canonical apply demo - `DC-06` correctly says direct `SOUL.md` edits are runtime artifact edits, not canonical identity changes, unless DB rows plus render/sync proof exist +- Telegram-visible direct-claim benchmark is now ready to request with exact scope, but not run: + - target is Telegram group `Leo` (`-5146042086`) + - exact raw no-context messages are `DC-01` through `DC-06` from the benchmark catalog + - required proof after authorization is before/after DB counts, Telegram-visible replies, scorer output, final screenshot/accessibility proof, and service readback + - artifact: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.md` + - current status remains `telegram_visible_messages_sent=false` - Strict apply canary is proven: one strict `add_edge` proposal became a canonical `public.claim_edges` row, and read-only plan verification now passes with after-readback checks. - Guarded approval helper is installed on VPS at `/opt/teleo-eval/workspaces/deploy-infra/scripts/approve_proposal.py`, checksum `c650b8b24b7a89a9e23859c04d850db441483d407fc81580e250ccad65a61d83`, imports successfully, and rejects re-approval of the already-applied canary. - Chrome/Telegram Web is authenticated and read/write-proven for bot testing in the `Leo` group through Computer Use. @@ -62,7 +68,7 @@ A working Leo, for Cory/m3taversal's current expectation, is a Telegram-facing a ## What Is Still Not Proven -- The no-context direct-claim behavior Cory is likely to use is now handler-proven for `DC-01` through `DC-06` on the VPS GatewayRunner path. This was not a Telegram-visible public/group post; it used a temporary profile and left DB counts/service state unchanged. +- The no-context direct-claim behavior Cory is likely to use is handler-proven for `DC-01` through `DC-06` on the VPS GatewayRunner path, and an exact Telegram-visible authorization packet is ready. It has still not been sent as a public/group post. - Old Cory-approved mapped rich proposals are still not fully directly applyable: - `14fa5ecc-ac7a-41c1-807d-a2e85b936617` - `ac036c9d-20a0-4ffe-881f-57d6b7bacf22` diff --git a/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md b/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md index 09ce6b2..593e6f4 100644 --- a/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md +++ b/docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md @@ -1,6 +1,6 @@ # Working Leo Execution Plan - 2026-07-09 -Claim ceiling: VPS Telegram memory/KB audit, full live-safe open-ended Telegram suite, full mixed-evidence Cory-style outcome/direct-claim sandbox benchmark, live VPS GatewayRunner no-context direct-claim suite, Telegram-to-`kb_stage` proposal staging, and strict existing-ID apply are working. The live VPS handler suite for exact `DC-01` through `DC-06` prompts strict-scores `6/6` after the authorized-session-plan deploy and scorer refresh, with no Telegram post, no live profile write, no production DB apply, unchanged DB counts, temp profile cleanup, and unchanged gateway service state during the harness. The full prepared DB packet set is integrated-clone-proven in dependency order: mapped rich base, cross-surface anchor, Rio strategy context, governance/concept schema, and Helmer 7 Powers. A read-only live production preflight baseline now passes `5/5`, an authorized apply operator runbook is generated, an authorized apply session plan maps every apply-window output path with authorization gates, a production apply authorization packet is ready-to-request, the authorization verifier refuses the current apply window because no exact authorization text/operator identity is present, and the receipt assembler refuses final receipt emission because post-apply session artifacts/counts are absent. None of those packets are production-applied or authorized. GCP parity remains pending. +Claim ceiling: VPS Telegram memory/KB audit, full live-safe open-ended Telegram suite, full mixed-evidence Cory-style outcome/direct-claim sandbox benchmark, live VPS GatewayRunner no-context direct-claim suite, Telegram-visible DC authorization packet, Telegram-to-`kb_stage` proposal staging, and strict existing-ID apply are working. The live VPS handler suite for exact `DC-01` through `DC-06` prompts strict-scores `6/6` after the authorized-session-plan deploy and scorer refresh, with no Telegram post, no live profile write, no production DB apply, unchanged DB counts, temp profile cleanup, and unchanged gateway service state during the harness. The Telegram-visible DC suite is exact-scoped and ready to request authorization, but not sent. The full prepared DB packet set is integrated-clone-proven in dependency order: mapped rich base, cross-surface anchor, Rio strategy context, governance/concept schema, and Helmer 7 Powers. A read-only live production preflight baseline now passes `5/5`, an authorized apply operator runbook is generated, an authorized apply session plan maps every apply-window output path with authorization gates, a production apply authorization packet is ready-to-request, the authorization verifier refuses the current apply window because no exact authorization text/operator identity is present, and the receipt assembler refuses final receipt emission because post-apply session artifacts/counts are absent. None of those packets are production-applied or authorized. GCP parity remains pending. ## WL-EXEC-01 - VPS Telegram-visible memory + KB audit - Status: `done` @@ -13,14 +13,15 @@ Claim ceiling: VPS Telegram memory/KB audit, full live-safe open-ended Telegram - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-live-open-ended-suite-score-current.md` ## WL-EXEC-01B - Cory-style outcome and direct-claim benchmark -- Status: `sandbox_fixture_done_live_handler_done` -- Result: Full mixed-evidence Cory-style benchmark scores `20/20`: `OE-01` through `OE-05` are retained live Telegram read-only replies; `CS-01` through `CS-09` are sandbox expected-answer fixtures covering broader regular-use cases; `DC-01` through `DC-06` are no-context direct-claim expected-answer fixtures requiring the correct Cory-style follow-up action. A live VPS GatewayRunner handler run of the exact `DC-01` through `DC-06` prompts now returns `6/6` replies and strict-scores `6/6` with no Telegram post, no live profile write, no production DB apply, unchanged DB counts, and unchanged service state during the harness. -- Not done condition: Reopen if `DC-02` stops recognizing approved Helmer staging row `a64df080-8502-42e2-98f4-9bbdecb8da73` as approved/unapplied, `DC-03` infers decision-matrix approval while `matrix_voters`, `proposal_votes`, and `proposal_decisions` tables are absent, any direct-claim answer omits the `Next Cory-style follow-up:` line, or the handler mutates DB/service state during a read-only suite. +- Status: `sandbox_fixture_done_live_handler_done_telegram_visible_auth_ready` +- Result: Full mixed-evidence Cory-style benchmark scores `20/20`: `OE-01` through `OE-05` are retained live Telegram read-only replies; `CS-01` through `CS-09` are sandbox expected-answer fixtures covering broader regular-use cases; `DC-01` through `DC-06` are no-context direct-claim expected-answer fixtures requiring the correct Cory-style follow-up action. A live VPS GatewayRunner handler run of the exact `DC-01` through `DC-06` prompts now returns `6/6` replies and strict-scores `6/6` with no Telegram post, no live profile write, no production DB apply, unchanged DB counts, and unchanged service state during the harness. The Telegram-visible DC run now has an exact authorization packet for the raw `DC-01` through `DC-06` group messages, proof paths, scorer command, DB/service readbacks, and screenshot/accessibility capture; it has not been sent. +- Not done condition: Reopen if `DC-02` stops recognizing approved Helmer staging row `a64df080-8502-42e2-98f4-9bbdecb8da73` as approved/unapplied, `DC-03` infers decision-matrix approval while `matrix_voters`, `proposal_votes`, and `proposal_decisions` tables are absent, any direct-claim answer omits the `Next Cory-style follow-up:` line, the handler mutates DB/service state during a read-only suite, or the Telegram-visible DC authorization packet drifts from the raw benchmark catalog. - Evidence: - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-cory-outcome-sandbox-results-current.md` - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/working-leo-cory-outcome-sandbox-score-current.md` - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-review-current.md` - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.md` + - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-visible-direct-claim-authorization-packet-current.md` ## WL-EXEC-02 - VPS runtime stability - Status: `done` diff --git a/scripts/build_telegram_visible_direct_claim_authorization_packet.py b/scripts/build_telegram_visible_direct_claim_authorization_packet.py new file mode 100644 index 0000000..5dbe572 --- /dev/null +++ b/scripts/build_telegram_visible_direct_claim_authorization_packet.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Build an exact authorization packet for a Telegram-visible DC benchmark run. + +The packet is a planner, not a sender. It records the raw no-context DC-01..DC-06 +messages, the target Telegram group, the expected proof artifacts, and the +required authorization sentence. It never sends Telegram messages and never +mutates the database. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import working_leo_open_ended_benchmark as benchmark + +REPORT_DIR = Path("docs/reports/leo-working-state-20260709") +DEFAULT_OUT = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.json" +DEFAULT_MARKDOWN_OUT = REPORT_DIR / "telegram-visible-direct-claim-authorization-packet-current.md" +DEFAULT_LIVE_OPEN_SCORE = REPORT_DIR / "telegram-live-open-ended-suite-score-current.json" +DEFAULT_HANDLER_SUITE = REPORT_DIR / "telegram-handler-direct-claim-suite-current.json" +DEFAULT_HANDLER_SCORE = REPORT_DIR / "telegram-handler-direct-claim-suite-score-current.json" +DEFAULT_GCP_PROBE = REPORT_DIR / "gcp-db-parity-current-probe-current.json" +DEFAULT_CHAT_ID = "-5146042086" +DEFAULT_CHAT_LABEL = "Leo" +DEFAULT_PROOF_DIR = "outputs/telegram-visible-direct-claim-suite-20260710" + + +def _load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _path_status(path: Path) -> dict[str, Any]: + exists = path.exists() + return {"path": str(path), "exists": exists, "bytes": path.stat().st_size if exists else 0} + + +def current_git_sha(repo_root: Path = Path(".")) -> str: + proc = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + text=True, + capture_output=True, + check=False, + ) + return proc.stdout.strip() if proc.returncode == 0 else "" + + +def direct_claim_messages() -> list[dict[str, Any]]: + return [ + { + "prompt_id": prompt["id"], + "dimension": prompt["dimension"], + "message": prompt["message"], + "expected_answer": prompt["expected_answer"], + "expected_follow_up": prompt["expected_follow_up"], + "required_signals": list(prompt["required_signals"]), + } + for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS + ] + + +def _score_passes(score: dict[str, Any], *, expected_count: int) -> bool: + score_block = score.get("score", {}) + return ( + score_block.get("pass") is True + and score_block.get("passes") == score_block.get("expected_prompt_count") == expected_count + and score_block.get("failures") == [] + and score_block.get("missing_prompt_ids") == [] + and score_block.get("unexpected_prompt_ids") == [] + ) + + +def _live_open_score_passes(score: dict[str, Any]) -> bool: + score_block = score.get("score", {}) + return score_block.get("pass") is True and score_block.get("passes") == score_block.get( + "expected_prompt_count" + ) + + +def _handler_suite_is_safe(suite: dict[str, Any], *, expected_count: int) -> bool: + return ( + suite.get("pass_runtime") is True + and suite.get("posted_to_telegram") is False + and suite.get("changed_live_profile") is False + and suite.get("mutates_kb_by_harness") is False + and suite.get("db_counts_changed") is False + and suite.get("temp_profile_removed") is True + and len(suite.get("results", [])) == expected_count + ) + + +def build_packet( + *, + live_open_score_path: Path = DEFAULT_LIVE_OPEN_SCORE, + handler_suite_path: Path = DEFAULT_HANDLER_SUITE, + handler_score_path: Path = DEFAULT_HANDLER_SCORE, + gcp_probe_path: Path = DEFAULT_GCP_PROBE, + chat_id: str = DEFAULT_CHAT_ID, + chat_label: str = DEFAULT_CHAT_LABEL, + proof_dir: str = DEFAULT_PROOF_DIR, + git_sha: str | None = None, +) -> dict[str, Any]: + messages = direct_claim_messages() + live_open_score = _load_json(live_open_score_path) + handler_suite = _load_json(handler_suite_path) + handler_score = _load_json(handler_score_path) + gcp_probe = _load_json(gcp_probe_path) if gcp_probe_path.exists() else {} + sha = git_sha if git_sha is not None else current_git_sha() + expected_ids = [item["prompt_id"] for item in messages] + + evidence_paths = { + "live_open_score": _path_status(live_open_score_path), + "handler_suite": _path_status(handler_suite_path), + "handler_score": _path_status(handler_score_path), + "gcp_probe": _path_status(gcp_probe_path), + } + expected_proof_paths = { + "results_json": f"{proof_dir}/telegram-visible-direct-claim-suite-results-current.json", + "score_json": f"{proof_dir}/telegram-visible-direct-claim-suite-score-current.json", + "score_markdown": f"{proof_dir}/telegram-visible-direct-claim-suite-score-current.md", + "final_screenshot": f"{proof_dir}/telegram-visible-direct-claim-suite-final.png", + "final_accessibility": f"{proof_dir}/telegram-visible-direct-claim-suite-final-accessibility.txt", + "db_before_after": f"{proof_dir}/telegram-visible-direct-claim-suite-db-before-after.json", + "service_readback": f"{proof_dir}/telegram-visible-direct-claim-suite-service-readback.txt", + } + + checks = { + "live_open_telegram_suite_passed": _live_open_score_passes(live_open_score), + "handler_direct_claim_suite_safe": _handler_suite_is_safe(handler_suite, expected_count=len(messages)), + "handler_direct_claim_score_passed": _score_passes(handler_score, expected_count=len(messages)), + "all_evidence_paths_exist": all(item["exists"] and item["bytes"] > 0 for item in evidence_paths.values()), + "raw_dc_messages_preserved": [item["message"] for item in messages] + == [prompt["message"] for prompt in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS], + "telegram_visible_messages_not_sent_by_packet": True, + "production_apply_not_authorized_by_packet": True, + "gcp_parity_not_required_for_vps_telegram_test": gcp_probe.get("ready_for_gcp_readonly_parity") is not True, + } + ready_to_request = all(checks.values()) + authorization_text = ( + "I authorize a Telegram-visible read-only direct-claim benchmark in the Leo group " + f"`{chat_label}` (`{chat_id}`) using exactly the six raw messages DC-01 through DC-06 listed in " + "`telegram-visible-direct-claim-authorization-packet-current.json`. Capture before/after DB counts, " + "Telegram-visible replies, scorer output, final screenshot/accessibility proof, and service readback. " + "Do not run production DB apply, do not edit SOUL.md, do not post any extra messages beyond those six " + "benchmark prompts unless separately authorized." + ) + + return { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "mode": "telegram_visible_direct_claim_authorization_packet_not_sent", + "packet_generation_git_sha": sha, + "ready_to_request_authorization": ready_to_request, + "authorization_gate_status": "ready_to_request_explicit_authorization" if ready_to_request else "not_ready", + "telegram_visible_messages_sent": False, + "telegram_visible_message_authorization_present": False, + "production_apply_executed": False, + "production_apply_authorization_present": False, + "mutates_db": False, + "target": { + "platform": "telegram", + "chat_label": chat_label, + "chat_id": chat_id, + "message_count": len(messages), + "expected_prompt_ids": expected_ids, + }, + "exact_messages": messages, + "checks": checks, + "evidence_paths": evidence_paths, + "expected_proof_paths_after_authorized_run": expected_proof_paths, + "explicit_operator_authorization_text": authorization_text, + "clear_CTA": ( + "To run the Telegram-visible DC benchmark, reply with the exact authorization text above. After that, " + "Codex should send only the six listed messages to the Leo group, capture the retained proof paths, " + "score the retained replies with `scripts/working_leo_open_ended_benchmark.py --results-json ... " + "--include-direct-claim-followups`, and read back DB/service stability." + ), + "next_non_user_action_after_authorization": [ + "Read current service state and DB counts before sending.", + "Send exactly the six raw DC-01..DC-06 messages to Telegram group Leo.", + "Capture each Telegram-visible Leo reply plus a final screenshot/accessibility readback.", + "Read DB counts and service state after sending.", + "Score the retained replies and update the current truth index.", + ], + "do_not_run_without": [ + "Exact operator authorization text naming the Telegram group and six-message scope.", + "Authenticated Telegram UI/API route that can send to the Leo group.", + "Before-count and after-count readbacks for public.* and kb_stage.kb_proposals.", + ], + "claim_ceiling": ( + "This packet makes the Telegram-visible DC benchmark executable and auditable. It does not send " + "Telegram messages, does not mutate the DB, does not authorize production apply, and does not prove the " + "DC prompts are Telegram-visible until the authorized run is actually completed and scored." + ), + } + + +def write_json(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_markdown(path: Path, data: dict[str, Any]) -> None: + lines = [ + "# Telegram-Visible Direct-Claim Authorization Packet", + "", + f"Generated UTC: `{data['generated_at_utc']}`", + f"Mode: `{data['mode']}`", + f"Ready to request authorization: `{data['ready_to_request_authorization']}`", + f"Authorization gate status: `{data['authorization_gate_status']}`", + f"Telegram-visible messages sent: `{data['telegram_visible_messages_sent']}`", + f"Telegram-visible authorization present: `{data['telegram_visible_message_authorization_present']}`", + f"Production apply executed: `{data['production_apply_executed']}`", + f"Mutates DB: `{data['mutates_db']}`", + "", + "## Target", + "", + f"- Chat: `{data['target']['chat_label']}` (`{data['target']['chat_id']}`)", + f"- Message count: `{data['target']['message_count']}`", + f"- Prompt IDs: `{', '.join(data['target']['expected_prompt_ids'])}`", + "", + "## Exact Messages", + "", + ] + for item in data["exact_messages"]: + lines.extend( + [ + f"### {item['prompt_id']} / {item['dimension']}", + "", + item["message"], + "", + f"- Required signals: `{', '.join(item['required_signals'])}`", + "", + ] + ) + lines.extend(["## Checks", ""]) + for key, value in sorted(data["checks"].items()): + lines.append(f"- `{key}`: `{value}`") + lines.extend( + [ + "", + "## Explicit Operator Authorization Text", + "", + data["explicit_operator_authorization_text"], + "", + "## Clear CTA", + "", + data["clear_CTA"], + "", + "## Next Non-User Action After Authorization", + "", + ] + ) + lines.extend(f"- {item}" for item in data["next_non_user_action_after_authorization"]) + lines.extend(["", "## Do Not Run Without", ""]) + lines.extend(f"- {item}" for item in data["do_not_run_without"]) + lines.extend(["", "## Claim Ceiling", "", data["claim_ceiling"], ""]) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines), encoding="utf-8") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--live-open-score", type=Path, default=DEFAULT_LIVE_OPEN_SCORE) + parser.add_argument("--handler-suite", type=Path, default=DEFAULT_HANDLER_SUITE) + parser.add_argument("--handler-score", type=Path, default=DEFAULT_HANDLER_SCORE) + parser.add_argument("--gcp-probe", type=Path, default=DEFAULT_GCP_PROBE) + parser.add_argument("--chat-id", default=DEFAULT_CHAT_ID) + parser.add_argument("--chat-label", default=DEFAULT_CHAT_LABEL) + parser.add_argument("--proof-dir", default=DEFAULT_PROOF_DIR) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + data = build_packet( + live_open_score_path=args.live_open_score, + handler_suite_path=args.handler_suite, + handler_score_path=args.handler_score, + gcp_probe_path=args.gcp_probe, + chat_id=args.chat_id, + chat_label=args.chat_label, + proof_dir=args.proof_dir, + ) + write_json(args.out, data) + write_markdown(args.markdown_out, data) + print( + json.dumps( + { + "out": str(args.out), + "markdown_out": str(args.markdown_out), + "ready_to_request_authorization": data["ready_to_request_authorization"], + "telegram_visible_messages_sent": data["telegram_visible_messages_sent"], + "authorization_gate_status": data["authorization_gate_status"], + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if data["ready_to_request_authorization"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_build_telegram_visible_direct_claim_authorization_packet.py b/tests/test_build_telegram_visible_direct_claim_authorization_packet.py new file mode 100644 index 0000000..2da9e5a --- /dev/null +++ b/tests/test_build_telegram_visible_direct_claim_authorization_packet.py @@ -0,0 +1,84 @@ +"""Tests for scripts/build_telegram_visible_direct_claim_authorization_packet.py.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import build_telegram_visible_direct_claim_authorization_packet as packet # noqa: E402 +import working_leo_open_ended_benchmark as benchmark # noqa: E402 + + +def test_packet_ready_but_never_sends_messages(): + report = packet.build_packet(git_sha="262ddb9") + + assert report["mode"] == "telegram_visible_direct_claim_authorization_packet_not_sent" + assert report["ready_to_request_authorization"] is True + assert report["authorization_gate_status"] == "ready_to_request_explicit_authorization" + assert report["telegram_visible_messages_sent"] is False + assert report["telegram_visible_message_authorization_present"] is False + assert report["production_apply_executed"] is False + assert report["production_apply_authorization_present"] is False + assert report["mutates_db"] is False + assert report["target"]["chat_id"] == "-5146042086" + assert report["target"]["chat_label"] == "Leo" + assert report["target"]["expected_prompt_ids"] == ["DC-01", "DC-02", "DC-03", "DC-04", "DC-05", "DC-06"] + assert all(report["checks"].values()) + assert "I authorize a Telegram-visible read-only direct-claim benchmark" in report[ + "explicit_operator_authorization_text" + ] + assert "Do not run production DB apply" in report["explicit_operator_authorization_text"] + assert "does not send Telegram messages" in report["claim_ceiling"] + + +def test_packet_preserves_raw_no_context_dc_messages(): + report = packet.build_packet(git_sha="262ddb9") + + packet_messages = [item["message"] for item in report["exact_messages"]] + catalog_messages = [item["message"] for item in benchmark.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS] + + assert packet_messages == catalog_messages + assert packet_messages[0] == "Did we actually update the knowledge base, or is it still just proposals?" + assert packet_messages[1] == "Is Helmer's 7 Powers in Leo now?" + assert "read-only benchmark" not in packet_messages[0].lower() + assert "Do not change the KB" not in "\n".join(packet_messages) + + +def test_packet_markdown_contains_exact_messages_and_cta(tmp_path: Path): + report = packet.build_packet(git_sha="262ddb9") + out = tmp_path / "telegram-dc-auth.md" + + packet.write_markdown(out, report) + + text = out.read_text(encoding="utf-8") + assert "Telegram-Visible Direct-Claim Authorization Packet" in text + assert "Telegram-visible messages sent: `False`" in text + assert "Did we actually update the knowledge base" in text + assert "If we changed SOUL.md" in text + assert "Explicit Operator Authorization Text" in text + assert "Do Not Run Without" in text + + +def test_packet_not_ready_when_handler_score_fails(monkeypatch): + real_load = packet._load_json + + def fake_load(path: Path): + data = real_load(path) + if path == packet.DEFAULT_HANDLER_SCORE: + data = dict(data) + data["score"] = dict(data["score"]) + data["score"]["pass"] = False + data["score"]["passes"] = 5 + data["score"]["failures"] = [{"prompt_id": "DC-03"}] + return data + + monkeypatch.setattr(packet, "_load_json", fake_load) + + report = packet.build_packet(git_sha="262ddb9") + + assert report["ready_to_request_authorization"] is False + assert report["authorization_gate_status"] == "not_ready" + assert report["checks"]["handler_direct_claim_score_passed"] is False