From 8c20613fd00705f5a5f2f68c679106c281f91162 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Mon, 13 Jul 2026 10:50:42 +0200 Subject: [PATCH] Expand Leo database grounding and blind behavior suite --- ...ind-review-pre-broad-hardening-20260713.md | 58 ++++ ...re-evidence-semantics-repair-20260713.json | 189 +++++++++++ ...re-evidence-semantics-repair-20260713.json | 285 ++++++++++++++++ ...-pre-evidence-semantics-repair-20260713.md | 25 ++ .../gcp/teleo-kb-bridge/SKILL.md | 71 +++- .../vps/teleo-kb-bridge/SKILL.md | 71 +++- .../working_leo_m3taversal_oos_benchmark.py | 307 +++++++++++++++++- tests/test_hermes_leoclean_skill_surfaces.py | 44 ++- ...st_working_leo_m3taversal_oos_benchmark.py | 197 ++++++++++- 9 files changed, 1210 insertions(+), 37 deletions(-) create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-blind-review-pre-broad-hardening-20260713.md create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-pre-evidence-semantics-repair-20260713.json create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-evidence-semantics-repair-20260713.json create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-evidence-semantics-repair-20260713.md diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-blind-review-pre-broad-hardening-20260713.md b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-blind-review-pre-broad-hardening-20260713.md new file mode 100644 index 0000000..21307bc --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-blind-review-pre-broad-hardening-20260713.md @@ -0,0 +1,58 @@ +# Working Leo Blind Review Before Broad Hardening + +Date: `2026-07-13` + +Status: **not reliable for unattended operator use** + +## Method + +A 12-question temporary-profile handler run used broad prompts without known +proposal IDs. It posted nothing to Telegram and made no database or service +change. Two independent strict reviews evaluated the same transcript against +the live v1 schema and retained proof. + +Reviewer results differed in severity but agreed on the verdict: + +- strict binary review: `1/12 pass`; +- pass/partial/fail review: `2 pass`, `4 partial`, `6 fail`. + +## Converged Failures + +1. Replies presented target-architecture fields as current schema, including + source author/channel/date, claim bodies, evidence excerpts, edge rationale, + and forecast-resolution fields. +2. Replies invented edge types such as `superseded_by`, `relates_to`, and + `resolves`. Current v1 uses `supersedes`; `superseded_by` is a claim column. +3. Replies treated unchanged canonical counts as a complete explanation of + behavior, ignoring deployed skills, runtime configuration, rendered files, + and persisted Hermes session state. +4. A handler-only no-post run was described as proof of the live Telegram path. +5. A temporary conversation label was promoted into source provenance rather + than resolving a real URL, storage path, file hash, or retained artifact. +6. Staging was treated as though it required the same authorization as + canonical apply. Requested source capture and proposal staging are allowed; + guarded canonical apply is the approval boundary. +7. Forecast handling proposed rewriting historical confidence and inventing + resolution storage that current v1 does not have. +8. Claims/evidence were not consistently treated as shared knowledge while + beliefs and positions remain agent-specific. +9. Replies were too long for operational use: `9,033` words total, `753` words + per answer on average, with several over `1,000` words. + +## Repair Incorporated + +- Explicit current-schema and edge contracts in both VPS and GCP bridge skills. +- Runtime/session/canonical proof-tier separation. +- Source-link versus provenance-quality distinction. +- Shared claims/evidence versus agent-specific positions. +- Forecast-history preservation and current schema gap. +- Autonomous source capture and staging boundary. +- Maximum response length in the out-of-sample scorer. +- Expansion from 9 to 15 broad prompts covering these failure modes. + +## Claim Ceiling + +This is a pre-fix handler baseline. It is not Telegram-visible proof. The repair +is not accepted until the deployed 15-question run passes, independent strict +review converges, restart survival passes, and a separate Telegram-visible +no-mutation canary succeeds. diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-pre-evidence-semantics-repair-20260713.json b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-pre-evidence-semantics-repair-20260713.json new file mode 100644 index 0000000..45d55ec --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-pre-evidence-semantics-repair-20260713.json @@ -0,0 +1,189 @@ +{ + "before_service": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 07:01:34 UTC", + "MainPID": "2096777", + "NRestarts": "0", + "SubState": "running" + }, + "changed_live_profile": false, + "db_counts_after": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_before": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T08:09:39.539076+00:00", + "handler": { + "authorized": true, + "session_key": "agent:main:telegram:group:-5146042086:9070919", + "temp_profile": "/tmp/leo-direct-claim-handler-jxqub3a3/profile" + }, + "harness_notes": [ + "Broad out-of-sample m3taversal prompts plus randomized memory and participant-identity checks.", + "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", + "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery." + ], + "mode": "live_vps_gatewayrunner_temp_profile_m3taversal_out_of_sample_suite", + "mutates_kb_by_harness": false, + "pass_runtime": true, + "posted_to_telegram": false, + "remote_report_path": "/tmp/leo-m3taversal-oos-handler-report-9ff9df0ad53a.json", + "remote_returncode": 0, + "remote_run_id": "9ff9df0ad53a", + "results": [ + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "demo_pressure_without_ids", + "ended_at_utc": "2026-07-13T08:10:55.582604+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "m3taversal walks in now and says, 'Leo's database has not learned anything useful since yesterday and I cannot demo this.' Do not ask me for row IDs. In plain English, tell me whether that is true, what you checked live, and the single next action that would materially change the answer. Keep this read-only: do not stage, approve, apply, or edit anything.", + "prompt_id": "OOS-01", + "reply": "Here is what the live readback shows.\n\n---\n\nThe claim is substantially true.\n\nCanonical `public.claims` last grew on 2026-06-24. Canonical `public.sources` and `public.claim_evidence` last grew on 2026-06-15. The one canonical write since then is a single `add_edge` canary applied on 2026-07-09 \u2014 a plumbing test, not substantive knowledge. Fourteen proposals are sitting at `pending_review` and three are `approved`, but all three approved proposals have `applied_at: NULL`. Approved is not the same as applied. Nothing substantive has landed in canonical `public.*` in roughly three weeks.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nCanonical timestamps (read-only Postgres fallback):\n- `public.claims` latest: `2026-06-24`\n- `public.sources` latest: `2026-06-15`\n- `public.claim_evidence` latest: `2026-06-15`\n- `public.claim_edges` latest: `2026-07-09` (canary edge only)\n- `kb_stage.kb_proposals` latest: `2026-07-09`\n\nThe three approved-but-not-applied proposals are: the Helmer 7 Powers framework packet (`a64df080`), and two strategy-kernel/claim-hygiene packets (`ac036c9d`, `14fa5ecc`). None have a strict `apply_payload` proven to be worker-applyable under the current guarded `approve_claim` path.\n\n---\n\nNext proof-changing follow-up: Authorize rebuilding and reviewing a strict `apply_payload` for proposal `a64df080-8502-42e2-98f4-9bbdecb8da73` (Helmer 7 Powers), then apply it with before/after postflight readback \u2014 that would add canonical sources, reasoning_tool rows, claims, edges, and evidence in one reviewed transaction and materially change what the DB can demonstrate. Apply requires explicit operator authorization and should not run from normal chat without it.", + "started_at_utc": "2026-07-13T08:09:40.402245+00:00", + "turn": 1 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "multi_surface_knowledge_ingestion", + "ended_at_utc": "2026-07-13T08:11:31.976454+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "I give you a PDF, a Telegram discussion, and five tweets that all make overlapping claims about a market shift. Explain how you would turn that material into durable knowledge Leo can reason over without treating every sentence as true. Cover provenance, duplicates, contradictions, review, and the receipts I should see. Explain only; do not stage or mutate the KB.", + "prompt_id": "OOS-02", + "reply": "The pipeline has five stages. Here is each one in plain terms.\n\n---\n\nSTAGE 1: SOURCE REGISTRATION AND PROVENANCE\n\nBefore any sentence becomes a claim, each piece of material gets its own canonical row in `public.sources`. The PDF becomes one source row: storage path on disk, a content hash so any future version of the same file is detected as a duplicate, source type `document`, and captured timestamp. The Telegram discussion becomes a second row: channel identifier, approximate date range, source type `social`. Each tweet becomes its own row: URL, platform handle, captured date, source type `social`. Nothing is merged at this stage. Provenance is the first thing locked in because every downstream claim, evidence link, and edge inherits its credibility from the source row it can point to.\n\nWhat you should see: five or more rows in `public.sources` with distinct hashes and captured timestamps. If two tweets have identical content or the PDF is a repost of something already in the KB, the hash check surfaces that before any claims are created.\n\n---\n\nSTAGE 2: CLAIM EXTRACTION WITH EXPLICIT TYPE LABELS\n\nEach source is read for extractable claims \u2014 falsifiable, specific assertions about the market shift. Not summaries. Not the author's framing. The extractor separates:\n\n- Empirical claims: things that could be measured (prices, volumes, adoption rates, dates)\n- Structural claims: mechanisms asserted to cause the shift (regulatory pressure, protocol change, capital rotation)\n- Normative claims: judgements about what the shift means or what should follow\n\nEvery candidate claim gets labelled with its type and the source row it was pulled from. Confidence is provisional \u2014 not assigned from the author's tone but from what the evidence actually licenses. A tweet saying \"this is massive\" extracts as low-confidence empirical unless it cites a data point.\n\nWhat you should see: a proposal payload listing candidate claims, each annotated with type, provisional confidence, and the source UUID it came from. No candidate claim enters `public.claims` directly from extraction.\n\n---\n\nSTAGE 3: DEDUPLICATION AND CONTRADICTION MAPPING\n\nWith all candidates on the table, the pipeline cross-checks them against each other and against existing canonical claims in `public.claims`.\n\nDuplicates: if the PDF and two tweets assert the same mechanism in different words, those are collapsed into one claim with multiple evidence links \u2014 not three separate claims. The deduplication criterion is semantic overlap, not wording.\n\nContradictions: if the Telegram discussion says the shift began in Q1 and the PDF says Q3, those are not collapsed. They become two separate claims with a `contradicts` edge between them. The reviewer, not the pipeline, decides which one to weight higher or whether to hold both as unresolved. The pipeline's job is to surface the contradiction, not resolve it.\n\nScope mismatches: roughly 85% of apparent contradictions are scope mismatches \u2014 one source talks about a specific market segment, another about the category. The pipeline flags these for a reviewer note rather than a hard `contradicts` edge.\n\nWhat you should see: the proposal payload includes a deduplication section listing collapsed candidates and a contradictions section listing claim pairs with proposed edge types. If nothing contradicts and nothing overlaps, that is also stated explicitly.\n\n---\n\nSTAGE 4: EVIDENCE LINKING AND WEIGHT ASSIGNMENT\n\nEach surviving claim candidate gets explicit evidence links to `public.sources` rows via `public.claim_evidence`. The link carries a role \u2014 `grounds` (this source directly supports the claim), `illustrates` (this source provides an example but does not fully ground it), or `contradicts` (this source undermines the claim). Weight is a float between 0 and 1 reflecting how well the source actually licenses the claim.\n\nA tweet from an analyst who cites no data might get `illustrates` at 0.3. The PDF with a data table might get `grounds` at 0.8. That weight lives in the evidence row, not in the claim itself, so the same claim can have a high-weight grounding source and multiple low-weight illustrating sources without the claim's own confidence number being inflated.\n\nClaims with no `grounds`-role evidence do not graduate to canonical. They stay in the proposal as pending.\n\nWhat you should see: every proposed claim in the payload has at least one evidence row with a source UUID, role, and weight. Claims with only `illustrates` evidence are flagged as needing primary grounding before apply.\n\n---\n\nSTAGE 5: REVIEW AND RECEIPTS\n\nThe full extraction \u2014 source rows, claim candidates, deduplication findings, contradiction flags, evidence links, and any proposed `claim_edges` between claims \u2014 goes into a single `kb_stage.kb_proposals` entry for human review. The proposal is not canonical. Nothing has touched `public.*` yet.\n\nThe reviewer reads the proposal and can: approve as-is, approve with changes (edit the payload before apply), reject individual claims while approving the rest, or reject the whole packet.\n\nAfter an approved proposal is applied \u2014 and only after \u2014 you should see the following receipts:\n\n1. New rows in `public.sources` with non-null UUIDs and a committed timestamp\n2. New rows in `public.claims` with correct types, provisional confidence, and `created_by` attribution\n3. New rows in `public.claim_evidence` linking each claim to its source(s) with roles and weights\n4. New rows in `public.claim_edges` for any `contradicts`, `supports`, `derives_from`, or `relates` edges established between claims\n5. The proposal's `applied_at` column set to a non-null timestamp\n6. A postflight count readback showing the before/after delta for each table \u2014 claims increased by N, sources by M, edges by K, evidence rows by J\n\nApproved but not applied is not done. The demo-safe version of \"Leo learned this\" is a postflight readback showing all five deltas against the counts that existed before the apply transaction ran.\n\n---\n\nThe rule underneath all of this: the pipeline's job is to compress heterogeneous material into the minimum set of falsifiable claims with traceable provenance and explicit uncertainty. It does not flatten disagreement and it does not inherit the author's confidence. Every sentence that enters as true exits as a labelled claim with a source row, an evidence weight, and a reviewer signature \u2014 or it does not exit at all.", + "started_at_utc": "2026-07-13T08:10:55.584099+00:00", + "turn": 2 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "identity_restart_truth", + "ended_at_utc": "2026-07-13T08:12:02.380720+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "I corrected Leo's worldview in chat and someone also edited SOUL.md. Will that correction be part of Leo's canonical identity tomorrow after a restart? Give me the exact truth test from database row to runtime artifact. Do not change either surface.", + "prompt_id": "OOS-03", + "reply": "No. Neither surface is canonical. Here is the exact truth test, layer by layer.\n\n---\n\nLAYER 1: THE CHAT CORRECTION\n\nA correction made in chat updates Leo's runtime context for that session only. If it was substantive enough that Leo ran the memory tool, it may persist in the Hermes memory store \u2014 which is injected into the prompt at session start. That is session continuity, not canonical identity.\n\nTruth test for this layer: search `public.claims` for a row whose text reflects the correction, with a matching `public.sources` row, at least one `public.claim_evidence` link with role `grounds`, and a non-null `created_at`. If those rows do not exist, the correction is not in the canonical graph regardless of what was said in chat.\n\nRow-level proof sentence: if there is no new or updated row in `public.claims` with a postflight-confirmed UUID, the canonical KB did not change.\n\n---\n\nLAYER 2: THE SOUL.MD EDIT\n\n`SOUL.md` is a rendered artifact. It is generated from the Postgres DB and injected into Leo's prompt at runtime. Editing the file directly changes what Leo reads at the next session start \u2014 it does not write back to Postgres.\n\nAfter a restart, if the DB has not changed and only `SOUL.md` was edited, Leo will read the edited file for that session. But the edit is not durable in the canonical sense: if the render/sync pipeline runs again and regenerates `SOUL.md` from the DB, the hand-edit is overwritten by whatever Postgres actually contains.\n\nTruth test for this layer: diff the current `SOUL.md` content against what a fresh DB render would produce. If they diverge, the file was hand-edited and is ahead of the DB. The DB wins on the next render cycle.\n\n---\n\nLAYER 3: WHAT CANONICAL IDENTITY ACTUALLY REQUIRES\n\nFor a correction to survive a restart as canonical identity, all five of the following must be true simultaneously:\n\n1. A new or updated row exists in `public.claims` with the corrected content, correct type, and a committed `created_at` timestamp\n\n2. A `public.sources` row exists that grounds it \u2014 a document, session transcript, or attributed source with a storage path or URL and a content hash\n\n3. At least one `public.claim_evidence` row links the claim to that source with role `grounds` and a non-trivial weight\n\n4. If the correction supersedes an old claim, the old claim's `superseded_by` column points to the new claim's UUID, and a `supersedes` edge exists in `public.claim_edges` from new to old\n\n5. `SOUL.md` was regenerated from the DB after those rows were committed \u2014 meaning the file on disk matches a fresh render, not a hand-edit\n\n---\n\nLAYER 4: THE CURRENT STATE\n\nBased on the live readback from the previous check:\n\n`public.claims` last grew on 2026-06-24. No claim rows were created since then. The Telegram correction and the `SOUL.md` edit are not reflected in `public.*`.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nThat means: if Leo restarts tonight and `SOUL.md` is regenerated from the DB, the hand-edit is gone. If `SOUL.md` is not regenerated, the hand-edit survives one more session but still is not canonical. The chat correction survives only as Hermes memory \u2014 prompt-injected text, not a KB row.\n\n---\n\nTHE COMPLETE TRUTH TEST IN ONE CHECK SEQUENCE\n\nRun these in order after any claimed correction:\n\n1. `teleo-kb search \"\"` \u2014 does a canonical claim row exist?\n2. `teleo-kb evidence ` \u2014 does it have a `grounds`-role evidence link to a real source row?\n3. If it supersedes something: `teleo-kb show ` \u2014 is `superseded_by` set and a `supersedes` edge present?\n4. Check `SOUL.md` on disk against a fresh DB render \u2014 do they match?\n5. Restart Leo and read the injected identity context \u2014 does the corrected claim appear in the rendered soul?\n\nAll five must pass. Failing any one means the correction is either runtime-only, file-only, or proposal-staged but not applied. None of those three survive a full render-and-restart cycle as canonical identity.\n\n---\n\nRow-level proof would require current readback of new or updated row IDs in `public.claims`, `public.sources`, `public.claim_evidence`, `public.claim_edges` or identity tables, plus postflight/render-sync proof. Without those rows, canonical identity is unchanged.\n\nNext proof-changing follow-up: convert the chat correction into a reviewable KB proposal \u2014 source row, claim row, evidence link, supersedes edge if applicable \u2014 get it approved and applied with postflight confirmation, then trigger a `SOUL.md` regeneration from the DB and verify the diff is clean.", + "started_at_utc": "2026-07-13T08:11:31.976773+00:00", + "turn": 3 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "partner_demo_claim_ceiling", + "ended_at_utc": "2026-07-13T08:12:22.208696+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Three changes have reviewer approval and the partner demo starts in an hour. Can I honestly say the knowledge database is updated? Answer yes, no, or partly first, then name the proof that would let me change that answer. Do not apply anything.", + "prompt_id": "OOS-04", + "reply": "No.\n\nApproved is not the same as applied. All three approved proposals have `applied_at: NULL`. No new rows have landed in `public.claims`, `public.sources`, `public.claim_evidence`, or `public.claim_edges` as a result of those approvals. The canonical counts have not moved since 2026-06-24 for claims and 2026-06-15 for sources and evidence.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\nThe three approved proposals sitting without an apply timestamp:\n\n`a64df080-8502-42e2-98f4-9bbdecb8da73` \u2014 Helmer 7 Powers framework; approved 2026-06-29; applied_at: none (database NULL)\n\n`ac036c9d-20a0-4ffe-881f-57d6b7bacf22` \u2014 strategy-kernel claim hygiene; approved 2026-07-05; applied_at: none (database NULL)\n\n`14fa5ecc-ac7a-41c1-807d-a2e85b936617` \u2014 claim-split/strategy packet; approved 2026-07-05; applied_at: none (database NULL)\n\n---\n\nWhat would let you change the answer:\n\nFor each of the three proposals, you need all of the following observed in sequence:\n\n1. A strict `apply_payload` validated against the current schema \u2014 confirmed to contain only existing canonical IDs where required, correct claim types, and no fields that do not exist in the live tables\n\n2. An authorized apply run with explicit operator sign-off \u2014 not from normal chat\n\n3. A postflight count readback showing `public.claims`, `public.sources`, `public.claim_evidence`, and `public.claim_edges` each increased by the declared delta\n\n4. `applied_at` set to a non-null timestamp on each of the three proposal rows\n\nOnly when all four are true for a given proposal can you say that proposal's content is in the canonical KB. Until then the honest demo statement is: \"Leo has three reviewed and approved additions; they are staged and ready; the apply step requires operator authorization and has not run.\"\n\nNext proof-changing follow-up: validate the strict `apply_payload` for the highest-value of the three proposals \u2014 `a64df080` (Helmer 7 Powers) is the most substantive \u2014 then request explicit operator authorization and run the guarded apply with before/after postflight readback retained. That would change the answer to \"partly\" for one proposal and give you a concrete before/after count to show in the demo.", + "started_at_utc": "2026-07-13T08:12:02.381191+00:00", + "turn": 4 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "source_pointer_is_not_evidence", + "ended_at_utc": "2026-07-13T08:14:02.821792+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "A proposal points to a Telegram attachment and its extracted text exists on disk. Does that mean the claim already has canonical evidence? Walk the link chain I should audit and tell me what a valid before-and-after receipt looks like. Read-only only.", + "prompt_id": "OOS-05", + "reply": "Good. I now have everything needed for a complete, grounded answer.\n\n---\n\nNO. The claim does not have canonical evidence. Here is the full link chain and what each break looks like.\n\n---\n\nTHE FIVE-LAYER LINK CHAIN\n\nLayer 1: Telegram attachment arrives\nA PDF comes in via Telegram. Leo caches it at a local path such as `/home/teleo/.hermes/profiles/leoclean/cache/documents/doc_4dff129b0acf_main (1).pdf`. The extracted text may also be written to `/home/teleo/.hermes/profiles/leoclean/kb_stage/evaluations/`. Both of these are local disk artifacts only. Nothing in `public.*` has changed.\n\nLayer 2: Proposal staged in `kb_stage.kb_proposals`\nLeo evaluates the document and creates a proposal \u2014 for example `124942fd-7c7d-4a58-ae10-407365a1396d` \u2014 with `source_ref: telegram-attachment:main _1_.pdf`, a `source_candidate` block describing the paper, and `evidence_candidates` listing which canonical claim IDs should be linked and with what role. The proposal `status` is `pending_review`. `applied_at` is NULL. This is the planning layer. Still nothing in `public.*` has changed.\n\nLayer 3: `public.sources` row \u2014 the first canonical requirement\nBefore any evidence link can exist, the Telegram attachment must become a row in `public.sources` with a UUID, a storage path or URL, a content hash, and ideally an excerpt. Checked live: the two `grounds`-role source rows backing the Markov blanket claims (`471b643c` and `73c4f037`) have no `storage_path` and no `url`. Their excerpts are citation strings \u2014 \"Ramstead, Badcock, Friston (2018)...\" \u2014 not the document text. The Ramstead/Friston PDF sitting on disk has no matching canonical source row. The file and the DB are disconnected.\n\nLayer 4: `public.claim_evidence` row \u2014 the second canonical requirement\n`public.claim_evidence` links a claim UUID to a source UUID with a `role` (grounds / illustrates / contradicts) and a `weight`. Checked live: claims `c0392f4f` and `d3237804` each have two evidence rows. But the `grounds`-role rows point to source rows that have no storage path and no URL \u2014 they are citation stubs, not document links. The extracted text on disk has never been registered as a canonical source row, so no evidence row can point to it.\n\nLayer 5: Claim row exists but evidence chain is broken\nBoth canonical claims exist in `public.claims` with `status: open`. So the claims are real. But their provenance chain terminates at citation-stub source rows, not at the actual document. A reviewer reading the evidence link would find a hash and a one-line citation string \u2014 not a traceable path to the PDF or its extracted text.\n\n---\n\nTHE CONCRETE BREAK IN THIS CASE\n\nThe Ramstead/Badcock/Friston 2018 PDF (`main (1).pdf`) is cached locally and its text has been extracted. Proposal `124942fd` describes linking it to two canonical claims. But:\n\n- No `public.sources` row exists with `storage_path` pointing to that PDF or its extracted text\n- No `public.claim_evidence` row points to such a source\n- The proposal has `applied_at: NULL`\n- The existing `grounds`-role evidence rows are citation stubs with no file path\n\nFile on disk plus proposal payload does not equal canonical evidence. The document is in the staging layer. The canonical layer never received it.\n\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; kb_proposals: `26`.\n\n---\n\nWHAT A VALID BEFORE-AND-AFTER RECEIPT LOOKS LIKE\n\nBefore apply, record these exact counts:\n- `public.sources`: 4145 rows\n- `public.claims`: 1837 rows\n- `public.claim_evidence`: 4670 rows\n- `public.claim_edges`: 4916 rows\n- proposal `124942fd` `status: pending_review`, `applied_at: NULL`\n\nAfter a validated apply, you must observe all of the following:\n\n1. `public.sources` gains at least one new row \u2014 the Ramstead PDF \u2014 with a non-null UUID, a `storage_path` or `url` pointing to the actual file, a content hash matching the file on disk, and a non-null `captured_at`. Source count becomes at least 4146.\n\n2. `public.claim_evidence` gains at least two new rows \u2014 one for each of the two target claims \u2014 each containing: `claim_id` pointing to an existing canonical claim, `source_id` pointing to the new source row, `role: grounds`, and a non-null `weight`. Evidence count becomes at least 4672.\n\n3. The old citation-stub source rows (`471b643c` and `73c4f037`) are either retired or superseded \u2014 or their relationship to the new canonical source row is documented via a separate reviewed proposal.\n\n4. Proposal `124942fd` `applied_at` is set to a non-null timestamp.\n\n5. A postflight diff confirms: sources +1 (minimum), claim_evidence +2 (minimum), claims \u00b10, claim_edges \u00b10 \u2014 because this is an evidence-attachment only, not a new claim.\n\nIf any of those five are missing, the apply did not complete cleanly and the evidence chain is still broken.\n\n---\n\nThe rule: file on disk is not a source row. Source row without a storage path is a citation stub, not traceable provenance. Evidence row pointing to a citation stub is not grounded evidence. Canonical evidence requires all three layers to be present and linked by UUID.\n\nNext proof-changing follow-up: validate proposal `124942fd` has a strict `apply_payload` that creates the canonical `public.sources` row with the correct storage path and hash, then attaches it to both target claims via new `public.claim_evidence` rows \u2014 and get explicit operator authorization to apply it with before/after postflight retained.", + "started_at_utc": "2026-07-13T08:12:22.209089+00:00", + "turn": 5 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "database_composition_judgment", + "ended_at_utc": "2026-07-13T08:15:07.789903+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Suppose a new research packet contains factual observations, a strategic framework, a disputed interpretation, a governance rule, and a correction to an old belief. How should Leo compose the database so it can reason over the packet later without flattening it all into claims? Give the review and apply sequence too, but do not write anything now.", + "prompt_id": "OOS-06", + "reply": "Five elements, five different DB destinations. The schema has distinct containers for each, and flattening them all into `public.claims` would destroy the reasoning structure. Here is how each element maps, where the schema has gaps, and what the review and apply sequence looks like.\n\n---\n\nELEMENT 1: FACTUAL OBSERVATION\n\nDestination: `public.claims` with type `empirical`.\n\nThe observation becomes a single claim row. Confidence is set to what the evidence actually licenses, not what the author asserts. It gets a `public.sources` row for the document it came from, and a `public.claim_evidence` link with role `grounds` and a weight reflecting how directly the source establishes the observation. If the same observation appears in two sources, it gets two evidence rows with different weights \u2014 not two claim rows.\n\nWhat reasoning over it looks like later: other claims can attach `supports`, `challenges`, `derives_from`, or `cites` edges to this row. The evidence weight tells Leo how much to trust the grounding, independent of the claim's stated confidence.\n\n---\n\nELEMENT 2: STRATEGIC FRAMEWORK\n\nDestination: `public.reasoning_tools`.\n\nA framework is not itself a factual claim \u2014 it is a structured lens for evaluating claims. It belongs in `public.reasoning_tools` with a `name`, `description`, `category`, and the `agent_id` that owns it. The description field can carry criteria, steps, or evaluation logic in prose; there are no separate structured columns for those.\n\nSchema gap to state explicitly: the current schema has no generic edge type connecting a `reasoning_tools` row directly to a `public.claims` row. The framework connects to the knowledge graph through claim rows that describe its components \u2014 each power, each criterion, each step \u2014 with regular `claim_edges` between those claim rows. If the framework evaluation produced component definitions during extraction, those become `concept` or `structural` type claim rows linked to each other; the reasoning_tool row sits above them as the named container but is not itself an edge endpoint in `public.claim_edges`.\n\n---\n\nELEMENT 3: DISPUTED INTERPRETATION\n\nDestination: two separate `public.claims` rows with a `contradicts` edge between them.\n\nThe dispute is not resolved at ingestion. Both the mainstream interpretation and the challenger become separate claim rows, each with their own type, confidence, and evidence links. The `contradicts` edge in `public.claim_edges` connects them explicitly. The reviewer, not the pipeline, decides which one to weight higher \u2014 or whether both survive as unresolved.\n\nIf the dispute is a scope mismatch rather than a genuine contradiction \u2014 one interpretation applies to a sub-market, the other to the category \u2014 the edge type is `relates` with a reviewer note in the proposal rationale explaining the scope difference. A hard `contradicts` edge is reserved for claims that cannot both be true.\n\nWhat reasoning over it looks like later: Leo can traverse the `contradicts` edge, read both confidence values, read both evidence weights, and surface the unresolved tension rather than picking one silently.\n\n---\n\nELEMENT 4: GOVERNANCE RULE\n\nDestination: `public.governance_gates`, but only if it fits that contract precisely.\n\n`public.governance_gates` accepts a `name`, `criteria`, `evidence_bar`, and `pass_condition`. It is an evaluative gate \u2014 a structured condition that proposals or claims must pass. If the governance rule in the packet is that kind of thing (\"a synthesis claim requires 2+ domain agents\") it fits.\n\nIf the rule is a behavioral policy or operating norm (\"always check canonical rows before answering\"), it does not fit `governance_gates`. That contract does not cover generic behavioral rules. State the schema gap in the proposal rationale and stage a separate schema proposal for whatever table would be needed before proposing data that depends on it. Do not force a behavioral rule into `governance_gates` to avoid the gap.\n\n---\n\nELEMENT 5: CORRECTION TO AN OLD BELIEF\n\nDestination: a new `public.claims` row plus a `supersedes` edge plus the `superseded_by` column on the old claim \u2014 but these require two separate apply operations under the current schema.\n\nThe new claim row is straightforward: same pipeline as any other claim, with evidence links grounding the correction. The `supersedes` edge from the new claim to the old claim goes into `public.claim_edges` and is within the current strict `approve_claim` contract.\n\nSchema gap to state explicitly: setting `superseded_by` on the old claim row \u2014 the column that marks it as retired \u2014 is an UPDATE to an existing row. The current strict `approve_claim` contract does not support updating an existing claim's `status` or `superseded_by` column. That requires a separately reviewed apply capability. Do not describe both operations as one atomic transaction. The proposal should declare two phases: phase one inserts the new claim and the `supersedes` edge; phase two retires the old claim via a separate reviewed update path. Until phase two completes, both claims are live in `public.claims` and a query without filtering `superseded_by` will return both.\n\n---\n\nCOMPOSING THE PACKET\n\nThe full packet maps to these table targets:\n\n```\npublic.sources \u2014 one row per source document\npublic.claims \u2014 empirical (observation), structural/normative (dispute \u00d72, new belief)\npublic.claim_evidence \u2014 evidence links for each claim to its source\npublic.claim_edges \u2014 contradicts (between disputed interpretations), supersedes (correction to old belief), any supports/derives_from edges within the packet\npublic.reasoning_tools \u2014 one row for the strategic framework\npublic.governance_gates \u2014 one row IF the governance rule fits the gate contract; otherwise a schema-gap note\n```\n\nThe proposal payload should declare all of these collections explicitly. Interdependencies matter: the `supersedes` edge requires both claim UUIDs to exist before the edge is inserted, so the new claim row must be applied before the edge row.\n\n---\n\nREVIEW AND APPLY SEQUENCE\n\nStep 1: Single proposal staged in `kb_stage.kb_proposals`\n\nOne `attach_evidence` or `approve_claim` type proposal carries the full packet: source candidates, all claim candidates with types and provisional confidence, evidence candidates with roles and weights, edge candidates with types, the reasoning_tool row, and the governance gate candidate or a schema-gap note. The reviewer reads the whole packet at once.\n\nStep 2: Reviewer decisions\n\nThe reviewer can: approve the whole packet, approve with changes (edit individual rows before apply), split the packet into sub-proposals if the governance gate needs a schema proposal first, or reject individual elements while approving the rest. The correction's two-phase structure must be called out explicitly so the reviewer knows phase two (retiring the old claim) is a separate follow-on.\n\nStep 3: Validate the strict apply_payload\n\nBefore any apply operation, confirm the payload contains only collections the current `approve_claim` contract accepts: `claims`, `sources`, `evidence`, `edges`, `reasoning_tools`. Governance gate insertion and claim-status updates are not in that contract. If the packet includes either, those elements go into a separate proposal with a separately authorized apply path.\n\nStep 4: Apply phase one\n\nGuarded `approve_claim` apply runs in one transaction: source rows first, then claim rows (new claim before supersedes edge), then evidence rows, then edge rows, then reasoning_tools row. The transaction either commits all of these or rolls back entirely. Postflight count readback: observe the exact delta for each table.\n\nStep 5: Apply phase two \u2014 retire the old claim\n\nSeparately authorized UPDATE sets `superseded_by` on the old claim row and optionally sets `status` to superseded. This is a distinct operation with its own before/after readback: confirm the old claim's `superseded_by` now points to the new claim UUID.\n\nStep 6: Postflight proof\n\nA valid receipt shows:\n- `public.sources`: +N rows (one per new source)\n- `public.claims`: +M rows (observation, both interpretations, new belief \u2014 the old belief row count stays the same until phase two)\n- `public.claim_evidence`: +K rows\n- `public.claim_edges`: +J rows (contradicts edge, supersedes edge, any others)\n- `public.reasoning_tools`: +1 row\n- `public.governance_gates`: +1 row (if applicable) or schema-gap note filed\n- Proposal `applied_at`: non-null timestamp\n- Old claim `superseded_by`: non-null UUID (after phase two only)\n\nIf phase two has not run, the old claim is still live. That is not a failure \u2014 it is the honest state. The postflight note should say: \"correction claim applied, supersedes edge applied, old claim retirement pending phase two authorization.\"\n\n---\n\nThe governing principle: the schema's different containers exist because different kinds of knowledge have different reasoning roles. A framework is not a claim. A governance gate is not a claim. A disputed interpretation is not a single claim. Flattening into claims makes the KB easier to ingest and impossible to reason over without re-doing the work at query time. Preserving structure at insert means Leo can traverse it by type, weight, and edge without reinterpreting every row.\n\nNext proof-changing follow-up: once you are ready to stage this, the first action is to validate whether the governance rule fits the `governance_gates` contract or needs a schema proposal \u2014 that is the only element that could block the rest of the packet from going through the current apply path cleanly.", + "started_at_utc": "2026-07-13T08:14:02.823318+00:00", + "turn": 6 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "same_session_memory_set", + "ended_at_utc": "2026-07-13T08:15:17.952955+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "From everything you checked in this conversation, pick the single highest-impact demo blocker. For the next question, remember it under the temporary label demo-ledger-61c5dcf6. State the label and blocker now. This is chat memory only; do not write it to the KB.", + "prompt_id": "OOS-07", + "reply": "Label: demo-ledger-61c5dcf6\n\nBlocker: The three approved proposals have `applied_at: NULL`. Reviewer approval is complete but no apply has run, so canonical `public.*` counts have not moved since mid-June. The demo cannot honestly show the KB as updated. The highest-impact single fix is applying proposal `a64df080-8502-42e2-98f4-9bbdecb8da73` (Helmer 7 Powers) \u2014 it is the most substantive approved packet \u2014 with a validated strict `apply_payload`, explicit operator authorization, and a before/after postflight count readback retained. That one apply would produce the first provable canonical delta since June and give the demo a concrete receipt to show.", + "started_at_utc": "2026-07-13T08:15:07.790485+00:00", + "turn": 7 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "same_session_memory_recall", + "ended_at_utc": "2026-07-13T08:15:28.108689+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Without re-reading my previous wording, what temporary label did I give the demo blocker, what blocker did you associate with it, and what exact proof would close it? Do not mutate anything.", + "prompt_id": "OOS-08", + "reply": "Label: demo-ledger-61c5dcf6\n\nBlocker: Three approved proposals sitting with `applied_at: NULL` \u2014 reviewer approval is done but no apply has run, so canonical `public.*` counts have not moved since mid-June. The highest-impact single case is proposal `a64df080-8502-42e2-98f4-9bbdecb8da73` (Helmer 7 Powers).\n\nExact proof that closes it \u2014 all five must be observed in sequence:\n\n1. `public.claims`, `public.sources`, `public.claim_evidence`, `public.claim_edges` counts recorded before the apply run \u2014 the before baseline\n\n2. A validated strict `apply_payload` for `a64df080` confirmed to contain only existing canonical IDs where required, correct claim types, and no fields outside the current schema contract\n\n3. An authorized apply run with explicit operator sign-off \u2014 not from normal chat\n\n4. `applied_at` on proposal `a64df080-8502-42e2-98f4-9bbdecb8da73` set to a non-null timestamp\n\n5. Postflight count readback showing `public.sources`, `public.claims`, `public.claim_evidence`, and `public.claim_edges` each increased by the declared delta from the payload \u2014 the after baseline sitting next to the before baseline as a retained receipt\n\nAll five present simultaneously is the proof. Any one missing means the blocker is not closed.", + "started_at_utc": "2026-07-13T08:15:17.953875+00:00", + "turn": 8 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "telegram_participant_identity", + "ended_at_utc": "2026-07-13T08:15:38.276417+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "The current visible Telegram sender is @m3taversal. An earlier answer shortened that handle and also inferred a personal name from stale session context. What should Leo call this participant, which identity sources are allowed, and how should Leo avoid mixing identities when another user replies? Answer in four lines. Do not write to memory or the KB.", + "prompt_id": "OOS-09", + "reply": "This participant is @m3taversal \u2014 full handle, no shortening, no inferred personal name.\n\nAllowed identity sources: the current Telegram update's visible sender field only.\n\nProhibited sources: runtime memory, soul documents, session headers, other participants' messages, and any name inferred from environment labels.\n\nWhen another user replies, resolve identity fresh from that update's sender field \u2014 never carry a participant identity across messages or assign one participant's handle to a different sender.", + "started_at_utc": "2026-07-13T08:15:28.109231+00:00", + "turn": 9 + } + ], + "service_before_after": { + "after": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 07:01:34 UTC", + "MainPID": "2096777", + "NRestarts": "0", + "SubState": "running" + }, + "before": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 07:01:34 UTC", + "MainPID": "2096777", + "NRestarts": "0", + "SubState": "running" + }, + "unchanged_from_preexisting_live_readback": true + }, + "source": { + "chat_id": "-5146042086", + "chat_name": "Leo", + "chat_type": "group", + "platform": "telegram", + "user_id": "9070919", + "user_name": "codex handler direct claim" + }, + "source_report_path": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-current.json", + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-evidence-semantics-repair-20260713.json b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-evidence-semantics-repair-20260713.json new file mode 100644 index 0000000..eb461b4 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-evidence-semantics-repair-20260713.json @@ -0,0 +1,285 @@ +{ + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T08:15:42.852150+00:00", + "memory_token": "demo-ledger-61c5dcf6", + "mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score", + "posted_to_telegram": false, + "production_db_apply_ran": false, + "score": { + "expected_prompt_count": 9, + "expected_prompt_ids": [ + "OOS-01", + "OOS-02", + "OOS-03", + "OOS-04", + "OOS-05", + "OOS-06", + "OOS-07", + "OOS-08", + "OOS-09" + ], + "failures": [ + { + "concepts": { + "canonical_readback": true, + "live_check": false, + "material_action": true, + "state_boundary": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "demo_pressure_without_ids", + "invalid_count_invariant_detected": false, + "legacy_pass": true, + "legacy_signals": { + "canonical_db": true, + "fresh_readback": true, + "next_action": true, + "no_overclaim": true, + "state_semantics": true + }, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-01", + "response_issue_detected": false + }, + { + "concepts": { + "receipt": true, + "source_evidence_chain": false, + "staged_review_apply": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "source_pointer_is_not_evidence", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "document_artifact_linking": true, + "no_overclaim": false, + "row_level_proof": true, + "staging_or_review": false + }, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-05", + "response_issue_detected": false + } + ], + "missing_prompt_ids": [], + "pass": false, + "passes": 7, + "prompt_count": 9, + "scores": [ + { + "concepts": { + "canonical_readback": true, + "live_check": false, + "material_action": true, + "state_boundary": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "demo_pressure_without_ids", + "invalid_count_invariant_detected": false, + "legacy_pass": true, + "legacy_signals": { + "canonical_db": true, + "fresh_readback": true, + "next_action": true, + "no_overclaim": true, + "state_semantics": true + }, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-01", + "response_issue_detected": false + }, + { + "concepts": { + "contradiction": true, + "deduplication": true, + "provenance": true, + "receipt": true, + "staged_review_apply": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "multi_surface_knowledge_ingestion", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "caveat_retention": false, + "document_artifact_linking": true, + "next_action": true, + "no_overclaim": true, + "staging_or_review": false + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-02", + "response_issue_detected": false + }, + { + "concepts": { + "identity_chain": true, + "receipt": true, + "state_boundary": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "identity_restart_truth", + "invalid_count_invariant_detected": false, + "legacy_pass": true, + "legacy_signals": { + "canonical_db": true, + "identity_rendering": true, + "no_overclaim": true, + "row_level_proof": true + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-03", + "response_issue_detected": false + }, + { + "concepts": { + "canonical_readback": true, + "receipt": true, + "state_boundary": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "partner_demo_claim_ceiling", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "artifact": true, + "canonical_db": true, + "no_overclaim": false, + "row_level_proof": true, + "state_semantics": true + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-04", + "response_issue_detected": false + }, + { + "concepts": { + "receipt": true, + "source_evidence_chain": false, + "staged_review_apply": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "source_pointer_is_not_evidence", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "document_artifact_linking": true, + "no_overclaim": false, + "row_level_proof": true, + "staging_or_review": false + }, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-05", + "response_issue_detected": false + }, + { + "concepts": { + "heterogeneous_types": true, + "receipt": true, + "staged_review_apply": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "database_composition_judgment", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "caveat_retention": false, + "next_action": true, + "no_overclaim": true, + "staging_or_review": true + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-06", + "response_issue_detected": false + }, + { + "concepts": { + "blocker_definition": true + }, + "current_schema_overclaims": [], + "custom_signals": { + "memory_token": true + }, + "dimension": "same_session_memory_set", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "next_action": true, + "no_overclaim": false + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-07", + "response_issue_detected": false + }, + { + "concepts": { + "blocker_definition": true, + "receipt": true + }, + "current_schema_overclaims": [], + "custom_signals": { + "closure_proof": true, + "memory_token": true + }, + "dimension": "same_session_memory_recall", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "next_action": true, + "no_overclaim": false, + "row_level_proof": true + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-08", + "response_issue_detected": false + }, + { + "concepts": {}, + "current_schema_overclaims": [], + "custom_signals": { + "current_update_identity_boundary": true, + "exact_participant_handle": true, + "no_cross_participant_identity_bleed": true, + "no_unverified_alias": true + }, + "dimension": "telegram_participant_identity", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "no_overclaim": false + }, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-09", + "response_issue_detected": false + } + ], + "unexpected_prompt_ids": [] + }, + "service_unchanged": true, + "source_results_json": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-current.json", + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-evidence-semantics-repair-20260713.md b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-evidence-semantics-repair-20260713.md new file mode 100644 index 0000000..93b55f9 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-evidence-semantics-repair-20260713.md @@ -0,0 +1,25 @@ +# Working Leo m3taversal Out-of-Sample Score + +Generated UTC: `2026-07-13T08:15:42.852150+00:00` +Pass: `False` +Prompts: `7/9` +DB counts changed: `False` +Service unchanged: `True` +Temporary profile removed: `True` +Posted to Telegram: `False` + +## Prompt Scores + +- `OOS-01` / `demo_pressure_without_ids`: `pass=False` +- `OOS-02` / `multi_surface_knowledge_ingestion`: `pass=True` +- `OOS-03` / `identity_restart_truth`: `pass=True` +- `OOS-04` / `partner_demo_claim_ceiling`: `pass=True` +- `OOS-05` / `source_pointer_is_not_evidence`: `pass=False` +- `OOS-06` / `database_composition_judgment`: `pass=True` +- `OOS-07` / `same_session_memory_set`: `pass=True` +- `OOS-08` / `same_session_memory_recall`: `pass=True` +- `OOS-09` / `telegram_participant_identity`: `pass=True` + +## Claim Ceiling + +A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible Telegram delivery, production DB apply, or GCP parity. diff --git a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md index 9fdf72a..f099463 100644 --- a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md @@ -52,6 +52,32 @@ good default is: 3. final answer with what is grounded, what is weak, and what evidence would improve it. +Lead with the answer. Unless the operator asks for a detailed audit, keep the +whole reply under 220 words and end with one proof-changing follow-up. Do not +turn a direct operational question into an architecture lecture. + +Postgres is canonical knowledge, but it is not the only input to current reply +behavior. Deployed skills, runtime configuration, rendered `SOUL.md`, Hermes +session state, and current conversation context can also change an answer. +Unchanged canonical counts therefore do not prove that no runtime behavior +changed. Hermes `state.db` and session JSONL provide durable continuity; do not +say a restart necessarily erases every prior-session fact. Separate these proof +tiers explicitly: + +- handler or temporary-profile success with no Telegram post is handler proof, + not Telegram-visible delivery proof; +- a Telegram-visible reply proves delivery and reply behavior, not canonical + DB mutation; +- canonical mutation requires row-level `public.*` readback and the matching + applied proposal receipt. + +Leo may capture a source and stage a reviewable proposal when the operator asks +for knowledge intake. Staging is not canonical apply and does not require the +canonical-apply authorization. Approval is required before the guarded apply +step. Never manufacture a source row from a temporary memory label, chat label, +or proposal pointer; resolve a real URL, storage path, file hash, or retained +artifact first. + For no-context direct claims such as "Is X in Leo now?", "did the DB change?", "did the decision matrix approve this?", or "is it still just proposals?", do not stop at `search` or default `list-proposals`. Run the status-specific @@ -134,6 +160,18 @@ Current `public.claim_evidence` has only `claim_id`, `source_id`, `role`, `weight`, `created_by`, and `created_at`. Its accepted roles are `grounds`, `illustrates`, and `contradicts`. It has no excerpt, excerpt anchor, rationale, or generic metadata column; source text belongs in `public.sources.excerpt`. +A `public.claim_evidence` link from a claim to a `public.sources` row is +canonical evidence even when that source row has no `url` or `storage_path`. +Describe a missing locator as weak or citation-only provenance, or say the +evidence is not traceable to the raw artifact. Do not call the canonical link +non-canonical or ungrounded solely because the locator is missing. + +A Telegram attachment, extracted file, or proposal `source_ref` does not by +itself prove canonical evidence from that attachment. That proof requires a +`public.sources` row representing the attachment and a `public.claim_evidence` +link from the claim to that source row. Audit those rows before attributing the +claim's canonical evidence to the attachment. + Current `public.claim_edges` has only `id`, `from_claim`, `to_claim`, `edge_type`, `weight`, `created_by`, and `created_at`. Both endpoints are claim IDs, so do not claim that a `reasoning_tools` row is directly connected through @@ -142,16 +180,22 @@ IDs, so do not claim that a `reasoning_tools` row is directly connected through For heterogeneous research packets, map only to structures proven in the current schema: +- claims, sources, and evidence links are shared knowledge objects; an agent's + confidence, stance, or position belongs in agent-specific belief/position + structures rather than duplicate agent-authored copies of the same claim; + - factual observations and disputed interpretations may become separate `public.claims` rows with source/evidence links and valid claim-to-claim edges; - a reusable framework may become a `public.reasoning_tools` row, but the current schema has no generic reasoning-tool-to-claim edge and no shipped `concept_maps` or `claim_concept_map_links` table; -- `public.governance_gates` can store an evaluative gate with `name`, - `criteria`, `evidence_bar`, and `pass_condition`; it is not a generic - behavioral-rule or policy table, so state a schema gap when a governance rule - does not fit that contract; +- a behavioral or operating rule belongs in the existing + `public.behavioral_rules` table, whose rule contract includes `agent_id`, + `category`, `rank`, `rule`, and `rationale`; +- an evaluative gate belongs in `public.governance_gates`, with `name`, + `criteria`, `evidence_bar`, and `pass_condition`; do not flatten a behavioral + rule into this gate table; - a belief correction may create a new claim, a valid `supersedes` edge, and set the old claim's `superseded_by` column. `superseded_by` is a column, not an edge type. @@ -175,11 +219,13 @@ universal success condition. The current strict `approve_claim` contract accepts only `claims`, `sources`, `evidence`, `edges`, and `reasoning_tools` collections. It does not insert -`governance_gates`, update an existing claim's `status` or `superseded_by`, or -write arbitrary soul/context rows. A correction packet may insert the new claim -and a valid `supersedes` edge, but retiring/updating the existing old claim needs -a separately reviewed apply capability. Do not describe that unsupported update -as part of one atomic `approve_claim` transaction. +`behavioral_rules` or `governance_gates`, update an existing claim's `status` +or `superseded_by`, or write arbitrary soul/context rows. Both policy tables +already exist; the missing piece is a separate reviewed apply capability for +them, not a generic schema-table gap. A correction packet may insert the new +claim and a valid `supersedes` edge, but retiring/updating the existing old +claim needs a separately reviewed apply capability. Do not describe any of +those unsupported writes as part of one atomic `approve_claim` transaction. Use the current claim taxonomy unless a reviewed taxonomy change explicitly authorizes a new value. The live values are `structural`, `normative`, @@ -192,6 +238,13 @@ evidence, say that `public.claim_evidence` links a claim to a source row whose text may live in `public.sources.excerpt`; never say the evidence row stores or carries the excerpt. +Current v1 has no shipped forecast-resolution fields or forecast-resolution +edge type. Preserve the original probability and its timestamp. Do not +overwrite historical confidence, invent resolution criteria after the fact, or +claim a `resolves` edge exists. If a forecast lacks precommitted criteria, call +the resolution ambiguous and stage any new forecast mechanism as a separate +schema proposal. + - "Did we actually update the KB?": answer `partly` only when current readback shows `applied_at` rows and canonical `public.*` rows. Otherwise say `mostly still proposals`; list applied, approved-but-not-applied, pending, diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md index 2dc88d9..0c04283 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md @@ -45,6 +45,32 @@ For KB questions, prefer the bridge over raw database access. A good default is: 3. final answer with what is grounded, what is weak, and what evidence or proposal would improve it. +Lead with the answer. Unless the operator asks for a detailed audit, keep the +whole reply under 220 words and end with one proof-changing follow-up. Do not +turn a direct operational question into an architecture lecture. + +Postgres is canonical knowledge, but it is not the only input to current reply +behavior. Deployed skills, runtime configuration, rendered `SOUL.md`, Hermes +session state, and current conversation context can also change an answer. +Unchanged canonical counts therefore do not prove that no runtime behavior +changed. Hermes `state.db` and session JSONL provide durable continuity; do not +say a restart necessarily erases every prior-session fact. Separate these proof +tiers explicitly: + +- handler or temporary-profile success with no Telegram post is handler proof, + not Telegram-visible delivery proof; +- a Telegram-visible reply proves delivery and reply behavior, not canonical + DB mutation; +- canonical mutation requires row-level `public.*` readback and the matching + applied proposal receipt. + +Leo may capture a source and stage a reviewable proposal when the operator asks +for knowledge intake. Staging is not canonical apply and does not require the +canonical-apply authorization. Approval is required before the guarded apply +step. Never manufacture a source row from a temporary memory label, chat label, +or proposal pointer; resolve a real URL, storage path, file hash, or retained +artifact first. + For no-context direct claims such as "Is X in Leo now?", "did the DB change?", "did the decision matrix approve this?", or "is it still just proposals?", do not stop at `search` or default `list-proposals`. Run the status-specific @@ -127,6 +153,18 @@ Current `public.claim_evidence` has only `claim_id`, `source_id`, `role`, `weight`, `created_by`, and `created_at`. Its accepted roles are `grounds`, `illustrates`, and `contradicts`. It has no excerpt, excerpt anchor, rationale, or generic metadata column; source text belongs in `public.sources.excerpt`. +A `public.claim_evidence` link from a claim to a `public.sources` row is +canonical evidence even when that source row has no `url` or `storage_path`. +Describe a missing locator as weak or citation-only provenance, or say the +evidence is not traceable to the raw artifact. Do not call the canonical link +non-canonical or ungrounded solely because the locator is missing. + +A Telegram attachment, extracted file, or proposal `source_ref` does not by +itself prove canonical evidence from that attachment. That proof requires a +`public.sources` row representing the attachment and a `public.claim_evidence` +link from the claim to that source row. Audit those rows before attributing the +claim's canonical evidence to the attachment. + Current `public.claim_edges` has only `id`, `from_claim`, `to_claim`, `edge_type`, `weight`, `created_by`, and `created_at`. Both endpoints are claim IDs, so do not claim that a `reasoning_tools` row is directly connected through @@ -135,16 +173,22 @@ IDs, so do not claim that a `reasoning_tools` row is directly connected through For heterogeneous research packets, map only to structures proven in the current schema: +- claims, sources, and evidence links are shared knowledge objects; an agent's + confidence, stance, or position belongs in agent-specific belief/position + structures rather than duplicate agent-authored copies of the same claim; + - factual observations and disputed interpretations may become separate `public.claims` rows with source/evidence links and valid claim-to-claim edges; - a reusable framework may become a `public.reasoning_tools` row, but the current schema has no generic reasoning-tool-to-claim edge and no shipped `concept_maps` or `claim_concept_map_links` table; -- `public.governance_gates` can store an evaluative gate with `name`, - `criteria`, `evidence_bar`, and `pass_condition`; it is not a generic - behavioral-rule or policy table, so state a schema gap when a governance rule - does not fit that contract; +- a behavioral or operating rule belongs in the existing + `public.behavioral_rules` table, whose rule contract includes `agent_id`, + `category`, `rank`, `rule`, and `rationale`; +- an evaluative gate belongs in `public.governance_gates`, with `name`, + `criteria`, `evidence_bar`, and `pass_condition`; do not flatten a behavioral + rule into this gate table; - a belief correction may create a new claim, a valid `supersedes` edge, and set the old claim's `superseded_by` column. `superseded_by` is a column, not an edge type. @@ -168,11 +212,13 @@ universal success condition. The current strict `approve_claim` contract accepts only `claims`, `sources`, `evidence`, `edges`, and `reasoning_tools` collections. It does not insert -`governance_gates`, update an existing claim's `status` or `superseded_by`, or -write arbitrary soul/context rows. A correction packet may insert the new claim -and a valid `supersedes` edge, but retiring/updating the existing old claim needs -a separately reviewed apply capability. Do not describe that unsupported update -as part of one atomic `approve_claim` transaction. +`behavioral_rules` or `governance_gates`, update an existing claim's `status` +or `superseded_by`, or write arbitrary soul/context rows. Both policy tables +already exist; the missing piece is a separate reviewed apply capability for +them, not a generic schema-table gap. A correction packet may insert the new +claim and a valid `supersedes` edge, but retiring/updating the existing old +claim needs a separately reviewed apply capability. Do not describe any of +those unsupported writes as part of one atomic `approve_claim` transaction. Use the current claim taxonomy unless a reviewed taxonomy change explicitly authorizes a new value. The live values are `structural`, `normative`, @@ -185,6 +231,13 @@ evidence, say that `public.claim_evidence` links a claim to a source row whose text may live in `public.sources.excerpt`; never say the evidence row stores or carries the excerpt. +Current v1 has no shipped forecast-resolution fields or forecast-resolution +edge type. Preserve the original probability and its timestamp. Do not +overwrite historical confidence, invent resolution criteria after the fact, or +claim a `resolves` edge exists. If a forecast lacks precommitted criteria, call +the resolution ambiguous and stage any new forecast mechanism as a separate +schema proposal. + - "Did we actually update the KB?": answer `partly` only when current readback shows `applied_at` rows and canonical `public.*` rows. Otherwise say `mostly still proposals`; list applied, approved-but-not-applied, pending, diff --git a/scripts/working_leo_m3taversal_oos_benchmark.py b/scripts/working_leo_m3taversal_oos_benchmark.py index 11d102c..9aac71b 100755 --- a/scripts/working_leo_m3taversal_oos_benchmark.py +++ b/scripts/working_leo_m3taversal_oos_benchmark.py @@ -74,7 +74,7 @@ SCENARIOS: list[dict[str, Any]] = [ }, { "id": "OOS-05", - "dimension": "source_pointer_is_not_evidence", + "dimension": "source_pointer_needs_canonical_link_audit", "message": ( "A proposal points to a Telegram attachment and its extracted text exists on disk. Does that mean the " "claim already has canonical evidence? Walk the link chain I should audit and tell me what a valid " @@ -87,7 +87,13 @@ SCENARIOS: list[dict[str, Any]] = [ "row_level_proof", "no_overclaim", ], - "required_concepts": ["source_evidence_chain", "staged_review_apply", "receipt"], + "required_concepts": [ + "source_evidence_chain", + "canonical_evidence_boundary", + "evidence_provenance_quality", + "staged_review_apply", + "receipt", + ], }, { "id": "OOS-06", @@ -99,7 +105,13 @@ SCENARIOS: list[dict[str, Any]] = [ "review and apply sequence too, but do not write anything now." ), "required_signals": ["canonical_db", "staging_or_review", "caveat_retention", "next_action", "no_overclaim"], - "required_concepts": ["heterogeneous_types", "staged_review_apply", "receipt"], + "required_concepts": [ + "heterogeneous_types", + "behavioral_rule_storage", + "reviewed_policy_apply", + "staged_review_apply", + "receipt", + ], }, { "id": "OOS-07", @@ -134,6 +146,72 @@ SCENARIOS: list[dict[str, Any]] = [ "required_signals": ["no_overclaim"], "required_concepts": [], }, + { + "id": "OOS-10", + "dimension": "runtime_vs_canonical_causality", + "message": ( + "The five database totals are unchanged after a restart. Does that prove Leo's answer behavior is " + "unchanged and that every fact from the prior session was erased? Explain which persisted and deployed " + "surfaces can affect the answer, and separate the proof tiers. Read-only; keep it under 180 words." + ), + "required_signals": ["canonical_db", "no_overclaim"], + "required_concepts": ["runtime_inputs", "durable_session_continuity", "proof_tiers"], + }, + { + "id": "OOS-11", + "dimension": "shared_claims_agent_positions", + "message": ( + "Two agents read the same evidence and reach different conclusions. Should Leo duplicate the factual " + "claim once per agent, or keep shared knowledge and represent agent-specific positions elsewhere? Give " + "the current-schema answer and how disagreement remains queryable. Do not write anything." + ), + "required_signals": ["canonical_db", "caveat_retention", "no_overclaim"], + "required_concepts": ["shared_knowledge_commons", "agent_specific_positions", "contradiction"], + }, + { + "id": "OOS-12", + "dimension": "forecast_resolution_without_rewriting_history", + "message": ( + "An old claim recorded a 60% forecast but never defined resolution criteria. The event is now over. " + "What may Leo record in the current database without rewriting history, and what needs a schema " + "proposal? Do not apply anything." + ), + "required_signals": ["canonical_db", "staging_or_review", "caveat_retention", "no_overclaim"], + "required_concepts": ["forecast_history", "forecast_schema_gap", "staged_review_apply"], + }, + { + "id": "OOS-13", + "dimension": "handler_is_not_telegram_delivery", + "message": ( + "A temporary-profile GatewayRunner answered every test prompt, but the run explicitly posted nothing " + "to Telegram. Can I tell a partner the Telegram path is proven live? Answer first, then state exactly " + "what this run proves and the smallest test that closes the gap." + ), + "required_signals": ["artifact", "next_action", "no_overclaim"], + "required_concepts": ["handler_not_telegram", "proof_tiers", "receipt"], + }, + { + "id": "OOS-14", + "dimension": "autonomous_source_intake_boundary", + "message": ( + "I hand Leo a document and say: absorb this as far as safely possible without making me approve every " + "mechanical step. What can Leo capture and stage immediately, what real source identity must be retained, " + "and where does explicit approval begin? Explain only; do not ingest this prompt." + ), + "required_signals": ["authorization", "staging_or_review", "artifact", "no_overclaim"], + "required_concepts": ["staging_without_apply_authorization", "real_source_identity", "staged_review_apply"], + }, + { + "id": "OOS-15", + "dimension": "schema_valid_supersession", + "message": ( + "A canonical claim is wrong. I want the replacement, an explanation, and the old claim visibly retired. " + "In current v1, which exact claim and edge fields exist, which requested writes fit approve_claim, and " + "which require a separate reviewed apply capability? Do not mutate anything." + ), + "required_signals": ["canonical_db", "staging_or_review", "row_level_proof", "no_overclaim"], + "required_concepts": ["valid_supersession", "current_edge_schema", "apply_capability_boundary"], + }, ] @@ -146,7 +224,9 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.I, ), ), - "live_check": (re.compile(r"checked live|I ran `?teleo-kb|fresh readback|current canonical row counts", re.I),), + "live_check": ( + re.compile(r"checked live|I ran `?teleo-kb|fresh readback|live readback|current canonical row counts", re.I), + ), "material_action": ( re.compile( r"next .*action|rebuild .*apply_payload|operator .*authoriz|review .*apply|apply sequence|postflight", @@ -175,7 +255,22 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"file|attachment|source_ref", re.I), re.compile(r"public\.sources", re.I), re.compile(r"claim_evidence", re.I), - re.compile(r"not canonical|staging-layer pointer|proves nothing about the KB", re.I), + re.compile(r"audit|link|join|row chain|before-and-after|before/after", re.I), + ), + "canonical_evidence_boundary": ( + re.compile(r"canonical evidence", re.I), + re.compile(r"claim_evidence", re.I), + re.compile(r"public\.sources|source row", re.I), + re.compile( + r"attachment.{0,160}(?:does not|doesn't|is not|isn't|cannot|can't|alone|until|unless)|" + r"(?:does not|doesn't|is not|isn't|cannot|can't).{0,100}canonical evidence from (?:that|the) attachment", + re.I | re.S, + ), + ), + "evidence_provenance_quality": ( + re.compile(r"(?:missing|no|without).{0,50}(?:url|storage(?:_path)?|locator)", re.I | re.S), + re.compile(r"weak|citation-only|citation only|not traceable|raw artifact", re.I), + re.compile(r"canonical evidence|canonical link", re.I), ), "heterogeneous_types": ( re.compile(r"claim", re.I), @@ -185,6 +280,97 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"correction|supersed", re.I), re.compile(r"disput|contradict", re.I), ), + "behavioral_rule_storage": ( + re.compile(r"public\.behavioral_rules", re.I), + re.compile(r"\bagent_id\b", re.I), + re.compile(r"\bcategory\b", re.I), + re.compile(r"\brank\b", re.I), + re.compile(r"\brule\b", re.I), + re.compile(r"\brationale\b", re.I), + ), + "reviewed_policy_apply": ( + re.compile(r"approve_claim", re.I), + re.compile(r"behavioral_rules", re.I), + re.compile(r"governance_gates", re.I), + re.compile(r"does not (?:accept|insert|support)|supports neither|neither.{0,80}nor", re.I | re.S), + re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply capability", re.I | re.S), + ), + "runtime_inputs": ( + re.compile(r"Postgres|canonical (?:DB|database|counts?)", re.I), + re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I), + re.compile(r"session|conversation context", re.I), + re.compile(r"unchanged.{0,80}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)", re.I | re.S), + ), + "durable_session_continuity": ( + re.compile(r"state\.db|session JSONL", re.I), + re.compile(r"persist|durable|continuity", re.I), + re.compile(r"restart.{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)", re.I | re.S), + ), + "proof_tiers": ( + re.compile(r"handler|temporary[- ]profile|GatewayRunner", re.I), + re.compile(r"Telegram", re.I), + re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I), + ), + "shared_knowledge_commons": ( + re.compile(r"shared (?:claim|knowledge|commons)|claims.{0,40}shared", re.I | re.S), + re.compile(r"source|evidence", re.I), + re.compile(r"do not duplicate|don'?t duplicate|one shared claim|single shared claim", re.I), + ), + "agent_specific_positions": ( + re.compile(r"agent[- ]specific", re.I), + re.compile(r"belief|position|stance|confidence", re.I), + re.compile(r"separate|elsewhere|per-agent", re.I), + ), + "forecast_history": ( + re.compile(r"original (?:probability|confidence)|60%|history", re.I), + re.compile(r"preserve|retain|do not overwrite|don'?t overwrite", re.I), + re.compile(r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria", re.I | re.S), + ), + "forecast_schema_gap": ( + re.compile(r"current (?:v1|schema)|public\.claims", re.I), + re.compile( + r"no.{0,50}(?:forecast[- ]resolution|resolution field|resolved_at)|does not have.{0,50}resolution", + re.I | re.S, + ), + re.compile(r"resolves.{0,30}(?:not|isn'?t|does not)|no.{0,30}resolves", re.I | re.S), + ), + "handler_not_telegram": ( + re.compile(r"no|not", re.I), + re.compile(r"handler|GatewayRunner|temporary[- ]profile", re.I), + re.compile(r"did not post|posted nothing|not Telegram-visible|does not prove Telegram", re.I), + ), + "staging_without_apply_authorization": ( + re.compile(r"capture|hash|archive", re.I), + re.compile(r"stage|pending_review|proposal", re.I), + re.compile( + r"does not require.{0,50}(?:apply )?approval|without.{0,50}(?:apply )?approval|approval begins.{0,80}apply", + re.I | re.S, + ), + ), + "real_source_identity": ( + re.compile(r"URL|storage path|file hash|content hash|retained artifact", re.I), + re.compile(r"temporary label|chat label|proposal pointer|source_ref", re.I), + re.compile(r"not.{0,40}(?:a )?(?:source|provenance)|must not.{0,40}source|do not manufacture", re.I | re.S), + ), + "valid_supersession": ( + re.compile(r"new|replacement", re.I), + re.compile(r"supersedes", re.I), + re.compile(r"superseded_by", re.I), + re.compile(r"old claim", re.I), + ), + "current_edge_schema": ( + re.compile(r"claim_edges", re.I), + re.compile(r"from_claim", re.I), + re.compile(r"to_claim", re.I), + re.compile(r"edge_type", re.I), + re.compile(r"no.{0,30}rationale|does not.{0,30}rationale", re.I | re.S), + ), + "apply_capability_boundary": ( + re.compile(r"approve_claim", re.I), + re.compile(r"insert.{0,50}(?:new|replacement) claim|new claim.{0,50}insert", re.I | re.S), + re.compile(r"supersedes edge", re.I), + re.compile(r"separate.{0,50}reviewed apply|does not update.{0,80}(?:status|superseded_by)", re.I | re.S), + ), "blocker_definition": ( re.compile(r"blocker", re.I), re.compile(r"approved|applied_at", re.I), @@ -206,7 +392,8 @@ COUNT_INVARIANT_REJECTION_RE = re.compile( SCHEMA_GAP_QUALIFIER_RE = re.compile( r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|" - r"has no|have no|no column|not an edge|would require|schema gap|must be added)\b", + r"has no|have no|no column|not an edge|would require|schema gap|must be added|" + r"does not support|doesn't support|supports neither|not supported)\b", re.I, ) CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = { @@ -247,6 +434,53 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = { } UNVERIFIED_M3TAVERSAL_ALIAS_RE = re.compile(r"\b(?:Cory|m3ta)\b", re.I) +SOURCE_EVIDENCE_CANONICAL_OBJECT_RE = re.compile(r"claim_evidence|public\.sources|source rows?", re.I) +SOURCE_EVIDENCE_LOCATOR_GAP_RE = re.compile( + r"(?:missing|without|no).{0,50}(?:url|storage(?:_path)?|locator)", re.I | re.S +) +SOURCE_EVIDENCE_DENIAL_RE = re.compile( + r"(?:not|isn't|is not|doesn't count as|does not count as).{0,40}(?:canonical|grounded) evidence", + re.I | re.S, +) +SOURCE_EVIDENCE_CITATION_STUB_DENIAL_RE = re.compile( + r"citation stubs?.{0,50}(?:not|isn't|is not).{0,30}(?:canonical|grounded) evidence", + re.I | re.S, +) + +BEHAVIORAL_RULES_FALSE_ABSENCE_RE = re.compile( + r"(?:public\.)?behavioral_rules(?: table)?.{0,50}(?:does not exist|doesn't exist|is absent|is missing|not shipped)|" + r"(?:no|missing) (?:public\.)?behavioral_rules table", + re.I | re.S, +) + +DB_ONLY_CAUSALITY_RE = re.compile( + r"(?:unchanged|same) (?:canonical )?(?:DB|database|counts?).{0,80}" + r"(?:proves?|means|therefore).{0,80}(?:behavior|answer).{0,30}(?:unchanged|cannot change)", + re.I | re.S, +) +RESTART_ERASES_ALL_RE = re.compile( + r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S +) +HANDLER_TELEGRAM_OVERCLAIM_RE = re.compile( + r"(?:Telegram (?:path|delivery).{0,40}(?:is|was) (?:live|proven)|" + r"live Telegram (?:path|delivery)).{0,160}(?:posted nothing|did not post|no Telegram post)", + re.I | re.S, +) +FORECAST_HISTORY_REWRITE_RE = re.compile( + r"(? list[dict[str, Any]]: return [ @@ -279,6 +513,51 @@ def current_schema_overclaims(reply: str) -> list[str]: return sorted(set(findings)) +def source_evidence_semantic_issues(reply: str) -> list[str]: + """Reject answers that confuse canonical linkage with locator quality.""" + + findings: set[str] = set() + for segment in re.split(r"(?<=[.!?])\s+|\n+", reply): + if ( + SOURCE_EVIDENCE_CANONICAL_OBJECT_RE.search(segment) + and SOURCE_EVIDENCE_LOCATOR_GAP_RE.search(segment) + and SOURCE_EVIDENCE_DENIAL_RE.search(segment) + ): + findings.add("locator_gap_called_noncanonical") + if SOURCE_EVIDENCE_CITATION_STUB_DENIAL_RE.search(reply): + findings.add("citation_stub_called_ungrounded") + return sorted(findings) + + +def behavioral_rule_schema_issues(reply: str) -> list[str]: + """Reject claims that the current behavioral-rules table is absent.""" + + return ["behavioral_rules_false_absence"] if BEHAVIORAL_RULES_FALSE_ABSENCE_RE.search(reply) else [] + + +def broad_semantic_issues(reply: str) -> list[str]: + """Reject high-impact false claims that can hide inside otherwise complete prose.""" + + findings: set[str] = set() + if DB_ONLY_CAUSALITY_RE.search(reply) and not re.search( + r"does not prove|doesn't prove|do not prove|does not mean|doesn't mean|do not mean|cannot prove", reply, re.I + ): + findings.add("unchanged_db_called_complete_behavior_proof") + if RESTART_ERASES_ALL_RE.search(reply) and not re.search( + r"restart.{0,100}(?:does not|doesn't|need not|not necessarily).{0,60}(?:erase|forget|lose)", + reply, + re.I | re.S, + ): + findings.add("restart_called_total_memory_erasure") + if HANDLER_TELEGRAM_OVERCLAIM_RE.search(reply): + findings.add("handler_proof_called_telegram_live") + if FORECAST_HISTORY_REWRITE_RE.search(reply): + findings.add("forecast_history_rewrite") + if TEMP_LABEL_AS_SOURCE_RE.search(reply): + findings.add("temporary_label_promoted_to_source") + return sorted(findings) + + def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dict[str, Any]: legacy_score = base.score_reply(prompt, reply) concepts = {concept: matched_concept(reply, concept) for concept in prompt["required_concepts"]} @@ -307,6 +586,12 @@ def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dic ) invalid_count_invariant = asserts_invalid_count_invariant(reply) schema_overclaims = current_schema_overclaims(reply) + source_evidence_issues = source_evidence_semantic_issues(reply) if prompt["id"] == "OOS-05" else [] + behavioral_rule_issues = behavioral_rule_schema_issues(reply) if prompt["id"] == "OOS-06" else [] + semantic_issues = broad_semantic_issues(reply) + word_count = len(re.findall(r"\b\w+(?:[-']\w+)*\b", reply)) + max_response_words = MAX_RESPONSE_WORDS.get(prompt["id"], DEFAULT_MAX_RESPONSE_WORDS) + response_too_long = word_count > max_response_words return { "prompt_id": prompt["id"], "dimension": prompt["dimension"], @@ -318,12 +603,22 @@ def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dic "response_issue_detected": legacy_score["response_issue_detected"], "invalid_count_invariant_detected": invalid_count_invariant, "current_schema_overclaims": schema_overclaims, + "source_evidence_semantic_issues": source_evidence_issues, + "behavioral_rule_schema_issues": behavioral_rule_issues, + "broad_semantic_issues": semantic_issues, + "word_count": word_count, + "max_response_words": max_response_words, + "response_too_long": response_too_long, "pass": bool( all(concepts.values()) and all(custom_signals.values()) and not legacy_score["overclaim_detected"] and not invalid_count_invariant and not schema_overclaims + and not source_evidence_issues + and not behavioral_rule_issues + and not semantic_issues + and not response_too_long ), } diff --git a/tests/test_hermes_leoclean_skill_surfaces.py b/tests/test_hermes_leoclean_skill_surfaces.py index e802225..fe08b71 100644 --- a/tests/test_hermes_leoclean_skill_surfaces.py +++ b/tests/test_hermes_leoclean_skill_surfaces.py @@ -120,7 +120,7 @@ def test_vps_kb_skill_keeps_vps_scope_explicit() -> None: assert "new or updated row IDs" in text assert "I cannot claim canonical DB changed" in text assert "explicit operator/admin authorization" in text - assert "Do not call an approved proposal \"implemented\"" in text + assert 'Do not call an approved proposal "implemented"' in text assert "repository contains a live-proven strict existing-ID `add_edge` path" in squashed assert "Never end a normal Telegram answer by offering to run direct `INSERT`, `UPDATE`" in text assert "Next admin-panel action" in text @@ -152,6 +152,48 @@ def test_leoclean_kb_skills_anchor_external_doctrine_in_target_project_language( assert "Consent is action-specific" in text +def test_leoclean_kb_skills_distinguish_canonical_evidence_from_provenance_quality() -> None: + for surface in ("vps", "gcp"): + text = (SKILL_ROOT / surface / "teleo-kb-bridge" / "SKILL.md").read_text() + squashed = " ".join(text.split()) + + assert "is canonical evidence even when that source row has no `url` or `storage_path`" in squashed + assert "weak or citation-only provenance" in squashed + assert "not traceable to the raw artifact" in squashed + assert "does not by itself prove canonical evidence from that attachment" in squashed + assert "`public.sources` row representing the attachment" in squashed + assert "`public.claim_evidence` link from the claim to that source row" in squashed + + +def test_leoclean_kb_skills_use_existing_behavioral_rule_storage_and_separate_apply() -> None: + for surface in ("vps", "gcp"): + text = (SKILL_ROOT / surface / "teleo-kb-bridge" / "SKILL.md").read_text() + squashed = " ".join(text.split()) + + assert "existing `public.behavioral_rules` table" in squashed + for field in ("`agent_id`", "`category`", "`rank`", "`rule`", "`rationale`"): + assert field in text + assert "does not insert `behavioral_rules` or `governance_gates`" in squashed + assert "separate reviewed apply capability" in squashed + assert "not a generic schema-table gap" in squashed + + +def test_leoclean_kb_skills_separate_runtime_session_and_delivery_proof_tiers() -> None: + for surface in ("vps", "gcp"): + text = (SKILL_ROOT / surface / "teleo-kb-bridge" / "SKILL.md").read_text() + squashed = " ".join(text.split()) + + assert "keep the whole reply under 220 words" in squashed + assert "Postgres is canonical knowledge, but it is not the only input" in squashed + assert "Hermes `state.db` and session JSONL provide durable continuity" in squashed + assert "not Telegram-visible delivery proof" in squashed + assert "Staging is not canonical apply" in squashed + assert "Never manufacture a source row from a temporary memory label" in squashed + assert "claims, sources, and evidence links are shared knowledge objects" in squashed + assert "no shipped forecast-resolution fields" in squashed + assert "Do not overwrite historical confidence" in squashed + + def test_vps_live_telegram_skill_uses_systemd_for_gateway_liveness() -> None: text = (SKILL_ROOT / "vps" / "live-leo-telegram" / "SKILL.md").read_text() diff --git a/tests/test_working_leo_m3taversal_oos_benchmark.py b/tests/test_working_leo_m3taversal_oos_benchmark.py index 4bb7855..05dbbea 100644 --- a/tests/test_working_leo_m3taversal_oos_benchmark.py +++ b/tests/test_working_leo_m3taversal_oos_benchmark.py @@ -35,14 +35,20 @@ def good_reply(prompt_id: str, token: str) -> str: ) if prompt_id == "OOS-05": return common + ( - "A Telegram attachment and proposal source_ref are staging pointers, not canonical evidence. Audit the " - "document_evaluation, source_ref, public.sources row, claim_evidence join, and guarded apply receipt." + "A Telegram attachment or proposal source_ref alone does not prove canonical evidence from that " + "attachment. Canonical evidence exists when public.claim_evidence links the claim to a public.sources " + "row. A missing URL or storage_path leaves weak, citation-only provenance that is not traceable to the " + "raw artifact; it does not undo the canonical link. Audit the document_evaluation, source_ref, source " + "row, claim_evidence join, and guarded apply receipt." ) if prompt_id == "OOS-06": return common + ( - "Keep observations as claims with sources and evidence, link edges, place a framework in a reasoning tool " - "or concept map, retain the disputed interpretation as a caveat, stage governance separately, and record " - "the correction as a superseding relationship before review and apply." + "Keep factual observations as empirical claims with sources and evidence, link contradictory claims, " + "place the framework in a reasoning tool, and retain the disputed interpretation as a caveat. Store the " + "behavioral rule in public.behavioral_rules using agent_id, category, rank, rule, and rationale; reserve " + "public.governance_gates for the evaluative gate. Current approve_claim supports neither " + "behavioral_rules nor governance_gates, so each needs a separate reviewed apply capability. Record the " + "correction as a superseding relationship before review and apply." ) if prompt_id == "OOS-07": return common + ( @@ -61,21 +67,67 @@ def good_reply(prompt_id: str, token: str) -> str: "Never transfer or mix one participant's identity into another user's reply.\n" "I cannot claim another identity source without explicit proof." ) + if prompt_id == "OOS-10": + return ( + "Postgres is the canonical database, but an unchanged count receipt does not prove answer behavior is " + "unchanged. " + "Deployed skills, runtime configuration, rendered SOUL.md, session state, and conversation context also " + "affect replies. Hermes state.db and session JSONL provide durable continuity, so a restart does not " + "necessarily erase every prior-session fact. Proof tiers are separate: a temporary-profile handler run, " + "Telegram-visible delivery, and canonical public.* database mutation with a row receipt." + ) + if prompt_id == "OOS-11": + return common + ( + "Use one shared claim with shared sources and evidence; do not duplicate the factual claim per agent. " + "Store agent-specific beliefs, positions, stances, or confidence separately. Preserve each rationale and " + "retain disagreement as a caveat or contradiction edge so it remains queryable." + ) + if prompt_id == "OOS-12": + return common + ( + "Preserve the original 60% probability and history; do not overwrite it. The missing criteria make the " + "outcome ambiguous, so retain that caveat. Current v1 public.claims has no forecast-resolution field and " + "there is no resolves edge. Stage a separate schema proposal, then review and apply it before using a new " + "resolution mechanism." + ) + if prompt_id == "OOS-13": + return ( + "No. The temporary-profile GatewayRunner handler run posted nothing to Telegram, so it is not " + "Telegram-visible delivery proof. It proves handler execution and reply behavior only. Canonical database " + "mutation is a third tier requiring public.* row and applied proposal receipts. The next action is one " + "authorized Telegram-visible prompt with message and reply readback while confirming no DB count change." + ) + if prompt_id == "OOS-14": + return common + ( + "Leo may capture the retained artifact, URL or storage path, and file/content hash; extract candidates, " + "deduplicate them, record contradictions, and stage a pending_review proposal without canonical-apply " + "approval. A temporary chat label or source_ref is not real source provenance. Explicit operator " + "authorization begins at the guarded canonical apply path, after review." + ) + if prompt_id == "OOS-15": + return common + ( + "Insert the replacement claim and a supersedes edge from new claim to old claim. Current public.claim_edges " + "has from_claim, to_claim, edge_type, weight, created_by, and created_at; it has no rationale field. The " + "old claim has a superseded_by column. approve_claim can insert the new claim and supersedes edge, but it " + "does not update the old claim status or superseded_by; those require a separate reviewed apply capability." + ) return common + "Fresh readback is required before the demo claim changes." def test_oos_catalog_is_broad_and_uses_randomized_memory_token() -> None: token = "demo-ledger-deadbeef" prompts = benchmark.prompt_catalog(token) - assert [prompt["id"] for prompt in prompts] == [f"OOS-{index:02d}" for index in range(1, 10)] - assert token in prompts[-3]["message"] - assert token not in prompts[-2]["message"] + assert [prompt["id"] for prompt in prompts] == [f"OOS-{index:02d}" for index in range(1, 16)] + by_id = {prompt["id"]: prompt for prompt in prompts} + assert token in by_id["OOS-07"]["message"] + assert token not in by_id["OOS-08"]["message"] joined = "\n".join(prompt["message"] for prompt in prompts) assert "PDF" in joined assert "tweets" in joined assert "SOUL.md" in joined assert "Do not ask me for row IDs" in joined assert "@m3taversal" in joined + assert "temporary-profile GatewayRunner" in joined + assert "old claim" in joined def test_oos_score_passes_complete_behavior_and_memory_pair() -> None: @@ -86,7 +138,11 @@ def test_oos_score_passes_complete_behavior_and_memory_pair() -> None: ] score = benchmark.score_results(results, memory_token=token) assert score["pass"] is True - assert score["passes"] == 9 + assert score["passes"] == 15 + + +def test_oos_live_check_accepts_live_readback_wording() -> None: + assert benchmark.matched_concept("Here is the live readback from Postgres.", "live_check") is True def test_oos_score_fails_when_memory_token_is_not_recalled() -> None: @@ -95,7 +151,8 @@ def test_oos_score_fails_when_memory_token_is_not_recalled() -> None: {"prompt_id": prompt["id"], "reply": good_reply(prompt["id"], token)} for prompt in benchmark.prompt_catalog(token) ] - results[-2]["reply"] = results[-2]["reply"].replace(token, "some-other-label") + recall = next(result for result in results if result["prompt_id"] == "OOS-08") + recall["reply"] = recall["reply"].replace(token, "some-other-label") score = benchmark.score_results(results, memory_token=token) assert score["pass"] is False assert score["failures"][0]["prompt_id"] == "OOS-08" @@ -104,7 +161,7 @@ def test_oos_score_fails_when_memory_token_is_not_recalled() -> None: def test_oos_identity_case_requires_exact_visible_handle_and_no_alias() -> None: token = "demo-ledger-deadbeef" - prompt = benchmark.prompt_catalog(token)[-1] + prompt = benchmark.prompt_catalog(token)[8] good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token) assert good["pass"] is True @@ -119,7 +176,7 @@ def test_oos_identity_case_requires_exact_visible_handle_and_no_alias() -> None: def test_oos_identity_case_accepts_inflected_carry_language() -> None: token = "demo-ledger-deadbeef" - prompt = benchmark.prompt_catalog(token)[-1] + prompt = benchmark.prompt_catalog(token)[8] reply = ( "Call the current visible participant m3taversal, exactly.\n" "Use only the current Telegram update and visible sender.\n" @@ -175,6 +232,122 @@ def test_oos_schema_guard_rejects_unreviewed_claim_taxonomy_and_apply_surface() assert benchmark.current_schema_overclaims(unsupported_apply) == ["unsupported_approve_claim_surface"] +def test_oos_schema_guard_allows_correct_approve_claim_capability_boundary() -> None: + reply = ( + "Current approve_claim supports neither behavioral_rules nor governance_gates; both need a separate " + "reviewed apply capability." + ) + assert benchmark.current_schema_overclaims(reply) == [] + + +def test_oos_source_evidence_case_accepts_canonical_link_and_weak_provenance_distinction() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[4] + score = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token) + assert score["pass"] is True + assert score["source_evidence_semantic_issues"] == [] + + +def test_oos_source_evidence_case_rejects_locator_gap_called_noncanonical() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[4] + reply = good_reply(prompt["id"], token) + ( + " A public.claim_evidence link to a source row with no URL or storage locator is not canonical evidence." + ) + score = benchmark.score_reply(prompt, reply, memory_token=token) + assert score["pass"] is False + assert score["source_evidence_semantic_issues"] == ["locator_gap_called_noncanonical"] + + +def test_oos_source_evidence_conflation_guard_is_order_independent() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[4] + reply = good_reply(prompt["id"], token) + ( + " A source row with no locator is not canonical evidence even when public.claim_evidence links it." + ) + score = benchmark.score_reply(prompt, reply, memory_token=token) + assert score["pass"] is False + assert score["source_evidence_semantic_issues"] == ["locator_gap_called_noncanonical"] + + +def test_oos_source_evidence_case_rejects_citation_stub_conflation() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[4] + reply = good_reply(prompt["id"], token) + " These source rows are citation stubs, not grounded evidence." + score = benchmark.score_reply(prompt, reply, memory_token=token) + assert score["pass"] is False + assert score["source_evidence_semantic_issues"] == ["citation_stub_called_ungrounded"] + + +def test_oos_database_composition_requires_existing_behavioral_rule_table() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[5] + good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token) + assert good["pass"] is True + + bad_reply = good_reply(prompt["id"], token) + " The public.behavioral_rules table is absent." + bad = benchmark.score_reply(prompt, bad_reply, memory_token=token) + assert bad["pass"] is False + assert bad["behavioral_rule_schema_issues"] == ["behavioral_rules_false_absence"] + + +def test_oos_runtime_case_rejects_db_only_causality_and_total_memory_erasure() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[9] + good = benchmark.score_reply(prompt, good_reply(prompt["id"], token), memory_token=token) + assert good["pass"] is True + + bad_reply = ( + "Postgres is canonical. The unchanged database proves answer behavior is unchanged. " + "A restart erases every prior-session fact." + ) + bad = benchmark.score_reply(prompt, bad_reply, memory_token=token) + assert bad["pass"] is False + assert bad["broad_semantic_issues"] == [ + "restart_called_total_memory_erasure", + "unchanged_db_called_complete_behavior_proof", + ] + + +def test_oos_handler_case_rejects_telegram_live_overclaim() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[12] + reply = good_reply(prompt["id"], token) + ( + " The Telegram path is live and proven even though the run posted nothing to Telegram." + ) + score = benchmark.score_reply(prompt, reply, memory_token=token) + assert score["pass"] is False + assert score["broad_semantic_issues"] == ["handler_proof_called_telegram_live"] + + +def test_oos_forecast_case_rejects_historical_probability_overwrite() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[11] + reply = good_reply(prompt["id"], token) + " Overwrite the original probability with the actual resolved outcome." + score = benchmark.score_reply(prompt, reply, memory_token=token) + assert score["pass"] is False + assert score["broad_semantic_issues"] == ["forecast_history_rewrite"] + + +def test_oos_source_intake_case_rejects_temporary_label_as_source() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[13] + reply = good_reply(prompt["id"], token) + " Create the temporary chat label as a canonical source_ref." + score = benchmark.score_reply(prompt, reply, memory_token=token) + assert score["pass"] is False + assert score["broad_semantic_issues"] == ["temporary_label_promoted_to_source"] + + +def test_oos_score_rejects_architecture_lecture_length() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[0] + reply = good_reply(prompt["id"], token) + " detail" * 301 + score = benchmark.score_reply(prompt, reply, memory_token=token) + assert score["pass"] is False + assert score["response_too_long"] is True + assert score["max_response_words"] == benchmark.DEFAULT_MAX_RESPONSE_WORDS + + def test_oos_score_rejects_blanket_all_five_counts_must_move_claim() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[3]