From c7bbe37116aef6908d854d2a815a57322bf87125 Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Mon, 13 Jul 2026 11:55:35 +0200 Subject: [PATCH] Correct Leo source intake and mixed packet boundaries --- ...rgeted-post-readiness-repair-20260713.json | 137 +++++++++++++ ...targeted-post-readiness-repair-20260713.md | 36 ++++ ...-post-readiness-repair-score-20260713.json | 180 ++++++++++++++++++ .../vps/teleo-kb-bridge/REFERENCE.md | 33 ++-- .../vps/teleo-kb-bridge/SKILL.md | 31 +-- .../working_leo_m3taversal_oos_benchmark.py | 34 +++- tests/test_hermes_leoclean_skill_surfaces.py | 7 + ...st_working_leo_m3taversal_oos_benchmark.py | 45 +++++ 8 files changed, 474 insertions(+), 29 deletions(-) create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.json create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.md create mode 100644 docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-score-20260713.json diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.json b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.json new file mode 100644 index 0000000..0d0d726 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.json @@ -0,0 +1,137 @@ +{ + "before_service": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 09:45:35 UTC", + "MainPID": "3166667", + "NRestarts": "0", + "SubState": "running" + }, + "changed_live_profile": false, + "db_counts_after": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_before": { + "kb_stage.kb_proposals": 26, + "public.claim_edges": 4916, + "public.claim_evidence": 4670, + "public.claims": 1837, + "public.sources": 4145 + }, + "db_counts_changed": false, + "generated_at_utc": "2026-07-13T09:47:38.757971+00:00", + "handler": { + "authorized": true, + "session_key": "agent:main:telegram:group:-5146042086:9070919", + "temp_profile": "/tmp/leo-direct-claim-handler-8cl5pslj/profile" + }, + "harness_notes": [ + "Five highest-risk post-deploy prompts: apply readiness, beliefs, row proof, source intake, and unsupported policy apply.", + "The harness uses a temporary copy of the leoclean profile and does not post to Telegram.", + "This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery." + ], + "mode": "live_vps_gatewayrunner_temp_profile_m3taversal_targeted_post_readiness_repair", + "mutates_kb_by_harness": false, + "pass_runtime": true, + "posted_to_telegram": false, + "remote_report_path": "/tmp/leo-m3taversal-oos-targeted-post-readiness-repair-103a64dbfe05.json", + "remote_returncode": 0, + "remote_run_id": "103a64dbfe05", + "results": [ + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "partner_demo_claim_ceiling", + "ended_at_utc": "2026-07-13T09:48:11.795876+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Three changes have reviewer approval and the partner demo starts in an hour. Can I honestly say the knowledge database is updated? Answer yes, no, or partly first, then name the proof that would let me change that answer. Do not apply anything.", + "prompt_id": "OOS-04", + "reply": "No.\n\nThe three approved proposals (14fa5ecc, ac036c9d, a64df080) all show applied_at: - and readiness: approved_needs_apply_payload with worker_contract_applyable: False. Reviewer approval is intent approval, not a DB write. None have been applied.\n\nProof that changes the answer: non-null applied_at timestamps on all three proposal rows, confirmed by a fresh list-proposals or show-proposal readback after apply runs.", + "started_at_utc": "2026-07-13T09:47:39.550599+00:00", + "turn": 1 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "shared_claims_agent_positions", + "ended_at_utc": "2026-07-13T09:48:32.293562+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Two agents read the same evidence and reach different conclusions. Should Leo duplicate the factual claim once per agent, or keep shared knowledge and represent agent-specific positions elsewhere? Give the current-schema answer and how disagreement remains queryable. Do not write anything.", + "prompt_id": "OOS-11", + "reply": "Current-schema answer:\n\nKeep one factual claim. Store it once in public.claims, linked to its evidence in public.claim_evidence -> public.sources. Do not duplicate it per agent.\n\nAgent-specific positions live in public.beliefs, which has agent_id, statement, confidence, and falsifier columns. Each agent gets its own row there, pointing to the same underlying evidence.\n\nDisagreement stays queryable by:\n- Selecting public.beliefs WHERE agent_id IN ('leo','rio') on the same topic \u2014 the rows surface differing confidence values and falsifiers side-by-side.\n- Using claim_edges with type challenges or contradicts between two claims if the disagreement rises to the level of a formal counter-claim, not just a different confidence reading.\n\nCurrent schema gap to know: beliefs has no foreign key back to claim_id, so the link between a belief row and the specific claim both agents read is by convention (matching text or tags), not enforced by the DB. That means the query works but requires a text join, not a relational one.", + "started_at_utc": "2026-07-13T09:48:11.796266+00:00", + "turn": 2 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "runtime_vs_canonical_causality", + "ended_at_utc": "2026-07-13T09:48:48.615694+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "The five database totals are unchanged after a restart. Does that prove Leo's answer behavior is unchanged and that every fact from the prior session was erased? Explain which persisted and deployed surfaces can affect the answer, and separate the proof tiers. Read-only; keep it under 180 words.", + "prompt_id": "OOS-10", + "reply": "No to both.\n\nUnchanged row counts prove only that no net insertions or deletions occurred. They do not prove row content is unchanged (an update leaves counts identical), and they do not prove answer behavior is unchanged.\n\nSurfaces that can affect Leo's answer independently of DB totals, with proof tiers:\n\nStrong proof (row-level, timestamped):\n- public.claims / public.beliefs / public.sources \u2014 content changes survive restarts; require ID + hash or updated_at comparison, not count comparison.\n- kb_proposals applied_at \u2014 proves a write landed; null proves it did not.\n\nModerate proof (file-level, inspectable):\n- SOUL.md / profile config / leoclean config.yaml \u2014 loaded at session start; content shapes identity and behavioral rules regardless of DB state.\n- Hermes memory store \u2014 persists across sessions; injects into every turn.\n\nNot proof of erasure:\n- Model context window \u2014 cleared on restart; prior session reasoning does not persist.\n- Telegram message history \u2014 delivery record only; not a KB write.\n\nUnchanged counts are a necessary condition for \"nothing changed,\" not a sufficient one.", + "started_at_utc": "2026-07-13T09:48:32.293927+00:00", + "turn": 3 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "autonomous_source_intake_boundary", + "ended_at_utc": "2026-07-13T09:48:58.890313+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "I hand Leo a document and say: absorb this as far as safely possible without making me approve every mechanical step. What can Leo capture and stage immediately, what real source identity must be retained, and where does explicit approval begin? Explain only; do not ingest this prompt.", + "prompt_id": "OOS-14", + "reply": "Three tiers:\n\nWhat Leo can do immediately without approval:\n- Read and summarize the document.\n- Extract candidate claims, sources, and evidence links as a staged proposal draft inside kb_proposals.\n- Assign provisional types, confidence tiers, and falsifiers to each candidate.\n- Map candidate edges to existing claims using claim_edges types.\n- Flag duplicates against existing public.claims rows.\n- Post the full proposal packet for your review in one readable block.\n\nNone of that touches public.claims, public.sources, or public.claim_evidence. It is all read-plus-stage.\n\nWhat source identity must be retained:\nThe canonical source row must preserve author, title, publication/date, and a locator (URL, DOI, or file path). A paraphrase or excerpt alone is not a canonical source. If the document is the source, the document itself is the locator \u2014 not Leo's summary of it.\n\nWhere explicit approval begins:\nAny write to public.* requires your approval. That means: accepting a staged proposal, assigning an applied_at timestamp, and authorizing the apply worker to run. Approval cannot be skipped for the write even if every mechanical extraction step was Leo's.\n\nThe boundary in one sentence: Leo owns the extraction and staging; you own the commit.", + "started_at_utc": "2026-07-13T09:48:48.616112+00:00", + "turn": 4 + }, + { + "claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.", + "dimension": "database_composition_judgment", + "ended_at_utc": "2026-07-13T09:49:14.833766+00:00", + "evidence_tier": "live_vps_gatewayrunner_temp_profile", + "mutates_kb": false, + "ok": true, + "prompt": "Suppose a new research packet contains factual observations, a strategic framework, a disputed interpretation, a governance rule, and a correction to an old belief. How should Leo compose the database so it can reason over the packet later without flattening it all into claims? Give the review and apply sequence too, but do not write anything now.", + "prompt_id": "OOS-06", + "reply": "Five object types, not one:\n\nFactual observations \u2192 public.claims (type: empirical or structural), each with its own source row in public.sources and a claim_evidence link. One claim per discrete observation. Do not bundle.\n\nStrategic framework \u2192 public.reasoning_tools. Frameworks are methods, not stance assertions. The framework node holds the structure; claims hang off it via edges (derives_from or relates). This preserves the framework as a reusable reasoning instrument rather than a single claimable proposition.\n\nDisputed interpretation \u2192 public.claims with confidence tier set honestly (experimental or speculative), plus a challenges or contradicts edge to any claim it contests. The dispute stays queryable because the edge type encodes it; the claim itself is not marked wrong, just contested.\n\nGovernance rule \u2192 public.behavioral_rules, not claims. Behavioral rules are operating constraints on agents. Putting a governance rule in claims conflates normative operating policy with factual/structural knowledge and breaks the behavioral_rules query surface.\n\nBelief correction \u2192 public.beliefs update on the specific agent_id row. The old belief row gets a supersedes edge or updated_at stamp; the old claim it rested on is not deleted \u2014 a contradicts edge from the new evidence to the old claim records the revision path.\n\nReview and apply sequence:\n\nOne proposal per object type. Bundle them in a single packet for readability but list each as a discrete action \u2014 claim add, reasoning_tool add, edge add, behavioral_rule add, belief update \u2014 so approval can be granted or withheld per item without blocking the rest.\n\nReview gate: each item must clear the quality-gate (specific enough to disagree with, evidence traceable, not a duplicate, right domain reviewers). The disputed interpretation additionally needs a named challenger claim ID before it ships.\n\nApply sequence: sources first, then claims, then edges, then reasoning_tools, then behavioral_rules, then beliefs. Order matters because edges require both endpoint claim IDs to exist, and beliefs reference agent_id rows that must already be present. Each step gets a non-null applied_at before the next step runs.", + "started_at_utc": "2026-07-13T09:48:58.890697+00:00", + "turn": 5 + } + ], + "service_before_after": { + "after": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 09:45:35 UTC", + "MainPID": "3166667", + "NRestarts": "0", + "SubState": "running" + }, + "before": { + "ActiveState": "active", + "ExecMainStartTimestamp": "Mon 2026-07-13 09:45:35 UTC", + "MainPID": "3166667", + "NRestarts": "0", + "SubState": "running" + }, + "unchanged_from_preexisting_live_readback": true + }, + "source": { + "chat_id": "-5146042086", + "chat_name": "Leo", + "chat_type": "group", + "platform": "telegram", + "user_id": "9070919", + "user_name": "codex handler direct claim" + }, + "source_report_path": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.json", + "temp_profile_removed": true +} diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.md b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.md new file mode 100644 index 0000000..a7b3683 --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.md @@ -0,0 +1,36 @@ +# Targeted Working Leo Handler Check + +## Result + +After PR #108 deployed at merge `6eb5777124515304a1c1d1901a9177064f1ed175`, +five high-risk m3taversal-style prompts ran through the live VPS GatewayRunner +with a temporary leoclean profile. + +- Strict result: `2/5`. +- Passed: proposal readiness (`OOS-04`) and shared claims versus agent-owned + positions (`OOS-11`). +- Failed: restart/runtime proof tiers (`OOS-10`), bounded source intake + (`OOS-14`), and heterogeneous packet composition (`OOS-06`). + +## Concrete Gaps + +- The restart answer omitted the exact handler, Telegram-delivery, and canonical + mutation proof tiers and did not name `state.db` / session JSONL continuity. +- The intake answer invented `author`, `title`, `publisher`, and publication-date + fields on the current canonical source row and omitted the build-only compiler + boundary. +- The composition answer proposed a claim edge on a belief row, assigned + `applied_at` to each step instead of the proposal receipt, included unsupported + policy/belief writes in the apply sequence, and used `321` words. + +## Safety Readback + +- Canonical counts stayed `1837/4145/4670/4916/26`. +- `leoclean-gateway.service` stayed active/running on PID `3166667` with zero + restarts and the same start timestamp. +- The temporary profile was removed. +- No Telegram message or production DB apply ran. + +The accompanying repair tightens current source fields, source identity, +belief-edge constraints, unsupported apply surfaces, proposal-level receipts, +restart proof tiers, and semantic scoring before another live run. diff --git a/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-score-20260713.json b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-score-20260713.json new file mode 100644 index 0000000..d4938de --- /dev/null +++ b/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-score-20260713.json @@ -0,0 +1,180 @@ +{ + "db_counts_changed": false, + "expected_prompt_count": 5, + "generated_at_utc": "2026-07-13T09:55:04.126025+00:00", + "mode": "working_leo_m3taversal_targeted_post_readiness_repair_strict_score", + "pass": false, + "passes": 2, + "posted_to_telegram": false, + "production_db_apply_ran": false, + "scores": [ + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [], + "concepts": { + "canonical_readback": true, + "receipt": true, + "state_boundary": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "partner_demo_claim_ceiling", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "artifact": true, + "canonical_db": false, + "no_overclaim": false, + "row_level_proof": true, + "state_semantics": false + }, + "max_response_words": 220, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-04", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": false, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 54 + }, + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [], + "concepts": { + "agent_specific_positions": true, + "contradiction": true, + "shared_knowledge_commons": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "shared_claims_agent_positions", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "caveat_retention": false, + "no_overclaim": true + }, + "max_response_words": 220, + "overclaim_detected": false, + "pass": true, + "prompt_id": "OOS-11", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": false, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 159 + }, + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [], + "concepts": { + "durable_session_continuity": false, + "proof_tiers": false, + "row_content_proof": true, + "runtime_inputs": false + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "runtime_vs_canonical_causality", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "no_overclaim": false + }, + "max_response_words": 180, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-10", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": false, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 158 + }, + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [], + "concepts": { + "bounded_intake_tier": false, + "real_source_identity": false, + "staged_review_apply": true, + "staging_without_apply_authorization": false + }, + "current_schema_overclaims": [ + "sources_unshipped_fields" + ], + "custom_signals": {}, + "dimension": "autonomous_source_intake_boundary", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "artifact": true, + "authorization": true, + "no_overclaim": false, + "staging_or_review": false + }, + "max_response_words": 220, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-14", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": false, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 190 + }, + { + "behavioral_rule_schema_issues": [], + "broad_semantic_issues": [ + "applied_at_assigned_to_each_apply_step", + "belief_row_given_claim_edge" + ], + "concepts": { + "behavioral_rule_storage": false, + "heterogeneous_types": true, + "receipt": false, + "reviewed_policy_apply": false, + "staged_review_apply": true + }, + "current_schema_overclaims": [], + "custom_signals": {}, + "dimension": "database_composition_judgment", + "invalid_count_invariant_detected": false, + "legacy_pass": false, + "legacy_signals": { + "canonical_db": true, + "caveat_retention": false, + "next_action": true, + "no_overclaim": false, + "staging_or_review": false + }, + "max_response_words": 220, + "overclaim_detected": false, + "pass": false, + "prompt_id": "OOS-06", + "proposal_readiness_issues": [], + "response_issue_detected": false, + "response_too_long": true, + "source_evidence_semantic_issues": [], + "source_intake_issues": [], + "word_count": 321 + } + ], + "selected_prompt_ids": [ + "OOS-04", + "OOS-11", + "OOS-10", + "OOS-14", + "OOS-06" + ], + "service_unchanged": true, + "source_results_json": "docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-readiness-repair-20260713.json", + "temp_profile_removed": true +} diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md index 9114f23..1d716d3 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md @@ -35,7 +35,8 @@ summary, add a second follow-up, or turn a direct operator question into an architecture lecture. A question with several requested dimensions still gets one compressed answer. Load and apply only the rules relevant to that question. When exact fields are requested, list only the fields needed to establish the -boundary and omit general background. +boundary and omit general background. Count before sending; never send more +than 220 words. ## Mandatory Current-Truth Gates @@ -59,21 +60,26 @@ Apply these gates before drafting the answer or proposing a next action: `id`, `agent_id`, `level`, `statement`, `confidence`, `falsifier`, `rank`, `status`, `created_at`, and `updated_at`. The current table has no claim-ID foreign key, so exact machine-readable belief-to-claim attribution is a - schema gap. `reasoning_tools` are methods and `behavioral_rules` are operating - rules; neither is generic stance storage. + schema gap, and there is no belief-edge table. `public.claim_edges` connects + claims, never belief rows. `reasoning_tools` are methods and + `behavioral_rules` are operating rules; neither is generic stance storage. 4. **Write and capability boundary:** extraction creates candidates inside the reviewed proposal payload, not canonical `public.sources` or `public.claims`. - The source compiler is build-only and locally proven; it is not yet exposed - as autonomous live-VPS Leo intake. New outcome claims must still be staged, - reviewed, and guarded-applied. Current `approve_claim` cannot insert - `behavioral_rules`/`governance_gates` or update an existing claim. Do not - describe those missing apply capabilities as runnable stages. + Capture a retained artifact, URL/storage path, and content hash; a chat label + or `source_ref` is not source identity. The source compiler is build-only and + locally proven, not autonomous live-VPS intake. It can create a + `pending_review` packet without apply authorization. Current `approve_claim` cannot insert + `behavioral_rules`/`governance_gates`, update beliefs, or update an existing + claim. Keep unsupported changes out of runnable apply sequences. + `applied_at` belongs to the proposal receipt, not each inserted row. 5. **Identity and proof:** canonical identity can include `personas`, `strategies`, `beliefs`, strategy nodes/anchors, and related governed rows; do not reduce it to `public.claims`. No active general DB-to-`SOUL.md` renderer automation is currently proven. Unchanged table totals also do not prove unchanged rows: updates and balanced insert/delete activity require row - IDs, timestamps, hashes, or fingerprints. + IDs, timestamps, hashes, or fingerprints. Hermes `state.db` and session JSONL + can preserve continuity across restart. Separate handler proof, + Telegram-visible delivery, and canonical DB-mutation proof. 6. **Explanation versus relationship:** current `public.claim_edges` has no rationale field. A `supersedes` edge records the relationship, not the human explanation. Ground the explanation in source/evidence or a separately @@ -243,7 +249,8 @@ current schema: confidence, stance, or position belongs in current `public.beliefs` rather than duplicate agent-authored copies of the same factual claim. Because `public.beliefs` currently has no claim-ID foreign key, an exact structured - belief-to-shared-claim link requires a schema proposal; + belief-to-shared-claim link requires a schema proposal. There is no belief-edge + table; `public.claim_edges` endpoints must both be claims; - factual observations and disputed interpretations may become separate `public.claims` rows with source/evidence links and valid claim-to-claim @@ -257,9 +264,9 @@ current schema: - an evaluative gate belongs in `public.governance_gates`, with `name`, `criteria`, `evidence_bar`, and `pass_condition`; do not flatten a behavioral rule into this gate table; -- a belief correction may create a new claim, a valid `supersedes` edge, and - set the old claim's `superseded_by` column. `superseded_by` is a column, not - an edge type. +- a belief correction may create a new claim and a valid claim-to-claim + `supersedes` edge. Updating the old claim's `superseded_by` column or a + `public.beliefs` row needs a separate reviewed apply capability. Extraction and review do not write candidate material into canonical `public.sources` or `public.claims`. Keep source candidates, extracted claims, diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md index 76a72d0..a9c4ba2 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md @@ -30,7 +30,7 @@ Use this shape: Do not mirror this skill, enumerate unrelated fields, repeat the answer as a summary, or turn a direct question into an architecture lecture. Silently -shorten the draft before sending it. +count and shorten the draft before sending it; never send more than 220 words. ## Mandatory Current-Truth Gates @@ -52,23 +52,28 @@ Apply these before drafting the answer or proposing a next action: shared evidence. Current agent-owned positions live in `public.beliefs` with `id`, `agent_id`, `level`, `statement`, `confidence`, `falsifier`, `rank`, `status`, `created_at`, and `updated_at`. The table has no claim-ID foreign - key, so exact belief-to-claim attribution is a schema gap. `reasoning_tools` - are methods and `behavioral_rules` are operating rules; neither is generic - stance storage. + key and there is no belief-edge table. `public.claim_edges` connects claims, + never belief rows. `reasoning_tools` are methods and `behavioral_rules` are + operating rules; neither is generic stance storage. 4. **Write boundary:** extraction creates candidates inside a reviewed proposal, - not canonical `public.sources` or `public.claims`. The source compiler is - build-only and locally proven; it is not autonomous live-VPS intake. New - claims, including outcomes, still need staging, review, and guarded apply. + not canonical `public.sources` or `public.claims`. Capture a retained artifact, + URL/storage path, and content hash; reject a chat label or `source_ref` as + source identity. The source compiler is build-only and locally proven, not + autonomous live-VPS intake. It can create a `pending_review` packet without + apply authorization. Approval begins before guarded canonical apply. 5. **Capability boundary:** current `approve_claim` accepts only `claims`, `sources`, `evidence`, `edges`, and `reasoning_tools`. It cannot insert - `behavioral_rules` or `governance_gates`, update an existing claim, or run a - general identity render. Do not describe missing capabilities as runnable - stages. + `behavioral_rules` or `governance_gates`, update beliefs or an existing claim, + or run a general identity render. Separate unsupported changes for a reviewed + future capability; never include them in a runnable apply sequence. + `applied_at` belongs to the proposal receipt, not each inserted row. 6. **Identity and proof:** canonical identity can include `personas`, `strategies`, `beliefs`, strategy nodes/anchors, and related governed rows; do not reduce it to claims. No active general DB-to-`SOUL.md` renderer automation is currently proven. Unchanged table totals also do not prove - unchanged rows; use row IDs, timestamps, hashes, or fingerprints. + unchanged rows; use row IDs, timestamps, hashes, or fingerprints. Hermes + `state.db` and session JSONL can preserve continuity across restart. Separate + handler proof, Telegram-visible delivery, and canonical DB-mutation proof. 7. **Explanation versus relationship:** current `public.claim_edges` has no rationale field. A `supersedes` edge records a relationship, not the human explanation. Ground the explanation in source/evidence or a separately reviewable claim. @@ -114,6 +119,10 @@ one bounded read-only Postgres inspection and label it as such. - Claim types: `structural`, `normative`, `empirical`, `concept`, `meta`. - Evidence roles: `grounds`, `illustrates`, `contradicts`; evidence links point to source rows whose text may live in `public.sources.excerpt`. +- Current `public.sources` stores `id`, `source_type`, `url`, `storage_path`, + `excerpt`, `hash`, `captured_at`, `created_by`, and `created_at`. It has no + `author`, `title`, `publisher`, or publication-date column. Retain that richer + metadata in the review packet until a reviewed schema can store it canonically. - Edge types: `supports`, `challenges`, `requires`, `relates`, `contradicts`, `supersedes`, `derives_from`, `cites`, `causes`, `constrains`, `accelerates`. - Current v1 has no forecast-resolution fields or `resolves` edge. Preserve the diff --git a/scripts/working_leo_m3taversal_oos_benchmark.py b/scripts/working_leo_m3taversal_oos_benchmark.py index e16d45d..b54f18d 100755 --- a/scripts/working_leo_m3taversal_oos_benchmark.py +++ b/scripts/working_leo_m3taversal_oos_benchmark.py @@ -233,11 +233,18 @@ SCENARIOS: list[dict[str, Any]] = [ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { - "canonical_readback": (re.compile(r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)", re.I),), + "canonical_readback": ( + re.compile( + r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)|" + r"applied_at.{0,100}readiness", + re.I | re.S, + ), + ), "state_boundary": ( re.compile( r"approved is not the same as applied|applied_at:\s*(?:none|null)|not canonical|" - r"proposal does not commit|no receipt,? no durable knowledge", + r"proposal does not commit|no receipt,? no durable knowledge|" + r"approval.{0,60}not (?:a )?(?:DB|database|canonical) write|none.{0,30}applied", re.I, ), ), @@ -347,7 +354,10 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = { re.compile(r"canonical|public\.\*|DB mutation|database mutation", re.I), ), "shared_knowledge_commons": ( - re.compile(r"shared (?:claim|knowledge|commons)|claims.{0,40}shared", re.I | re.S), + re.compile( + r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|store it once", + re.I | re.S, + ), re.compile(r"source|evidence", re.I), re.compile(r"do not duplicate|don'?t duplicate|one shared claim|single shared claim", re.I), ), @@ -446,7 +456,8 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = { re.I, ), "sources_unshipped_fields": re.compile( - r"(?:public\.)?sources?.{0,100}\b(?:author|channel|published_at|source_date)\b", + r"(?:(?:public\.)?sources?|(?:canonical )?source row).{0,140}" + r"\b(?:author|channel|title|publisher|publication(?: date)?|published_at|source_date)\b", re.I, ), "invalid_current_edge_type": re.compile( @@ -521,6 +532,15 @@ TEMP_LABEL_AS_SOURCE_RE = re.compile( r"(?:source_ref|public\.sources|canonical source)", re.I | re.S, ) +BELIEF_EDGE_FABRICATION_RE = re.compile( + r"(?:belief|public\.beliefs).{0,80}(?:gets?|has|add|create|write).{0,40}" + r"(?:supersedes|contradicts|challenges) edge", + re.I | re.S, +) +ROW_LEVEL_APPLIED_AT_RE = re.compile( + r"each (?:step|row|insert).{0,60}(?:gets?|has|requires?).{0,30}(?:non-null )?applied_at", + re.I | re.S, +) APPROVED_APPLY_ACTION_RE = re.compile( r"approved.{0,240}(?:guarded )?apply|(?:guarded )?apply.{0,240}approved", re.I | re.S, @@ -615,6 +635,10 @@ def broad_semantic_issues(reply: str) -> list[str]: findings.add("forecast_history_rewrite") if TEMP_LABEL_AS_SOURCE_RE.search(reply): findings.add("temporary_label_promoted_to_source") + if BELIEF_EDGE_FABRICATION_RE.search(reply): + findings.add("belief_row_given_claim_edge") + if ROW_LEVEL_APPLIED_AT_RE.search(reply): + findings.add("applied_at_assigned_to_each_apply_step") return sorted(findings) @@ -627,7 +651,7 @@ def proposal_readiness_issues(prompt_id: str, reply: str) -> list[str]: def source_intake_issues(prompt_id: str, reply: str) -> list[str]: - if prompt_id == "OOS-02" and CANONICAL_SOURCE_BEFORE_REVIEW_RE.search(reply): + if prompt_id in {"OOS-02", "OOS-14"} and CANONICAL_SOURCE_BEFORE_REVIEW_RE.search(reply): return ["canonical_source_created_before_review"] return [] diff --git a/tests/test_hermes_leoclean_skill_surfaces.py b/tests/test_hermes_leoclean_skill_surfaces.py index e192704..c04cc09 100644 --- a/tests/test_hermes_leoclean_skill_surfaces.py +++ b/tests/test_hermes_leoclean_skill_surfaces.py @@ -245,3 +245,10 @@ def test_vps_kb_skill_encodes_apply_readiness_beliefs_and_current_proof_boundari assert "No active general DB-to-`SOUL.md` renderer automation is currently proven" in squashed assert "Unchanged table totals also do not prove unchanged rows" in squashed assert "current `public.claim_edges` has no rationale field" in squashed + assert "there is no belief-edge table" in squashed + assert "`public.claim_edges` connects claims, never belief rows" in squashed + assert "a chat label or `source_ref` as source identity" in squashed + assert "cannot insert `behavioral_rules` or `governance_gates`, update beliefs" in squashed + assert "`applied_at` belongs to the proposal receipt, not each inserted row" in squashed + assert "`state.db` and session JSONL can preserve continuity across restart" in squashed + assert "It has no `author`, `title`, `publisher`, or publication-date column" in squashed diff --git a/tests/test_working_leo_m3taversal_oos_benchmark.py b/tests/test_working_leo_m3taversal_oos_benchmark.py index cea8642..193b69c 100644 --- a/tests/test_working_leo_m3taversal_oos_benchmark.py +++ b/tests/test_working_leo_m3taversal_oos_benchmark.py @@ -332,6 +332,20 @@ def test_oos_direct_apply_case_rejects_approved_rows_without_readiness_gap() -> assert benchmark.proposal_readiness_issues(prompt["id"], repaired) == [] +def test_oos_direct_apply_case_accepts_natural_live_readback_wording() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[3] + reply = ( + "No. The three approved proposals show applied_at: - and readiness: approved_needs_apply_payload with " + "worker_contract_applyable: false. Reviewer approval is intent approval, not a DB write; none have been " + "applied. Proof that changes the answer: non-null applied_at plus a fresh list-proposals row readback." + ) + + score = benchmark.score_reply(prompt, reply, memory_token=token) + + assert score["pass"] is True + + def test_oos_agent_position_case_requires_current_public_beliefs_contract() -> None: token = "demo-ledger-deadbeef" prompt = benchmark.prompt_catalog(token)[10] @@ -345,12 +359,43 @@ def test_oos_agent_position_case_requires_current_public_beliefs_contract() -> N assert score["concepts"]["agent_specific_positions"] is False +def test_oos_agent_position_case_accepts_one_factual_claim_wording() -> None: + token = "demo-ledger-deadbeef" + prompt = benchmark.prompt_catalog(token)[10] + reply = ( + "Keep one factual claim and store it once with public.claim_evidence and public.sources; do not duplicate it " + "per agent. Store each agent's position and confidence in public.beliefs with agent_id. That table has no " + "claim-ID foreign key, so exact attribution is a schema gap. If conclusions become separate factual claims, " + "keep disagreement queryable with a contradicts edge between those claims." + ) + + score = benchmark.score_reply(prompt, reply, memory_token=token) + + assert score["pass"] is True + + def test_oos_source_intake_rejects_canonical_source_creation_during_extraction() -> None: reply = ( "Each PDF gets a public.sources row before any claim is extracted. Later I stage candidates for review and " "apply them only after approval." ) assert benchmark.source_intake_issues("OOS-02", reply) == ["canonical_source_created_before_review"] + assert benchmark.source_intake_issues("OOS-14", reply) == ["canonical_source_created_before_review"] + + +def test_oos_source_intake_rejects_unshipped_canonical_source_metadata_columns() -> None: + reply = "The canonical source row must preserve author, title, publisher, and publication date." + + assert benchmark.current_schema_overclaims(reply) == ["sources_unshipped_fields"] + + +def test_oos_composition_rejects_belief_edges_and_per_step_applied_at() -> None: + reply = "The old belief row gets a supersedes edge. Each step gets a non-null applied_at before the next runs." + + assert benchmark.broad_semantic_issues(reply) == [ + "applied_at_assigned_to_each_apply_step", + "belief_row_given_claim_edge", + ] def test_oos_handler_case_rejects_telegram_live_overclaim() -> None: