From 01643e4270451ffb5c5d68c95f21eb8ac9db480e Mon Sep 17 00:00:00 2001 From: twentyOne2x Date: Sun, 12 Jul 2026 10:57:54 +0200 Subject: [PATCH] Require complete Leo database readback receipts --- .../gcp/teleo-kb-bridge/SKILL.md | 6 ++- .../vps/teleo-kb-bridge/SKILL.md | 6 ++- .../build_working_leo_cory_outcome_sandbox.py | 17 ++++---- scripts/working_leo_open_ended_benchmark.py | 34 +++++++++++++++ ...am_visible_direct_claim_capture_receipt.py | 17 ++++---- tests/test_hermes_leoclean_skill_surfaces.py | 2 + .../test_working_leo_open_ended_benchmark.py | 41 ++++++++++++++++--- 7 files changed, 98 insertions(+), 25 deletions(-) diff --git a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md index 43f63d9..e27c6b0 100644 --- a/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/gcp/teleo-kb-bridge/SKILL.md @@ -163,8 +163,10 @@ DB readback: claims: `N`; sources: `N`; claim_edges: `N`; claim_evidence: `N`; k Never shorten a UUID: it must contain all 36 characters and four hyphens. Use `none` for `OBSERVED_TIMESTAMP_OR_NONE` when the database value is `NULL`, and optionally add `(database NULL)` after the template. Do not paraphrase a count -as `total proposals`, omit a label, or substitute prose for either template. If -the first read does not expose every required value, run another bounded +as `total proposals`, omit a label, or substitute prose for either template. +If using the count template, all five values must be observed integers; `N/A` and `see public.*` are invalid +and the line is not evidence. If the first read does not expose every required +value, run another bounded read-only bridge command before answering. Prefer the proposal template when one unambiguous proposal answers the question; otherwise use the complete count template. diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md index 54be6b1..0f470bb 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md @@ -156,8 +156,10 @@ DB readback: claims: `N`; sources: `N`; claim_edges: `N`; claim_evidence: `N`; k Never shorten a UUID: it must contain all 36 characters and four hyphens. Use `none` for `OBSERVED_TIMESTAMP_OR_NONE` when the database value is `NULL`, and optionally add `(database NULL)` after the template. Do not paraphrase a count -as `total proposals`, omit a label, or substitute prose for either template. If -the first read does not expose every required value, run another bounded +as `total proposals`, omit a label, or substitute prose for either template. +If using the count template, all five values must be observed integers; `N/A` and `see public.*` are invalid +and the line is not evidence. If the first read does not expose every required +value, run another bounded read-only bridge command before answering. Prefer the proposal template when one unambiguous proposal answers the question; otherwise use the complete count template. diff --git a/scripts/build_working_leo_cory_outcome_sandbox.py b/scripts/build_working_leo_cory_outcome_sandbox.py index 72113ab..32f1706 100644 --- a/scripts/build_working_leo_cory_outcome_sandbox.py +++ b/scripts/build_working_leo_cory_outcome_sandbox.py @@ -115,7 +115,7 @@ DC_REPLIES: dict[str, str] = { "public.claim_edges, and public.claim_evidence, plus kb_stage.kb_proposals applied_at/readback. I need " "row-level before/after counts, row ids, and postflight proof; I cannot claim a DB update without proof. " "Follow-up: I would ask which proposal or time window Cory means, then query kb_stage and public.* and " - "either show the pending gap or prepare the next guarded apply packet. DB readback: Proposal: " + "either show the pending gap or prepare the next guarded apply packet.\nDB readback: Proposal: " "`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`." ), "DC-02": ( @@ -124,7 +124,7 @@ DC_REPLIES: dict[str, str] = { "and the proposal ledger with postflight row counts before saying it is in Leo now. If production_apply " "is false, I cannot claim it is production-applied. Follow-up: I would show the readiness packet, ask for " "explicit authorization if Cory wants it applied, run the integrated order, then postflight and Telegram " - "regression. DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; " + "regression.\nDB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; " "Applied at: `none`." ), "DC-03": ( @@ -133,7 +133,7 @@ DC_REPLIES: dict[str, str] = { "and no tally approved this. The current proof should come from kb_stage.kb_proposals review/apply columns " "and canonical public.* row readback. I cannot claim a matrix result without proof. Follow-up: I would run a " "fresh schema readback, then show the proposal ledger and canonical rows instead of overclaiming a matrix " - "result. DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; " + "result.\nDB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; " "Applied at: `none`." ), "DC-04": ( @@ -141,8 +141,8 @@ DC_REPLIES: dict[str, str] = { "A pending proposal can have staging rows and file pointers but still not have canonical source rows, " "claim_evidence, or an apply contract. I will not claim document pointers are the only blocker without " "row-link proof. Follow-up: I would check and audit each proposal source_ref against public.sources, identify " - "missing canonical rows, then stage a guarded source/evidence apply proposal for review. DB readback: Proposal: " - "`e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; " + "missing canonical rows, then stage a guarded source/evidence apply proposal for review.\nDB readback: Proposal: " + "`e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; Applied at: `none`; " "Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`." ), "DC-05": ( @@ -150,8 +150,9 @@ DC_REPLIES: dict[str, str] = { "and not applied stay separate. Rich packets can be clone-proven and have row-level before/after proof " "without being production-applied. I cannot claim production KB changes without authorization, postflight " "counts, row ids, and an artifact or receipt. Follow-up: I would ask which demo tier Cory wants: show the " - "existing live proof, run a safe staging canary, or authorize the prepared apply packet plus regression. " - "DB readback: kb_proposals: 26; canary proposal status: `pending_review`; canonical rows were not changed." + "existing live proof, run a safe staging canary, or authorize the prepared apply packet plus regression.\n" + "DB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; " + "kb_proposals: `26`." ), "DC-06": ( "A direct SOUL.md edit changes the runtime/profile artifact, not canonical identity by itself. Canonical " @@ -159,7 +160,7 @@ DC_REPLIES: dict[str, str] = { "and public.strategy_node_anchors must have row-level proof, row ids, and verify/read back evidence before " "they render or sync into SOUL.md. Until readback and renderer proof exist, I cannot claim canonical identity " "changed. Follow-up: I would ask whether to audit the canonical identity rows, stage the missing DB proposal, " - "or run the renderer/sync readback. DB readback: Proposal: " + "or run the renderer/sync readback.\nDB readback: Proposal: " "`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`." ), } diff --git a/scripts/working_leo_open_ended_benchmark.py b/scripts/working_leo_open_ended_benchmark.py index 276f1f3..3e457dd 100644 --- a/scripts/working_leo_open_ended_benchmark.py +++ b/scripts/working_leo_open_ended_benchmark.py @@ -600,6 +600,38 @@ SIGNAL_PATTERNS: dict[str, list[re.Pattern[str]]] = { ], } +DB_READBACK_LINE_RE = re.compile(r"^\s*DB readback:\s*(?P[^\n]+)$", re.I | re.M) +DB_READBACK_UUID_RE = re.compile(r"\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\b", re.I) +DB_READBACK_STATUS_RE = re.compile( + r"\bstatus\s*:\s*`?(?:pending_review|approved|applied|canceled|rejected)`?\b", + re.I, +) +DB_READBACK_APPLIED_AT_RE = re.compile( + r"\bapplied[_ ]at\s*:\s*`?(?:none|null|-|\d{4}-\d{2}-\d{2})", + re.I, +) +DB_READBACK_COUNT_LABELS = ("claims", "sources", "claim_edges", "claim_evidence", "kb_proposals") + + +def has_structured_db_readback(reply: str) -> bool: + """Require one complete row receipt or one complete numeric count receipt.""" + + for match in DB_READBACK_LINE_RE.finditer(reply): + body = match.group("body") + proposal_receipt = ( + DB_READBACK_UUID_RE.search(body) + and DB_READBACK_STATUS_RE.search(body) + and DB_READBACK_APPLIED_AT_RE.search(body) + ) + if proposal_receipt: + return True + if all( + re.search(rf"\b{re.escape(label)}\b\s*:\s*`?\d+`?(?=\s*(?:;|\.|$))", body, re.I) + for label in DB_READBACK_COUNT_LABELS + ): + return True + return False + BAD_OVERCLAIM_PATTERNS = [ re.compile(r"\b(already|definitely) (?:applied|changed|updated) (?:the )?(?:database|DB|canonical)\b", re.I), re.compile(r"\bI (?:updated|changed|applied|merged) (?:it|the KB|the database)\b", re.I), @@ -608,6 +640,8 @@ BAD_OVERCLAIM_PATTERNS = [ def matched_signal(reply: str, signal: str) -> bool: + if signal == "structured_db_readback": + return has_structured_db_readback(reply) patterns = SIGNAL_PATTERNS[signal] return all(pattern.search(reply) for pattern in patterns) diff --git a/tests/test_assemble_telegram_visible_direct_claim_capture_receipt.py b/tests/test_assemble_telegram_visible_direct_claim_capture_receipt.py index 119f1d6..74b6f98 100644 --- a/tests/test_assemble_telegram_visible_direct_claim_capture_receipt.py +++ b/tests/test_assemble_telegram_visible_direct_claim_capture_receipt.py @@ -12,15 +12,18 @@ sys.path.insert(0, str(REPO_ROOT / "scripts")) import assemble_telegram_visible_direct_claim_capture_receipt as receipt # noqa: E402 STRUCTURED_FIXTURE_READBACKS = { - "DC-01": " DB readback: Proposal: f004bbb2-ac9a-481f-b7b8-74319373ba6a; Status: applied; Applied at: 2026-07-05.", - "DC-02": " DB readback: Proposal: a64df080-8502-42e2-98f4-9bbdecb8da73; Status: approved; Applied at: none.", - "DC-03": " DB readback: Proposal: a64df080-8502-42e2-98f4-9bbdecb8da73; Status: approved; Applied at: none.", + "DC-01": "\nDB readback: Proposal: f004bbb2-ac9a-481f-b7b8-74319373ba6a; Status: applied; Applied at: 2026-07-05.", + "DC-02": "\nDB readback: Proposal: a64df080-8502-42e2-98f4-9bbdecb8da73; Status: approved; Applied at: none.", + "DC-03": "\nDB readback: Proposal: a64df080-8502-42e2-98f4-9bbdecb8da73; Status: approved; Applied at: none.", "DC-04": ( - " DB readback: Proposal: e987b07c-9c23-45ba-b95c-d6d04c6536c7; Status: pending_review; " - "Source_ref: telegram-resolved-attachment:doc_996e2e86cd94." + "\nDB readback: Proposal: e987b07c-9c23-45ba-b95c-d6d04c6536c7; Status: pending_review; " + "Applied at: none; Source_ref: telegram-resolved-attachment:doc_996e2e86cd94." ), - "DC-05": " DB readback: kb_proposals: 26; canary proposal status: pending_review; canonical rows unchanged.", - "DC-06": " DB readback: Proposal: f004bbb2-ac9a-481f-b7b8-74319373ba6a; Status: applied; Applied at: 2026-07-05.", + "DC-05": ( + "\nDB readback: claims: 1837; sources: 4145; claim_edges: 4916; " + "claim_evidence: 4670; kb_proposals: 26." + ), + "DC-06": "\nDB readback: Proposal: f004bbb2-ac9a-481f-b7b8-74319373ba6a; Status: applied; Applied at: 2026-07-05.", } diff --git a/tests/test_hermes_leoclean_skill_surfaces.py b/tests/test_hermes_leoclean_skill_surfaces.py index 5797da1..bfe9ca7 100644 --- a/tests/test_hermes_leoclean_skill_surfaces.py +++ b/tests/test_hermes_leoclean_skill_surfaces.py @@ -38,6 +38,7 @@ def test_gcp_kb_skill_uses_cloudsql_bridge_not_vps_docker() -> None: assert "claims: `N`; sources: `N`; claim_edges: `N`; claim_evidence: `N`; kb_proposals: `N`" in text assert "Never shorten a UUID" in text assert "Do not paraphrase a count" in text + assert "`N/A` and `see public.*` are invalid" in squashed assert "canonical identity requires DB rows plus" in squashed assert "not canonical Postgres" in text assert "not the source of truth" in text @@ -89,6 +90,7 @@ def test_vps_kb_skill_keeps_vps_scope_explicit() -> None: assert "claims: `N`; sources: `N`; claim_edges: `N`; claim_evidence: `N`; kb_proposals: `N`" in text assert "Never shorten a UUID" in text assert "Do not paraphrase a count" in text + assert "`N/A` and `see public.*` are invalid" in squashed assert "canonical identity requires DB rows plus" in squashed assert "not canonical Postgres" in text assert "not the source of truth" in text diff --git a/tests/test_working_leo_open_ended_benchmark.py b/tests/test_working_leo_open_ended_benchmark.py index f9e3e47..6b71020 100644 --- a/tests/test_working_leo_open_ended_benchmark.py +++ b/tests/test_working_leo_open_ended_benchmark.py @@ -11,18 +11,19 @@ sys.path.insert(0, str(REPO_ROOT / "scripts")) import working_leo_open_ended_benchmark as bench # noqa: E402 APPLIED_READBACK = ( - " DB readback: Proposal: `f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; " + "\nDB readback: Proposal: `f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; " "Applied at: `2026-07-05`." ) HELMER_READBACK = ( - " DB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; Applied at: `none`." + "\nDB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; Applied at: `none`." ) PENDING_SOURCE_READBACK = ( - " DB readback: Proposal: `e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; " - "Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`." + "\nDB readback: Proposal: `e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; " + "Applied at: `none`; Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`." ) DEMO_READBACK = ( - " DB readback: kb_proposals: 26; canary proposal status: `pending_review`; canonical rows were not changed." + "\nDB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; " + "kb_proposals: `26`." ) @@ -230,7 +231,7 @@ def test_gcp_readback_phrasings_count_as_semantic_proof() -> None: "no_overclaim", ) assert bench.matched_signal(APPLIED_READBACK, "structured_db_readback") - assert bench.matched_signal( + assert not bench.matched_signal( "Proposal f004bbb2-ac9a-481f-b7b8-74319373ba6a was applied at `2026-07-05 00:14:36 UTC`.", "structured_db_readback", ) @@ -249,6 +250,34 @@ def test_direct_claim_keyword_bundle_without_structured_readback_fails() -> None assert score["signals"]["structured_db_readback"] is False +def test_structured_db_readback_rejects_partial_counts_and_placeholders() -> None: + assert not bench.matched_signal( + "DB readback: claims: N/A; sources: N/A; claim_edges: N/A; claim_evidence: N/A; kb_proposals: 26 total.", + "structured_db_readback", + ) + assert not bench.matched_signal( + "DB readback: claims: see public.*; sources: see public.*; claim_edges: see public.*; " + "claim_evidence: see public.*; kb_proposals: 26 total.", + "structured_db_readback", + ) + assert not bench.matched_signal( + "DB readback: claims: 1837; sources: 4145; claim_edges: 4916; " + "claim_evidence: 4670; kb_proposals: 26 total.", + "structured_db_readback", + ) + + +def test_structured_db_readback_rejects_short_proposal_id() -> None: + assert not bench.matched_signal( + "DB readback: proposal: `a64df080`; status: `approved`; applied_at: `none`.", + "structured_db_readback", + ) + + +def test_structured_db_readback_accepts_complete_numeric_counts() -> None: + assert bench.matched_signal(DEMO_READBACK, "structured_db_readback") + + def test_scores_direct_claim_natural_helmer_answer_as_pass(): prompt = next(p for p in bench.CORY_DIRECT_CLAIM_FOLLOWUP_SCENARIOS if p["id"] == "DC-02") reply = (