1499 lines
70 KiB
Python
Executable file
1499 lines
70 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Out-of-sample m3taversal-style benchmark for the live Leo handler.
|
|
|
|
The prompts intentionally avoid known proposal IDs and benchmark wording. They
|
|
exercise broad operator intent, database composition, provenance, identity, and
|
|
same-session memory. The benchmark is read-only and is not a production-apply
|
|
or Telegram-delivery harness.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import working_leo_open_ended_benchmark as base
|
|
|
|
SCENARIOS: list[dict[str, Any]] = [
|
|
{
|
|
"id": "OOS-01",
|
|
"dimension": "demo_pressure_without_ids",
|
|
"message": (
|
|
"m3taversal walks in now and says, 'Leo's database has not learned anything useful since yesterday and I "
|
|
"cannot demo this.' Do not ask me for row IDs. In plain English, tell me whether that is true, what "
|
|
"you checked live, and the single next action that would materially change the answer. Keep this "
|
|
"read-only: do not stage, approve, apply, or edit anything."
|
|
),
|
|
"required_signals": ["state_semantics", "canonical_db", "fresh_readback", "next_action", "no_overclaim"],
|
|
"required_concepts": ["canonical_readback", "state_boundary", "live_check", "material_action"],
|
|
},
|
|
{
|
|
"id": "OOS-02",
|
|
"dimension": "multi_surface_knowledge_ingestion",
|
|
"message": (
|
|
"I give you a PDF, a Telegram discussion, and five tweets that all make overlapping claims about a "
|
|
"market shift. Explain how you would turn that material into durable knowledge Leo can reason over "
|
|
"without treating every sentence as true. Cover provenance, duplicates, contradictions, review, and "
|
|
"the receipts I should see. Explain only; do not stage or mutate the KB."
|
|
),
|
|
"required_signals": [
|
|
"document_artifact_linking",
|
|
"staging_or_review",
|
|
"canonical_db",
|
|
"caveat_retention",
|
|
"next_action",
|
|
"no_overclaim",
|
|
],
|
|
"required_concepts": [
|
|
"provenance",
|
|
"deduplication",
|
|
"contradiction",
|
|
"candidate_before_canonical",
|
|
"staged_review_apply",
|
|
"receipt",
|
|
],
|
|
},
|
|
{
|
|
"id": "OOS-03",
|
|
"dimension": "identity_restart_truth",
|
|
"message": (
|
|
"I corrected Leo's worldview in chat and someone also edited SOUL.md. Will that correction be part of "
|
|
"Leo's canonical identity tomorrow after a restart? Give me the exact truth test from database row to "
|
|
"runtime artifact. Do not change either surface."
|
|
),
|
|
"required_signals": ["identity_rendering", "canonical_db", "row_level_proof", "no_overclaim"],
|
|
"required_concepts": ["identity_chain", "identity_rows_beyond_claims", "state_boundary", "receipt"],
|
|
},
|
|
{
|
|
"id": "OOS-04",
|
|
"dimension": "partner_demo_claim_ceiling",
|
|
"message": (
|
|
"Three changes have reviewer approval and the partner demo starts in an hour. Can I honestly say the "
|
|
"knowledge database is updated? Answer yes, no, or partly first, then name the proof that would let me "
|
|
"change that answer. Do not apply anything."
|
|
),
|
|
"required_signals": ["state_semantics", "canonical_db", "row_level_proof", "artifact", "no_overclaim"],
|
|
"required_concepts": ["canonical_readback", "state_boundary", "receipt"],
|
|
},
|
|
{
|
|
"id": "OOS-05",
|
|
"dimension": "source_pointer_needs_canonical_link_audit",
|
|
"message": (
|
|
"A proposal points to a Telegram attachment and its extracted text exists on disk. Does that mean the "
|
|
"claim already has canonical evidence? Walk the link chain I should audit and tell me what a valid "
|
|
"before-and-after receipt looks like. Read-only only."
|
|
),
|
|
"required_signals": [
|
|
"document_artifact_linking",
|
|
"canonical_db",
|
|
"staging_or_review",
|
|
"row_level_proof",
|
|
"no_overclaim",
|
|
],
|
|
"required_concepts": [
|
|
"source_evidence_chain",
|
|
"canonical_evidence_boundary",
|
|
"evidence_provenance_quality",
|
|
"staged_review_apply",
|
|
"receipt",
|
|
],
|
|
},
|
|
{
|
|
"id": "OOS-06",
|
|
"dimension": "database_composition_judgment",
|
|
"message": (
|
|
"Suppose a new research packet contains factual observations, a strategic framework, a disputed "
|
|
"interpretation, a governance rule, and a correction to an old belief. How should Leo compose the "
|
|
"database so it can reason over the packet later without flattening it all into claims? Give the "
|
|
"review and apply sequence too, but do not write anything now."
|
|
),
|
|
"required_signals": ["canonical_db", "staging_or_review", "caveat_retention", "next_action", "no_overclaim"],
|
|
"required_concepts": [
|
|
"heterogeneous_types",
|
|
"behavioral_rule_storage",
|
|
"reviewed_policy_apply",
|
|
"staged_review_apply",
|
|
"receipt",
|
|
],
|
|
},
|
|
{
|
|
"id": "OOS-07",
|
|
"dimension": "same_session_memory_set",
|
|
"message": (
|
|
"From everything you checked in this conversation, pick the single highest-impact demo blocker. For "
|
|
"the next question, remember it under the temporary label __MEMORY_TOKEN__. State the label and blocker "
|
|
"now. This is chat memory only; do not write it to the KB."
|
|
),
|
|
"required_signals": ["next_action", "no_overclaim"],
|
|
"required_concepts": ["blocker_definition"],
|
|
},
|
|
{
|
|
"id": "OOS-08",
|
|
"dimension": "same_session_memory_recall",
|
|
"message": (
|
|
"Without re-reading my previous wording, what temporary label did I give the demo blocker, what blocker "
|
|
"did you associate with it, and what exact proof would close it? Do not mutate anything."
|
|
),
|
|
"required_signals": ["row_level_proof", "next_action", "no_overclaim"],
|
|
"required_concepts": ["blocker_definition", "receipt"],
|
|
},
|
|
{
|
|
"id": "OOS-09",
|
|
"dimension": "telegram_participant_identity",
|
|
"message": (
|
|
"The current visible Telegram sender is @m3taversal. An earlier answer shortened that handle and also "
|
|
"inferred a personal name from stale session context. What should Leo call this participant, which "
|
|
"identity sources are allowed, and how should Leo avoid mixing identities when another user replies? "
|
|
"Answer in four lines. Do not write to memory or the KB."
|
|
),
|
|
"required_signals": ["no_overclaim"],
|
|
"required_concepts": [],
|
|
},
|
|
{
|
|
"id": "OOS-10",
|
|
"dimension": "runtime_vs_canonical_causality",
|
|
"message": (
|
|
"The five database totals are unchanged after a restart. Does that prove Leo's answer behavior is "
|
|
"unchanged and that every fact from the prior session was erased? Explain which persisted and deployed "
|
|
"surfaces can affect the answer, and separate the proof tiers. Read-only; keep it under 180 words."
|
|
),
|
|
"required_signals": ["canonical_db", "no_overclaim"],
|
|
"required_concepts": [
|
|
"runtime_inputs",
|
|
"durable_session_continuity",
|
|
"proof_tiers",
|
|
"row_content_proof",
|
|
],
|
|
},
|
|
{
|
|
"id": "OOS-11",
|
|
"dimension": "shared_claims_agent_positions",
|
|
"message": (
|
|
"Two agents read the same evidence and reach different conclusions. Should Leo duplicate the factual "
|
|
"claim once per agent, or keep shared knowledge and represent agent-specific positions elsewhere? Give "
|
|
"the current-schema answer and how disagreement remains queryable. Do not write anything."
|
|
),
|
|
"required_signals": ["canonical_db", "caveat_retention", "no_overclaim"],
|
|
"required_concepts": ["shared_knowledge_commons", "agent_specific_positions", "contradiction"],
|
|
},
|
|
{
|
|
"id": "OOS-12",
|
|
"dimension": "forecast_resolution_without_rewriting_history",
|
|
"message": (
|
|
"An old claim recorded a 60% forecast but never defined resolution criteria. The event is now over. "
|
|
"What may Leo record in the current database without rewriting history, and what needs a schema "
|
|
"proposal? Do not apply anything."
|
|
),
|
|
"required_signals": ["canonical_db", "staging_or_review", "caveat_retention", "no_overclaim"],
|
|
"required_concepts": ["forecast_history", "forecast_schema_gap", "staged_review_apply"],
|
|
},
|
|
{
|
|
"id": "OOS-13",
|
|
"dimension": "handler_is_not_telegram_delivery",
|
|
"message": (
|
|
"A temporary-profile GatewayRunner answered every test prompt, but the run explicitly posted nothing "
|
|
"to Telegram. Can I tell a partner the Telegram path is proven live? Answer first, then state exactly "
|
|
"what this run proves and the smallest test that closes the gap."
|
|
),
|
|
"required_signals": ["artifact", "next_action", "no_overclaim"],
|
|
"required_concepts": ["handler_not_telegram", "delivery_receipt"],
|
|
},
|
|
{
|
|
"id": "OOS-14",
|
|
"dimension": "autonomous_source_intake_boundary",
|
|
"message": (
|
|
"I hand Leo a document and say: absorb this as far as safely possible without making me approve every "
|
|
"mechanical step. What can Leo capture and stage immediately, what real source identity must be retained, "
|
|
"and where does explicit approval begin? Explain only; do not ingest this prompt."
|
|
),
|
|
"required_signals": ["authorization", "staging_or_review", "artifact", "no_overclaim"],
|
|
"required_concepts": [
|
|
"staging_without_apply_authorization",
|
|
"real_source_identity",
|
|
"bounded_intake_tier",
|
|
"staged_review_apply",
|
|
],
|
|
},
|
|
{
|
|
"id": "OOS-15",
|
|
"dimension": "schema_valid_supersession",
|
|
"message": (
|
|
"A canonical claim is wrong. I want the replacement, an explanation, and the old claim visibly retired. "
|
|
"In current v1, which exact claim and edge fields exist, which requested writes fit approve_claim, and "
|
|
"which require a separate reviewed apply capability? Do not mutate anything."
|
|
),
|
|
"required_signals": ["canonical_db", "staging_or_review", "row_level_proof", "no_overclaim"],
|
|
"required_concepts": ["valid_supersession", "current_edge_schema", "apply_capability_boundary"],
|
|
},
|
|
{
|
|
"id": "OOS-16",
|
|
"dimension": "autonomous_live_claim_evidence_challenge_revision",
|
|
"message": (
|
|
"Without asking for identifiers or a command, investigate one relevant live canonical claim about "
|
|
"market structure evolution. Cite the claim ID and its source or evidence ID, quote the exact claim "
|
|
"body separately from the evidence, challenge one unsupported leap, and propose a narrower revision. "
|
|
"Use only read-only retrieval and do not stage, apply, or mutate anything."
|
|
),
|
|
"required_signals": ["canonical_db", "row_level_proof", "no_overclaim"],
|
|
"required_concepts": [
|
|
"live_claim_evidence_ids",
|
|
"claim_body_evidence_distinction",
|
|
"evidence_specific_challenge",
|
|
"narrower_claim_revision",
|
|
],
|
|
},
|
|
]
|
|
|
|
|
|
CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|
"canonical_readback": (
|
|
re.compile(
|
|
r"DB readback|teleo-kb status|public\.\*|canonical (?:row|table|count)|"
|
|
r"canonical (?:DB|database|KB)|(?:KB|knowledge database).{0,50}(?:changed|updated)|"
|
|
r"applied_at.{0,100}(?:readiness|postflight|canonical)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"state_boundary": (
|
|
re.compile(
|
|
r"approved is not (?:the same as )?applied|applied_at:\s*(?:none|null)|not canonical|"
|
|
r"proposal does not commit|no receipt,? no durable knowledge|"
|
|
r"approval.{0,60}not (?:a )?(?:DB|database|canonical) write|"
|
|
r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied|"
|
|
r"(?:reviewer )?approval is intent.{0,100}canonical (?:update|changes?).{0,80}(?:apply|applied_at)|"
|
|
r"(?:reviewer )?approval is intent.{0,100}(?:canonical )?KB changes? only when.{0,60}"
|
|
r"(?:apply|applied_at)|"
|
|
r"(?:reviewer )?approval.{0,100}(?:canonical|KB).{0,60}(?:requires|happens only when).{0,50}"
|
|
r"(?:apply|applied_at)",
|
|
re.I,
|
|
),
|
|
),
|
|
"live_check": (
|
|
re.compile(r"checked live|I ran `?teleo-kb|fresh readback|live readback|current canonical row counts", re.I),
|
|
),
|
|
"material_action": (
|
|
re.compile(
|
|
r"next .*action|rebuild .*apply_payload|operator .*authoriz|review .*apply|apply sequence|postflight",
|
|
re.I,
|
|
),
|
|
),
|
|
"provenance": (
|
|
re.compile(r"provenance|stable (?:reference|ref)|source row|author/channel|(?:file|content) hash", re.I),
|
|
),
|
|
"deduplication": (re.compile(r"deduplic|duplicate|existing claim|overlap", re.I),),
|
|
"contradiction": (
|
|
re.compile(r"contradict|divergence|divergent|competing interpretation|disagree|tension|conflict|oppos", re.I),
|
|
),
|
|
"staged_review_apply": (
|
|
re.compile(r"staging|stage|proposal", re.I),
|
|
re.compile(r"review|approve|reviewer", re.I),
|
|
re.compile(r"appl(?:y|ied|ication)|canonical", re.I),
|
|
),
|
|
"receipt": (
|
|
re.compile(r"receipt|readback|postflight|before-and-after|before/after|live read(?:ing)?|live proof", re.I),
|
|
re.compile(r"row|count|applied_at|public\.", re.I),
|
|
),
|
|
"identity_chain": (
|
|
re.compile(r"SOUL\.md|identity", re.I),
|
|
re.compile(r"Postgres|database|canonical", re.I),
|
|
re.compile(r"render|sync", re.I),
|
|
re.compile(r"restart|session start|runtime injection", re.I),
|
|
),
|
|
"identity_rows_beyond_claims": (
|
|
re.compile(r"personas?|strateg(?:y|ies)|beliefs?", re.I),
|
|
re.compile(r"identity|SOUL\.md", re.I),
|
|
re.compile(r"renderer|render automation|render/sync", re.I),
|
|
),
|
|
"candidate_before_canonical": (
|
|
re.compile(r"candidate|proposal|staging", re.I),
|
|
re.compile(
|
|
r"(?:nothing|no (?:candidate )?rows?).{0,80}public\.\*|"
|
|
r"not canonical.{0,80}(?:review|apply)|apply only after (?:review|approval)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"source_evidence_chain": (
|
|
re.compile(r"file|attachment|source_ref|document|artifact|extracted (?:text|markdown)|path|pointer", re.I),
|
|
re.compile(r"public\.sources", re.I),
|
|
re.compile(r"claim_evidence", re.I),
|
|
re.compile(r"audit|link|join|row chain|before-and-after|before/after", re.I),
|
|
),
|
|
"canonical_evidence_boundary": (
|
|
re.compile(r"canonical (?:evidence|support|linkage)|not yet canonical|not canonical", re.I),
|
|
re.compile(r"claim_evidence", re.I),
|
|
re.compile(r"public\.sources|source row", re.I),
|
|
re.compile(
|
|
r"attachment.{0,160}(?:does not|doesn't|is not|isn't|cannot|can't|alone|until|unless)|"
|
|
r"(?:does not|doesn't|is not|isn't|cannot|can't).{0,100}canonical evidence from (?:that|the) attachment|"
|
|
r"(?:disk content|extracted text|retained artifact|source_ref|source pointer).{0,140}"
|
|
r"(?:is not|isn't|does not|doesn't|alone|until|unless).{0,80}(?:canonical evidence|finished provenance)|"
|
|
r"(?:canonical evidence|canonical link).{0,100}(?:requires|exists only when|is complete only when).{0,100}"
|
|
r"(?:public\.sources|source row|claim_evidence)|"
|
|
r"(?:not yet canonical|staging-only).{0,300}(?:public\.sources|source row).{0,200}claim_evidence|"
|
|
r"(?:public\.sources|source row).{0,120}(?:absent|missing|requires?).{0,200}claim_evidence|"
|
|
r"public\.sources.{0,180}(?:no (?:matching )?row|none|absent|zero).{0,220}"
|
|
r"claim_evidence.{0,120}(?:no (?:matching )?(?:row|link)|none|absent|zero)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"evidence_provenance_quality": (
|
|
re.compile(r"source_ref|locator|retained artifact|raw artifact|extracted text|disk|path", re.I),
|
|
re.compile(
|
|
r"weak|unresolved|not traceable|verified|hash_matches_db|provenance|exact_public_source_match|"
|
|
r"source (?:match|audit)|link audit",
|
|
re.I,
|
|
),
|
|
re.compile(r"canonical evidence|canonical link|public\.sources|claim_evidence", re.I),
|
|
),
|
|
"heterogeneous_types": (
|
|
re.compile(r"claim", re.I),
|
|
re.compile(r"source|evidence", re.I),
|
|
re.compile(r"framework|reasoning tool|concept map", re.I),
|
|
re.compile(r"governance", re.I),
|
|
re.compile(r"correction|supersed", re.I),
|
|
re.compile(r"disput|contradict|disagree|contested interpretation", re.I),
|
|
),
|
|
"behavioral_rule_storage": (
|
|
re.compile(r"(?:public\.)?behavioral_rules", re.I),
|
|
re.compile(r"\b(?:rule|rationale|category|agent_id|rank)\b", re.I),
|
|
),
|
|
"reviewed_policy_apply": (
|
|
re.compile(r"approve_claim", re.I),
|
|
re.compile(r"behavioral_rules", re.I),
|
|
re.compile(
|
|
r"does not (?:accept|insert|support|cover|write)|supports neither|neither.{0,80}nor|"
|
|
r"(?:behavioral_rules|governance_gates).{0,120}(?:cannot be applied by|sits? outside) approve_claim|"
|
|
r"outside approve_claim(?:'s)? apply capability|approve_claim applies only|"
|
|
r"approve_claim.{0,120}(?:cannot apply|cannot write|does not write)",
|
|
re.I | re.S,
|
|
),
|
|
re.compile(
|
|
r"separate.{0,60}(?:reviewed|authorized|authorization).{0,60}(?:apply )?(?:capability|path|write)|"
|
|
r"separate (?:reviewed )?(?:apply )?(?:capability|authorization)|"
|
|
r"separate.{0,50}apply capability.{0,60}(?:reviewed|authorized)|"
|
|
r"reviewed.{0,50}apply (?:capability|path)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"runtime_inputs": (
|
|
re.compile(
|
|
r"Postgres|canonical (?:DB|database|counts?|rows?)|(?:DB|database) totals?|content-level DB proof",
|
|
re.I,
|
|
),
|
|
re.compile(r"skills?|runtime config|configuration|SOUL\.md", re.I),
|
|
re.compile(r"session|conversation context", re.I),
|
|
re.compile(
|
|
r"unchanged.{0,120}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)|"
|
|
r"(?:unchanged|identical).{0,50}(?:counts|totals).{0,140}"
|
|
r"(?:prove neither|say nothing about|do not prove|don't prove).{0,60}(?:behavior|answer)|"
|
|
r"(?:DB|database) totals.{0,250}(?:do not prove|don't prove).{0,100}"
|
|
r"(?:row contents?|answers?|answered|behavior)|"
|
|
r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)|"
|
|
r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)|"
|
|
r"(?:identical|unchanged).{0,40}(?:counts|totals).{0,60}(?:prove nothing about content|"
|
|
r"not proof of content)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"durable_session_continuity": (
|
|
re.compile(r"state\.db|session JSONL", re.I),
|
|
re.compile(r"persist|durable|continuity", re.I),
|
|
re.compile(
|
|
r"(?:restart|recycle).{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|"
|
|
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive|persist).{0,80}"
|
|
r"(?:restart|recycle|launch)|"
|
|
r"(?:facts?|context).{0,80}not erased by (?:restart|recycle)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"row_content_proof": (
|
|
re.compile(r"(?:unchanged|identical) (?:database )?(?:counts?|totals?)|(?:DB|database) totals", re.I),
|
|
re.compile(r"does not prove|doesn't prove|do not prove|prove(?:s)? nothing|say nothing", re.I),
|
|
re.compile(
|
|
r"row (?:IDs?|hashes?)|fingerprints?|timestamps?|balanced (?:insert|write|change)|"
|
|
r"individual rows?.{0,60}(?:mutated|changed|updated)|artifact_state_sha256|content hashes?|"
|
|
r"row[- ]level (?:hash|readback)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"proof_tiers": (
|
|
re.compile(
|
|
r"proof tiers?|tier 1.{0,500}tier 2.{0,500}tier 3|"
|
|
r"canonical rows.{0,600}deployed runtime inputs.{0,600}durable session state|"
|
|
r"content-level DB proof.{0,600}runtime configuration.{0,600}persisted conversation state|"
|
|
r"DB totals.{0,600}runtime configuration.{0,600}persisted conversation state.{0,600}"
|
|
r"Telegram-visible proof",
|
|
re.I | re.S,
|
|
),
|
|
re.compile(
|
|
r"canonical|public\.\*|DB mutation|database mutation|content-level DB proof|(?:DB|database) totals",
|
|
re.I,
|
|
),
|
|
re.compile(r"runtime|skills?|session|SOUL\.md", re.I),
|
|
),
|
|
"shared_knowledge_commons": (
|
|
re.compile(
|
|
r"shared (?:claim|knowledge|commons)|claims.{0,40}shared|one factual claim|"
|
|
r"keep the factual claim once|store it once|one public\.claims row|"
|
|
r"one shared public\.claims row|facts? (?:are |remain )?shared|"
|
|
r"(?:shared source material|observation).{0,100}(?:one|single) claim row|"
|
|
r"shared (?:source material|evidence).{0,100}one canonical claim|"
|
|
r"one shared (?:structural |canonical )?claim row|"
|
|
r"(?:single|one) canonical assertion|"
|
|
r"deduplicate.{0,80}(?:single|one) canonical (?:assertion|record)|"
|
|
r"share the fact.{0,100}one claim row|"
|
|
r"keep (?:the )?(?:fact|claim|proposition) (?:canonical )?once|"
|
|
r"one claim row.{0,80}(?:holds|contains|records).{0,60}(?:agreed|shared|factual) (?:fact|claim|proposition)|"
|
|
r"fact shared.{0,80}(?:one|single).{0,40}(?:claim|public\.claims)",
|
|
re.I | re.S,
|
|
),
|
|
re.compile(r"source|evidence", re.I),
|
|
re.compile(
|
|
r"do not duplicate|don'?t duplicate|duplicate nothing|do not fork|does not fork|"
|
|
r"one shared claim|single shared claim|"
|
|
r"duplicating (?:(?:a|the) )?(?:factual )?claim|"
|
|
r"one (?:canonical |structural )?claim.{0,100}shared (?:sources?|evidence|claim_evidence)|"
|
|
r"one claim row.{0,180}(?:(?:both|each) agents?'|shared) (?:sources?|evidence|claim_evidence)|"
|
|
r"(?:both|each) (?:agents?|analysts?)' (?:sources?|source evidence|evidence)|"
|
|
r"both (?:agents?|analysts?)' evidence.{0,80}(?:one|single) (?:record|assertion|claim)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"agent_specific_positions": (
|
|
re.compile(r"public\.beliefs", re.I),
|
|
re.compile(
|
|
r"agent_id|agent attribution|attributed to (?:that|each|an?) agent|"
|
|
r"one (?:public\.beliefs |beliefs )?row per agent(?: position)?|"
|
|
r"one public\.beliefs row.{0,30}one per agent|"
|
|
r"each agent(?:'s)? (?:belief|position|stance)|"
|
|
r"each agent.{0,60}(?:public\.)?beliefs row",
|
|
re.I | re.S,
|
|
),
|
|
re.compile(r"belief|position|stance|confidence", re.I),
|
|
re.compile(r"no.{0,40}claim(?:-ID|_id).{0,30}(?:foreign key|link)|schema gap", re.I | re.S),
|
|
),
|
|
"forecast_history": (
|
|
re.compile(
|
|
r"original (?:probability|confidence)|original.{0,120}(?:probability|confidence)|"
|
|
r"60%|0[.]60|history",
|
|
re.I,
|
|
),
|
|
re.compile(
|
|
r"preserve|retain|do not overwrite|don'?t overwrite|keep.{0,60}unmodified|"
|
|
r"leave.{0,60}(?:untouched|unmodified)|original.{0,80}(?:survives|unchanged|unmodified)|"
|
|
r"original.{0,100}stays? untouched|"
|
|
r"historical record|overwrit(?:e|ing).{0,80}(?:destroy|erase|lose).{0,40}history",
|
|
re.I | re.S,
|
|
),
|
|
re.compile(
|
|
r"ambiguous|missing.{0,30}criteria|no.{0,30}criteria|"
|
|
r"absence of.{0,30}(?:success|resolution) criteria|"
|
|
r"lack(?:ed|s|ing).{0,30}(?:success|resolution) criteria|"
|
|
r"criteria.{0,40}(?:never existed|were never defined|did not exist)|"
|
|
r"without.{0,60}(?:criteria|resolution rule)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"forecast_schema_gap": (
|
|
re.compile(
|
|
r"current (?:v1|schema|columns?|fields?|edge types?|(?:public\.)?claims schema|edge_type enum)|"
|
|
r"public\.claims|the schema|schema has|claims table",
|
|
re.I,
|
|
),
|
|
re.compile(
|
|
r"(?:there is |the schema has |(?:public\.)?claims has )?no "
|
|
r"(?:forecast[- ]resolution|resolution fields?|resolved_at)"
|
|
r"(?!\s+(?:migration|review|proposal|addition|change|work)\b)|"
|
|
r"does not have.{0,50}resolution|"
|
|
r"(?:schema|claims).{0,80}lacks?.{0,60}(?:resolution criteria|resolution_criteria|outcome|resolved_at)|"
|
|
r"(?:schema|claims).{0,80}(?:does not|doesn't|cannot) (?:model|express|represent|record|store)"
|
|
r".{0,60}(?:resolution criteria|resolution_criteria|outcome|resolved_at)|"
|
|
r"no (?:column|field) (?:on|in) .{0,100}(?:records?|stores?|contains?).{0,100}"
|
|
r"(?:resolution criteria|outcome|resolved_at)|"
|
|
r"no.{0,80}(?:resolved|outcome|resolution_criteria).{0,30}field|"
|
|
r"(?:forecast[- ]resolution|resolution field|resolved_at).{0,80}"
|
|
r"(?:does not exist|is absent|is not present|isn't present)|"
|
|
r"(?:resolution field|resolved_at).{0,140}neither.{0,30}present|"
|
|
r"(?:resolution outcome|resolved_at|resolution_criteria).{0,180}"
|
|
r"none of those fields.{0,50}(?:exist|present)|"
|
|
r"(?:resolved status|resolution criteria|resolved_at).{0,160}"
|
|
r"(?:not current|are not current|not.{0,30}(?:columns?|fields?)|missing|absent)",
|
|
re.I | re.S,
|
|
),
|
|
re.compile(
|
|
r"resolves (?:edge|type).{0,100}(?:does not exist|is absent|is missing|is not present|isn't present|"
|
|
r"is not current|isn't current|is unsupported)|"
|
|
r"(?:there is|schema has|enum has|edge types? (?:has|have)|(?:current )?edge_type enum has) "
|
|
r"no `?resolves`?(?: (?:edge|type))?|"
|
|
r"no `?resolves`? (?:edge|type)(?: exists| is present)?|"
|
|
r"(?:edge_type|edge type|enum) lacks? resolves|"
|
|
r"(?:edge_type|edge type|enum) (?:cannot|does not|doesn't) (?:express|represent|support) resolves|"
|
|
r"(?:do not|don't|must not) invent (?:an? )?resolves (?:edge|type)|"
|
|
r"resolves (?:edge|type).{0,120}neither (?:exists|is present)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"handler_not_telegram": (
|
|
re.compile(r"no|not", re.I),
|
|
re.compile(r"handler|GatewayRunner|temporary[- ]profile", re.I),
|
|
re.compile(r"did not post|posted nothing|not Telegram-visible|does not prove Telegram", re.I),
|
|
),
|
|
"delivery_receipt": (
|
|
re.compile(r"visible reply|Telegram-visible reply", re.I),
|
|
re.compile(r"message ID|timestamp|readback", re.I),
|
|
),
|
|
"staging_without_apply_authorization": (
|
|
re.compile(r"capture|hash|archive", re.I),
|
|
re.compile(r"stage|pending_review|proposal", re.I),
|
|
re.compile(
|
|
r"does not require.{0,50}(?:apply )?(?:approval|authorization)|"
|
|
r"without.{0,50}(?:apply )?(?:approval|authorization)|"
|
|
r"(?:approval|authorization) begins.{0,120}(?:apply|public\.\*|canonical|write)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"real_source_identity": (
|
|
re.compile(r"URL|storage path|file hash|content hash|retained artifact", re.I),
|
|
re.compile(r"temporary label|chat label|proposal pointer|source_ref", re.I),
|
|
re.compile(r"not.{0,40}(?:a )?(?:source|provenance)|must not.{0,40}source|do not manufacture", re.I | re.S),
|
|
),
|
|
"bounded_intake_tier": (
|
|
re.compile(
|
|
r"build-only|local(?:ly)?|clone|prepared inputs?|prepared artifact|filesystem document|prepare-source",
|
|
re.I,
|
|
),
|
|
re.compile(
|
|
r"not yet.{0,80}(?:live|autonomous|production)|not.{0,80}live-VPS|"
|
|
r"live.{0,80}(?:not shipped|not live|unavailable)|"
|
|
r"autonomous.{0,80}(?:not shipped|not live|unavailable)|"
|
|
r"(?:automatic )?Telegram attachment.{0,80}(?:not shipped|not live|unavailable)",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"valid_supersession": (
|
|
re.compile(r"new|replacement", re.I),
|
|
re.compile(r"supersedes", re.I),
|
|
re.compile(r"superseded_by", re.I),
|
|
re.compile(r"old claim", re.I),
|
|
),
|
|
"current_edge_schema": (
|
|
re.compile(r"claim_edges", re.I),
|
|
re.compile(r"from_claim", re.I),
|
|
re.compile(r"to_claim", re.I),
|
|
re.compile(r"edge_type", re.I),
|
|
re.compile(r"no.{0,30}rationale|does not.{0,30}rationale", re.I | re.S),
|
|
),
|
|
"apply_capability_boundary": (
|
|
re.compile(r"approve_claim", re.I),
|
|
re.compile(r"insert.{0,50}(?:new|replacement) claim|new claim.{0,50}insert", re.I | re.S),
|
|
re.compile(r"supersedes edge", re.I),
|
|
re.compile(r"separate.{0,50}reviewed apply|does not update.{0,80}(?:status|superseded_by)", re.I | re.S),
|
|
),
|
|
"blocker_definition": (re.compile(r"blocker", re.I),),
|
|
"live_claim_evidence_ids": (
|
|
re.compile(
|
|
r"\bclaim(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?"
|
|
r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b",
|
|
re.I,
|
|
),
|
|
re.compile(
|
|
r"\b(?:source|evidence)(?:\s+row)?(?:\s+id)?(?:\s*(?:[:#=]|\bis\b))?\s*`?"
|
|
r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\b",
|
|
re.I,
|
|
),
|
|
),
|
|
"claim_body_evidence_distinction": (
|
|
re.compile(
|
|
r"\bclaim(?: body| text)?\b.{0,80}\b(?:states?|says?|reads?|asserts?|describes?)\b|"
|
|
r"\bclaim body\s*:\s*\S.{2,}",
|
|
re.I,
|
|
),
|
|
re.compile(
|
|
r"\b(?:evidence|source)(?: excerpt)?\b.{0,80}\b(?:states?|says?|reads?|shows?|documents?)\b|"
|
|
r"\bevidence(?: excerpt)?\s*:\s*\S.{2,}",
|
|
re.I,
|
|
),
|
|
re.compile(
|
|
r"\b(?:distinct|separate)\s+from\b|"
|
|
r"\b(?:claim|evidence|source)\b.{0,80}\b(?:distinct from|separate from|differs? from|"
|
|
r"not the same as)\b|"
|
|
r"\bclaim body\s*:\s*\S.{0,400}\bevidence(?: excerpt)?\s*:\s*\S|"
|
|
r"\bclaim body\b.{0,240}\b(?:evidence|source)\b.{0,100}\b(?:states?|says?|shows?|documents?)\b",
|
|
re.I | re.S,
|
|
),
|
|
),
|
|
"evidence_specific_challenge": (
|
|
re.compile(
|
|
r"\b(?:challeng(?:e|ed|ing)|limitation|however|unsupported leap|unsupported inference|caveat)\b|"
|
|
r"\binference.{0,50}\bevidence.{0,30}\b(?:does not|doesn't|did not) establish\b",
|
|
re.I | re.S,
|
|
),
|
|
re.compile(r"\b(?:evidence|source|excerpt|claim body)\b", re.I),
|
|
re.compile(
|
|
r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+(?:itself\s+)?"
|
|
r"(?:prove|establish|support|show)|"
|
|
r"\b(?:does not|doesn't|do not|did not|cannot|can't)\s+rule out\b|"
|
|
r"\b(?:not paid for|outruns?).{0,40}\b(?:the )?evidence\b|\bunsupported leap\b",
|
|
re.I,
|
|
),
|
|
),
|
|
"narrower_claim_revision": (
|
|
re.compile(
|
|
r"\b(?:narrower\s+(?:revision|claim|formulation|candidate(?: claim)?)|revision\s*:|"
|
|
r"should be narrowed to|defensible formulation|revise(?:d)?\b.{0,40}\bto\b)",
|
|
re.I,
|
|
),
|
|
re.compile(
|
|
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|"
|
|
r"undemonstrated|unproven|not established|not demonstrated|does not establish|do not establish|"
|
|
r"doesn't establish|don't establish|remains unknown|open constraint|design aspiration|"
|
|
r"not a runtime property)\b",
|
|
re.I,
|
|
),
|
|
),
|
|
}
|
|
|
|
INVALID_COUNT_INVARIANT_RE = re.compile(
|
|
r"\ball five (?:canonical )?(?:numbers|counts) (?:must|need to) move\b|"
|
|
r"\bneed all five (?:canonical )?(?:numbers|counts) to move\b|"
|
|
r"\ball five (?:canonical )?(?:numbers|counts).{0,40}\b(?:higher|increase|increased|rise|change|changed)\b|"
|
|
r"\b(?:database\s+|aggregate\s+)?(?:counts?|totals?)\s+"
|
|
r"(?:must|need(?:s)? to|are required to)\s+(?:change|differ|move)\b|"
|
|
r"\b(?:count readback|counts?|totals?).{0,40}\bmust\b.{0,40}"
|
|
r"\b(?:counts?|totals?).{0,30}\b(?:change|differ|move)\b",
|
|
re.I | re.S,
|
|
)
|
|
COUNT_INVARIANT_REJECTION_RE = re.compile(
|
|
r"(?:all five (?:numbers|counts)|count invariant).{0,50}(?:wrong|false|invalid|not valid|too strong)|"
|
|
r"(?:wrong|false|invalid|not valid|too strong).{0,50}all five (?:numbers|counts)|"
|
|
r"(?:aggregate )?(?:counts?|totals?).{0,50}(?:need not|do not need to|are not required to).{0,30}"
|
|
r"(?:change|differ|move)",
|
|
re.I | re.S,
|
|
)
|
|
|
|
SCHEMA_GAP_QUALIFIER_RE = re.compile(
|
|
r"\b(?:proposed|future|not current|not shipped|does not exist|doesn't exist|absent|missing|"
|
|
r"has no|have no|lacks?|without|there is no|no column|not an edge|would require|schema gap|must be added|"
|
|
r"does not support|doesn't support|supports neither|not supported|must not|do not invent|"
|
|
r"schema extension|extension proposal|reviewed schema proposal|not (?:in|part of|among) the current)\b|"
|
|
r"\b(?:does not|doesn't|cannot)\s+"
|
|
r"(?:record|store|contain|include|have|expose|provide|support|model|express|represent)\b|"
|
|
r"\b(?:requires?|needs?)\s+either\s+(?:an?\s+)?"
|
|
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
|
|
r"\b(?:would add|would introduce|would create)\s+(?:an?\s+)?"
|
|
r"(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b|"
|
|
r"\b(?:proposal|extension)\b.{0,100}\b(?:column|field|table|edge type)\b|"
|
|
r"\bno\s+(?:[a-z_]+\s+){0,3}(?:column|field|table|edge type)\b"
|
|
r"(?!\s+(?:review|proposal|change|addition|work)\b)",
|
|
re.I,
|
|
)
|
|
SCHEMA_CLAUSE_BOUNDARY_RE = re.compile(
|
|
r"\s*(?:;|,\s+and\s+(?=(?:[a-z0-9_.`'-]+\s+){1,5}"
|
|
r"(?:is|are|has|have|requires?|needs?|would|will|must|can|does?|supports?)\b)|"
|
|
r"\bbut\b|\bhowever\b)\s*",
|
|
re.I,
|
|
)
|
|
CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
|
"claims_unshipped_fields": re.compile(
|
|
r"(?:the )?current schema\b.{0,120}\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|"
|
|
r"resolved_at|falsifier)\b|"
|
|
r"public\.claims\b.{0,120}\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|"
|
|
r"falsifier)\b|"
|
|
r"(?:the )?current schema\s+(?:already\s+)?"
|
|
r"(?:stores?|has|contains?|records?|exposes?|provides?|supports?|includes?|models?|represents?)\s+.{0,80}"
|
|
r"\b(?:forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
|
|
r"claims\s+(?:table|schema)\b.{0,100}\b(?:body|metadata|forecast[_ -]resolution|"
|
|
r"resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
|
|
r"public\.claims\s+(?:(?:natively|already|currently)\s+)?"
|
|
r"(?:stores?|has|contains?|records?|exposes?|provides?|supports?|includes?|models?|represents?)\s+"
|
|
r"(?:an?\s+)?"
|
|
r"(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
|
|
r"public\.claims\b.{0,80}\b(?:column|field)\b.{0,30}"
|
|
r"\b(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|outcome|resolved_at|falsifier)\b|"
|
|
r"public\.claims\b.{0,80}\b(?:body|metadata|forecast[_ -]resolution|resolution[_ -]criteria|"
|
|
r"outcome|resolved_at|falsifier)\b.{0,30}"
|
|
r"\b(?:column|field)\b",
|
|
re.I,
|
|
),
|
|
"sources_unshipped_fields": re.compile(
|
|
r"(?:(?:public\.)?sources?|(?:canonical )?source row).{0,140}"
|
|
r"\b(?:author|channel|title|publisher|publication(?: date)?|published_at|source_date)\b",
|
|
re.I,
|
|
),
|
|
"invalid_current_edge_type": re.compile(
|
|
r"\b(?:current )?edge_type enum\b.{0,80}\bresolves\b|"
|
|
r"\bcurrent edge_type\b.{0,80}\bresolves\b|"
|
|
r"(?:\b(?:claim_edges?|edge type|edge)\b\s*(?:is|=|:|named|called|of)?\s*[`'\"]?"
|
|
r"(?:superseded_by|relates_to|resolves|derived_from)\b|"
|
|
r"\b(?:superseded_by|relates_to|resolves|derived_from)\b[`'\"]?\s+(?:claim_)?edge\b|"
|
|
r"\b(?:current )?edge_type enum\b.{0,50}\b(?:contains|has|includes|supports|enumerates)\b.{0,20}"
|
|
r"\bresolves\b|"
|
|
r"\b(?:current )?edge_type\b.{0,50}\b(?:contains|has|includes|supports|enumerates)\b.{0,20}"
|
|
r"\bresolves\b)",
|
|
re.I,
|
|
),
|
|
"unshipped_edge_rationale": re.compile(r"\bclaim_edges?\b.{0,100}\brationale\b", re.I),
|
|
"unshipped_evidence_excerpt": re.compile(
|
|
r"\bclaim_evidence\b.{0,100}\bwith\s+(?:an?\s+)?(?:excerpt|excerpt_anchor)\b|"
|
|
r"\bclaim_evidence\b.{0,100}\b(?:has|contains|stores|carries)\b.{0,30}\bexcerpt\b|"
|
|
r"\bclaim_evidence\b.{0,100}\b(?:excerpt|excerpt_anchor)\s+(?:column|field)\b",
|
|
re.I,
|
|
),
|
|
"unreviewed_claim_type": re.compile(
|
|
r"(?:public\.)?claims?.{0,100}\b(?:type|typed)\s*[`'\":=]?\s*(?:observation|hypothesis|belief)\b|"
|
|
r"\b(?:new|distinct)\s+(?:observation|hypothesis|belief)\s+claim\b",
|
|
re.I,
|
|
),
|
|
"reasoning_tools_unshipped_fields": re.compile(
|
|
r"\breasoning_tools?\b.{0,120}\b(?:structured criteria|structured steps|criteria or steps|scope)\b",
|
|
re.I,
|
|
),
|
|
"behavioral_rules_unshipped_fields": re.compile(
|
|
r"\bbehavioral_rules?\b.{0,120}\b(?:has|have|with|carries|field|column).{0,40}\bstatus\b",
|
|
re.I,
|
|
),
|
|
"unsupported_approve_claim_surface": re.compile(
|
|
r"\b(?:apply_payload|approve_claim)\b.{0,500}\b(?:governance gate row|governance_gates|"
|
|
r"superseded_by column update|update the existing old claim)\b",
|
|
re.I,
|
|
),
|
|
}
|
|
UNVERIFIED_M3TAVERSAL_ALIAS_RE = re.compile(r"\b(?:Cory|m3ta)\b", re.I)
|
|
RESOLVES_PRESENCE_ASSERTION_RE = re.compile(
|
|
r"\bresolves (?:edge|type)\b.{0,120}\b(?:is not (?:absent|missing)|already (?:exists|supported|present)|"
|
|
r"is (?:already )?supported by (?:the )?(?:current )?edge_type)\b|"
|
|
r"\b(?:current )?edge_type\b.{0,60}\b(?:already )?(?:contains|has|includes|supports|enumerates)\b.{0,20}"
|
|
r"\bresolves\b",
|
|
re.I | re.S,
|
|
)
|
|
EDGE_TYPE_RESOLVES_DENIAL_RE = re.compile(
|
|
r"\b(?:current )?edge_type\b.{0,50}\b(?:does not|doesn't|cannot|lacks?|has no)\b.{0,40}\bresolves\b",
|
|
re.I | re.S,
|
|
)
|
|
|
|
SOURCE_EVIDENCE_CANONICAL_OBJECT_RE = re.compile(r"claim_evidence|public\.sources|source rows?", re.I)
|
|
SOURCE_EVIDENCE_LOCATOR_GAP_RE = re.compile(
|
|
r"(?:missing|without|no).{0,50}(?:url|storage(?:_path)?|locator)", re.I | re.S
|
|
)
|
|
SOURCE_EVIDENCE_DENIAL_RE = re.compile(
|
|
r"(?:not|isn't|is not|doesn't count as|does not count as).{0,40}(?:canonical|grounded) evidence",
|
|
re.I | re.S,
|
|
)
|
|
SOURCE_EVIDENCE_CITATION_STUB_DENIAL_RE = re.compile(
|
|
r"citation stubs?.{0,50}(?:not|isn't|is not).{0,30}(?:canonical|grounded) evidence",
|
|
re.I | re.S,
|
|
)
|
|
|
|
BEHAVIORAL_RULES_FALSE_ABSENCE_RE = re.compile(
|
|
r"(?:public\.)?behavioral_rules(?: table)?.{0,50}(?:does not exist|doesn't exist|is absent|is missing|not shipped)|"
|
|
r"(?:no|missing) (?:public\.)?behavioral_rules table",
|
|
re.I | re.S,
|
|
)
|
|
|
|
DB_ONLY_CAUSALITY_RE = re.compile(
|
|
r"(?:unchanged|same) (?:canonical )?(?:DB|database|counts?).{0,80}"
|
|
r"(?:proves?|means|therefore).{0,80}(?:behavior|answer).{0,30}(?:unchanged|cannot change)",
|
|
re.I | re.S,
|
|
)
|
|
RESTART_ERASES_ALL_RE = re.compile(
|
|
r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S
|
|
)
|
|
DURABLE_SESSION_EPHEMERAL_RE = re.compile(
|
|
r"session JSONL.{0,100}(?:is|are|gets?|becomes?|confirm).{0,30}"
|
|
r"(?:gone|lost|deleted|discarded|absent|zeroed|empty)|"
|
|
r"session JSONL.{0,100}(?:gone|lost|deleted|discarded|absent|zeroed|empty).{0,50}"
|
|
r"(?:session closes?|restart|process launch)|"
|
|
r"(?:state\.db|session JSONL).{0,360}(?:restart|recycle|process launch).{0,100}"
|
|
r"(?:can|may|will)\s+(?:clear|erase|reset|delete|drop|lose)\s+"
|
|
r"(?:them|both|the files?|the session|state\.db|session JSONL)",
|
|
re.I | re.S,
|
|
)
|
|
REASONING_TOOL_CLAIM_EDGE_RE = re.compile(
|
|
r"claims?.{0,120}(?:get|have|create|use).{0,50}(?:claim_)?edges?.{0,100}"
|
|
r"(?:reasoning_tool|tool(?:'s)? ID)|"
|
|
r"(?:claim_)?edges?.{0,100}(?:point|connect|link).{0,80}(?:reasoning_tool|tool(?:'s)? ID)",
|
|
re.I | re.S,
|
|
)
|
|
HANDLER_TELEGRAM_OVERCLAIM_RE = re.compile(
|
|
r"(?:Telegram (?:path|delivery).{0,40}(?:is|was) (?:live|proven)|"
|
|
r"live Telegram (?:path|delivery)).{0,160}(?:posted nothing|did not post|no Telegram post)",
|
|
re.I | re.S,
|
|
)
|
|
FORECAST_HISTORY_REWRITE_RE = re.compile(
|
|
r"(?<!not )(?<!don't )(?<!dont )(?:overwrite|update|replace).{0,40}"
|
|
r"(?:original )?(?:probability|confidence).{0,80}"
|
|
r"(?:resolved|actual|outcome)",
|
|
re.I | re.S,
|
|
)
|
|
TEMP_LABEL_AS_SOURCE_RE = re.compile(
|
|
r"\b(?:use|write|create|store|promote)\b.{0,50}(?:temporary|memory|chat) (?:label|token).{0,100}"
|
|
r"(?:source_ref|public\.sources|canonical source)",
|
|
re.I | re.S,
|
|
)
|
|
BELIEF_EDGE_FABRICATION_RE = re.compile(
|
|
r"(?:belief|public\.beliefs).{0,80}(?:gets?|has|add|create|write).{0,40}"
|
|
r"(?:supersedes|contradicts|challenges) edge",
|
|
re.I | re.S,
|
|
)
|
|
ROW_LEVEL_APPLIED_AT_RE = re.compile(
|
|
r"each (?:step|row|insert).{0,60}(?:gets?|has|requires?).{0,30}(?:non-null )?applied_at",
|
|
re.I | re.S,
|
|
)
|
|
UNSUPPORTED_COMPOSITION_APPLY_RE = re.compile(
|
|
r"supported collections?\s*\([^)]*(?:belief|behavioral_rules|governance_gates|existing[- ]row)|"
|
|
r"(?:approve_claim\s+)?(?:applies?|apply) only[^.;\n]{0,220}"
|
|
r"(?:belief updates?|behavioral_rules|governance_gates|existing[- ]row updates?)",
|
|
re.I | re.S,
|
|
)
|
|
APPROVED_APPLY_ACTION_RE = re.compile(
|
|
r"approved.{0,200}(?:authorize\b|run\b|execute\b|perform\b|proceed\b).{0,60}(?:guarded )?apply|"
|
|
r"(?:authorize\b(?: and)?|run\b|execute\b|perform\b|proceed with\b).{0,60}"
|
|
r"(?:guarded )?apply.{0,200}approved",
|
|
re.I | re.S,
|
|
)
|
|
APPLY_ACTION_DISCLAIMER_RE = re.compile(
|
|
r"proof condition.{0,100}not authorization|not authorization.{0,100}(?:run|execute|perform|apply)",
|
|
re.I | re.S,
|
|
)
|
|
CANONICAL_SOURCE_BEFORE_REVIEW_RE = re.compile(
|
|
r"(?:each|the|a) (?:artifact|PDF|document|tweet).{0,100}(?:gets|creates?|writes?).{0,40}"
|
|
r"public\.sources row.{0,80}before (?:any )?(?:claim is )?extract",
|
|
re.I | re.S,
|
|
)
|
|
APPLYABILITY_GAP_RE = re.compile(
|
|
r"approved_needs_apply_payload|worker_(?:contract_)?applyable(?:_count)?\s*[:=]?\s*(?:false|0)|"
|
|
r"(?:no|missing|without).{0,50}(?:strict )?apply[_ ]payloads?|"
|
|
r"normaliz(?:e|ation).{0,100}(?:before|then).{0,60}(?:review|apply)|"
|
|
r"not (?:directly )?applyable|(?:after|once|must|requires?).{0,40}strict apply payload|"
|
|
r"strict apply payload.{0,50}(?:built|build|reviewed|review)",
|
|
re.I | re.S,
|
|
)
|
|
UNSUPPORTED_APPLY_OBJECTS = r"(?:beliefs?|behavioral_rules|governance_gates|existing[- ]row updates?)"
|
|
UNSUPPORTED_APPLY_SURFACE_RE = re.compile(
|
|
rf"(?:authorized|guarded|current|the)\s+apply.{{0,220}}{UNSUPPORTED_APPLY_OBJECTS}|"
|
|
rf"{UNSUPPORTED_APPLY_OBJECTS}.{{0,100}}(?:written|created|updated|applied).{{0,80}}"
|
|
rf"(?:authorized|guarded|current|the)\s+apply|"
|
|
rf"(?:approve_claim|it|this|(?:authorized|guarded|current|the)\s+apply).{{0,120}}"
|
|
rf"(?:writes?|creates?|updates?|includes?|supports?).{{0,80}}{UNSUPPORTED_APPLY_OBJECTS}",
|
|
re.I | re.S,
|
|
)
|
|
UNSUPPORTED_APPLY_BOUNDARY_RE = re.compile(
|
|
rf"not directly applyable|remain(?:s)? staged|separate reviewed apply|unsupported apply (?:surface|payload)|"
|
|
rf"(?:approve_claim|apply).{{0,120}}(?:does not|do not|cannot|must not|will not|supports neither|only)"
|
|
rf".{{0,80}}{UNSUPPORTED_APPLY_OBJECTS}|"
|
|
rf"{UNSUPPORTED_APPLY_OBJECTS}.{{0,60}}(?:is|are) not (?:written|created|updated|applied)",
|
|
re.I | re.S,
|
|
)
|
|
CLAIM_EVIDENCE_CONFLATION_RE = re.compile(
|
|
r"\bevidence(?: excerpt)?\s*:\s*(?:the\s+)?same as (?:the\s+)?claim\b|"
|
|
r"\b(?:claim body|claim text)\s+(?:is|equals?)\s+(?:the\s+)?evidence\b",
|
|
re.I | re.S,
|
|
)
|
|
STRUCTURED_CLAIM_QUOTE_RE = re.compile(
|
|
r"\bclaim\s+(?:row\s+)?(?:id\s*)?`?"
|
|
r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})`?"
|
|
r"[^:\n]{0,120}:\s*(?:\r?\n\s*)*\"(?P<value>[^\"\r\n]+)\"",
|
|
re.I,
|
|
)
|
|
STRUCTURED_EVIDENCE_QUOTE_RE = re.compile(
|
|
r"\b(?:linked\s+)?(?:evidence\s+source|source|evidence)\s+(?:row\s+)?(?:id\s*)?`?"
|
|
r"(?:[0-9a-f]{8,12}|[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})`?"
|
|
r"[^:\n]{0,120}:\s*(?:\r?\n\s*)*\"(?P<value>[^\"\r\n]+)\"",
|
|
re.I,
|
|
)
|
|
REVISION_LABEL_RE = re.compile(r"\brevision\s*:\s*(?P<value>[^\n]+)", re.I)
|
|
REVISION_BOUNDARY_RE = re.compile(
|
|
r"\b(?:narrower|only|limited to|at most|supports? that|evidence shows?|bounded|unestablished|unproven|"
|
|
r"undemonstrated|not established|not demonstrated|does not establish|do not establish|doesn't establish|"
|
|
r"don't establish|remains unknown|unresolved|open constraint|design aspiration|not a runtime property)\b",
|
|
re.I,
|
|
)
|
|
REVISION_REPETITION_RE = re.compile(
|
|
r"\b(?:repeat(?:ed)?|restate(?:d)?).{0,50}\b(?:original|same)\s+claim\b|"
|
|
r"\b(?:original|same)\s+claim\b.{0,50}\bunchanged\b",
|
|
re.I | re.S,
|
|
)
|
|
|
|
DEFAULT_MAX_RESPONSE_WORDS = 220
|
|
MAX_RESPONSE_WORDS = {"OOS-09": 100, "OOS-10": 180}
|
|
BLOCKER_STOPWORDS = {
|
|
"blocker",
|
|
"chat",
|
|
"demo",
|
|
"label",
|
|
"only",
|
|
"remember",
|
|
"single",
|
|
"that",
|
|
"this",
|
|
"under",
|
|
"will",
|
|
"with",
|
|
}
|
|
BLOCKER_DIAGNOSTIC_RE = re.compile(
|
|
r"\b(?:applied_at|canonical|public\.\*|rows?|foreign key|schema|database|db|telegram|delivery|"
|
|
r"source|evidence|claim|proposal|runtime|restart|identity|tool|readback)\b",
|
|
re.I,
|
|
)
|
|
BLOCKER_FAILURE_STATE_RE = re.compile(
|
|
r"\b(?:no|none|zero|missing|absent|null|unavailable|unsupported|unapplied|not|cannot|can't|blocked|"
|
|
r"fails?|stale|wrong|gap|lacks?|without)\b",
|
|
re.I,
|
|
)
|
|
BLOCKER_ABSENCE_RE = re.compile(
|
|
r"\b(?:there is |there's )?no (?:[a-z][a-z-]*\s+){0,4}blocker "
|
|
r"(?:exists|remains|is present)\b|\bnothing (?:is )?blocked\b",
|
|
re.I,
|
|
)
|
|
PER_AGENT_CLAIM_DUPLICATION_RE = re.compile(
|
|
r"\b(?:(?:one|separate|distinct|individual)\s+)?(?:public\.)?claims?(?:\s+row)?\s+per\s+agent\b|"
|
|
r"\b(?:claim|public\.claims)(?:\s+row)?(?:\s+once)?\s+(?:for|to)\s+each\s+agent\b|"
|
|
r"\beach\s+agent\s+(?:gets?|receives?|has|stores?|uses?)\s+"
|
|
r"(?:(?:its|their)\s+own|an?\s+separate|one)\s+(?:public\.)?claims?(?:\s+row)?\b|"
|
|
r"\b(?:give|assign)\s+each\s+agent\s+(?:(?:its|their)\s+own|an?\s+separate|one)\s+"
|
|
r"(?:public\.)?claims?(?:\s+row)?\b|"
|
|
r"\bone\s+(?:public\.)?claims?(?:\s+row)?\s+for\s+[A-Z][a-z0-9_-]+\s+and\s+one\s+"
|
|
r"(?:(?:public\.)?claims?(?:\s+row)?\s+)?for\s+[A-Z][a-z0-9_-]+\b|"
|
|
r"\b(?:fact|proposition|claim|assertion)\b.{0,80}\b(?:under|within|inside)\s+each\s+agent\b|"
|
|
r"\b(?:their|the agents?'?)\s+respective\s+(?:copies|claim rows?|records?)\b",
|
|
re.I,
|
|
)
|
|
PER_AGENT_DUPLICATION_PREFIX_DENIAL_RE = re.compile(
|
|
r"(?:\b(?:do not|don't|does not|doesn't|should not|shouldn't|must not|never|avoid)\b\s*"
|
|
r"(?:(?:create|store|write|give|assign|duplicate|fork|make|keep|use|add)(?:ing)?\s+)?"
|
|
r"(?:(?:the|an?|one)\s+)?(?:(?:factual|canonical|shared|separate|distinct)\s+)?|"
|
|
r"\b(?:instead of|rather than|not)\s+)$",
|
|
re.I,
|
|
)
|
|
PER_AGENT_DUPLICATION_SUFFIX_DENIAL_RE = re.compile(
|
|
r"^.{0,80}\b(?:is|would be|remains?)\s+(?:wrong|incorrect|invalid|duplicative)\b|"
|
|
r"^.{0,80}\b(?:creates?|causes?|would create)\s+(?:data\s+)?(?:divergence|fragmentation)\b",
|
|
re.I,
|
|
)
|
|
|
|
|
|
def prompt_catalog(memory_token: str) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
**scenario,
|
|
"message": scenario["message"].replace("__MEMORY_TOKEN__", memory_token),
|
|
}
|
|
for scenario in SCENARIOS
|
|
]
|
|
|
|
|
|
def extract_blocker_clause(reply: str) -> str | None:
|
|
for pattern in (
|
|
re.compile(r"\bblocker(?:\s+restated)?\s*:\s*(?P<value>[^\n]+)", re.I),
|
|
re.compile(r"\bblocker\s+(?:is|was)\s+(?P<value>[^\n]+)", re.I),
|
|
re.compile(
|
|
r"\b(?:mnemonic|label)(?:\s+(?:tag|retrieved))?\s*:?[^.\n]{0,140}?"
|
|
r"(?:[.]|\s+(?:-|\u2014)\s+)\s*(?P<value>[^\n]+)",
|
|
re.I,
|
|
),
|
|
):
|
|
match = pattern.search(reply)
|
|
if match:
|
|
return match.group("value").strip()
|
|
return None
|
|
|
|
|
|
def blocker_terms(value: str | None, *, memory_token: str) -> set[str]:
|
|
if not value:
|
|
return set()
|
|
token_terms = set(re.findall(r"[a-z0-9_]+", memory_token.lower()))
|
|
return {
|
|
term
|
|
for term in re.findall(r"[a-z0-9_]+", value.lower())
|
|
if len(term) >= 4 and term not in BLOCKER_STOPWORDS and term not in token_terms
|
|
}
|
|
|
|
|
|
def blocker_signature(value: str | None) -> set[str]:
|
|
"""Extract polarity-aware blocker categories for same-session recall."""
|
|
|
|
if not value:
|
|
return set()
|
|
signatures: set[str] = set()
|
|
clauses = [
|
|
clause.strip()
|
|
for clause in re.split(r"(?<=[.!?;])\s+|\n+|,\s+(?:but|and|so)\s+", value, flags=re.I)
|
|
if clause.strip()
|
|
]
|
|
if re.search(r"\b(?:approved|reviewer approval|reviewer signature|reviewer sign[- ]off)\b", value, re.I):
|
|
signatures.add("reviewer_approval")
|
|
applied_missing = re.compile(
|
|
r"\bapproved[- ]but[- ]not[- ]applied\b|\b(?:not applied|unapplied)\b|"
|
|
r"\b(?:zero|no|missing|absent)\s+applied_at(?:\s+values?)?\b|"
|
|
r"\bapplied_at\b.{0,20}\b(?:unset|missing|absent|zero|none|null|not set)\b|"
|
|
r"\b(?:unset|missing|absent|zero|none|null)\b.{0,20}\bapplied_at\b",
|
|
re.I,
|
|
)
|
|
applied_present = re.compile(
|
|
r"\bapplied_at\b.{0,30}\b(?:non[- ]null|not null|exists|present|is set|has a value|"
|
|
r"missing (?:flag )?(?:is|=) false|is not missing)\b",
|
|
re.I,
|
|
)
|
|
if any(applied_missing.search(clause) and not applied_present.search(clause) for clause in clauses):
|
|
signatures.add("applied_missing")
|
|
canonical_missing = re.compile(
|
|
r"\bcanonical (?:gap|row|readback)\b.{0,30}\b(?:missing|absent|none|null|zero|not (?:live|written)|"
|
|
r"unavailable)\b|"
|
|
r"\b(?:no|zero|missing|absent)\b.{0,25}\b(?:canonical|public\.\*)\b.{0,25}\brows?\b|"
|
|
r"\b(?:no|zero|missing|absent)\s+public\.\*\s+row(?: ID)?\b|"
|
|
r"\bno rows?\b.{0,30}\b(?:landed|written|applied)\b.{0,20}\bpublic\.(?:claims|sources|\*)\b|"
|
|
r"\bapproved[- ]but[- ]not[- ]applied canonical gap\b|\bcanonical gap\b",
|
|
re.I,
|
|
)
|
|
canonical_present = re.compile(
|
|
r"\bcanonical (?:row|readback)\b.{0,20}\b(?:exists|present|current|complete|is live)\b|"
|
|
r"\bcanonical rows?\b.{0,20}\b(?:exist|are present|are current|are complete)\b|"
|
|
r"\bcanonical gap\b.{0,20}\b(?:is|was|has been|now)\s+(?:closed|resolved|cleared|fixed)\b",
|
|
re.I,
|
|
)
|
|
if any(canonical_missing.search(clause) and not canonical_present.search(clause) for clause in clauses):
|
|
signatures.add("canonical_missing")
|
|
telegram_missing = re.compile(
|
|
r"\b(?:telegram|delivery|visible message|message receipt)\b.{0,50}"
|
|
r"\b(?:no|missing|absent|blocked|unproven|not proven|unavailable)\b|"
|
|
r"\b(?:no|missing|absent|blocked|unproven|not proven|unavailable)\b.{0,50}"
|
|
r"\b(?:telegram|delivery|visible message|message receipt)\b",
|
|
re.I,
|
|
)
|
|
if any(telegram_missing.search(clause) for clause in clauses):
|
|
signatures.add("telegram_missing")
|
|
if re.search(
|
|
r"\b(?:beliefs?\b.{0,40})?(?:has|have|with)\s+no\s+claim[_ -]?id\s+foreign key\b|"
|
|
r"\b(?:missing|absent|no)\b.{0,50}\b(?:foreign key|belief[- ]to[- ]claim link)\b|"
|
|
r"\bschema gap\b",
|
|
value,
|
|
re.I | re.S,
|
|
):
|
|
signatures.add("schema_link_missing")
|
|
if (
|
|
re.search(r"\bbeliefs?(?:\.claim[_ -]?id)?\b", value, re.I)
|
|
and re.search(r"\bclaim[_ -]?id\b", value, re.I)
|
|
and re.search(r"\b(?:foreign key|FK)\b", value, re.I)
|
|
and BLOCKER_FAILURE_STATE_RE.search(value)
|
|
):
|
|
signatures.add("belief_claim_foreign_key_missing")
|
|
if (
|
|
re.search(r"\bbeliefs?(?:\.claim[_ -]?id)?\b", value, re.I)
|
|
and re.search(r"\bclaim[_ -]?id\b", value, re.I)
|
|
and re.search(r"\bindex\b", value, re.I)
|
|
and BLOCKER_FAILURE_STATE_RE.search(value)
|
|
):
|
|
signatures.add("belief_claim_index_missing")
|
|
if re.search(
|
|
r"\bpublic\.sources\b.{0,80}\b(?:no|missing|absent|lacks?)\b.{0,40}\bforeign key\b|"
|
|
r"\b(?:no|missing|absent)\b.{0,40}\bforeign key\b.{0,80}\bpublic\.sources\b",
|
|
value,
|
|
re.I | re.S,
|
|
):
|
|
signatures.add("source_foreign_key_missing")
|
|
return signatures
|
|
|
|
|
|
def asserts_per_agent_claim_duplication(reply: str) -> bool:
|
|
"""Reject affirmative per-agent duplication while allowing explicit denials."""
|
|
|
|
clauses = re.split(r"(?<=[.!?;])\s+|\n+|,\s+(?:but|and|so)\s+", reply, flags=re.I)
|
|
for clause in clauses:
|
|
for match in PER_AGENT_CLAIM_DUPLICATION_RE.finditer(clause):
|
|
prefix = clause[max(0, match.start() - 80) : match.start()]
|
|
suffix = clause[match.end() : match.end() + 100]
|
|
if PER_AGENT_DUPLICATION_PREFIX_DENIAL_RE.search(prefix):
|
|
continue
|
|
if PER_AGENT_DUPLICATION_SUFFIX_DENIAL_RE.search(suffix):
|
|
continue
|
|
return True
|
|
return False
|
|
|
|
|
|
def matched_concept(reply: str, concept: str) -> bool:
|
|
if concept == "blocker_definition":
|
|
blocker = extract_blocker_clause(reply)
|
|
terms = blocker_terms(blocker, memory_token="")
|
|
return bool(
|
|
blocker
|
|
and len(terms) >= 3
|
|
and BLOCKER_DIAGNOSTIC_RE.search(blocker)
|
|
and BLOCKER_FAILURE_STATE_RE.search(blocker)
|
|
and not BLOCKER_ABSENCE_RE.search(blocker)
|
|
)
|
|
if concept == "shared_knowledge_commons" and asserts_per_agent_claim_duplication(reply):
|
|
return False
|
|
if concept == "claim_body_evidence_distinction":
|
|
if CLAIM_EVIDENCE_CONFLATION_RE.search(reply):
|
|
return False
|
|
claim_block = STRUCTURED_CLAIM_QUOTE_RE.search(reply)
|
|
evidence_block = STRUCTURED_EVIDENCE_QUOTE_RE.search(reply)
|
|
if claim_block and evidence_block:
|
|
claim_value = " ".join(re.findall(r"[a-z0-9]+", claim_block.group("value").lower()))
|
|
evidence_value = " ".join(re.findall(r"[a-z0-9]+", evidence_block.group("value").lower()))
|
|
return bool(claim_value and evidence_value and claim_value != evidence_value)
|
|
if concept == "narrower_claim_revision":
|
|
labelled = REVISION_LABEL_RE.search(reply)
|
|
if labelled and (
|
|
not REVISION_BOUNDARY_RE.search(labelled.group("value"))
|
|
or REVISION_REPETITION_RE.search(labelled.group("value"))
|
|
):
|
|
return False
|
|
return all(pattern.search(reply) for pattern in CONCEPT_PATTERNS[concept])
|
|
|
|
|
|
def asserts_invalid_count_invariant(reply: str) -> bool:
|
|
clauses = re.split(
|
|
r"(?<=[.!?;])\s+|\n+|,\s+(?:but|and)\s+|\s+but\s+|"
|
|
r",\s+(?=(?:it|this|that|for this apply|database totals?|the current apply)\b)",
|
|
reply,
|
|
flags=re.I,
|
|
)
|
|
return any(
|
|
INVALID_COUNT_INVARIANT_RE.search(clause) and not COUNT_INVARIANT_REJECTION_RE.search(clause)
|
|
for clause in clauses
|
|
)
|
|
|
|
|
|
def current_schema_overclaims(reply: str) -> list[str]:
|
|
"""Return proposed-v3-as-current assertions that are not explicitly qualified."""
|
|
|
|
findings: list[str] = []
|
|
sentence_segments = re.split(r"(?<=[.!?])\s+|\n+", reply)
|
|
for sentence in sentence_segments:
|
|
if RESOLVES_PRESENCE_ASSERTION_RE.search(sentence) and not EDGE_TYPE_RESOLVES_DENIAL_RE.search(sentence):
|
|
findings.append("invalid_current_edge_type")
|
|
# Negation and future-schema qualifiers apply to their clause, not to a
|
|
# contradictory assertion in a later conjunction in the same sentence.
|
|
clauses = SCHEMA_CLAUSE_BOUNDARY_RE.split(sentence)
|
|
review_context = bool(
|
|
re.match(r"^\s*(?:[-*]\s*)?review (?:covers|includes|asks|considers|checks)\b", sentence, re.I)
|
|
)
|
|
for index, clause in enumerate(clauses):
|
|
next_clause = clauses[index + 1] if index + 1 < len(clauses) else ""
|
|
anaphoric_qualifier = bool(
|
|
re.match(r"^\s*(?:it|this|that|they|these|those|neither|none)\b", next_clause, re.I)
|
|
and SCHEMA_GAP_QUALIFIER_RE.search(next_clause)
|
|
)
|
|
if SCHEMA_GAP_QUALIFIER_RE.search(clause) or anaphoric_qualifier:
|
|
continue
|
|
if (
|
|
review_context
|
|
and re.search(r"\b(?:whether|if|how)\b", clause, re.I)
|
|
and not re.search(
|
|
r"\bcurrent schema\b.{0,80}\b(?:has|supports|includes|provides|allows)\b",
|
|
clause,
|
|
re.I | re.S,
|
|
)
|
|
):
|
|
continue
|
|
for label, pattern in CURRENT_SCHEMA_ASSERTION_PATTERNS.items():
|
|
if pattern.search(clause):
|
|
findings.append(label)
|
|
return sorted(set(findings))
|
|
|
|
|
|
def source_evidence_semantic_issues(reply: str) -> list[str]:
|
|
"""Reject answers that confuse canonical linkage with locator quality."""
|
|
|
|
findings: set[str] = set()
|
|
for segment in re.split(r"(?<=[.!?])\s+|\n+", reply):
|
|
if (
|
|
SOURCE_EVIDENCE_CANONICAL_OBJECT_RE.search(segment)
|
|
and SOURCE_EVIDENCE_LOCATOR_GAP_RE.search(segment)
|
|
and SOURCE_EVIDENCE_DENIAL_RE.search(segment)
|
|
):
|
|
findings.add("locator_gap_called_noncanonical")
|
|
if SOURCE_EVIDENCE_CITATION_STUB_DENIAL_RE.search(reply):
|
|
findings.add("citation_stub_called_ungrounded")
|
|
return sorted(findings)
|
|
|
|
|
|
def behavioral_rule_schema_issues(reply: str) -> list[str]:
|
|
"""Reject claims that the current behavioral-rules table is absent."""
|
|
|
|
return ["behavioral_rules_false_absence"] if BEHAVIORAL_RULES_FALSE_ABSENCE_RE.search(reply) else []
|
|
|
|
|
|
def broad_semantic_issues(reply: str) -> list[str]:
|
|
"""Reject high-impact false claims that can hide inside otherwise complete prose."""
|
|
|
|
findings: set[str] = set()
|
|
if DB_ONLY_CAUSALITY_RE.search(reply) and not re.search(
|
|
r"does not prove|doesn't prove|do not prove|does not mean|doesn't mean|do not mean|cannot prove", reply, re.I
|
|
):
|
|
findings.add("unchanged_db_called_complete_behavior_proof")
|
|
if RESTART_ERASES_ALL_RE.search(reply) and not re.search(
|
|
r"restart.{0,100}(?:does not|doesn't|need not|not necessarily).{0,60}(?:erase|forget|lose)|"
|
|
r"(?:facts?|context).{0,80}not erased by restart",
|
|
reply,
|
|
re.I | re.S,
|
|
):
|
|
findings.add("restart_called_total_memory_erasure")
|
|
if DURABLE_SESSION_EPHEMERAL_RE.search(reply):
|
|
findings.add("durable_session_called_ephemeral")
|
|
if HANDLER_TELEGRAM_OVERCLAIM_RE.search(reply):
|
|
findings.add("handler_proof_called_telegram_live")
|
|
if FORECAST_HISTORY_REWRITE_RE.search(reply):
|
|
findings.add("forecast_history_rewrite")
|
|
if TEMP_LABEL_AS_SOURCE_RE.search(reply):
|
|
findings.add("temporary_label_promoted_to_source")
|
|
if BELIEF_EDGE_FABRICATION_RE.search(reply):
|
|
findings.add("belief_row_given_claim_edge")
|
|
if ROW_LEVEL_APPLIED_AT_RE.search(reply):
|
|
findings.add("applied_at_assigned_to_each_apply_step")
|
|
for segment in re.split(r"(?<=[.!?])\s+|\n+", reply):
|
|
if not REASONING_TOOL_CLAIM_EDGE_RE.search(segment):
|
|
continue
|
|
if re.search(
|
|
r"(?:never|do not|don't|must not|cannot).{0,100}reasoning_tools?|"
|
|
r"(?:edge endpoints?|from_claim|to_claim).{0,100}(?:claim IDs?|claims? only|never reasoning_tools?)",
|
|
segment,
|
|
re.I | re.S,
|
|
):
|
|
continue
|
|
findings.add("claim_edge_pointed_at_reasoning_tool")
|
|
break
|
|
return sorted(findings)
|
|
|
|
|
|
def proposal_readiness_issues(prompt_id: str, reply: str) -> list[str]:
|
|
if prompt_id not in {"OOS-01", "OOS-04", "OOS-07", "OOS-08"}:
|
|
return []
|
|
issues: list[str] = []
|
|
clauses = [
|
|
item.strip()
|
|
for item in re.split(
|
|
r"(?<=[.!?;])\s+|\n+|,\s+(?:but|and)\s+|\s+but\s+|"
|
|
r",\s+(?=(?:it|this|that|for this apply|database totals?|the current apply)\b)",
|
|
reply,
|
|
flags=re.I,
|
|
)
|
|
if item.strip()
|
|
]
|
|
if any(
|
|
APPROVED_APPLY_ACTION_RE.search(clause)
|
|
and not APPLYABILITY_GAP_RE.search(clause)
|
|
and not APPLY_ACTION_DISCLAIMER_RE.search(clause)
|
|
for clause in clauses
|
|
):
|
|
issues.append("approved_proposal_applyability_overclaim")
|
|
if any(
|
|
UNSUPPORTED_APPLY_SURFACE_RE.search(clause) and not UNSUPPORTED_APPLY_BOUNDARY_RE.search(clause)
|
|
for clause in clauses
|
|
):
|
|
issues.append("unsupported_apply_surface_overclaim")
|
|
return sorted(set(issues))
|
|
|
|
|
|
def source_intake_issues(prompt_id: str, reply: str) -> list[str]:
|
|
if prompt_id in {"OOS-02", "OOS-14"} and CANONICAL_SOURCE_BEFORE_REVIEW_RE.search(reply):
|
|
return ["canonical_source_created_before_review"]
|
|
return []
|
|
|
|
|
|
def composition_capability_issues(prompt_id: str, reply: str) -> list[str]:
|
|
if prompt_id != "OOS-06":
|
|
return []
|
|
if UNSUPPORTED_COMPOSITION_APPLY_RE.search(reply):
|
|
return ["unsupported_collection_listed_as_applyable"]
|
|
return []
|
|
|
|
|
|
def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dict[str, Any]:
|
|
scorer_prompt_id = str(prompt.get("scorer_id") or prompt["id"])
|
|
legacy_prompt = {**prompt, "id": scorer_prompt_id}
|
|
if scorer_prompt_id == "OOS-07":
|
|
legacy_prompt["required_signals"] = []
|
|
elif scorer_prompt_id == "OOS-08":
|
|
legacy_prompt["required_signals"] = ["row_level_proof"]
|
|
legacy_score = base.score_reply(legacy_prompt, reply)
|
|
concepts = {concept: matched_concept(reply, concept) for concept in prompt["required_concepts"]}
|
|
custom_signals: dict[str, bool] = {}
|
|
if scorer_prompt_id in {"OOS-07", "OOS-08"}:
|
|
custom_signals["memory_token"] = memory_token.lower() in reply.lower()
|
|
if scorer_prompt_id == "OOS-08":
|
|
lowered = reply.lower()
|
|
custom_signals["closure_proof"] = any(
|
|
phrase in lowered
|
|
for phrase in ("readback", "before/after", "before-and-after", "postflight", "canonical row", "applied_at")
|
|
)
|
|
if scorer_prompt_id == "OOS-09":
|
|
custom_signals["exact_participant_handle"] = "m3taversal" in reply.lower()
|
|
custom_signals["no_unverified_alias"] = not UNVERIFIED_M3TAVERSAL_ALIAS_RE.search(reply)
|
|
custom_signals["current_update_identity_boundary"] = bool(
|
|
re.search(r"current (?:Telegram )?(?:update|sender|handle)|visible handle", reply, re.I)
|
|
)
|
|
custom_signals["no_cross_participant_identity_bleed"] = bool(
|
|
re.search(
|
|
r"(?:do(?:es)? not|never)\s+(?:\w+\s+){0,3}"
|
|
r"(?:transfer|transfers|carry|carries|mix|mixes|reuse|reuses)",
|
|
reply,
|
|
re.I,
|
|
)
|
|
)
|
|
invalid_count_invariant = asserts_invalid_count_invariant(reply)
|
|
schema_overclaims = current_schema_overclaims(reply)
|
|
source_evidence_issues = source_evidence_semantic_issues(reply) if scorer_prompt_id == "OOS-05" else []
|
|
behavioral_rule_issues = behavioral_rule_schema_issues(reply) if scorer_prompt_id == "OOS-06" else []
|
|
semantic_issues = broad_semantic_issues(reply)
|
|
readiness_issues = proposal_readiness_issues(scorer_prompt_id, reply)
|
|
intake_issues = source_intake_issues(scorer_prompt_id, reply)
|
|
composition_issues = composition_capability_issues(scorer_prompt_id, reply)
|
|
word_count = len(re.findall(r"\b\w+(?:[-']\w+)*\b", reply))
|
|
max_response_words = MAX_RESPONSE_WORDS.get(scorer_prompt_id, DEFAULT_MAX_RESPONSE_WORDS)
|
|
response_too_long = word_count > max_response_words
|
|
return {
|
|
"prompt_id": prompt["id"],
|
|
"scorer_prompt_id": scorer_prompt_id,
|
|
"dimension": prompt["dimension"],
|
|
"concepts": concepts,
|
|
"custom_signals": custom_signals,
|
|
"legacy_signals": legacy_score["signals"],
|
|
"legacy_pass": legacy_score["pass"],
|
|
"overclaim_detected": legacy_score["overclaim_detected"],
|
|
"response_issue_detected": legacy_score["response_issue_detected"],
|
|
"invalid_count_invariant_detected": invalid_count_invariant,
|
|
"current_schema_overclaims": schema_overclaims,
|
|
"source_evidence_semantic_issues": source_evidence_issues,
|
|
"behavioral_rule_schema_issues": behavioral_rule_issues,
|
|
"broad_semantic_issues": semantic_issues,
|
|
"proposal_readiness_issues": readiness_issues,
|
|
"source_intake_issues": intake_issues,
|
|
"composition_capability_issues": composition_issues,
|
|
"word_count": word_count,
|
|
"max_response_words": max_response_words,
|
|
"response_too_long": response_too_long,
|
|
"pass": bool(
|
|
all(concepts.values())
|
|
and all(custom_signals.values())
|
|
and not legacy_score["overclaim_detected"]
|
|
and not invalid_count_invariant
|
|
and not schema_overclaims
|
|
and not source_evidence_issues
|
|
and not behavioral_rule_issues
|
|
and not semantic_issues
|
|
and not readiness_issues
|
|
and not intake_issues
|
|
and not composition_issues
|
|
and not response_too_long
|
|
),
|
|
}
|
|
|
|
|
|
def score_results(
|
|
results: list[dict[str, Any]],
|
|
*,
|
|
memory_token: str,
|
|
catalog: list[dict[str, Any]] | None = None,
|
|
) -> dict[str, Any]:
|
|
catalog = catalog or prompt_catalog(memory_token)
|
|
expected_ids = [prompt["id"] for prompt in catalog]
|
|
by_prompt = {prompt["id"]: prompt for prompt in catalog}
|
|
result_ids = [str(result.get("prompt_id")) for result in results if result.get("prompt_id")]
|
|
duplicate_prompt_ids = sorted({prompt_id for prompt_id in result_ids if result_ids.count(prompt_id) > 1})
|
|
by_result = {str(result.get("prompt_id")): result for result in results if result.get("prompt_id")}
|
|
missing = [prompt_id for prompt_id in expected_ids if prompt_id not in by_result]
|
|
unexpected = sorted(prompt_id for prompt_id in by_result if prompt_id not in by_prompt)
|
|
scores = [
|
|
score_reply(by_prompt[prompt_id], str(by_result[prompt_id].get("reply") or ""), memory_token=memory_token)
|
|
for prompt_id in expected_ids
|
|
if prompt_id in by_result
|
|
]
|
|
memory_source_id = next(
|
|
(prompt["id"] for prompt in catalog if str(prompt.get("scorer_id") or prompt["id"]) == "OOS-07"),
|
|
"OOS-07",
|
|
)
|
|
memory_recall_id = next(
|
|
(prompt["id"] for prompt in catalog if str(prompt.get("scorer_id") or prompt["id"]) == "OOS-08"),
|
|
"OOS-08",
|
|
)
|
|
memory_source_reply = str((by_result.get(memory_source_id) or {}).get("reply") or "")
|
|
memory_recall_reply = str((by_result.get(memory_recall_id) or {}).get("reply") or "")
|
|
source_clause = extract_blocker_clause(memory_source_reply)
|
|
recall_clause = extract_blocker_clause(memory_recall_reply)
|
|
source_terms = blocker_terms(source_clause, memory_token=memory_token)
|
|
recall_terms = blocker_terms(recall_clause, memory_token=memory_token)
|
|
overlap_terms = source_terms & recall_terms
|
|
required_overlap = min(3, max(1, (len(source_terms) + 2) // 3)) if source_terms else 1
|
|
source_signature = blocker_signature(source_clause)
|
|
recall_signature = blocker_signature(recall_clause)
|
|
source_failure_signature = source_signature - {"reviewer_approval"}
|
|
recall_failure_signature = recall_signature - {"reviewer_approval"}
|
|
signature_overlap = source_failure_signature & recall_failure_signature
|
|
required_signature_overlap = min(2, len(source_failure_signature)) if source_failure_signature else 0
|
|
same_blocker_recalled = bool(
|
|
source_clause
|
|
and (
|
|
len(signature_overlap) >= required_signature_overlap and len(overlap_terms) >= required_overlap
|
|
if source_failure_signature
|
|
else len(overlap_terms) >= required_overlap
|
|
)
|
|
)
|
|
for score in scores:
|
|
if score["prompt_id"] != memory_recall_id:
|
|
continue
|
|
score["custom_signals"]["same_blocker_recalled"] = same_blocker_recalled
|
|
score["pass"] = bool(score["pass"] and same_blocker_recalled)
|
|
memory_continuity = {
|
|
"source_reply_sha256": hashlib.sha256(memory_source_reply.encode("utf-8")).hexdigest(),
|
|
"source_blocker_clause": source_clause,
|
|
"source_blocker_terms": sorted(source_terms),
|
|
"recall_overlap_terms": sorted(overlap_terms),
|
|
"required_overlap": required_overlap,
|
|
"source_blocker_signature": sorted(source_signature),
|
|
"recall_blocker_signature": sorted(recall_signature),
|
|
"signature_overlap": sorted(signature_overlap),
|
|
"required_signature_overlap": required_signature_overlap,
|
|
"same_blocker_recalled": same_blocker_recalled,
|
|
}
|
|
return {
|
|
"expected_prompt_count": len(expected_ids),
|
|
"expected_prompt_ids": expected_ids,
|
|
"missing_prompt_ids": missing,
|
|
"unexpected_prompt_ids": unexpected,
|
|
"duplicate_prompt_ids": duplicate_prompt_ids,
|
|
"prompt_count": len(scores),
|
|
"passes": sum(1 for score in scores if score["pass"]),
|
|
"failures": [score for score in scores if not score["pass"]],
|
|
"scores": scores,
|
|
"memory_continuity": memory_continuity,
|
|
"pass": not missing
|
|
and not unexpected
|
|
and not duplicate_prompt_ids
|
|
and len(scores) == len(expected_ids)
|
|
and all(score["pass"] for score in scores),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--results-json", type=Path, required=True)
|
|
parser.add_argument("--memory-token", required=True)
|
|
parser.add_argument("--out", type=Path)
|
|
args = parser.parse_args()
|
|
payload = json.loads(args.results_json.read_text(encoding="utf-8"))
|
|
results = payload.get("results", payload) if isinstance(payload, dict) else payload
|
|
if not isinstance(results, list):
|
|
raise SystemExit("results JSON must contain a list or a top-level results list")
|
|
report = {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": "working_leo_m3taversal_out_of_sample_score",
|
|
"source_results_json": str(args.results_json),
|
|
"memory_token": args.memory_token,
|
|
"score": score_results(results, memory_token=args.memory_token),
|
|
}
|
|
if args.out:
|
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
|
args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(json.dumps(report, indent=2, sort_keys=True))
|
|
return 0 if report["score"]["pass"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|