Merge pull request #111 from living-ip/codex/leo-deterministic-context-contracts-20260713
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
Drive Leo answers from live database contracts
This commit is contained in:
twentyOne2x
GitHub
commit
e41c02f18d
11 changed files with 758 additions and 8 deletions
|
|
@ -0,0 +1,40 @@
|
|||
# Deterministic Context Contract Live Canary
|
||||
|
||||
## Canary
|
||||
|
||||
The proposed `kb_tool.py` was streamed to `python3 - --local` on the VPS and ran
|
||||
two read-only `context` queries against the real `teleo-pg` Postgres database.
|
||||
The proposed file was not installed during this canary.
|
||||
For schema-sensitive contracts, the bridge read the current columns from
|
||||
`information_schema.columns`; it did not rely on the fallback code constants.
|
||||
|
||||
## Source Intake Contract
|
||||
|
||||
The document/intake query returned:
|
||||
|
||||
- reply target `170`, hard maximum `220`;
|
||||
- required lead: local compiler proven; live VPS/chat intake not shipped;
|
||||
- build-local compiler capability tier;
|
||||
- retained artifact plus URL/storage path plus content hash;
|
||||
- pending-review candidate packet before canonical apply;
|
||||
- exact current `public.sources` columns and explicit rejection of invented
|
||||
author/title/publisher/publication-date columns.
|
||||
|
||||
## Mixed Packet Contract
|
||||
|
||||
The heterogeneous-packet query returned:
|
||||
|
||||
- facts/disputes -> claims plus sources/evidence;
|
||||
- reusable framework -> `reasoning_tools`;
|
||||
- operating rule -> `behavioral_rules`;
|
||||
- evaluative gate -> `governance_gates`;
|
||||
- agent position -> `beliefs`;
|
||||
- exact supported `approve_claim` collections and exact current columns;
|
||||
- explicit guards against reasoning-tool/belief edge endpoints, unshipped
|
||||
fields, per-row `applied_at`, and unsupported writes in the apply sequence.
|
||||
|
||||
## Safety Readback
|
||||
|
||||
After both queries, canonical counts remained `1837/4145/4670/4916/26`.
|
||||
`leoclean-gateway.service` remained active/running on PID `3166667` with zero
|
||||
restarts; the deployed checkout remained clean at `ee06d0d`.
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
{
|
||||
"before_service": {
|
||||
"ActiveState": "active",
|
||||
"ExecMainStartTimestamp": "Mon 2026-07-13 09:45:35 UTC",
|
||||
"MainPID": "3166667",
|
||||
"NRestarts": "0",
|
||||
"SubState": "running"
|
||||
},
|
||||
"changed_live_profile": false,
|
||||
"db_counts_after": {
|
||||
"kb_stage.kb_proposals": 26,
|
||||
"public.claim_edges": 4916,
|
||||
"public.claim_evidence": 4670,
|
||||
"public.claims": 1837,
|
||||
"public.sources": 4145
|
||||
},
|
||||
"db_counts_before": {
|
||||
"kb_stage.kb_proposals": 26,
|
||||
"public.claim_edges": 4916,
|
||||
"public.claim_evidence": 4670,
|
||||
"public.claims": 1837,
|
||||
"public.sources": 4145
|
||||
},
|
||||
"db_counts_changed": false,
|
||||
"generated_at_utc": "2026-07-13T10:08:46.596883+00:00",
|
||||
"handler": {
|
||||
"authorized": true,
|
||||
"session_key": "agent:main:telegram:group:-5146042086:9070919",
|
||||
"temp_profile": "/tmp/leo-direct-claim-handler-uatcn7c_/profile"
|
||||
},
|
||||
"harness_notes": [
|
||||
"Third unchanged five-prompt canary after reply-budget and capability-adherence repair.",
|
||||
"The harness uses a temporary copy of the leoclean profile and does not post to Telegram.",
|
||||
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery."
|
||||
],
|
||||
"mode": "live_vps_gatewayrunner_temp_profile_m3taversal_targeted_post_runtime_adherence_repair",
|
||||
"mutates_kb_by_harness": false,
|
||||
"pass_runtime": true,
|
||||
"posted_to_telegram": false,
|
||||
"remote_report_path": "/tmp/leo-m3taversal-oos-targeted-post-runtime-adherence-repair-c16aa2e1268b.json",
|
||||
"remote_returncode": 0,
|
||||
"remote_run_id": "c16aa2e1268b",
|
||||
"results": [
|
||||
{
|
||||
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
||||
"dimension": "partner_demo_claim_ceiling",
|
||||
"ended_at_utc": "2026-07-13T10:09:12.047793+00:00",
|
||||
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
||||
"mutates_kb": false,
|
||||
"ok": true,
|
||||
"prompt": "Three changes have reviewer approval and the partner demo starts in an hour. Can I honestly say the knowledge database is updated? Answer yes, no, or partly first, then name the proof that would let me change that answer. Do not apply anything.",
|
||||
"prompt_id": "OOS-04",
|
||||
"reply": "No.\n\nThe DB shows 3 proposals at status approved, 0 of the three have a non-null applied_at. All three read approved_needs_apply_payload \u2014 reviewer sign-off is intent, not a canonical write.\n\nDB readback: claims: 1837; sources: 4145; claim_edges: 4916; claim_evidence: 4670; kb_proposals: 26 (applied: 2, approved: 3, pending: 14).\n\nProof that changes the answer: non-null applied_at timestamps on all three approved proposal rows, confirmed by a fresh teleo-kb status showing the row counts moved. Nothing short of that is a canonical DB update.",
|
||||
"started_at_utc": "2026-07-13T10:08:47.075688+00:00",
|
||||
"turn": 1
|
||||
},
|
||||
{
|
||||
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
||||
"dimension": "shared_claims_agent_positions",
|
||||
"ended_at_utc": "2026-07-13T10:09:27.654528+00:00",
|
||||
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
||||
"mutates_kb": false,
|
||||
"ok": true,
|
||||
"prompt": "Two agents read the same evidence and reach different conclusions. Should Leo duplicate the factual claim once per agent, or keep shared knowledge and represent agent-specific positions elsewhere? Give the current-schema answer and how disagreement remains queryable. Do not write anything.",
|
||||
"prompt_id": "OOS-11",
|
||||
"reply": "Current-schema answer, read-only:\n\nOne factual claim, stored once in public.claims with shared evidence in public.claim_evidence pointing to public.sources. Do not duplicate it.\n\nAgent-specific positions live in public.beliefs: each row carries agent_id, statement, confidence, falsifier, level, and status. Two agents who read the same evidence and disagree each get their own beliefs row \u2014 same underlying claim, different belief rows, different agent_id values.\n\nDisagreement is queryable by selecting from public.beliefs where the statement references the same subject and comparing across agent_id values. Confidence and falsifier fields make the divergence precise: you can see not just that they disagree but how confident each is and what would change their mind.\n\nCurrent constraint to be honest about: public.beliefs has no claim-ID foreign key and there is no belief-edge table. The schema does not enforce a direct link between a belief row and the specific claim it responds to \u2014 that relationship lives in the statement text or must be inferred. public.claim_edges connects claims to claims, never belief rows to anything. So the disagreement is queryable but the claim-to-belief join is not schema-guaranteed today.",
|
||||
"started_at_utc": "2026-07-13T10:09:12.048403+00:00",
|
||||
"turn": 2
|
||||
},
|
||||
{
|
||||
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
||||
"dimension": "runtime_vs_canonical_causality",
|
||||
"ended_at_utc": "2026-07-13T10:09:44.183239+00:00",
|
||||
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
||||
"mutates_kb": false,
|
||||
"ok": true,
|
||||
"prompt": "The five database totals are unchanged after a restart. Does that prove Leo's answer behavior is unchanged and that every fact from the prior session was erased? Explain which persisted and deployed surfaces can affect the answer, and separate the proof tiers. Read-only; keep it under 180 words.",
|
||||
"prompt_id": "OOS-10",
|
||||
"reply": "Neither claim holds.\n\nUnchanged totals prove only that no rows were inserted or deleted between the two reads. They do not prove rows were not updated \u2014 a modified claim, belief, or reasoning_tool leaves the count identical. Unchanged totals say nothing about answer behavior.\n\nProof tiers for what shapes an answer:\n\nCanonical DB (strongest): public.claims, public.beliefs, public.reasoning_tools, public.behavioral_rules. Persists across restarts. Unchanged totals give weak assurance here; row-level IDs, hashes, or updated_at timestamps give strong assurance.\n\nRuntime identity (injected each session): SOUL.md render, active Hermes profile config, model selection. Restart can change model or profile without touching DB totals. These directly affect answer behavior.\n\nSkill and memory layer (disk-persistent): Skills on disk and Hermes memory are loaded each session. Neither appears in DB totals. Both affect answers and survive restarts.\n\nSession continuity (weakest): Hermes state.db and session JSONL can carry context across a restart but are not canonical DB writes.\n\nFacts from the prior session are not erased by restart \u2014 all four tiers above persist. A restart clears only the in-memory conversation window.",
|
||||
"started_at_utc": "2026-07-13T10:09:27.654813+00:00",
|
||||
"turn": 3
|
||||
},
|
||||
{
|
||||
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
||||
"dimension": "autonomous_source_intake_boundary",
|
||||
"ended_at_utc": "2026-07-13T10:09:54.867871+00:00",
|
||||
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
||||
"mutates_kb": false,
|
||||
"ok": true,
|
||||
"prompt": "I hand Leo a document and say: absorb this as far as safely possible without making me approve every mechanical step. What can Leo capture and stage immediately, what real source identity must be retained, and where does explicit approval begin? Explain only; do not ingest this prompt.",
|
||||
"prompt_id": "OOS-14",
|
||||
"reply": "Three zones, hard boundaries between them.\n\nWhat Leo can do immediately without approval:\n\nRead and summarize the document. Extract candidate claims, identify which reasoning_tool or behavioral_rule neighborhood they belong to, draft a structured proposal packet with headline, body, evidence excerpt, proposed edge types, and evidence role. Stage that packet as a pending_review proposal in kb_proposals. All of this is local compilation \u2014 no canonical table is touched.\n\nWhat real source identity must be retained in that packet:\n\nA Telegram attachment, filename, or chat label is not a source identity. The packet must carry a retained artifact path or URL, a content hash, and the full excerpt text. Without those three the evidence link is unverifiable and the proposal will fail review. Richer metadata \u2014 author, title, date \u2014 should be retained in the packet even though current public.sources has no columns for them; they wait for a schema that can store them canonically.\n\nWhere explicit approval begins:\n\nAt the boundary between kb_proposals and public.claims / public.sources / public.claim_evidence. Nothing crosses that boundary without human review, an explicit apply authorization, a strict validated payload, a live apply worker, a committed transaction, and non-null applied_at confirmed by postflight row-count proof. Reviewer sign-off alone is intent, not a write. That line does not move regardless of how mechanical the steps feel.",
|
||||
"started_at_utc": "2026-07-13T10:09:44.183659+00:00",
|
||||
"turn": 4
|
||||
},
|
||||
{
|
||||
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
|
||||
"dimension": "database_composition_judgment",
|
||||
"ended_at_utc": "2026-07-13T10:10:09.239010+00:00",
|
||||
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
|
||||
"mutates_kb": false,
|
||||
"ok": true,
|
||||
"prompt": "Suppose a new research packet contains factual observations, a strategic framework, a disputed interpretation, a governance rule, and a correction to an old belief. How should Leo compose the database so it can reason over the packet later without flattening it all into claims? Give the review and apply sequence too, but do not write anything now.",
|
||||
"prompt_id": "OOS-06",
|
||||
"reply": "Five content types, five destinations.\n\nFactual observations \u2192 public.claims (type empirical or structural) with public.sources rows carrying artifact path, hash, excerpt, and public.claim_evidence linking them with role grounds or illustrates.\n\nStrategic framework \u2192 public.reasoning_tools. This is a reusable method or lens, not a stance. It gets a name, description, and scope. Claims that use it get edges of type derives_from or relates pointing at the tool's ID \u2014 that is how later reasoning can trace which framework shaped a claim.\n\nDisputed interpretation \u2192 public.claims (type normative or empirical) with an explicit confidence tier label and a falsifier field. If two agents read it differently, both get public.beliefs rows as described earlier. The claim stays singular; the divergence lives in beliefs.\n\nGovernance rule \u2192 public.behavioral_rules. Not a claim, not a reasoning_tool. It carries an agent scope, a rule text, and a status. Current approve_claim cannot insert behavioral_rules \u2014 this requires a separately reviewed capability and must be staged and held until that path is live.\n\nBelief correction \u2192 public.beliefs update on the existing row for the relevant agent_id. Current approve_claim also cannot update an existing belief row. Stage it separately; it cannot travel with the rest of the packet.\n\nReview and apply sequence:\n\nStage one reviewable proposal per supported type \u2014 claims/sources/evidence/edges/reasoning_tools in one packet, since approve_claim supports all five. Behavioral_rules correction and belief update go into separate staged proposals marked pending a future capability.\n\nReview confirms: each item maps to the right collection, source identity is complete, no self-merge, payload is strict and validated.\n\nApply executes only the supported packet first. Worker must be live, transaction committed, applied_at non-null confirmed by postflight row counts. The held proposals wait for their capability gates independently \u2014 they do not block the supported apply, and the supported apply does not authorize them.",
|
||||
"started_at_utc": "2026-07-13T10:09:54.868199+00:00",
|
||||
"turn": 5
|
||||
}
|
||||
],
|
||||
"service_before_after": {
|
||||
"after": {
|
||||
"ActiveState": "active",
|
||||
"ExecMainStartTimestamp": "Mon 2026-07-13 09:45:35 UTC",
|
||||
"MainPID": "3166667",
|
||||
"NRestarts": "0",
|
||||
"SubState": "running"
|
||||
},
|
||||
"before": {
|
||||
"ActiveState": "active",
|
||||
"ExecMainStartTimestamp": "Mon 2026-07-13 09:45:35 UTC",
|
||||
"MainPID": "3166667",
|
||||
"NRestarts": "0",
|
||||
"SubState": "running"
|
||||
},
|
||||
"unchanged_from_preexisting_live_readback": true
|
||||
},
|
||||
"source": {
|
||||
"chat_id": "-5146042086",
|
||||
"chat_name": "Leo",
|
||||
"chat_type": "group",
|
||||
"platform": "telegram",
|
||||
"user_id": "9070919",
|
||||
"user_name": "codex handler direct claim"
|
||||
},
|
||||
"source_report_path": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-runtime-adherence-repair-20260713.json",
|
||||
"temp_profile_removed": true
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# Post-Runtime-Adherence Targeted Handler Check
|
||||
|
||||
## Result
|
||||
|
||||
After PR #110 deployed at merge `ee06d0da9bb57987732edf657e5ca1b4e22ade65`,
|
||||
the unchanged five-prompt canary ran through the live VPS GatewayRunner.
|
||||
|
||||
- Strict result after semantic calibration: `3/5`.
|
||||
- Passed: proposal readiness (`OOS-04`), shared claims versus agent positions
|
||||
(`OOS-11`), and restart/persistence reasoning (`OOS-10`).
|
||||
- Source intake (`OOS-14`) stayed under the hard limit and preserved current
|
||||
source fields, but still implied immediate live compilation instead of stating
|
||||
that the compiler is build-local and not exposed to VPS/chat.
|
||||
- Mixed-packet composition (`OOS-06`) mapped the framework to
|
||||
`reasoning_tools`, but invented claim edges to a reasoning-tool ID, invented
|
||||
unshipped fields, and used `302` words.
|
||||
|
||||
## Safety Readback
|
||||
|
||||
- Canonical counts stayed `1837/4145/4670/4916/26`.
|
||||
- `leoclean-gateway.service` stayed active/running on PID `3166667` with zero
|
||||
restarts and the same start timestamp.
|
||||
- The temporary profile was removed.
|
||||
- No Telegram message or production DB apply ran.
|
||||
|
||||
Instruction-only tuning is not stable enough for the remaining two cases. The
|
||||
next repair moves their exact current capability and schema contracts into the
|
||||
`teleo-kb context` tool output selected from each operator question.
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
{
|
||||
"db_counts_changed": false,
|
||||
"expected_prompt_count": 5,
|
||||
"generated_at_utc": "2026-07-13T10:17:07.249420+00:00",
|
||||
"mode": "working_leo_m3taversal_targeted_post_runtime_adherence_repair_strict_score",
|
||||
"pass": false,
|
||||
"passes": 3,
|
||||
"posted_to_telegram": false,
|
||||
"production_db_apply_ran": false,
|
||||
"scores": [
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"canonical_readback": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "partner_demo_claim_ceiling",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"canonical_db": true,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true,
|
||||
"state_semantics": false
|
||||
},
|
||||
"max_response_words": 220,
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-04",
|
||||
"proposal_readiness_issues": [],
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"source_intake_issues": [],
|
||||
"word_count": 81
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"agent_specific_positions": true,
|
||||
"contradiction": true,
|
||||
"shared_knowledge_commons": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "shared_claims_agent_positions",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"no_overclaim": false
|
||||
},
|
||||
"max_response_words": 220,
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-11",
|
||||
"proposal_readiness_issues": [],
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"source_intake_issues": [],
|
||||
"word_count": 185
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"durable_session_continuity": true,
|
||||
"proof_tiers": true,
|
||||
"row_content_proof": true,
|
||||
"runtime_inputs": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "runtime_vs_canonical_causality",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"no_overclaim": true
|
||||
},
|
||||
"max_response_words": 180,
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-10",
|
||||
"proposal_readiness_issues": [],
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"source_intake_issues": [],
|
||||
"word_count": 178
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"bounded_intake_tier": false,
|
||||
"real_source_identity": true,
|
||||
"staged_review_apply": true,
|
||||
"staging_without_apply_authorization": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "autonomous_source_intake_boundary",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"authorization": true,
|
||||
"no_overclaim": false,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 220,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-14",
|
||||
"proposal_readiness_issues": [],
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"source_intake_issues": [],
|
||||
"word_count": 217
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [
|
||||
"claim_edge_pointed_at_reasoning_tool"
|
||||
],
|
||||
"concepts": {
|
||||
"behavioral_rule_storage": false,
|
||||
"heterogeneous_types": true,
|
||||
"receipt": true,
|
||||
"reviewed_policy_apply": false,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "database_composition_judgment",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": false,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 220,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-06",
|
||||
"proposal_readiness_issues": [],
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"source_intake_issues": [],
|
||||
"word_count": 302
|
||||
}
|
||||
],
|
||||
"selected_prompt_ids": [
|
||||
"OOS-04",
|
||||
"OOS-11",
|
||||
"OOS-10",
|
||||
"OOS-14",
|
||||
"OOS-06"
|
||||
],
|
||||
"service_unchanged": true,
|
||||
"source_results_json": "docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-targeted-post-runtime-adherence-repair-20260713.json",
|
||||
"temp_profile_removed": true
|
||||
}
|
||||
|
|
@ -22,6 +22,56 @@ from typing import Any
|
|||
|
||||
DEFAULT_CLAIM_BASE_URL = "https://leo.livingip.xyz"
|
||||
WORKER_SUPPORTED_PROPOSAL_TYPES = frozenset({"revise_strategy", "add_edge", "attach_evidence", "approve_claim"})
|
||||
CURRENT_PUBLIC_SCHEMA = {
|
||||
"behavioral_rules": ("id", "agent_id", "category", "rank", "rule", "rationale", "created_at"),
|
||||
"beliefs": (
|
||||
"id",
|
||||
"agent_id",
|
||||
"level",
|
||||
"statement",
|
||||
"confidence",
|
||||
"falsifier",
|
||||
"rank",
|
||||
"status",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
),
|
||||
"claim_edges": ("id", "from_claim", "to_claim", "edge_type", "weight", "created_by", "created_at"),
|
||||
"claims": (
|
||||
"id",
|
||||
"type",
|
||||
"text",
|
||||
"status",
|
||||
"confidence",
|
||||
"tags",
|
||||
"created_by",
|
||||
"superseded_by",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
),
|
||||
"governance_gates": (
|
||||
"id",
|
||||
"agent_id",
|
||||
"name",
|
||||
"criteria",
|
||||
"evidence_bar",
|
||||
"pass_condition",
|
||||
"rank",
|
||||
"created_at",
|
||||
),
|
||||
"reasoning_tools": ("id", "agent_id", "name", "description", "category", "created_at"),
|
||||
"sources": (
|
||||
"id",
|
||||
"source_type",
|
||||
"url",
|
||||
"storage_path",
|
||||
"excerpt",
|
||||
"hash",
|
||||
"captured_at",
|
||||
"created_by",
|
||||
"created_at",
|
||||
),
|
||||
}
|
||||
|
||||
STOPWORDS = {
|
||||
"a",
|
||||
|
|
@ -279,6 +329,167 @@ def with_proposal_readiness(proposal: dict[str, Any]) -> dict[str, Any]:
|
|||
return {**proposal, "readiness": classify_proposal_readiness(proposal)}
|
||||
|
||||
|
||||
def operational_contracts(
|
||||
query: str, *, current_schema: dict[str, list[str] | tuple[str, ...]] | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return question-specific current-runtime truth, not recalled architecture."""
|
||||
|
||||
lowered = query.lower()
|
||||
schema = current_schema if current_schema is not None else CURRENT_PUBLIC_SCHEMA
|
||||
contracts: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "reply_budget",
|
||||
"target_words": 170,
|
||||
"hard_max_words": 220,
|
||||
"shape": "direct answer; at most three short bullets; at most one proof-changing action",
|
||||
}
|
||||
]
|
||||
|
||||
source_terms = ("document", "pdf", "tweet", "attachment", "ingest", "intake", "absorb", "extract")
|
||||
if any(term in lowered for term in source_terms):
|
||||
contracts.append(
|
||||
{
|
||||
"id": "source_intake",
|
||||
"required_lead": "Local compiler proven; live VPS/chat intake not shipped.",
|
||||
"capability_tier": "build-local compiler only",
|
||||
"safe_without_apply_authorization": [
|
||||
"retain artifact plus URL or storage_path plus content hash",
|
||||
"extract candidates into a pending_review proposal packet",
|
||||
],
|
||||
"approval_begins": "before guarded canonical public.* apply, not before mechanical capture/staging",
|
||||
"current_public_sources_columns": list(schema.get("sources", ())),
|
||||
"must_not_claim": [
|
||||
"live VPS Leo can invoke the compiler from chat",
|
||||
"a chat label, filename, attachment ref, or source_ref is source identity",
|
||||
"author/title/publisher/publication date are current public.sources columns",
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
mixed_markers = (
|
||||
"packet",
|
||||
"factual observation",
|
||||
"strategic framework",
|
||||
"disputed interpretation",
|
||||
"governance rule",
|
||||
"old belief",
|
||||
"compose the database",
|
||||
)
|
||||
if sum(marker in lowered for marker in mixed_markers) >= 2:
|
||||
contracts.append(
|
||||
{
|
||||
"id": "mixed_packet_composition",
|
||||
"map": {
|
||||
"facts_and_disputes": "claims plus sources and evidence",
|
||||
"reusable_framework": "reasoning_tools",
|
||||
"operating_rule": "behavioral_rules",
|
||||
"evaluative_gate": "governance_gates",
|
||||
"agent_position": "beliefs",
|
||||
},
|
||||
"approve_claim_supported": ["claims", "sources", "evidence", "edges", "reasoning_tools"],
|
||||
"keep_staged_until_separate_capability": [
|
||||
"behavioral_rules",
|
||||
"governance_gates",
|
||||
"belief updates",
|
||||
"existing-row updates",
|
||||
],
|
||||
"schema_guards": [
|
||||
"claim_edges endpoints are claim IDs; never point an edge at a reasoning_tool or belief row",
|
||||
"reasoning_tools has no scope field",
|
||||
"claims has no falsifier field",
|
||||
"behavioral_rules has no status field",
|
||||
"applied_at belongs to the proposal receipt, not each row",
|
||||
],
|
||||
"current_columns": {
|
||||
name: list(schema.get(name, ()))
|
||||
for name in (
|
||||
"claims",
|
||||
"claim_edges",
|
||||
"reasoning_tools",
|
||||
"behavioral_rules",
|
||||
"governance_gates",
|
||||
"beliefs",
|
||||
)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if any(term in lowered for term in ("restart", "prior session", "erased", "answer behavior", "database totals")):
|
||||
contracts.append(
|
||||
{
|
||||
"id": "runtime_persistence",
|
||||
"count_boundary": "unchanged totals do not prove unchanged rows or unchanged answers",
|
||||
"persisted_inputs": [
|
||||
"Postgres rows",
|
||||
"skills",
|
||||
"runtime config",
|
||||
"SOUL.md",
|
||||
"state.db",
|
||||
"session JSONL",
|
||||
],
|
||||
"proof": "use row IDs/timestamps/hashes; separate handler, Telegram-visible delivery, and canonical mutation",
|
||||
}
|
||||
)
|
||||
|
||||
if any(
|
||||
term in lowered
|
||||
for term in ("two agents", "agent-specific", "different conclusions", "duplicate the factual claim")
|
||||
):
|
||||
contracts.append(
|
||||
{
|
||||
"id": "shared_claims_agent_positions",
|
||||
"fact_storage": "one shared claim plus shared sources/evidence",
|
||||
"position_storage": "one public.beliefs row per agent position",
|
||||
"schema_gap": "beliefs has no claim-ID foreign key and there is no belief-edge table",
|
||||
"edge_boundary": "claim_edges connects claims only",
|
||||
}
|
||||
)
|
||||
|
||||
if any(term in lowered for term in ("reviewer approval", "approved", "partner demo", "database updated")):
|
||||
contracts.append(
|
||||
{
|
||||
"id": "proposal_apply_readiness",
|
||||
"boundary": "reviewer approval is intent, not canonical apply",
|
||||
"required_readback": ["readiness", "applied_at", "row-level postflight"],
|
||||
"legacy_rule": "approved_needs_apply_payload requires strict normalization plus renewed review; do not apply the legacy row",
|
||||
}
|
||||
)
|
||||
|
||||
return contracts
|
||||
|
||||
|
||||
def load_current_public_schema(args: argparse.Namespace, table_names: set[str]) -> dict[str, list[str]]:
|
||||
if not table_names:
|
||||
return {}
|
||||
sql = f"""
|
||||
select jsonb_build_object(
|
||||
'table_name', table_name,
|
||||
'columns', jsonb_agg(column_name order by ordinal_position)
|
||||
)::text
|
||||
from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
and table_name = any({sql_array(sorted(table_names))})
|
||||
group by table_name
|
||||
order by table_name;
|
||||
"""
|
||||
return {row["table_name"]: row["columns"] for row in psql_json(args, sql)}
|
||||
|
||||
|
||||
def context_operational_contracts(args: argparse.Namespace, query: str) -> list[dict[str, Any]]:
|
||||
initial = operational_contracts(query)
|
||||
contract_ids = {contract["id"] for contract in initial}
|
||||
table_names: set[str] = set()
|
||||
if "source_intake" in contract_ids:
|
||||
table_names.add("sources")
|
||||
if "mixed_packet_composition" in contract_ids:
|
||||
table_names.update(
|
||||
{"claims", "claim_edges", "reasoning_tools", "behavioral_rules", "governance_gates", "beliefs"}
|
||||
)
|
||||
if not table_names:
|
||||
return initial
|
||||
return operational_contracts(query, current_schema=load_current_public_schema(args, table_names))
|
||||
|
||||
|
||||
def claim_base_url() -> str:
|
||||
return os.environ.get("TELEO_KB_CLAIM_BASE_URL", DEFAULT_CLAIM_BASE_URL).rstrip("/")
|
||||
|
||||
|
|
@ -955,6 +1166,7 @@ def bundle(args: argparse.Namespace, query: str, claim_limit: int, context_limit
|
|||
return {
|
||||
"query": query,
|
||||
"terms": query_terms(query),
|
||||
"operational_contracts": context_operational_contracts(args, query),
|
||||
"claims": [
|
||||
{
|
||||
**claim,
|
||||
|
|
@ -987,6 +1199,12 @@ def print_status(data: dict[str, Any]) -> None:
|
|||
|
||||
def print_claim_bundle(data: dict[str, Any]) -> None:
|
||||
print(f"# Teleo KB Context\n\nQuery: {data['query']}\n")
|
||||
contracts = data.get("operational_contracts") or []
|
||||
if contracts:
|
||||
print("## Current Runtime Contracts\n")
|
||||
for contract in contracts:
|
||||
print(f"- `{contract['id']}`: `{json.dumps(contract, sort_keys=True)}`")
|
||||
print()
|
||||
if data["context_rows"]:
|
||||
print("## Soul / Context Rows\n")
|
||||
for row in data["context_rows"]:
|
||||
|
|
@ -1114,6 +1332,7 @@ def main() -> int:
|
|||
data = {
|
||||
"query": args.query,
|
||||
"terms": query_terms(args.query),
|
||||
"operational_contracts": context_operational_contracts(args, args.query),
|
||||
"claims": find_claims(args, args.query, args.limit),
|
||||
"context_rows": find_context_rows(args, args.query, args.context_limit),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,6 +100,10 @@ question, run the local bridge:
|
|||
/home/teleo/.hermes/profiles/leoclean/bin/teleo-kb context "<question>"
|
||||
```
|
||||
|
||||
Treat the emitted `Current Runtime Contracts` / `operational_contracts` block as
|
||||
binding deployed-code readback. Follow its `required_lead`, schema guards,
|
||||
capability tier, and reply budget. It overrides recalled or generic architecture.
|
||||
|
||||
Use narrower bridge commands when needed:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -96,6 +96,10 @@ This is the VPS production leoclean surface. Start with:
|
|||
/home/teleo/.hermes/profiles/leoclean/bin/teleo-kb context "<question>"
|
||||
```
|
||||
|
||||
Treat the emitted `Current Runtime Contracts` / `operational_contracts` block as
|
||||
binding deployed-code readback. Follow its `required_lead`, schema guards,
|
||||
capability tier, and reply budget. It overrides recalled or generic architecture.
|
||||
|
||||
Use only the bounded follow-ups relevant to the question:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -245,7 +245,7 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
r"approved is not the same as applied|applied_at:\s*(?:none|null)|not canonical|"
|
||||
r"proposal does not commit|no receipt,? no durable knowledge|"
|
||||
r"approval.{0,60}not (?:a )?(?:DB|database|canonical) write|"
|
||||
r"approval is intent,? not apply|none.{0,30}applied",
|
||||
r"(?:approval|sign-off) is intent,? not (?:apply|a canonical write)|none.{0,30}applied",
|
||||
re.I,
|
||||
),
|
||||
),
|
||||
|
|
@ -339,7 +339,8 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(r"session|conversation context", re.I),
|
||||
re.compile(
|
||||
r"unchanged.{0,120}(?:does not|doesn't|do not).{0,80}(?:behavior|answer)|"
|
||||
r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)",
|
||||
r"(?:does not|doesn't|do not) prove.{0,60}(?:behavior|answer)|"
|
||||
r"unchanged (?:database )?(?:totals|counts).{0,60}say nothing about.{0,30}(?:behavior|answer)",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
|
|
@ -348,13 +349,14 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
re.compile(r"persist|durable|continuity", re.I),
|
||||
re.compile(
|
||||
r"restart.{0,80}(?:does not|doesn't|need not|not necessarily).{0,80}(?:erase|forget)|"
|
||||
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}restart",
|
||||
r"(?:state\.db|session JSONL).{0,120}(?:preserve|continuity|survive).{0,80}restart|"
|
||||
r"(?:facts?|context).{0,80}not erased by restart",
|
||||
re.I | re.S,
|
||||
),
|
||||
),
|
||||
"row_content_proof": (
|
||||
re.compile(r"unchanged (?:counts?|totals?)", re.I),
|
||||
re.compile(r"does not prove|doesn't prove|do not prove", re.I),
|
||||
re.compile(r"unchanged (?:database )?(?:counts?|totals?)", re.I),
|
||||
re.compile(r"does not prove|doesn't prove|do not prove|say nothing", re.I),
|
||||
re.compile(
|
||||
r"row (?:IDs?|hashes?)|fingerprints?|timestamps?|balanced (?:insert|write|change)|"
|
||||
r"individual rows?.{0,60}(?:mutated|changed|updated)",
|
||||
|
|
@ -470,7 +472,7 @@ SCHEMA_GAP_QUALIFIER_RE = re.compile(
|
|||
CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
||||
"claims_unshipped_fields": re.compile(
|
||||
r"(?:public\.)?claims?\b.{0,60}(?:stores?|has|have|with|column|field).{0,40}"
|
||||
r"\b(?:body|metadata|forecast[_ -]resolution|resolved_at)\b",
|
||||
r"\b(?:body|metadata|forecast[_ -]resolution|resolved_at|falsifier)\b",
|
||||
re.I,
|
||||
),
|
||||
"sources_unshipped_fields": re.compile(
|
||||
|
|
@ -496,7 +498,11 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
|||
re.I,
|
||||
),
|
||||
"reasoning_tools_unshipped_fields": re.compile(
|
||||
r"\breasoning_tools?\b.{0,120}\b(?:structured criteria|structured steps|criteria or steps)\b",
|
||||
r"\breasoning_tools?\b.{0,120}\b(?:structured criteria|structured steps|criteria or steps|scope)\b",
|
||||
re.I,
|
||||
),
|
||||
"behavioral_rules_unshipped_fields": re.compile(
|
||||
r"\bbehavioral_rules?\b.{0,120}\b(?:has|have|with|carries|field|column).{0,40}\bstatus\b",
|
||||
re.I,
|
||||
),
|
||||
"unsupported_approve_claim_surface": re.compile(
|
||||
|
|
@ -534,6 +540,12 @@ DB_ONLY_CAUSALITY_RE = re.compile(
|
|||
RESTART_ERASES_ALL_RE = re.compile(
|
||||
r"restart.{0,100}(?:erases?|forgets?|loses?).{0,40}(?:all|every|prior[- ]session)", re.I | re.S
|
||||
)
|
||||
REASONING_TOOL_CLAIM_EDGE_RE = re.compile(
|
||||
r"claims?.{0,120}(?:get|have|create|use).{0,50}(?:claim_)?edges?.{0,100}"
|
||||
r"(?:reasoning_tool|tool(?:'s)? ID)|"
|
||||
r"(?:claim_)?edges?.{0,100}(?:point|connect|link).{0,80}(?:reasoning_tool|tool(?:'s)? ID)",
|
||||
re.I | re.S,
|
||||
)
|
||||
HANDLER_TELEGRAM_OVERCLAIM_RE = re.compile(
|
||||
r"(?:Telegram (?:path|delivery).{0,40}(?:is|was) (?:live|proven)|"
|
||||
r"live Telegram (?:path|delivery)).{0,160}(?:posted nothing|did not post|no Telegram post)",
|
||||
|
|
@ -643,7 +655,8 @@ def broad_semantic_issues(reply: str) -> list[str]:
|
|||
):
|
||||
findings.add("unchanged_db_called_complete_behavior_proof")
|
||||
if RESTART_ERASES_ALL_RE.search(reply) and not re.search(
|
||||
r"restart.{0,100}(?:does not|doesn't|need not|not necessarily).{0,60}(?:erase|forget|lose)",
|
||||
r"restart.{0,100}(?:does not|doesn't|need not|not necessarily).{0,60}(?:erase|forget|lose)|"
|
||||
r"(?:facts?|context).{0,80}not erased by restart",
|
||||
reply,
|
||||
re.I | re.S,
|
||||
):
|
||||
|
|
@ -658,6 +671,8 @@ def broad_semantic_issues(reply: str) -> list[str]:
|
|||
findings.add("belief_row_given_claim_edge")
|
||||
if ROW_LEVEL_APPLIED_AT_RE.search(reply):
|
||||
findings.add("applied_at_assigned_to_each_apply_step")
|
||||
if REASONING_TOOL_CLAIM_EDGE_RE.search(reply):
|
||||
findings.add("claim_edge_pointed_at_reasoning_tool")
|
||||
return sorted(findings)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -194,6 +194,87 @@ def test_vps_bridge_proposal_readiness_separates_contract_from_live_worker_state
|
|||
assert "does not prove the production apply worker is enabled" in strict_approved["guidance"]
|
||||
|
||||
|
||||
def test_vps_bridge_operational_contracts_are_query_specific_and_schema_exact() -> None:
|
||||
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
||||
|
||||
source_contracts = {
|
||||
item["id"]: item
|
||||
for item in module.operational_contracts("I hand Leo a document. What can it absorb and stage before approval?")
|
||||
}
|
||||
source = source_contracts["source_intake"]
|
||||
assert source["required_lead"] == "Local compiler proven; live VPS/chat intake not shipped."
|
||||
assert source["capability_tier"] == "build-local compiler only"
|
||||
assert source["current_public_sources_columns"] == list(module.CURRENT_PUBLIC_SCHEMA["sources"])
|
||||
assert "author/title/publisher/publication date are current public.sources columns" in source["must_not_claim"]
|
||||
|
||||
mixed_contracts = {
|
||||
item["id"]: item
|
||||
for item in module.operational_contracts(
|
||||
"Compose a packet with a factual observation, strategic framework, governance rule, and old belief."
|
||||
)
|
||||
}
|
||||
mixed = mixed_contracts["mixed_packet_composition"]
|
||||
assert mixed["map"]["reusable_framework"] == "reasoning_tools"
|
||||
assert mixed["approve_claim_supported"] == ["claims", "sources", "evidence", "edges", "reasoning_tools"]
|
||||
assert "belief updates" in mixed["keep_staged_until_separate_capability"]
|
||||
assert "reasoning_tools has no scope field" in mixed["schema_guards"]
|
||||
assert mixed["current_columns"]["reasoning_tools"] == list(module.CURRENT_PUBLIC_SCHEMA["reasoning_tools"])
|
||||
assert mixed["current_columns"]["behavioral_rules"] == list(module.CURRENT_PUBLIC_SCHEMA["behavioral_rules"])
|
||||
|
||||
runtime_ids = {
|
||||
item["id"]
|
||||
for item in module.operational_contracts(
|
||||
"Do unchanged database totals after restart prove answer behavior and erase the prior session?"
|
||||
)
|
||||
}
|
||||
assert {"reply_budget", "runtime_persistence"} <= runtime_ids
|
||||
|
||||
|
||||
def test_vps_bridge_context_markdown_surfaces_runtime_contracts(capsys) -> None:
|
||||
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
||||
data = {
|
||||
"query": "absorb this document",
|
||||
"operational_contracts": module.operational_contracts("absorb this document"),
|
||||
"context_rows": [],
|
||||
"claims": [],
|
||||
}
|
||||
|
||||
module.print_claim_bundle(data)
|
||||
output = capsys.readouterr().out
|
||||
|
||||
assert "## Current Runtime Contracts" in output
|
||||
assert '"id": "source_intake"' in output
|
||||
assert "live VPS/chat intake not shipped" in output
|
||||
|
||||
|
||||
def test_vps_bridge_context_contract_reads_current_schema_from_postgres(monkeypatch) -> None:
|
||||
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
||||
captured_sql: list[str] = []
|
||||
|
||||
def fake_psql_json(_args, sql):
|
||||
captured_sql.append(sql)
|
||||
return [{"table_name": "sources", "columns": ["id", "source_type", "url", "hash", "new_live_column"]}]
|
||||
|
||||
monkeypatch.setattr(module, "psql_json", fake_psql_json)
|
||||
contracts = {
|
||||
item["id"]: item
|
||||
for item in module.context_operational_contracts(
|
||||
SimpleNamespace(), "Absorb this document into a staged packet."
|
||||
)
|
||||
}
|
||||
|
||||
assert contracts["source_intake"]["current_public_sources_columns"] == [
|
||||
"id",
|
||||
"source_type",
|
||||
"url",
|
||||
"hash",
|
||||
"new_live_column",
|
||||
]
|
||||
assert len(captured_sql) == 1
|
||||
assert "information_schema.columns" in captured_sql[0]
|
||||
assert "table_name = any" in captured_sql[0]
|
||||
|
||||
|
||||
def test_vps_bridge_decision_matrix_status_checks_schema_tables(monkeypatch) -> None:
|
||||
module = _load_module(BRIDGE_DIR / "kb_tool.py")
|
||||
captured_sql: list[str] = []
|
||||
|
|
|
|||
|
|
@ -257,3 +257,5 @@ def test_vps_kb_skill_encodes_apply_readiness_beliefs_and_current_proof_boundari
|
|||
assert "local compiler proven; live VPS/chat intake not shipped" in squashed
|
||||
assert "a reusable framework maps to `reasoning_tools`, not a claim graph" in squashed
|
||||
assert "never list them in the supported apply order" in squashed
|
||||
assert "Treat the emitted `Current Runtime Contracts` / `operational_contracts` block as binding" in squashed
|
||||
assert "It overrides recalled or generic architecture" in squashed
|
||||
|
|
|
|||
|
|
@ -361,6 +361,18 @@ def test_oos_direct_apply_case_accepts_strict_payload_as_precondition_wording()
|
|||
assert score["pass"] is True
|
||||
|
||||
|
||||
def test_oos_direct_apply_case_accepts_reviewer_signoff_boundary_wording() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[3]
|
||||
reply = (
|
||||
"No. All three read approved_needs_apply_payload with applied_at null; reviewer sign-off is intent, not a "
|
||||
"canonical write. DB readback shows 26 proposals. Proof that changes the answer is a non-null applied_at plus "
|
||||
"row-level postflight after strict normalization and renewed review."
|
||||
)
|
||||
|
||||
assert benchmark.score_reply(prompt, reply, memory_token=token)["pass"] is True
|
||||
|
||||
|
||||
def test_oos_agent_position_case_requires_current_public_beliefs_contract() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[10]
|
||||
|
|
@ -410,6 +422,25 @@ def test_oos_schema_guard_does_not_confuse_candidate_claims_with_source_metadata
|
|||
assert benchmark.current_schema_overclaims(reply) == []
|
||||
|
||||
|
||||
def test_oos_schema_guard_rejects_live_mixed_packet_field_hallucinations() -> None:
|
||||
reply = (
|
||||
"public.reasoning_tools has name, description, and scope. A public.claims row has a falsifier field. "
|
||||
"public.behavioral_rules carries a status column."
|
||||
)
|
||||
|
||||
assert benchmark.current_schema_overclaims(reply) == [
|
||||
"behavioral_rules_unshipped_fields",
|
||||
"claims_unshipped_fields",
|
||||
"reasoning_tools_unshipped_fields",
|
||||
]
|
||||
|
||||
|
||||
def test_oos_composition_rejects_claim_edge_to_reasoning_tool() -> None:
|
||||
reply = "Claims get claim_edges of type derives_from pointing at the reasoning_tool's ID."
|
||||
|
||||
assert benchmark.broad_semantic_issues(reply) == ["claim_edge_pointed_at_reasoning_tool"]
|
||||
|
||||
|
||||
def test_oos_runtime_case_accepts_explicit_persistence_tiers() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[9]
|
||||
|
|
@ -424,6 +455,18 @@ def test_oos_runtime_case_accepts_explicit_persistence_tiers() -> None:
|
|||
assert score["pass"] is True
|
||||
|
||||
|
||||
def test_oos_runtime_case_accepts_not_erased_by_restart_wording() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[9]
|
||||
reply = (
|
||||
"Neither. Unchanged database totals say nothing about answer behavior, and individual rows may be updated. "
|
||||
"Proof tiers: Tier 1 is canonical DB rows and hashes; Tier 2 is runtime skills, config and SOUL.md; Tier 3 is "
|
||||
"state.db and session JSONL, which preserve continuity. Facts are not erased by restart."
|
||||
)
|
||||
|
||||
assert benchmark.score_reply(prompt, reply, memory_token=token)["pass"] is True
|
||||
|
||||
|
||||
def test_oos_composition_rejects_belief_edges_and_per_step_applied_at() -> None:
|
||||
reply = "The old belief row gets a supersedes edge. Each step gets a non-null applied_at before the next runs."
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue