Merge pull request #102 from living-ip/codex/m3taversal-postdeploy-proof-20260713
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
Harden Leo against schema and receipt drift
This commit is contained in:
twentyOne2x
GitHub
commit
8d9fa1c2a1
7 changed files with 678 additions and 3 deletions
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,338 @@
|
|||
{
|
||||
"db_counts_changed": false,
|
||||
"generated_at_utc": "2026-07-13T07:10:12.532120+00:00",
|
||||
"memory_token": "demo-ledger-5a5ebcf0",
|
||||
"mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score",
|
||||
"posted_to_telegram": false,
|
||||
"production_db_apply_ran": false,
|
||||
"score": {
|
||||
"expected_prompt_count": 9,
|
||||
"expected_prompt_ids": [
|
||||
"OOS-01",
|
||||
"OOS-02",
|
||||
"OOS-03",
|
||||
"OOS-04",
|
||||
"OOS-05",
|
||||
"OOS-06",
|
||||
"OOS-07",
|
||||
"OOS-08",
|
||||
"OOS-09"
|
||||
],
|
||||
"failures": [
|
||||
{
|
||||
"concepts": {
|
||||
"identity_chain": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"invalid_current_edge_type"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "identity_restart_truth",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"identity_rendering": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-03",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"receipt": true,
|
||||
"source_evidence_chain": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"unshipped_evidence_excerpt"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "source_pointer_is_not_evidence",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"document_artifact_linking": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-05",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"heterogeneous_types": true,
|
||||
"receipt": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"invalid_current_edge_type"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "database_composition_judgment",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-06",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"current_update_identity_boundary": true,
|
||||
"exact_participant_handle": true,
|
||||
"no_cross_participant_identity_bleed": false,
|
||||
"no_unverified_alias": true
|
||||
},
|
||||
"dimension": "telegram_participant_identity",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"no_overclaim": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-09",
|
||||
"response_issue_detected": false
|
||||
}
|
||||
],
|
||||
"missing_prompt_ids": [],
|
||||
"pass": false,
|
||||
"passes": 5,
|
||||
"prompt_count": 9,
|
||||
"scores": [
|
||||
{
|
||||
"concepts": {
|
||||
"canonical_readback": true,
|
||||
"live_check": true,
|
||||
"material_action": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "demo_pressure_without_ids",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"fresh_readback": true,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"state_semantics": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-01",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"contradiction": true,
|
||||
"deduplication": true,
|
||||
"provenance": true,
|
||||
"receipt": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "multi_surface_knowledge_ingestion",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"document_artifact_linking": true,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-02",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"identity_chain": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"invalid_current_edge_type"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "identity_restart_truth",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"identity_rendering": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-03",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"canonical_readback": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "partner_demo_claim_ceiling",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"canonical_db": true,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true,
|
||||
"state_semantics": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-04",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"receipt": true,
|
||||
"source_evidence_chain": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"unshipped_evidence_excerpt"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "source_pointer_is_not_evidence",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"document_artifact_linking": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-05",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"heterogeneous_types": true,
|
||||
"receipt": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"invalid_current_edge_type"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "database_composition_judgment",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-06",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"blocker_definition": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"memory_token": true
|
||||
},
|
||||
"dimension": "same_session_memory_set",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"next_action": false,
|
||||
"no_overclaim": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-07",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"blocker_definition": true,
|
||||
"receipt": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"closure_proof": true,
|
||||
"memory_token": true
|
||||
},
|
||||
"dimension": "same_session_memory_recall",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"next_action": false,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-08",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"current_update_identity_boundary": true,
|
||||
"exact_participant_handle": true,
|
||||
"no_cross_participant_identity_bleed": false,
|
||||
"no_unverified_alias": true
|
||||
},
|
||||
"dimension": "telegram_participant_identity",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"no_overclaim": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-09",
|
||||
"response_issue_detected": false
|
||||
}
|
||||
],
|
||||
"unexpected_prompt_ids": []
|
||||
},
|
||||
"service_unchanged": true,
|
||||
"source_results_json": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-current.json",
|
||||
"temp_profile_removed": true
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
# Working Leo m3taversal Out-of-Sample Score
|
||||
|
||||
Generated UTC: `2026-07-13T07:10:12.532120+00:00`
|
||||
Pass: `False`
|
||||
Prompts: `5/9`
|
||||
DB counts changed: `False`
|
||||
Service unchanged: `True`
|
||||
Temporary profile removed: `True`
|
||||
Posted to Telegram: `False`
|
||||
|
||||
## Prompt Scores
|
||||
|
||||
- `OOS-01` / `demo_pressure_without_ids`: `pass=True`
|
||||
- `OOS-02` / `multi_surface_knowledge_ingestion`: `pass=True`
|
||||
- `OOS-03` / `identity_restart_truth`: `pass=False`
|
||||
- `OOS-04` / `partner_demo_claim_ceiling`: `pass=True`
|
||||
- `OOS-05` / `source_pointer_is_not_evidence`: `pass=False`
|
||||
- `OOS-06` / `database_composition_judgment`: `pass=False`
|
||||
- `OOS-07` / `same_session_memory_set`: `pass=True`
|
||||
- `OOS-08` / `same_session_memory_recall`: `pass=True`
|
||||
- `OOS-09` / `telegram_participant_identity`: `pass=False`
|
||||
|
||||
## Claim Ceiling
|
||||
|
||||
A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible Telegram delivery, production DB apply, or GCP parity.
|
||||
|
|
@ -130,6 +130,49 @@ as shipped. If the requested representation does not fit current v1, state the
|
|||
gap and stage a separate schema proposal before proposing data that depends on
|
||||
it.
|
||||
|
||||
Current `public.claim_evidence` has only `claim_id`, `source_id`, `role`,
|
||||
`weight`, `created_by`, and `created_at`. Its accepted roles are `grounds`,
|
||||
`illustrates`, and `contradicts`. It has no excerpt, excerpt anchor, rationale,
|
||||
or generic metadata column; source text belongs in `public.sources.excerpt`.
|
||||
Current `public.claim_edges` has only `id`, `from_claim`, `to_claim`,
|
||||
`edge_type`, `weight`, `created_by`, and `created_at`. Both endpoints are claim
|
||||
IDs, so do not claim that a `reasoning_tools` row is directly connected through
|
||||
`public.claim_edges`.
|
||||
|
||||
For heterogeneous research packets, map only to structures proven in the
|
||||
current schema:
|
||||
|
||||
- factual observations and disputed interpretations may become separate
|
||||
`public.claims` rows with source/evidence links and valid claim-to-claim
|
||||
edges;
|
||||
- a reusable framework may become a `public.reasoning_tools` row, but the
|
||||
current schema has no generic reasoning-tool-to-claim edge and no shipped
|
||||
`concept_maps` or `claim_concept_map_links` table;
|
||||
- `public.governance_gates` can store an evaluative gate with `name`,
|
||||
`criteria`, `evidence_bar`, and `pass_condition`; it is not a generic
|
||||
behavioral-rule or policy table, so state a schema gap when a governance rule
|
||||
does not fit that contract;
|
||||
- a belief correction may create a new claim, a valid `supersedes` edge, and
|
||||
set the old claim's `superseded_by` column. `superseded_by` is a column, not
|
||||
an edge type.
|
||||
|
||||
Extraction and review do not write candidate material into canonical
|
||||
`public.sources` or `public.claims`. Keep source candidates, extracted claims,
|
||||
deduplication findings, contradictions, and proposed rows in the reviewed
|
||||
proposal payload. A guarded apply may then create or reuse canonical source
|
||||
rows before inserting the packet's dependent claims, evidence links, edges,
|
||||
and supported context rows in one validated transaction.
|
||||
|
||||
Count receipts are packet-specific. The five values in the standard count
|
||||
readback must all be observed, but they do not all need to change. Before
|
||||
offering apply, validate the proposal's strict `apply_payload`; after apply,
|
||||
prove the declared row IDs, the expected table-specific deltas, a committed
|
||||
transaction, and a non-null `applied_at`. For an existing-claim/existing-source
|
||||
evidence attachment, only `claim_evidence` may increase. For an edge-only
|
||||
packet, only `claim_edges` may increase. Updating a proposal status need not
|
||||
increase the proposal count. Never use `all five counts increased` as a
|
||||
universal success condition.
|
||||
|
||||
- "Did we actually update the KB?": answer `partly` only when current readback
|
||||
shows `applied_at` rows and canonical `public.*` rows. Otherwise say
|
||||
`mostly still proposals`; list applied, approved-but-not-applied, pending,
|
||||
|
|
|
|||
|
|
@ -123,6 +123,49 @@ as shipped. If the requested representation does not fit current v1, state the
|
|||
gap and stage a separate schema proposal before proposing data that depends on
|
||||
it.
|
||||
|
||||
Current `public.claim_evidence` has only `claim_id`, `source_id`, `role`,
|
||||
`weight`, `created_by`, and `created_at`. Its accepted roles are `grounds`,
|
||||
`illustrates`, and `contradicts`. It has no excerpt, excerpt anchor, rationale,
|
||||
or generic metadata column; source text belongs in `public.sources.excerpt`.
|
||||
Current `public.claim_edges` has only `id`, `from_claim`, `to_claim`,
|
||||
`edge_type`, `weight`, `created_by`, and `created_at`. Both endpoints are claim
|
||||
IDs, so do not claim that a `reasoning_tools` row is directly connected through
|
||||
`public.claim_edges`.
|
||||
|
||||
For heterogeneous research packets, map only to structures proven in the
|
||||
current schema:
|
||||
|
||||
- factual observations and disputed interpretations may become separate
|
||||
`public.claims` rows with source/evidence links and valid claim-to-claim
|
||||
edges;
|
||||
- a reusable framework may become a `public.reasoning_tools` row, but the
|
||||
current schema has no generic reasoning-tool-to-claim edge and no shipped
|
||||
`concept_maps` or `claim_concept_map_links` table;
|
||||
- `public.governance_gates` can store an evaluative gate with `name`,
|
||||
`criteria`, `evidence_bar`, and `pass_condition`; it is not a generic
|
||||
behavioral-rule or policy table, so state a schema gap when a governance rule
|
||||
does not fit that contract;
|
||||
- a belief correction may create a new claim, a valid `supersedes` edge, and
|
||||
set the old claim's `superseded_by` column. `superseded_by` is a column, not
|
||||
an edge type.
|
||||
|
||||
Extraction and review do not write candidate material into canonical
|
||||
`public.sources` or `public.claims`. Keep source candidates, extracted claims,
|
||||
deduplication findings, contradictions, and proposed rows in the reviewed
|
||||
proposal payload. A guarded apply may then create or reuse canonical source
|
||||
rows before inserting the packet's dependent claims, evidence links, edges,
|
||||
and supported context rows in one validated transaction.
|
||||
|
||||
Count receipts are packet-specific. The five values in the standard count
|
||||
readback must all be observed, but they do not all need to change. Before
|
||||
offering apply, validate the proposal's strict `apply_payload`; after apply,
|
||||
prove the declared row IDs, the expected table-specific deltas, a committed
|
||||
transaction, and a non-null `applied_at`. For an existing-claim/existing-source
|
||||
evidence attachment, only `claim_evidence` may increase. For an edge-only
|
||||
packet, only `claim_edges` may increase. Updating a proposal status need not
|
||||
increase the proposal count. Never use `all five counts increased` as a
|
||||
universal success condition.
|
||||
|
||||
- "Did we actually update the KB?": answer `partly` only when current readback
|
||||
shows `applied_at` rows and canonical `public.*` rows. Otherwise say
|
||||
`mostly still proposals`; list applied, approved-but-not-applied, pending,
|
||||
|
|
|
|||
|
|
@ -193,7 +193,9 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
|
|||
}
|
||||
|
||||
INVALID_COUNT_INVARIANT_RE = re.compile(
|
||||
r"\ball five (?:numbers|counts) (?:must|need to) move\b|\bneed all five (?:numbers|counts) to move\b",
|
||||
r"\ball five (?:canonical )?(?:numbers|counts) (?:must|need to) move\b|"
|
||||
r"\bneed all five (?:canonical )?(?:numbers|counts) to move\b|"
|
||||
r"\ball five (?:canonical )?(?:numbers|counts).{0,40}\b(?:higher|increase|increased|rise|change|changed)\b",
|
||||
re.I,
|
||||
)
|
||||
COUNT_INVARIANT_REJECTION_RE = re.compile(
|
||||
|
|
@ -217,7 +219,9 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
|||
re.I,
|
||||
),
|
||||
"invalid_current_edge_type": re.compile(
|
||||
r"\b(?:claim_edges?|edge type|edge)\b.{0,100}\b(?:superseded_by|relates_to|resolves|derived_from)\b",
|
||||
r"(?:\b(?:claim_edges?|edge type|edge)\b\s*(?:is|=|:|named|called|of)?\s*[`'\"]?"
|
||||
r"(?:superseded_by|relates_to|resolves|derived_from)\b|"
|
||||
r"\b(?:superseded_by|relates_to|resolves|derived_from)\b[`'\"]?\s+(?:claim_)?edge\b)",
|
||||
re.I,
|
||||
),
|
||||
"unshipped_edge_rationale": re.compile(r"\bclaim_edges?\b.{0,100}\brationale\b", re.I),
|
||||
|
|
@ -276,7 +280,12 @@ def score_reply(prompt: dict[str, Any], reply: str, *, memory_token: str) -> dic
|
|||
re.search(r"current (?:Telegram )?(?:update|sender|handle)|visible handle", reply, re.I)
|
||||
)
|
||||
custom_signals["no_cross_participant_identity_bleed"] = bool(
|
||||
re.search(r"do not (?:transfer|carry|mix|reuse)|never (?:transfer|carry|mix|reuse)", reply, re.I)
|
||||
re.search(
|
||||
r"(?:do(?:es)? not|never)\s+(?:\w+\s+){0,3}"
|
||||
r"(?:transfer|transfers|carry|carries|mix|mixes|reuse|reuses)",
|
||||
reply,
|
||||
re.I,
|
||||
)
|
||||
)
|
||||
invalid_count_invariant = asserts_invalid_count_invariant(reply)
|
||||
schema_overclaims = current_schema_overclaims(reply)
|
||||
|
|
|
|||
|
|
@ -117,6 +117,19 @@ def test_oos_identity_case_requires_exact_visible_handle_and_no_alias() -> None:
|
|||
assert bad["custom_signals"]["no_unverified_alias"] is False
|
||||
|
||||
|
||||
def test_oos_identity_case_accepts_inflected_carry_language() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[-1]
|
||||
reply = (
|
||||
"Call the current visible participant m3taversal, exactly.\n"
|
||||
"Use only the current Telegram update and visible sender.\n"
|
||||
"Leo never carries a previous participant identity into another user's reply.\n"
|
||||
"No nickname or personal name is authorized."
|
||||
)
|
||||
score = benchmark.score_reply(prompt, reply, memory_token=token)
|
||||
assert score["pass"] is True
|
||||
|
||||
|
||||
def test_oos_score_rejects_proposed_schema_presented_as_current() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[1]
|
||||
|
|
@ -134,6 +147,16 @@ def test_oos_schema_guard_allows_explicit_future_schema_gap() -> None:
|
|||
assert benchmark.current_schema_overclaims(reply) == []
|
||||
|
||||
|
||||
def test_oos_schema_guard_distinguishes_superseded_by_column_from_edge_type() -> None:
|
||||
valid = (
|
||||
"Create a supersedes edge from the new claim to the old claim, then set the old claim's "
|
||||
"superseded_by pointer to the new claim ID."
|
||||
)
|
||||
invalid = "Create a superseded_by edge between the two claims."
|
||||
assert benchmark.current_schema_overclaims(valid) == []
|
||||
assert benchmark.current_schema_overclaims(invalid) == ["invalid_current_edge_type"]
|
||||
|
||||
|
||||
def test_oos_score_rejects_blanket_all_five_counts_must_move_claim() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[3]
|
||||
|
|
@ -143,6 +166,11 @@ def test_oos_score_rejects_blanket_all_five_counts_must_move_claim() -> None:
|
|||
assert score["invalid_count_invariant_detected"] is True
|
||||
|
||||
|
||||
def test_oos_score_rejects_blanket_all_five_counts_higher_claim() -> None:
|
||||
reply = "The proof is all five canonical counts higher than the baseline."
|
||||
assert benchmark.asserts_invalid_count_invariant(reply) is True
|
||||
|
||||
|
||||
def test_oos_score_does_not_penalize_an_explicit_retraction() -> None:
|
||||
reply = "The claim that all five counts must move is wrong; packet-specific row readback is required."
|
||||
assert benchmark.asserts_invalid_count_invariant(reply) is False
|
||||
|
|
|
|||
Loading…
Reference in a new issue