Merge pull request #103 from living-ip/codex/m3taversal-postdeploy-proof-20260713
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
Pin Leo composition to the shipped apply contract
This commit is contained in:
commit
b570db63b0
7 changed files with 554 additions and 1 deletions
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,265 @@
|
|||
{
|
||||
"db_counts_changed": false,
|
||||
"generated_at_utc": "2026-07-13T07:57:23.937950+00:00",
|
||||
"memory_token": "demo-ledger-84cf197e",
|
||||
"mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score",
|
||||
"posted_to_telegram": false,
|
||||
"production_db_apply_ran": false,
|
||||
"score": {
|
||||
"expected_prompt_count": 9,
|
||||
"expected_prompt_ids": [
|
||||
"OOS-01",
|
||||
"OOS-02",
|
||||
"OOS-03",
|
||||
"OOS-04",
|
||||
"OOS-05",
|
||||
"OOS-06",
|
||||
"OOS-07",
|
||||
"OOS-08",
|
||||
"OOS-09"
|
||||
],
|
||||
"failures": [
|
||||
{
|
||||
"concepts": {
|
||||
"heterogeneous_types": true,
|
||||
"receipt": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"unreviewed_claim_type"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "database_composition_judgment",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-06",
|
||||
"response_issue_detected": false
|
||||
}
|
||||
],
|
||||
"missing_prompt_ids": [],
|
||||
"pass": false,
|
||||
"passes": 8,
|
||||
"prompt_count": 9,
|
||||
"scores": [
|
||||
{
|
||||
"concepts": {
|
||||
"canonical_readback": true,
|
||||
"live_check": true,
|
||||
"material_action": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "demo_pressure_without_ids",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"fresh_readback": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"state_semantics": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-01",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"contradiction": true,
|
||||
"deduplication": true,
|
||||
"provenance": true,
|
||||
"receipt": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "multi_surface_knowledge_ingestion",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"document_artifact_linking": true,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-02",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"identity_chain": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "identity_restart_truth",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"identity_rendering": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-03",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"canonical_readback": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "partner_demo_claim_ceiling",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"canonical_db": true,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true,
|
||||
"state_semantics": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-04",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"receipt": true,
|
||||
"source_evidence_chain": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "source_pointer_is_not_evidence",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"document_artifact_linking": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-05",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"heterogeneous_types": true,
|
||||
"receipt": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"unreviewed_claim_type"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "database_composition_judgment",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-06",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"blocker_definition": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"memory_token": true
|
||||
},
|
||||
"dimension": "same_session_memory_set",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"next_action": false,
|
||||
"no_overclaim": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-07",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {
|
||||
"blocker_definition": true,
|
||||
"receipt": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"closure_proof": true,
|
||||
"memory_token": true
|
||||
},
|
||||
"dimension": "same_session_memory_recall",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"next_action": false,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-08",
|
||||
"response_issue_detected": false
|
||||
},
|
||||
{
|
||||
"concepts": {},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"current_update_identity_boundary": true,
|
||||
"exact_participant_handle": true,
|
||||
"no_cross_participant_identity_bleed": true,
|
||||
"no_unverified_alias": true
|
||||
},
|
||||
"dimension": "telegram_participant_identity",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"no_overclaim": false
|
||||
},
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-09",
|
||||
"response_issue_detected": false
|
||||
}
|
||||
],
|
||||
"unexpected_prompt_ids": []
|
||||
},
|
||||
"service_unchanged": true,
|
||||
"source_results_json": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-current.json",
|
||||
"temp_profile_removed": true
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
# Working Leo m3taversal Out-of-Sample Score
|
||||
|
||||
Generated UTC: `2026-07-13T07:57:23.937950+00:00`
|
||||
Pass: `False`
|
||||
Prompts: `8/9`
|
||||
DB counts changed: `False`
|
||||
Service unchanged: `True`
|
||||
Temporary profile removed: `True`
|
||||
Posted to Telegram: `False`
|
||||
|
||||
## Prompt Scores
|
||||
|
||||
- `OOS-01` / `demo_pressure_without_ids`: `pass=True`
|
||||
- `OOS-02` / `multi_surface_knowledge_ingestion`: `pass=True`
|
||||
- `OOS-03` / `identity_restart_truth`: `pass=True`
|
||||
- `OOS-04` / `partner_demo_claim_ceiling`: `pass=True`
|
||||
- `OOS-05` / `source_pointer_is_not_evidence`: `pass=True`
|
||||
- `OOS-06` / `database_composition_judgment`: `pass=False`
|
||||
- `OOS-07` / `same_session_memory_set`: `pass=True`
|
||||
- `OOS-08` / `same_session_memory_recall`: `pass=True`
|
||||
- `OOS-09` / `telegram_participant_identity`: `pass=True`
|
||||
|
||||
## Claim Ceiling
|
||||
|
||||
A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible Telegram delivery, production DB apply, or GCP parity.
|
||||
|
|
@ -173,6 +173,25 @@ packet, only `claim_edges` may increase. Updating a proposal status need not
|
|||
increase the proposal count. Never use `all five counts increased` as a
|
||||
universal success condition.
|
||||
|
||||
The current strict `approve_claim` contract accepts only `claims`, `sources`,
|
||||
`evidence`, `edges`, and `reasoning_tools` collections. It does not insert
|
||||
`governance_gates`, update an existing claim's `status` or `superseded_by`, or
|
||||
write arbitrary soul/context rows. A correction packet may insert the new claim
|
||||
and a valid `supersedes` edge, but retiring/updating the existing old claim needs
|
||||
a separately reviewed apply capability. Do not describe that unsupported update
|
||||
as part of one atomic `approve_claim` transaction.
|
||||
|
||||
Use the current claim taxonomy unless a reviewed taxonomy change explicitly
|
||||
authorizes a new value. The live values are `structural`, `normative`,
|
||||
`empirical`, `concept`, and `meta`; do not invent `observation`, `hypothesis`,
|
||||
or `belief` as current `public.claims.type` values. Current
|
||||
`public.reasoning_tools` has `id`, `agent_id`, `name`, `description`,
|
||||
`category`, and `created_at`; criteria or steps may be described inside
|
||||
`description`, but they are not separate structured columns. When describing
|
||||
evidence, say that `public.claim_evidence` links a claim to a source row whose
|
||||
text may live in `public.sources.excerpt`; never say the evidence row stores or
|
||||
carries the excerpt.
|
||||
|
||||
- "Did we actually update the KB?": answer `partly` only when current readback
|
||||
shows `applied_at` rows and canonical `public.*` rows. Otherwise say
|
||||
`mostly still proposals`; list applied, approved-but-not-applied, pending,
|
||||
|
|
|
|||
|
|
@ -166,6 +166,25 @@ packet, only `claim_edges` may increase. Updating a proposal status need not
|
|||
increase the proposal count. Never use `all five counts increased` as a
|
||||
universal success condition.
|
||||
|
||||
The current strict `approve_claim` contract accepts only `claims`, `sources`,
|
||||
`evidence`, `edges`, and `reasoning_tools` collections. It does not insert
|
||||
`governance_gates`, update an existing claim's `status` or `superseded_by`, or
|
||||
write arbitrary soul/context rows. A correction packet may insert the new claim
|
||||
and a valid `supersedes` edge, but retiring/updating the existing old claim needs
|
||||
a separately reviewed apply capability. Do not describe that unsupported update
|
||||
as part of one atomic `approve_claim` transaction.
|
||||
|
||||
Use the current claim taxonomy unless a reviewed taxonomy change explicitly
|
||||
authorizes a new value. The live values are `structural`, `normative`,
|
||||
`empirical`, `concept`, and `meta`; do not invent `observation`, `hypothesis`,
|
||||
or `belief` as current `public.claims.type` values. Current
|
||||
`public.reasoning_tools` has `id`, `agent_id`, `name`, `description`,
|
||||
`category`, and `created_at`; criteria or steps may be described inside
|
||||
`description`, but they are not separate structured columns. When describing
|
||||
evidence, say that `public.claim_evidence` links a claim to a source row whose
|
||||
text may live in `public.sources.excerpt`; never say the evidence row stores or
|
||||
carries the excerpt.
|
||||
|
||||
- "Did we actually update the KB?": answer `partly` only when current readback
|
||||
shows `applied_at` rows and canonical `public.*` rows. Otherwise say
|
||||
`mostly still proposals`; list applied, approved-but-not-applied, pending,
|
||||
|
|
|
|||
|
|
@ -225,7 +225,25 @@ CURRENT_SCHEMA_ASSERTION_PATTERNS: dict[str, re.Pattern[str]] = {
|
|||
re.I,
|
||||
),
|
||||
"unshipped_edge_rationale": re.compile(r"\bclaim_edges?\b.{0,100}\brationale\b", re.I),
|
||||
"unshipped_evidence_excerpt": re.compile(r"\bclaim_evidence\b.{0,100}\bexcerpt\b", re.I),
|
||||
"unshipped_evidence_excerpt": re.compile(
|
||||
r"\bclaim_evidence\b.{0,100}\bwith\s+(?:an?\s+)?(?:excerpt|excerpt_anchor)\b|"
|
||||
r"\bclaim_evidence\b.{0,100}\b(?:has|contains|stores|carries)\b.{0,30}\bexcerpt\b|"
|
||||
r"\bclaim_evidence\b.{0,100}\b(?:excerpt|excerpt_anchor)\s+(?:column|field)\b",
|
||||
re.I,
|
||||
),
|
||||
"unreviewed_claim_type": re.compile(
|
||||
r"(?:public\.)?claims?.{0,100}\b(?:type|typed)\s*[`'\":=]?\s*(?:observation|hypothesis|belief)\b",
|
||||
re.I,
|
||||
),
|
||||
"reasoning_tools_unshipped_fields": re.compile(
|
||||
r"\breasoning_tools?\b.{0,120}\b(?:structured criteria|structured steps|criteria or steps)\b",
|
||||
re.I,
|
||||
),
|
||||
"unsupported_approve_claim_surface": re.compile(
|
||||
r"\b(?:apply_payload|approve_claim)\b.{0,500}\b(?:governance gate row|governance_gates|"
|
||||
r"superseded_by column update|update the existing old claim)\b",
|
||||
re.I,
|
||||
),
|
||||
}
|
||||
UNVERIFIED_M3TAVERSAL_ALIAS_RE = re.compile(r"\b(?:Cory|m3ta)\b", re.I)
|
||||
|
||||
|
|
|
|||
|
|
@ -157,6 +157,24 @@ def test_oos_schema_guard_distinguishes_superseded_by_column_from_edge_type() ->
|
|||
assert benchmark.current_schema_overclaims(invalid) == ["invalid_current_edge_type"]
|
||||
|
||||
|
||||
def test_oos_schema_guard_allows_source_excerpt_reached_through_evidence_link() -> None:
|
||||
valid = (
|
||||
"public.claim_evidence links the claim to a public.sources row whose source excerpt grounds the observation."
|
||||
)
|
||||
invalid = "The public.claim_evidence row has an excerpt field."
|
||||
assert benchmark.current_schema_overclaims(valid) == []
|
||||
assert benchmark.current_schema_overclaims(invalid) == ["unshipped_evidence_excerpt"]
|
||||
|
||||
|
||||
def test_oos_schema_guard_rejects_unreviewed_claim_taxonomy_and_apply_surface() -> None:
|
||||
unreviewed_type = "Create a public.claims row with type observation."
|
||||
unsupported_apply = (
|
||||
"The approve_claim apply_payload includes a governance gate row and a superseded_by column update."
|
||||
)
|
||||
assert benchmark.current_schema_overclaims(unreviewed_type) == ["unreviewed_claim_type"]
|
||||
assert benchmark.current_schema_overclaims(unsupported_apply) == ["unsupported_approve_claim_surface"]
|
||||
|
||||
|
||||
def test_oos_score_rejects_blanket_all_five_counts_must_move_claim() -> None:
|
||||
token = "demo-ledger-deadbeef"
|
||||
prompt = benchmark.prompt_catalog(token)[3]
|
||||
|
|
|
|||
Loading…
Reference in a new issue