teleo-infrastructure/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-score-pre-concision-repair-20260713.json
2026-07-13 11:09:11 +02:00

792 lines
25 KiB
JSON

{
"db_counts_changed": false,
"generated_at_utc": "2026-07-13T09:01:32.681980+00:00",
"memory_token": "demo-ledger-fd596d14",
"mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score",
"posted_to_telegram": false,
"production_db_apply_ran": false,
"score": {
"expected_prompt_count": 15,
"expected_prompt_ids": [
"OOS-01",
"OOS-02",
"OOS-03",
"OOS-04",
"OOS-05",
"OOS-06",
"OOS-07",
"OOS-08",
"OOS-09",
"OOS-10",
"OOS-11",
"OOS-12",
"OOS-13",
"OOS-14",
"OOS-15"
],
"failures": [
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"canonical_readback": true,
"live_check": true,
"material_action": false,
"state_boundary": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "demo_pressure_without_ids",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"fresh_readback": false,
"next_action": true,
"no_overclaim": false,
"state_semantics": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-01",
"response_issue_detected": false,
"response_too_long": false,
"source_evidence_semantic_issues": [],
"word_count": 228
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"contradiction": true,
"deduplication": true,
"provenance": true,
"receipt": true,
"staged_review_apply": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "multi_surface_knowledge_ingestion",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"document_artifact_linking": true,
"next_action": true,
"no_overclaim": true,
"staging_or_review": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-02",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 806
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"identity_chain": true,
"receipt": true,
"state_boundary": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "identity_restart_truth",
"invalid_count_invariant_detected": false,
"legacy_pass": true,
"legacy_signals": {
"canonical_db": true,
"identity_rendering": true,
"no_overclaim": true,
"row_level_proof": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-03",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 752
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"canonical_evidence_boundary": true,
"evidence_provenance_quality": true,
"receipt": true,
"source_evidence_chain": true,
"staged_review_apply": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "source_pointer_needs_canonical_link_audit",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"document_artifact_linking": true,
"no_overclaim": true,
"row_level_proof": true,
"staging_or_review": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-05",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 735
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"behavioral_rule_storage": true,
"heterogeneous_types": true,
"receipt": true,
"reviewed_policy_apply": false,
"staged_review_apply": true
},
"current_schema_overclaims": [
"reasoning_tools_unshipped_fields",
"sources_unshipped_fields"
],
"custom_signals": {},
"dimension": "database_composition_judgment",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"next_action": true,
"no_overclaim": true,
"staging_or_review": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-06",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 1136
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"durable_session_continuity": false,
"proof_tiers": false,
"runtime_inputs": false
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "runtime_vs_canonical_causality",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"no_overclaim": false
},
"max_response_words": 180,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-10",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 191
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"agent_specific_positions": true,
"contradiction": true,
"shared_knowledge_commons": false
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "shared_claims_agent_positions",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"no_overclaim": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-11",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 311
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"forecast_history": true,
"forecast_schema_gap": false,
"staged_review_apply": false
},
"current_schema_overclaims": [
"invalid_current_edge_type"
],
"custom_signals": {},
"dimension": "forecast_resolution_without_rewriting_history",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"no_overclaim": false,
"staging_or_review": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-12",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 478
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"handler_not_telegram": false,
"proof_tiers": false,
"receipt": false
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "handler_is_not_telegram_delivery",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"artifact": true,
"next_action": false,
"no_overclaim": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-13",
"response_issue_detected": false,
"response_too_long": false,
"source_evidence_semantic_issues": [],
"word_count": 218
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"real_source_identity": true,
"staged_review_apply": true,
"staging_without_apply_authorization": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "autonomous_source_intake_boundary",
"invalid_count_invariant_detected": false,
"legacy_pass": true,
"legacy_signals": {
"artifact": true,
"authorization": true,
"no_overclaim": true,
"staging_or_review": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-14",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 663
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"apply_capability_boundary": false,
"current_edge_schema": false,
"valid_supersession": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "schema_valid_supersession",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"no_overclaim": false,
"row_level_proof": true,
"staging_or_review": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-15",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 532
}
],
"missing_prompt_ids": [],
"pass": false,
"passes": 4,
"prompt_count": 15,
"scores": [
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"canonical_readback": true,
"live_check": true,
"material_action": false,
"state_boundary": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "demo_pressure_without_ids",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"fresh_readback": false,
"next_action": true,
"no_overclaim": false,
"state_semantics": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-01",
"response_issue_detected": false,
"response_too_long": false,
"source_evidence_semantic_issues": [],
"word_count": 228
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"contradiction": true,
"deduplication": true,
"provenance": true,
"receipt": true,
"staged_review_apply": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "multi_surface_knowledge_ingestion",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"document_artifact_linking": true,
"next_action": true,
"no_overclaim": true,
"staging_or_review": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-02",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 806
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"identity_chain": true,
"receipt": true,
"state_boundary": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "identity_restart_truth",
"invalid_count_invariant_detected": false,
"legacy_pass": true,
"legacy_signals": {
"canonical_db": true,
"identity_rendering": true,
"no_overclaim": true,
"row_level_proof": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-03",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 752
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"canonical_readback": true,
"receipt": true,
"state_boundary": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "partner_demo_claim_ceiling",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"artifact": true,
"canonical_db": true,
"no_overclaim": false,
"row_level_proof": true,
"state_semantics": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": true,
"prompt_id": "OOS-04",
"response_issue_detected": false,
"response_too_long": false,
"source_evidence_semantic_issues": [],
"word_count": 241
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"canonical_evidence_boundary": true,
"evidence_provenance_quality": true,
"receipt": true,
"source_evidence_chain": true,
"staged_review_apply": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "source_pointer_needs_canonical_link_audit",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"document_artifact_linking": true,
"no_overclaim": true,
"row_level_proof": true,
"staging_or_review": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-05",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 735
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"behavioral_rule_storage": true,
"heterogeneous_types": true,
"receipt": true,
"reviewed_policy_apply": false,
"staged_review_apply": true
},
"current_schema_overclaims": [
"reasoning_tools_unshipped_fields",
"sources_unshipped_fields"
],
"custom_signals": {},
"dimension": "database_composition_judgment",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"next_action": true,
"no_overclaim": true,
"staging_or_review": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-06",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 1136
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"blocker_definition": true
},
"current_schema_overclaims": [],
"custom_signals": {
"memory_token": true
},
"dimension": "same_session_memory_set",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"next_action": false,
"no_overclaim": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": true,
"prompt_id": "OOS-07",
"response_issue_detected": false,
"response_too_long": false,
"source_evidence_semantic_issues": [],
"word_count": 102
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"blocker_definition": true,
"receipt": true
},
"current_schema_overclaims": [],
"custom_signals": {
"closure_proof": true,
"memory_token": true
},
"dimension": "same_session_memory_recall",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"next_action": true,
"no_overclaim": false,
"row_level_proof": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": true,
"prompt_id": "OOS-08",
"response_issue_detected": false,
"response_too_long": false,
"source_evidence_semantic_issues": [],
"word_count": 145
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {},
"current_schema_overclaims": [],
"custom_signals": {
"current_update_identity_boundary": true,
"exact_participant_handle": true,
"no_cross_participant_identity_bleed": true,
"no_unverified_alias": true
},
"dimension": "telegram_participant_identity",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"no_overclaim": false
},
"max_response_words": 100,
"overclaim_detected": false,
"pass": true,
"prompt_id": "OOS-09",
"response_issue_detected": false,
"response_too_long": false,
"source_evidence_semantic_issues": [],
"word_count": 67
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"durable_session_continuity": false,
"proof_tiers": false,
"runtime_inputs": false
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "runtime_vs_canonical_causality",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"no_overclaim": false
},
"max_response_words": 180,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-10",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 191
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"agent_specific_positions": true,
"contradiction": true,
"shared_knowledge_commons": false
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "shared_claims_agent_positions",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"no_overclaim": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-11",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 311
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"forecast_history": true,
"forecast_schema_gap": false,
"staged_review_apply": false
},
"current_schema_overclaims": [
"invalid_current_edge_type"
],
"custom_signals": {},
"dimension": "forecast_resolution_without_rewriting_history",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"no_overclaim": false,
"staging_or_review": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-12",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 478
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"handler_not_telegram": false,
"proof_tiers": false,
"receipt": false
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "handler_is_not_telegram_delivery",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"artifact": true,
"next_action": false,
"no_overclaim": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-13",
"response_issue_detected": false,
"response_too_long": false,
"source_evidence_semantic_issues": [],
"word_count": 218
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"real_source_identity": true,
"staged_review_apply": true,
"staging_without_apply_authorization": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "autonomous_source_intake_boundary",
"invalid_count_invariant_detected": false,
"legacy_pass": true,
"legacy_signals": {
"artifact": true,
"authorization": true,
"no_overclaim": true,
"staging_or_review": true
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-14",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 663
},
{
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"apply_capability_boundary": false,
"current_edge_schema": false,
"valid_supersession": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "schema_valid_supersession",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"no_overclaim": false,
"row_level_proof": true,
"staging_or_review": false
},
"max_response_words": 300,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-15",
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"word_count": 532
}
],
"unexpected_prompt_ids": []
},
"service_unchanged": true,
"source_results_json": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-current.json",
"temp_profile_removed": true
}