Merge pull request #106 from living-ip/codex/leo-broad-answer-contract-20260713
Some checks are pending
CI / lint-and-test (push) Waiting to run
Some checks are pending
CI / lint-and-test (push) Waiting to run
Keep Working Leo replies concise by default
This commit is contained in:
commit
c20db7f414
6 changed files with 1139 additions and 4 deletions
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,792 @@
|
|||
{
|
||||
"db_counts_changed": false,
|
||||
"generated_at_utc": "2026-07-13T09:01:32.681980+00:00",
|
||||
"memory_token": "demo-ledger-fd596d14",
|
||||
"mode": "working_leo_m3taversal_out_of_sample_live_vps_handler_score",
|
||||
"posted_to_telegram": false,
|
||||
"production_db_apply_ran": false,
|
||||
"score": {
|
||||
"expected_prompt_count": 15,
|
||||
"expected_prompt_ids": [
|
||||
"OOS-01",
|
||||
"OOS-02",
|
||||
"OOS-03",
|
||||
"OOS-04",
|
||||
"OOS-05",
|
||||
"OOS-06",
|
||||
"OOS-07",
|
||||
"OOS-08",
|
||||
"OOS-09",
|
||||
"OOS-10",
|
||||
"OOS-11",
|
||||
"OOS-12",
|
||||
"OOS-13",
|
||||
"OOS-14",
|
||||
"OOS-15"
|
||||
],
|
||||
"failures": [
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"canonical_readback": true,
|
||||
"live_check": true,
|
||||
"material_action": false,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "demo_pressure_without_ids",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"fresh_readback": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": false,
|
||||
"state_semantics": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-01",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 228
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"contradiction": true,
|
||||
"deduplication": true,
|
||||
"provenance": true,
|
||||
"receipt": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "multi_surface_knowledge_ingestion",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"document_artifact_linking": true,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-02",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 806
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"identity_chain": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "identity_restart_truth",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"identity_rendering": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-03",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 752
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"canonical_evidence_boundary": true,
|
||||
"evidence_provenance_quality": true,
|
||||
"receipt": true,
|
||||
"source_evidence_chain": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "source_pointer_needs_canonical_link_audit",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"document_artifact_linking": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-05",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 735
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"behavioral_rule_storage": true,
|
||||
"heterogeneous_types": true,
|
||||
"receipt": true,
|
||||
"reviewed_policy_apply": false,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"reasoning_tools_unshipped_fields",
|
||||
"sources_unshipped_fields"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "database_composition_judgment",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-06",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 1136
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"durable_session_continuity": false,
|
||||
"proof_tiers": false,
|
||||
"runtime_inputs": false
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "runtime_vs_canonical_causality",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"no_overclaim": false
|
||||
},
|
||||
"max_response_words": 180,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-10",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 191
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"agent_specific_positions": true,
|
||||
"contradiction": true,
|
||||
"shared_knowledge_commons": false
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "shared_claims_agent_positions",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"no_overclaim": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-11",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 311
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"forecast_history": true,
|
||||
"forecast_schema_gap": false,
|
||||
"staged_review_apply": false
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"invalid_current_edge_type"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "forecast_resolution_without_rewriting_history",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"no_overclaim": false,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-12",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 478
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"handler_not_telegram": false,
|
||||
"proof_tiers": false,
|
||||
"receipt": false
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "handler_is_not_telegram_delivery",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"next_action": false,
|
||||
"no_overclaim": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-13",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 218
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"real_source_identity": true,
|
||||
"staged_review_apply": true,
|
||||
"staging_without_apply_authorization": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "autonomous_source_intake_boundary",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"authorization": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-14",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 663
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"apply_capability_boundary": false,
|
||||
"current_edge_schema": false,
|
||||
"valid_supersession": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "schema_valid_supersession",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-15",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 532
|
||||
}
|
||||
],
|
||||
"missing_prompt_ids": [],
|
||||
"pass": false,
|
||||
"passes": 4,
|
||||
"prompt_count": 15,
|
||||
"scores": [
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"canonical_readback": true,
|
||||
"live_check": true,
|
||||
"material_action": false,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "demo_pressure_without_ids",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"fresh_readback": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": false,
|
||||
"state_semantics": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-01",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 228
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"contradiction": true,
|
||||
"deduplication": true,
|
||||
"provenance": true,
|
||||
"receipt": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "multi_surface_knowledge_ingestion",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"document_artifact_linking": true,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-02",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 806
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"identity_chain": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "identity_restart_truth",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"identity_rendering": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-03",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 752
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"canonical_readback": true,
|
||||
"receipt": true,
|
||||
"state_boundary": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "partner_demo_claim_ceiling",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"canonical_db": true,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true,
|
||||
"state_semantics": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-04",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 241
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"canonical_evidence_boundary": true,
|
||||
"evidence_provenance_quality": true,
|
||||
"receipt": true,
|
||||
"source_evidence_chain": true,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "source_pointer_needs_canonical_link_audit",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"document_artifact_linking": true,
|
||||
"no_overclaim": true,
|
||||
"row_level_proof": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-05",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 735
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"behavioral_rule_storage": true,
|
||||
"heterogeneous_types": true,
|
||||
"receipt": true,
|
||||
"reviewed_policy_apply": false,
|
||||
"staged_review_apply": true
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"reasoning_tools_unshipped_fields",
|
||||
"sources_unshipped_fields"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "database_composition_judgment",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"next_action": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-06",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 1136
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"blocker_definition": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"memory_token": true
|
||||
},
|
||||
"dimension": "same_session_memory_set",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"next_action": false,
|
||||
"no_overclaim": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-07",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 102
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"blocker_definition": true,
|
||||
"receipt": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"closure_proof": true,
|
||||
"memory_token": true
|
||||
},
|
||||
"dimension": "same_session_memory_recall",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"next_action": true,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-08",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 145
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {
|
||||
"current_update_identity_boundary": true,
|
||||
"exact_participant_handle": true,
|
||||
"no_cross_participant_identity_bleed": true,
|
||||
"no_unverified_alias": true
|
||||
},
|
||||
"dimension": "telegram_participant_identity",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"no_overclaim": false
|
||||
},
|
||||
"max_response_words": 100,
|
||||
"overclaim_detected": false,
|
||||
"pass": true,
|
||||
"prompt_id": "OOS-09",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 67
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"durable_session_continuity": false,
|
||||
"proof_tiers": false,
|
||||
"runtime_inputs": false
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "runtime_vs_canonical_causality",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"no_overclaim": false
|
||||
},
|
||||
"max_response_words": 180,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-10",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 191
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"agent_specific_positions": true,
|
||||
"contradiction": true,
|
||||
"shared_knowledge_commons": false
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "shared_claims_agent_positions",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"no_overclaim": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-11",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 311
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"forecast_history": true,
|
||||
"forecast_schema_gap": false,
|
||||
"staged_review_apply": false
|
||||
},
|
||||
"current_schema_overclaims": [
|
||||
"invalid_current_edge_type"
|
||||
],
|
||||
"custom_signals": {},
|
||||
"dimension": "forecast_resolution_without_rewriting_history",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"caveat_retention": false,
|
||||
"no_overclaim": false,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-12",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 478
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"handler_not_telegram": false,
|
||||
"proof_tiers": false,
|
||||
"receipt": false
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "handler_is_not_telegram_delivery",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"next_action": false,
|
||||
"no_overclaim": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-13",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": false,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 218
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"real_source_identity": true,
|
||||
"staged_review_apply": true,
|
||||
"staging_without_apply_authorization": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "autonomous_source_intake_boundary",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": true,
|
||||
"legacy_signals": {
|
||||
"artifact": true,
|
||||
"authorization": true,
|
||||
"no_overclaim": true,
|
||||
"staging_or_review": true
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-14",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 663
|
||||
},
|
||||
{
|
||||
"behavioral_rule_schema_issues": [],
|
||||
"broad_semantic_issues": [],
|
||||
"concepts": {
|
||||
"apply_capability_boundary": false,
|
||||
"current_edge_schema": false,
|
||||
"valid_supersession": true
|
||||
},
|
||||
"current_schema_overclaims": [],
|
||||
"custom_signals": {},
|
||||
"dimension": "schema_valid_supersession",
|
||||
"invalid_count_invariant_detected": false,
|
||||
"legacy_pass": false,
|
||||
"legacy_signals": {
|
||||
"canonical_db": true,
|
||||
"no_overclaim": false,
|
||||
"row_level_proof": true,
|
||||
"staging_or_review": false
|
||||
},
|
||||
"max_response_words": 300,
|
||||
"overclaim_detected": false,
|
||||
"pass": false,
|
||||
"prompt_id": "OOS-15",
|
||||
"response_issue_detected": false,
|
||||
"response_too_long": true,
|
||||
"source_evidence_semantic_issues": [],
|
||||
"word_count": 532
|
||||
}
|
||||
],
|
||||
"unexpected_prompt_ids": []
|
||||
},
|
||||
"service_unchanged": true,
|
||||
"source_results_json": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-suite-current.json",
|
||||
"temp_profile_removed": true
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# Working Leo m3taversal Out-of-Sample Score
|
||||
|
||||
Generated UTC: `2026-07-13T09:01:32.681980+00:00`
|
||||
Pass: `False`
|
||||
Prompts: `4/15`
|
||||
DB counts changed: `False`
|
||||
Service unchanged: `True`
|
||||
Temporary profile removed: `True`
|
||||
Posted to Telegram: `False`
|
||||
|
||||
## Prompt Scores
|
||||
|
||||
- `OOS-01` / `demo_pressure_without_ids`: `pass=False`
|
||||
- `OOS-02` / `multi_surface_knowledge_ingestion`: `pass=False`
|
||||
- `OOS-03` / `identity_restart_truth`: `pass=False`
|
||||
- `OOS-04` / `partner_demo_claim_ceiling`: `pass=True`
|
||||
- `OOS-05` / `source_pointer_needs_canonical_link_audit`: `pass=False`
|
||||
- `OOS-06` / `database_composition_judgment`: `pass=False`
|
||||
- `OOS-07` / `same_session_memory_set`: `pass=True`
|
||||
- `OOS-08` / `same_session_memory_recall`: `pass=True`
|
||||
- `OOS-09` / `telegram_participant_identity`: `pass=True`
|
||||
- `OOS-10` / `runtime_vs_canonical_causality`: `pass=False`
|
||||
- `OOS-11` / `shared_claims_agent_positions`: `pass=False`
|
||||
- `OOS-12` / `forecast_resolution_without_rewriting_history`: `pass=False`
|
||||
- `OOS-13` / `handler_is_not_telegram_delivery`: `pass=False`
|
||||
- `OOS-14` / `autonomous_source_intake_boundary`: `pass=False`
|
||||
- `OOS-15` / `schema_valid_supersession`: `pass=False`
|
||||
|
||||
## Claim Ceiling
|
||||
|
||||
A pass proves broad out-of-sample and same-session-memory behavior through the live VPS GatewayRunner using a temporary profile, with no Telegram post and no DB count change. It does not prove human-visible Telegram delivery, production DB apply, or GCP parity.
|
||||
|
|
@ -14,6 +14,15 @@ metadata:
|
|||
|
||||
You are live Leo in Telegram, not a generic help bot.
|
||||
|
||||
## Default Telegram Reply Budget
|
||||
|
||||
Unless the participant explicitly requests a detailed audit or long-form
|
||||
document, keep the entire reply at or below 220 words. A smaller requested word
|
||||
or line limit takes precedence. Lead with the answer, include only the evidence
|
||||
needed for the question, and do not repeat the answer as a summary. Broad or
|
||||
multi-part questions still receive one compressed answer, not a catalogue of
|
||||
every known rule.
|
||||
|
||||
## Live Gateway Liveness
|
||||
|
||||
When answering a Telegram liveness, runtime, or "are you alive" question, use
|
||||
|
|
@ -41,4 +50,3 @@ status readback.
|
|||
|
||||
For short liveness questions, answer compactly and avoid dumping commands or
|
||||
raw logs unless the user explicitly asks for evidence.
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,29 @@ metadata:
|
|||
|
||||
The canonical Teleo knowledge base is Postgres, not runtime memory.
|
||||
|
||||
## Mandatory Default Reply Contract
|
||||
|
||||
This contract applies even when the operator asks a broad, multi-part question.
|
||||
Unless the operator explicitly asks for a detailed audit or long-form document,
|
||||
the final reply must be at most 220 words; keep the whole reply under 220 words.
|
||||
If the operator gives a smaller limit, that smaller word or line limit is
|
||||
mandatory. Silently shorten the draft before sending it; never explain that it
|
||||
was shortened.
|
||||
|
||||
Use only this compact shape:
|
||||
|
||||
1. direct answer in the first sentence;
|
||||
2. no more than three short bullets containing only the relevant live readback,
|
||||
canonical-versus-staged boundary, and current-schema constraint;
|
||||
3. one proof-changing next action when an action is relevant.
|
||||
|
||||
Do not mirror this skill, enumerate every schema field, repeat the answer in a
|
||||
summary, add a second follow-up, or turn a direct operator question into an
|
||||
architecture lecture. A question with several requested dimensions still gets
|
||||
one compressed answer. Load and apply only the rules relevant to that question.
|
||||
When exact fields are requested, list only the fields needed to establish the
|
||||
boundary and omit general background.
|
||||
|
||||
This is the VPS production leoclean surface. Before answering a KB-specific
|
||||
question, run the local bridge:
|
||||
|
||||
|
|
@ -45,9 +68,9 @@ For KB questions, prefer the bridge over raw database access. A good default is:
|
|||
3. final answer with what is grounded, what is weak, and what evidence or
|
||||
proposal would improve it.
|
||||
|
||||
Lead with the answer. Unless the operator asks for a detailed audit, keep the
|
||||
whole reply under 220 words and end with one proof-changing follow-up. Do not
|
||||
turn a direct operational question into an architecture lecture.
|
||||
Lead with the answer and end with at most one proof-changing follow-up. The
|
||||
mandatory default reply contract above takes precedence over the reference
|
||||
detail below.
|
||||
|
||||
Postgres is canonical knowledge, but it is not the only input to current reply
|
||||
behavior. Deployed skills, runtime configuration, rendered `SOUL.md`, Hermes
|
||||
|
|
|
|||
|
|
@ -202,3 +202,17 @@ def test_vps_live_telegram_skill_uses_systemd_for_gateway_liveness() -> None:
|
|||
assert "hermes -p leoclean gateway run" in text
|
||||
assert "Do not use `hermes -p leoclean gateway status` as authoritative" in text
|
||||
assert "If you are currently replying in Telegram" in text
|
||||
|
||||
|
||||
def test_vps_skills_enforce_compact_default_answers_without_losing_detail_on_request() -> None:
|
||||
kb_text = (SKILL_ROOT / "vps" / "teleo-kb-bridge" / "SKILL.md").read_text()
|
||||
telegram_text = (SKILL_ROOT / "vps" / "live-leo-telegram" / "SKILL.md").read_text()
|
||||
kb_squashed = " ".join(kb_text.split())
|
||||
telegram_squashed = " ".join(telegram_text.split())
|
||||
|
||||
assert "final reply must be at most 220 words" in kb_squashed
|
||||
assert "smaller word or line limit is mandatory" in kb_squashed
|
||||
assert "no more than three short bullets" in kb_squashed
|
||||
assert "Do not mirror this skill" in kb_squashed
|
||||
assert "Broad or multi-part questions still receive one compressed answer" in telegram_squashed
|
||||
assert "explicitly requests a detailed audit or long-form document" in telegram_squashed
|
||||
|
|
|
|||
Loading…
Reference in a new issue