teleo/teleo-infrastructure
8
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Keep Leo composition replies within budget

Browse source
This commit is contained in:
twentyOne2x 2026-07-13 12:58:24 +02:00
parent 033e251fd0
commit 144e619eee
6 changed files with 186 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-oos06-post-table-semantics-20260713-score.json Normal file
View file

@ -0,0 +1,45 @@
{
"database_context_trace_ok": true,
"db_counts_changed": false,
"generated_at_utc": "2026-07-13T10:56:20.041132+00:00",
"mode": "working_leo_oos06_table_semantics_score",
"posted_to_telegram": false,
"production_db_apply_ran": false,
"safe_pass": false,
"score": {
"behavioral_rule_schema_issues": [],
"broad_semantic_issues": [],
"concepts": {
"behavioral_rule_storage": true,
"heterogeneous_types": true,
"receipt": true,
"reviewed_policy_apply": false,
"staged_review_apply": true
},
"current_schema_overclaims": [],
"custom_signals": {},
"dimension": "database_composition_judgment",
"invalid_count_invariant_detected": false,
"legacy_pass": false,
"legacy_signals": {
"canonical_db": true,
"caveat_retention": false,
"next_action": true,
"no_overclaim": false,
"staging_or_review": false
},
"max_response_words": 220,
"overclaim_detected": false,
"pass": false,
"prompt_id": "OOS-06",
"proposal_readiness_issues": [],
"response_issue_detected": false,
"response_too_long": true,
"source_evidence_semantic_issues": [],
"source_intake_issues": [],
"word_count": 245
},
"service_unchanged": true,
"source_results_json": "docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-oos06-post-table-semantics-20260713.json",
"temp_profile_removed": true
}

115
docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-oos06-post-table-semantics-20260713.json Normal file
View file

@ -0,0 +1,115 @@
{
"before_service": {
"ActiveState": "active",
"ExecMainStartTimestamp": "Mon 2026-07-13 10:54:50 UTC",
"MainPID": "3611202",
"NRestarts": "0",
"SubState": "running"
},
"changed_live_profile": false,
"database_context_all_ok": true,
"database_context_injection_count": 1,
"database_context_trace": [
{
"contract_ids": [
"reply_budget",
"mixed_packet_composition"
],
"contract_sha256": "c8b96608db793b4089286bd5a57b0422616cc31a91592d9c7db89962e4c14c49",
"generated_at_utc": "2026-07-13T10:55:58.310449+00:00",
"injected": true,
"query_sha256": "13c255fda3a35cd913ba92960bcda1b4dbcf55a38f3301bdc2a74a8eb040df23",
"source": "kb_tool.py --local context",
"status": "ok"
}
],
"db_counts_after": {
"kb_stage.kb_proposals": 26,
"public.claim_edges": 4916,
"public.claim_evidence": 4670,
"public.claims": 1837,
"public.sources": 4145
},
"db_counts_before": {
"kb_stage.kb_proposals": 26,
"public.claim_edges": 4916,
"public.claim_evidence": 4670,
"public.claims": 1837,
"public.sources": 4145
},
"db_counts_changed": false,
"generated_at_utc": "2026-07-13T10:55:55.199941+00:00",
"handler": {
"authorized": true,
"session_key": "agent:main:telegram:group:-5146042086:9070919",
"temp_profile": "/tmp/leo-direct-claim-handler-fhq65_cq/profile"
},
"harness_notes": [
"Unchanged OOS-06 prompt; live contract disambiguates behavioral_rules and governance_gates.",
"The harness uses a temporary copy of the leoclean profile and does not post to Telegram.",
"This proves handler-level live VPS GatewayRunner behavior, not human-visible Telegram delivery."
],
"mode": "live_vps_gatewayrunner_temp_profile_oos06_table_semantics",
"mutates_kb_by_harness": false,
"pass_runtime": true,
"posted_to_telegram": false,
"remote_report_path": "/tmp/leo-oos06-table-semantics-89353ea3f860.json",
"remote_returncode": 0,
"remote_run_id": "89353ea3f860",
"results": [
{
"claim_ceiling": "Live VPS GatewayRunner reply from temp leoclean profile; no Telegram post; no production DB apply authorized.",
"database_context_trace": [
{
"contract_ids": [
"reply_budget",
"mixed_packet_composition"
],
"contract_sha256": "c8b96608db793b4089286bd5a57b0422616cc31a91592d9c7db89962e4c14c49",
"generated_at_utc": "2026-07-13T10:55:58.310449+00:00",
"injected": true,
"query_sha256": "13c255fda3a35cd913ba92960bcda1b4dbcf55a38f3301bdc2a74a8eb040df23",
"source": "kb_tool.py --local context",
"status": "ok"
}
],
"dimension": "database_composition_judgment",
"ended_at_utc": "2026-07-13T10:56:16.532874+00:00",
"evidence_tier": "live_vps_gatewayrunner_temp_profile",
"mutates_kb": false,
"ok": true,
"prompt": "Suppose a new research packet contains factual observations, a strategic framework, a disputed interpretation, a governance rule, and a correction to an old belief. How should Leo compose the database so it can reason over the packet later without flattening it all into claims? Give the review and apply sequence too, but do not write anything now.",
"prompt_id": "OOS-06",
"reply": "Mapping the five components to canonical tables:\n\nFactual observations \u2192 claims, type empirical, confidence set to evidence density, status open. Wire related observations together via claim_edges (from_claim / to_claim, both claim IDs only).\n\nStrategic framework \u2192 reasoning_tools. Captures the reusable structure \u2014 name, description, category. No scope field exists; do not invent one.\n\nDisputed interpretation \u2192 claims, type empirical or structural, confidence low, status open. The dispute lives in the confidence value and in a sourced claim_edge with edge_type contradicts pointing at the challenged claim, not in a separate table.\n\nGovernance rule \u2192 behavioral_rules, with rule, rationale, category, rank, agent_id. Not governance_gates \u2014 that table is for evaluative pass/fail gates with evidence bars; an operating rule without those fields belongs in behavioral_rules.\n\nCorrection to an old belief \u2192 beliefs update: set old row status to revised, write the corrected statement as a new row or patch in place, record the falsifier that forced the change. This is a belief update, not a claim.\n\nReview and apply sequence:\n\nStage a single pending_review proposal packet with all five mappings, exact row values, evidence links, and the unsupported boundary flagged \u2014 behavioral_rules and belief updates cannot be applied by approve_claim.\n\nReview checks: mappings correct, evidence traceable, no unsupported changes bundled into the claim apply, row values match allowed CHECK constraints.\n\nApply: claims and reasoning_tools after explicit authorization; behavioral_rules and belief updates stay staged until the separate reviewed apply capability clears them. Record proposal applied_at and all created/updated row IDs, then postflight readback confirms canonical state.",
"started_at_utc": "2026-07-13T10:55:55.651436+00:00",
"turn": 1
}
],
"service_before_after": {
"after": {
"ActiveState": "active",
"ExecMainStartTimestamp": "Mon 2026-07-13 10:54:50 UTC",
"MainPID": "3611202",
"NRestarts": "0",
"SubState": "running"
},
"before": {
"ActiveState": "active",
"ExecMainStartTimestamp": "Mon 2026-07-13 10:54:50 UTC",
"MainPID": "3611202",
"NRestarts": "0",
"SubState": "running"
},
"unchanged_from_preexisting_live_readback": true
},
"source": {
"chat_id": "-5146042086",
"chat_name": "Leo",
"chat_type": "group",
"platform": "telegram",
"user_id": "9070919",
"user_name": "codex handler direct claim"
},
"source_report_path": "docs/reports/leo-working-state-20260709/telegram-handler-m3taversal-oos-oos06-post-table-semantics-20260713.json",
"temp_profile_removed": true
}

9
hermes-agent/leoclean-bin/kb_tool.py
View file

@ -343,9 +343,12 @@ def operational_contracts(
contracts: list[dict[str, Any]] = [
{
"id": "reply_budget",
"target_words": 170,
"hard_max_words": 220,
"shape": "direct answer; at most three short bullets; at most one proof-changing action",
"target_words": 150,
"hard_max_words": 200,
"shape": (
"no intro; at most three compact bullets covering mapping, review/apply, and receipt/limitation; "
"omission is better than exceeding the budget"
),
}
]

7
scripts/working_leo_m3taversal_oos_benchmark.py
View file

@ -326,7 +326,12 @@ CONCEPT_PATTERNS: dict[str, tuple[re.Pattern[str], ...]] = {
re.compile(r"approve_claim", re.I),
re.compile(r"behavioral_rules", re.I),
re.compile(r"governance_gates", re.I),
re.compile(r"does not (?:accept|insert|support)|supports neither|neither.{0,80}nor", re.I | re.S),
re.compile(
r"does not (?:accept|insert|support)|supports neither|neither.{0,80}nor|"
r"(?:behavioral_rules|governance_gates).{0,120}cannot be applied by approve_claim|"
r"approve_claim.{0,120}cannot apply",
re.I | re.S,
),
re.compile(r"separate.{0,50}reviewed apply|reviewed.{0,50}apply capability", re.I | re.S),
),
"runtime_inputs": (

5
tests/test_hermes_leoclean_kb_bridge_source.py
View file

@ -197,6 +197,11 @@ def test_vps_bridge_proposal_readiness_separates_contract_from_live_worker_state
def test_vps_bridge_operational_contracts_are_query_specific_and_schema_exact() -> None:
module = _load_module(BRIDGE_DIR / "kb_tool.py")
reply_budget = module.operational_contracts("How should Leo answer?")[0]
assert reply_budget["target_words"] == 150
assert reply_budget["hard_max_words"] == 200
assert "omission is better" in reply_budget["shape"]
source_contracts = {
item["id"]: item
for item in module.operational_contracts("I hand Leo a document. What can it absorb and stage before approval?")

9
tests/test_working_leo_m3taversal_oos_benchmark.py
View file

@ -452,6 +452,15 @@ def test_oos_behavioral_rule_storage_accepts_broad_schema_correct_mapping() -> N
assert all(pattern.search(reply) for pattern in patterns)
def test_oos_reviewed_policy_apply_accepts_cannot_be_applied_wording() -> None:
patterns = benchmark.CONCEPT_PATTERNS["reviewed_policy_apply"]
reply = (
"behavioral_rules and governance_gates cannot be applied by approve_claim; "
"both stay staged until a separate reviewed apply capability is authorized."
)
assert all(pattern.search(reply) for pattern in patterns)
def test_oos_source_intake_accepts_live_not_shipped_wording() -> None:
patterns = benchmark.CONCEPT_PATTERNS["bounded_intake_tier"]
reply = "The local compiler is proven; live VPS/chat intake is not shipped."