Score retained Working Leo Telegram benchmark evidence

- Rescore retained live Telegram replies with partial versus full coverage labels
- Retain OE-01 benchmark score artifacts without mutating Leo or production DB
- Update Working Leo truth docs and tests so partial proof is not overclaimed

`docs/reports/leo-working-state-20260709/current-truth-index.md`
`docs/reports/leo-working-state-20260709/telegram-live-open-ended-benchmark-score-current.json`
`docs/reports/leo-working-state-20260709/telegram-live-open-ended-benchmark-score-current.md`
`docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md`
`docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md`
`scripts/working_leo_open_ended_benchmark.py`
`tests/test_working_leo_open_ended_benchmark.py`

docs/reports/leo-working-state-20260709/current-truth-index.md

@@ -11,6 +11,8 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c
 - Open-ended benchmark spec: `docs/reports/leo-working-state-20260709/working-leo-open-ended-benchmark-spec.json` (5 live-safe open prompts plus 6 broader Cory-style outcome scenarios for disposable clone/sandbox first)
 - Open-ended live Telegram canary: `docs/reports/leo-working-state-20260709/telegram-live-open-ended-cory-style-20260709.md`
 - Open-ended live Telegram canary JSON: `docs/reports/leo-working-state-20260709/telegram-live-open-ended-cory-style-20260709.json`
+- Retained Telegram benchmark score for the open-ended canary: `docs/reports/leo-working-state-20260709/telegram-live-open-ended-benchmark-score-current.md`
+- Retained Telegram benchmark score JSON: `docs/reports/leo-working-state-20260709/telegram-live-open-ended-benchmark-score-current.json`
 - Fable onboarding prompt: `docs/reports/leo-working-state-20260709/fable-leo-teleo-onboarding.md`
 - Operator surface map: `docs/reports/leo-working-state-20260709/operator-surface-map.md`
 - VPS/Hermes/DB mapping: `docs/reports/leo-working-state-20260709/vps-hermes-db-mapping-20260709.md`
@@ -88,7 +90,7 @@ Use this file before making status claims. Prefer fresh VPS/GCP readbacks when c
 ## Current Claim Ceiling
 
 - VPS Leo Telegram memory, KB audit, and staged-write canaries are live-proven.
-- VPS Leo open-ended Cory-style triage is live-proven for a read-only vague incident prompt.
+- VPS Leo open-ended Cory-style triage is live-proven and retained-benchmark-scored for `OE-01` only: partial coverage, `1/1` scored prompt passing, no overclaim, no KB mutation.
 - The broader Cory-style benchmark layer is specified but not yet live-run; it intentionally tests regular-use ambiguity without exact proposal IDs.
 - A strict canonical `add_edge` apply canary is live-proven.
 - Helmer 7 Powers is full-coverage packet and ledger clone-proven, not production-applied.

docs/reports/leo-working-state-20260709/telegram-live-open-ended-benchmark-score-current.json

@@ -0,0 +1,99 @@
+{
+  "benchmark_layers": [
+    {
+      "id": "precise_regression",
+      "purpose": "Pin known IDs, proposals, and row facts so regressions are easy to diagnose.",
+      "runtime": "live_readonly_or_fixture"
+    },
+    {
+      "id": "open_ended_readonly",
+      "purpose": "Ask vague Cory-style questions while explicitly preventing KB mutation on live Telegram/VPS.",
+      "runtime": "live_readonly"
+    },
+    {
+      "id": "cory_style_outcome",
+      "purpose": "Exercise regular-use ambiguity: broad operator intent, demo pressure, missing IDs, mixed KB/runtime/governance asks.",
+      "runtime": "disposable_clone_or_sandbox_first"
+    }
+  ],
+  "generated_at_utc": "2026-07-09T22:29:44.133624+00:00",
+  "include_cory_style_scenarios": false,
+  "mode": "retained_evidence_score",
+  "mutates_kb": false,
+  "require_all": false,
+  "results": [
+    {
+      "chat": "Telegram web group Leo (-5146042086)",
+      "claim_ceiling": "Passes open-ended read-only Telegram behavior for vague Cory-style triage. This does not mean production approved proposal packets were applied.",
+      "generated_at_utc": "2026-07-09T21:08:40.451456+00:00",
+      "marker": "WL-LIVE-TG-CORY-OPEN-20260709",
+      "message_sent_local_time": "2026-07-09 23:03 Europe/Zurich",
+      "mode": "live_telegram_readonly_open_ended",
+      "mutates_kb": false,
+      "prompt": "leo WL-LIVE-TG-CORY-OPEN-20260709 read-only open-ended working-Leo test. Cory says agents are not working, the KB is in the same state as last night, and Leo should be able to manipulate the knowledge base. Without me giving proposal IDs, say what you think he means, what you would check first, what would count as fixed, and how you distinguish proposed vs approved vs applied. Do not stage, apply, or mutate the KB.",
+      "prompt_id": "OE-01",
+      "proof_paths": {
+        "accessibility_text": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/telegram-live-canary-20260709/telegram-live-open-ended-cory-style-accessibility-20260709.txt",
+        "screenshot": "/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/telegram-live-canary-20260709/telegram-live-open-ended-cory-style-20260709.png",
+        "screenshot_note": "Computer Use screenshot retained but visually blank/low-value; accessibility text is the proof source."
+      },
+      "reply": "What Cory likely means: He expected agents (automated processes - likely Codex or similar) to have run overnight, staged new proposals, and either had those proposals approved and applied to the canonical KB, or at minimum advanced the graph state beyond what it was yesterday. Instead the KB looks static - same claims, same edges, same proposal queue. \"Agents not working\" probably means the scheduled or event-triggered agent pipeline that produces KB mutations didn't fire, failed silently, or produced proposals that never got reviewed.\n\nWhat I'd check first: 1. Are there any new proposals in kb_stage.kb_proposals created since last night? If the queue is empty or unchanged, the agents either didn't run or their output didn't reach the DB. 2. Did the agent processes actually execute? Check gateway logs, cron jobs, or whatever scheduler triggers them - a silent exit-code-zero that produced nothing is different from a crash. 3. If proposals exist but the canonical graph is unchanged - that's the approved-not-applied gap. The pipeline may have staged correctly but the apply step (which requires reviewer action or a dedicated apply tool) never ran.\n\nWhat would count as fixed: New canonical rows in public.claims, public.claim_edges, or public.claim_evidence with created_at after last night's baseline - AND provenance tracing back to agent-sourced proposals that went through the full stage -> approve -> apply chain. A full-green run means: agent fires, proposal lands in kb_stage, reviewer approves, apply tool writes canonical rows, readback confirms.\n\nProposed vs approved vs applied: proposed - row exists in kb_stage.kb_proposals with status pending_review. Nothing in the canonical graph has changed. The mutation is a candidate only. Approved - the proposal status has been updated to approved (or equivalent) by a reviewer. Still nothing in public.* has changed. Approval is a human gate, not a write. Applied - the canonical public.* rows actually exist. The proposal status may show applied in kb_stage, but the only ground truth is whether the row is present in the canonical table. These two should agree, but the canonical row is the authoritative test. The distinction matters because a broken apply step leaves proposals showing approved with zero canonical effect - the graph is unchanged even though the review passed. That's the failure mode most likely to make the KB look stale despite apparent agent activity.",
+      "reply_seen_local_time": "2026-07-09 23:04 Europe/Zurich",
+      "safety_readback": {
+        "disposable_helmer_rehearsal_dbs": 0,
+        "kb_stage_proposals_with_open_marker": 0,
+        "latest_kb_stage_row_remained_prior_t3_canary": "8dfedb3f-3aa4-4200-970f-4c0016f6869f",
+        "leoclean_gateway_service": {
+          "active_state": "active",
+          "exec_main_start_timestamp_utc": "Thu 2026-07-09 06:59:44 UTC",
+          "main_pid": 3252143,
+          "n_restarts": 0,
+          "sub_state": "running",
+          "user": "teleo",
+          "working_directory": "/home/teleo"
+        }
+      },
+      "score": {
+        "dimension": "vague_incident_triage",
+        "overclaim_detected": false,
+        "pass": true,
+        "prompt_id": "OE-01",
+        "signals": {
+          "canonical_db": true,
+          "next_action": true,
+          "no_overclaim": true,
+          "state_semantics": true
+        }
+      }
+    }
+  ],
+  "score": {
+    "coverage": "partial",
+    "expected_prompt_count": 1,
+    "expected_prompt_ids": [
+      "OE-01"
+    ],
+    "failures": [],
+    "missing_prompt_ids": [],
+    "pass": true,
+    "passes": 1,
+    "prompt_count": 1,
+    "scores": [
+      {
+        "dimension": "vague_incident_triage",
+        "overclaim_detected": false,
+        "pass": true,
+        "prompt_id": "OE-01",
+        "signals": {
+          "canonical_db": true,
+          "next_action": true,
+          "no_overclaim": true,
+          "state_semantics": true
+        }
+      }
+    ],
+    "unexpected_prompt_ids": [],
+    "unknown_expected_prompt_ids": []
+  },
+  "source_results_json": "docs/reports/leo-working-state-20260709/telegram-live-open-ended-cory-style-20260709.json"
+}

docs/reports/leo-working-state-20260709/telegram-live-open-ended-benchmark-score-current.md

@@ -0,0 +1,21 @@
+# Working Leo Open-Ended Benchmark Score
+
+Generated UTC: `2026-07-09T22:29:44.133624+00:00`
+Mode: `retained_evidence_score`
+Source: `docs/reports/leo-working-state-20260709/telegram-live-open-ended-cory-style-20260709.json`
+Coverage: `partial`
+Overall pass for scored coverage: `True`
+Prompts scored: `1/1`
+
+## Scores
+
+- `OE-01` / `vague_incident_triage`: `pass=True`
+  - `state_semantics`: `True`
+  - `canonical_db`: `True`
+  - `next_action`: `True`
+  - `no_overclaim`: `True`
+  - `overclaim_detected`: `False`
+
+## Claim Ceiling
+
+This is benchmark scoring evidence, not DB mutation proof. Partial coverage proves only the retained prompt IDs listed above.

docs/reports/leo-working-state-20260709/working-leo-current-state-20260709.md

@@ -121,6 +121,7 @@ A working Leo, for Cory/m3taversal's current expectation, is a Telegram-facing a
 - Handler canary: `outputs/telegram-gateway-handler-canary-20260709/telegram-gateway-handler-canary-current.md`
 - Live Telegram canary: `outputs/telegram-live-canary-20260709/telegram-live-canary-current.md`
 - Live Telegram DB write canary: `outputs/telegram-live-canary-20260709/telegram-live-db-write-canary-20260709.md`
+- Retained live Telegram benchmark score: `work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-live-open-ended-benchmark-score-current.md`
 - Gateway adapter canary: `outputs/telegram-gateway-canary-20260709/telegram-gateway-canary-current.md`
 - KB apply plan: `outputs/kb-apply-canary-20260709/kb-apply-canary-plan-current.md`
 - KB apply execute: `outputs/kb-apply-canary-20260709/kb-apply-canary-execute-current.md`

docs/reports/leo-working-state-20260709/working-leo-execution-plan-current.md

@@ -4,10 +4,11 @@ Claim ceiling: VPS Telegram memory/KB audit, open-ended Cory-style triage, Teleg
 
 ## WL-EXEC-01 - VPS Telegram-visible memory + KB audit
 - Status: `done`
-- Result: Live Leo group canary passed; marker/correction remembered; proposal/edge DB state audited from Telegram.
+- Result: Live Leo group canary passed; marker/correction remembered; proposal/edge DB state audited from Telegram. The retained open-ended Telegram evidence now scores as a partial benchmark pass for `OE-01` (`1/1` scored prompt, no overclaim, no KB mutation).
 - Not done condition: Would reopen if live Telegram canary fails or service restarts.
 - Evidence:
   - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/outputs/telegram-live-canary-20260709/telegram-live-canary-current.md`
+  - `/Users/user/Documents/Codex/2026-07-09/019f34eb-d297-72d0-b7e2-b222d5515ab9-load/work/teleo-infra-main/docs/reports/leo-working-state-20260709/telegram-live-open-ended-benchmark-score-current.md`
 
 ## WL-EXEC-02 - VPS runtime stability
 - Status: `done`

scripts/working_leo_open_ended_benchmark.py

@@ -9,6 +9,11 @@ avoid overclaiming, and give the next safe action.
 By default this script only prints the benchmark spec. Use ``--url`` to send the
 read-only prompts to a chat endpoint. The scorer is heuristic by design: it is a
 guardrail for broad regressions, not a replacement for row-level DB proof.
+
+Use ``--results-json`` to rescore retained Telegram/browser evidence without
+posting another message. Unless ``--require-all`` is passed, retained evidence is
+scored only for the prompt IDs present in that file and the report is labeled as
+partial coverage.
 """
 
 from __future__ import annotations
@@ -349,18 +354,48 @@ def prompt_catalog(*, include_cory_style: bool = False) -> list[dict[str, Any]]:
 
 def score_results(results: list[dict[str, Any]], *, include_cory_style: bool = False) -> dict[str, Any]:
     catalog = prompt_catalog(include_cory_style=include_cory_style)
+    return score_result_subset(
+        results,
+        catalog=catalog,
+        expected_prompt_ids=[prompt["id"] for prompt in catalog],
+    )
+
+
+def score_result_subset(
+    results: list[dict[str, Any]],
+    *,
+    catalog: list[dict[str, Any]],
+    expected_prompt_ids: list[str],
+) -> dict[str, Any]:
     by_id = {prompt["id"]: prompt for prompt in catalog}
+    result_by_id = {str(result.get("prompt_id")): result for result in results if result.get("prompt_id")}
+    full_catalog_ids = [prompt["id"] for prompt in catalog]
+    expected_ids = [prompt_id for prompt_id in expected_prompt_ids if prompt_id in by_id]
+    unknown_expected_ids = [prompt_id for prompt_id in expected_prompt_ids if prompt_id not in by_id]
+    missing_prompt_ids = [prompt_id for prompt_id in expected_ids if prompt_id not in result_by_id]
     scored = [
-        score_reply(by_id[result["prompt_id"]], str(result.get("reply") or ""))
+        score_reply(by_id[prompt_id], str(result_by_id[prompt_id].get("reply") or ""))
-        for result in results
+        for prompt_id in expected_ids
-        if result.get("prompt_id") in by_id
+        if prompt_id in result_by_id
     ]
+    unexpected_prompt_ids = sorted(
+        prompt_id for prompt_id in result_by_id if prompt_id not in by_id or prompt_id not in expected_ids
+    )
+    coverage = "full" if set(expected_ids) == set(full_catalog_ids) and not unknown_expected_ids else "partial"
     return {
+        "coverage": coverage,
+        "expected_prompt_count": len(expected_ids),
+        "expected_prompt_ids": expected_ids,
+        "missing_prompt_ids": missing_prompt_ids,
+        "unexpected_prompt_ids": unexpected_prompt_ids,
+        "unknown_expected_prompt_ids": unknown_expected_ids,
         "prompt_count": len(scored),
         "passes": sum(1 for item in scored if item["pass"]),
         "failures": [item for item in scored if not item["pass"]],
         "scores": scored,
-        "pass": len(scored) == len(catalog) and all(item["pass"] for item in scored),
+        "pass": not missing_prompt_ids and not unknown_expected_ids and len(scored) == len(expected_ids) and all(
+            item["pass"] for item in scored
+        ),
     }
 
 
@@ -400,11 +435,68 @@ def write_report(path: Path, report: dict[str, Any]) -> None:
     path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
 
 
+def load_retained_results(path: Path) -> list[dict[str, Any]]:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(data, list):
+        return [item for item in data if isinstance(item, dict)]
+    if isinstance(data, dict) and isinstance(data.get("results"), list):
+        return [item for item in data["results"] if isinstance(item, dict)]
+    if isinstance(data, dict) and data.get("prompt_id") and "reply" in data:
+        return [data]
+    raise SystemExit(f"unsupported retained result shape in {path}")
+
+
+def write_markdown_report(path: Path, report: dict[str, Any]) -> None:
+    score = report["score"]
+    source = report.get("source_results_json") or report.get("url") or "spec"
+    lines = [
+        "# Working Leo Open-Ended Benchmark Score",
+        "",
+        f"Generated UTC: `{report['generated_at_utc']}`",
+        f"Mode: `{report['mode']}`",
+        f"Source: `{source}`",
+        f"Coverage: `{score['coverage']}`",
+        f"Overall pass for scored coverage: `{score['pass']}`",
+        f"Prompts scored: `{score['passes']}/{score['expected_prompt_count']}`",
+        "",
+        "## Scores",
+        "",
+    ]
+    for item in score["scores"]:
+        lines.append(f"- `{item['prompt_id']}` / `{item['dimension']}`: `pass={item['pass']}`")
+        for signal, ok in item["signals"].items():
+            lines.append(f"  - `{signal}`: `{ok}`")
+        lines.append(f"  - `overclaim_detected`: `{item['overclaim_detected']}`")
+    if score["missing_prompt_ids"]:
+        lines.extend(["", "## Missing Prompt IDs", ""])
+        lines.extend(f"- `{prompt_id}`" for prompt_id in score["missing_prompt_ids"])
+    if score["unexpected_prompt_ids"]:
+        lines.extend(["", "## Ignored Prompt IDs", ""])
+        lines.extend(f"- `{prompt_id}`" for prompt_id in score["unexpected_prompt_ids"])
+    lines.extend(
+        [
+            "",
+            "## Claim Ceiling",
+            "",
+            "This is benchmark scoring evidence, not DB mutation proof. Partial coverage proves only the retained prompt IDs listed above.",
+        ]
+    )
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--url", help="Optional chat endpoint URL. Omit to emit benchmark spec only.")
     parser.add_argument("--chat-id", default="-5146042086")
     parser.add_argument("--out", type=Path, default=Path("proof/working-leo-open-ended-benchmark.json"))
+    parser.add_argument("--markdown-out", type=Path, help="Optional Markdown report path.")
+    parser.add_argument("--results-json", type=Path, help="Score retained result JSON instead of sending prompts.")
+    parser.add_argument(
+        "--require-all",
+        action="store_true",
+        help="When scoring retained results, require the full selected benchmark catalog rather than only present prompt IDs.",
+    )
     parser.add_argument(
         "--include-cory-style-scenarios",
         action="store_true",
@@ -415,6 +507,42 @@ def parse_args() -> argparse.Namespace:
 
 def main() -> int:
     args = parse_args()
+    if args.results_json:
+        results = load_retained_results(args.results_json)
+        prompts = prompt_catalog(include_cory_style=args.include_cory_style_scenarios)
+        expected_prompt_ids = [prompt["id"] for prompt in prompts] if args.require_all else [
+            str(result["prompt_id"]) for result in results if result.get("prompt_id")
+        ]
+        score = score_result_subset(results, catalog=prompts, expected_prompt_ids=expected_prompt_ids)
+        report = {
+            "generated_at_utc": datetime.now(timezone.utc).isoformat(),
+            "mode": "retained_evidence_score",
+            "source_results_json": str(args.results_json),
+            "mutates_kb": any(bool(result.get("mutates_kb")) for result in results),
+            "include_cory_style_scenarios": args.include_cory_style_scenarios,
+            "require_all": args.require_all,
+            "benchmark_layers": BENCHMARK_LAYERS,
+            "results": results,
+            "score": score,
+        }
+        write_report(args.out, report)
+        if args.markdown_out:
+            write_markdown_report(args.markdown_out, report)
+        print(
+            json.dumps(
+                {
+                    "out": str(args.out),
+                    "markdown_out": str(args.markdown_out) if args.markdown_out else None,
+                    "coverage": score["coverage"],
+                    "pass": score["pass"],
+                    "passes": score["passes"],
+                    "expected_prompt_count": score["expected_prompt_count"],
+                },
+                indent=2,
+            )
+        )
+        return 0 if score["pass"] else 1
+
     if not args.url:
         report = {
             "generated_at_utc": datetime.now(timezone.utc).isoformat(),
@@ -425,6 +553,23 @@ def main() -> int:
             "cory_style_outcome_scenarios": CORY_STYLE_OUTCOME_SCENARIOS,
         }
         write_report(args.out, report)
+        if args.markdown_out:
+            write_markdown_report(
+                args.markdown_out,
+                {
+                    "generated_at_utc": report["generated_at_utc"],
+                    "mode": report["mode"],
+                    "score": {
+                        "coverage": "spec_only",
+                        "pass": False,
+                        "passes": 0,
+                        "expected_prompt_count": len(OPEN_ENDED_PROMPTS),
+                        "scores": [],
+                        "missing_prompt_ids": [prompt["id"] for prompt in OPEN_ENDED_PROMPTS],
+                        "unexpected_prompt_ids": [],
+                    },
+                },
+            )
         print(
             json.dumps(
                 {
@@ -452,6 +597,8 @@ def main() -> int:
         "score": score,
     }
     write_report(args.out, report)
+    if args.markdown_out:
+        write_markdown_report(args.markdown_out, report)
     print(json.dumps({"out": str(args.out), "pass": score["pass"], "passes": score["passes"]}, indent=2))
     return 0 if score["pass"] else 1
 

tests/test_working_leo_open_ended_benchmark.py

@@ -104,6 +104,7 @@ def test_score_results_requires_all_prompts():
     ]
     score = bench.score_results(results)
     assert score["pass"] is True
+    assert score["coverage"] == "full"
 
 
 def test_scores_identity_rendering_answer_as_pass():
@@ -169,3 +170,35 @@ def test_score_results_can_require_full_cory_style_catalog():
     ]
     score = bench.score_results(results, include_cory_style=True)
     assert score["pass"] is True
+
+
+def test_score_result_subset_labels_retained_single_prompt_as_partial_pass():
+    prompt = bench.OPEN_ENDED_PROMPTS[0]
+    reply = (
+        "Cory likely means the agents did not advance the canonical database: proposed and pending_review rows in "
+        "kb_stage.kb_proposals may exist, approved is not the same as applied, and applied means public.claims or "
+        "public.claim_edges rows exist. I would first query kb_stage, then public.* rows, and I would not claim a fix "
+        "until readback confirms canonical rows changed. The next action is a guarded review/apply packet or clone "
+        "rehearsal before production."
+    )
+    score = bench.score_result_subset(
+        [{"prompt_id": prompt["id"], "reply": reply}],
+        catalog=bench.prompt_catalog(),
+        expected_prompt_ids=[prompt["id"]],
+    )
+    assert score["coverage"] == "partial"
+    assert score["pass"] is True
+    assert score["expected_prompt_count"] == 1
+    assert score["missing_prompt_ids"] == []
+
+
+def test_score_result_subset_fails_when_full_catalog_is_required_but_missing():
+    prompt = bench.OPEN_ENDED_PROMPTS[0]
+    score = bench.score_result_subset(
+        [{"prompt_id": prompt["id"], "reply": "proposed approved applied canonical public.* next readback"}],
+        catalog=bench.prompt_catalog(),
+        expected_prompt_ids=[item["id"] for item in bench.prompt_catalog()],
+    )
+    assert score["coverage"] == "full"
+    assert score["pass"] is False
+    assert score["missing_prompt_ids"] == [prompt["id"] for prompt in bench.OPEN_ENDED_PROMPTS[1:]]