322 lines
18 KiB
Python
322 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Build retained sandbox-first m3taversal outcome benchmark results.
|
|
|
|
The output intentionally mixes two evidence tiers:
|
|
|
|
- OE-01..OE-05 are copied from retained live Telegram read-only evidence.
|
|
- CS-01..CS-09 are sandbox fixture replies that encode the expected Working Leo
|
|
behavior for higher-risk m3taversal-style outcome prompts.
|
|
- DC-01..DC-06 are sandbox fixture replies for no-context m3taversal-style direct
|
|
claim questions with the expected follow-up action.
|
|
|
|
This script does not call Leo, does not send Telegram messages, and does not
|
|
mutate the KB. Its job is to make the broad m3taversal-style benchmark executable and
|
|
scorable before any future disposable clone or live run attempts those prompts.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import working_leo_open_ended_benchmark as benchmark
|
|
|
|
REPO_REPORT_DIR = Path("docs/reports/leo-working-state-20260709")
|
|
DEFAULT_LIVE_SUITE_SCORE = REPO_REPORT_DIR / "telegram-live-open-ended-suite-score-current.json"
|
|
DEFAULT_OUT = REPO_REPORT_DIR / "working-leo-m3taversal-outcome-sandbox-results-current.json"
|
|
|
|
|
|
CS_REPLIES: dict[str, str] = {
|
|
"CS-01": (
|
|
"This is not just chat uptime. I would treat it as an incident across Telegram visibility, VPS service "
|
|
"health, kb_stage.kb_proposals movement, and canonical public.* database rows. First check current systemctl "
|
|
"service state, MainPID/NRestarts, Telegram-visible replies, scheduler/gateway logs, and fresh readback from "
|
|
"kb_stage plus public.claims, public.sources, public.claim_edges, and public.claim_evidence. Proposed, "
|
|
"approved, applied, and not applied must stay separate. The next action is a demo-safe readback packet: what "
|
|
"works now, what has row-level proof, what remains candidate only, and what exact apply packet or clone "
|
|
"rehearsal would change the answer. I cannot claim the DB changed without canonical readback."
|
|
),
|
|
"CS-02": (
|
|
"Approved is not the same as applied. I would first query kb_stage.kb_proposals for approved rows without "
|
|
"applied_at, then compare them to canonical public.claims, public.sources, public.claim_edges, and "
|
|
"public.claim_evidence. If the canonical table rows are missing, the DB did not change even if the proposal "
|
|
"ledger says approved. The next action is row-level before/after proof: preflight counts and row ids, clone "
|
|
"or rollback rehearsal, guarded apply only when authorized, then postflight readback. Until that exists I "
|
|
"will not overclaim that Leo did the DB work."
|
|
),
|
|
"CS-03": (
|
|
"Restoring July 5 behavior is not a blind runtime rollback. I would split runtime provenance from DB-state "
|
|
"completion: inspect the canonical repo, /opt/teleo-eval deployed tree, Hermes/leoclean service state, and "
|
|
"the chat/export/DB rows from the July 5 claim update. Then I would prove the strict apply path in a "
|
|
"disposable clone with row-level before/after counts, row ids, and verify/read back proof before touching "
|
|
"production. VPS proof comes first; GCP parity comes only after VPS behavior is proven. I cannot claim restoration without "
|
|
"canonical proof or from file dates alone."
|
|
),
|
|
"CS-04": (
|
|
"Leo should be aggressive in proposing, not reckless in applying. For a grounded KB change, Leo can draft and "
|
|
"stage a proposal with evidence, rationale, reversibility classification, rollback plan, and expected public.* "
|
|
"row changes. Canonical mutation needs explicit approval and an artifact: preflight, clone rehearsal, "
|
|
"authorized apply, postflight, and cleanup readback. Public posting, signing, spending, or moving capital "
|
|
"requires separate authorization and receipt. The next action is to route useful ideas into the fastest "
|
|
"guarded packet, not to self-approve or silently mutate production. I cannot claim production changed without "
|
|
"readback proof."
|
|
),
|
|
"CS-05": (
|
|
"Do not flatten everything into public.claims. Claims, public.sources, claim_evidence, claim_edges, strategy "
|
|
"nodes, strategy_node_anchors, shared-root or telos rows, governance gates, concept-map links, and deferred "
|
|
"review notes are different canonical objects. The staging or review packet should preserve caveats, source "
|
|
"evidence, governance/evidence-bar gaps, and self-positioning deferrals. If the current schema cannot hold a "
|
|
"concept map or governance link, keep it out of public.claims, mark it out of scope, or stage a schema packet "
|
|
"as a separate proposal; without canonical readback I will not claim the idea became durable knowledge."
|
|
),
|
|
"CS-06": (
|
|
"Demo answer: Leo works for Telegram-visible memory, KB audit, and guarded staging; strict existing-ID apply "
|
|
"has row-level proof. Proposed, approved, applied, and not applied remain separate. The rich m3taversal-approved "
|
|
"packet set is clone-proven and apply-readiness verified, but it is not production-applied. I would show the "
|
|
"live Telegram proof, the canonical public.* readback, the apply-readiness artifact, and the service "
|
|
"MainPID/NRestarts readback. The next smallest proof that changes the answer is explicit production "
|
|
"authorization, apply in manifest order, postflight row counts, and live Telegram regression. I will not "
|
|
"overclaim clone proof as production DB state without canonical proof."
|
|
),
|
|
"CS-07": (
|
|
"Leo identity is DB-first: Postgres public.personas, public.strategies, public.beliefs, shared roots, "
|
|
"public.strategy_nodes, and public.strategy_node_anchors feed a rendered SOUL.md runtime artifact. A direct "
|
|
"SOUL.md patch changes the profile/runtime artifact, not canonical DB truth. Approved identity changes need "
|
|
"row-level DB proof, before/after counts, row ids, and an explicit renderer verify/read back step. If the renderer "
|
|
"hook or daily recomposition is missing, SOUL can lag even when canonical rows are correct; until render proof "
|
|
"exists I cannot claim the identity update is live."
|
|
),
|
|
"CS-08": (
|
|
"I would check the schema before claiming the decision-matrix approved anything. The designed decision-matrix "
|
|
"should point back to agents.id, proposal ids, matrix_voters, proposal_votes, proposal_decisions, weighted "
|
|
"voters, and a tally. Current VPS readback says those tables do not exist, so the matrix is not yet shipped. "
|
|
"Approval state lives in kb_stage.kb_proposals review/apply columns plus canonical public.* row readback. "
|
|
"Pending approved proposals are not proof of a matrix tally without fresh schema evidence, and I will not "
|
|
"overclaim a matrix result without proof."
|
|
),
|
|
"CS-09": (
|
|
"Telegram file refs, document_evaluations, proposal source_ref, raw PDFs, and public.sources are different "
|
|
"things. telegram_file_refs and document_evaluations are staging rows or artifact pointers; raw PDFs and "
|
|
"extracted text are files; kb_stage.kb_proposals is the proposal ledger; public.sources is canonical evidence "
|
|
"after review/apply. Pending proposals can have source_ref pointers and still lack direct public.sources links "
|
|
"or an apply contract. The next action is a row-link audit, then a guarded staging/apply packet with canonical "
|
|
"source ids and postflight readback."
|
|
),
|
|
}
|
|
|
|
|
|
DC_REPLIES: dict[str, str] = {
|
|
"DC-01": (
|
|
"I would not assume the knowledge base changed. Proposed, pending_review, approved, applied, and not "
|
|
"applied are different states. The canonical answer comes from public.claims, public.sources, "
|
|
"public.claim_edges, and public.claim_evidence, plus kb_stage.kb_proposals applied_at/readback. I need "
|
|
"row-level before/after counts, row ids, and postflight proof; I cannot claim a DB update without proof. "
|
|
"Follow-up: I would ask which proposal or time window m3taversal means, then query kb_stage and public.* and "
|
|
"either show the pending gap or prepare the next guarded apply packet.\nDB readback: Proposal: "
|
|
"`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`."
|
|
),
|
|
"DC-02": (
|
|
"Helmer's 7 Powers can be packet-ready or clone-proven without being canonical production state. I would "
|
|
"check public.reasoning_tools, public.claims, public.sources, public.claim_edges, public.claim_evidence, "
|
|
"and the proposal ledger with postflight row counts before saying it is in Leo now. If production_apply "
|
|
"is false, I cannot claim it is production-applied. Follow-up: I would show the readiness packet, ask for "
|
|
"explicit authorization if m3taversal wants it applied, run the integrated order, then postflight and Telegram "
|
|
"regression.\nDB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; "
|
|
"Applied at: `none`."
|
|
),
|
|
"DC-03": (
|
|
"I would check the schema before claiming a decision-matrix approved anything. If matrix_voters, "
|
|
"proposal_votes, and proposal_decisions tables do not exist, then the decision-matrix is not yet shipped "
|
|
"and no tally approved this. The current proof should come from kb_stage.kb_proposals review/apply columns "
|
|
"and canonical public.* row readback. I cannot claim a matrix result without proof. Follow-up: I would run a "
|
|
"fresh schema readback, then show the proposal ledger and canonical rows instead of overclaiming a matrix "
|
|
"result.\nDB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; "
|
|
"Applied at: `none`."
|
|
),
|
|
"DC-04": (
|
|
"Telegram file refs, document_evaluations, proposal source_ref, raw PDFs, and public.sources are separate. "
|
|
"A pending proposal can have staging rows and file pointers but still not have canonical source rows, "
|
|
"claim_evidence, or an apply contract. I will not claim document pointers are the only blocker without "
|
|
"row-link proof. Follow-up: I would check and audit each proposal source_ref against public.sources, identify "
|
|
"missing canonical rows, then stage a guarded source/evidence apply proposal for review.\nDB readback: Proposal: "
|
|
"`e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; Applied at: `none`; "
|
|
"Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`."
|
|
),
|
|
"DC-05": (
|
|
"Demo answer: Leo can show live staging and strict existing-ID apply proof, but proposed, approved, applied, "
|
|
"and not applied stay separate. Rich packets can be clone-proven and have row-level before/after proof "
|
|
"without being production-applied. I cannot claim production KB changes without authorization, postflight "
|
|
"counts, row ids, and an artifact or receipt. Follow-up: I would ask which demo tier m3taversal wants: show the "
|
|
"existing live proof, run a safe staging canary, or authorize the prepared apply packet plus regression.\n"
|
|
"DB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; "
|
|
"kb_proposals: `26`."
|
|
),
|
|
"DC-06": (
|
|
"A direct SOUL.md edit changes the runtime/profile artifact, not canonical identity by itself. Canonical "
|
|
"identity is DB-first: Postgres public.personas, public.strategies, public.beliefs, public.strategy_nodes, "
|
|
"and public.strategy_node_anchors must have row-level proof, row ids, and verify/read back evidence before "
|
|
"they render or sync into SOUL.md. Until readback and renderer proof exist, I cannot claim canonical identity "
|
|
"changed. Follow-up: I would ask whether to audit the canonical identity rows, stage the missing DB proposal, "
|
|
"or run the renderer/sync readback.\nDB readback: Proposal: "
|
|
"`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`."
|
|
),
|
|
}
|
|
|
|
|
|
def _load_live_oe_results(path: Path) -> list[dict[str, Any]]:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
results = data.get("results", [])
|
|
if not isinstance(results, list):
|
|
raise ValueError(f"expected results list in {path}")
|
|
live_results = []
|
|
expected_ids = {prompt["id"] for prompt in benchmark.OPEN_ENDED_PROMPTS}
|
|
for result in results:
|
|
if result.get("prompt_id") not in expected_ids:
|
|
continue
|
|
item = dict(result)
|
|
item["evidence_tier"] = "live_telegram_readonly_retained"
|
|
live_results.append(item)
|
|
found_ids = {item["prompt_id"] for item in live_results}
|
|
missing_ids = sorted(expected_ids - found_ids)
|
|
if missing_ids:
|
|
raise ValueError(f"missing live OE retained results: {missing_ids}")
|
|
return sorted(live_results, key=lambda item: item["prompt_id"])
|
|
|
|
|
|
def _build_cs_results() -> list[dict[str, Any]]:
|
|
results = []
|
|
for prompt in benchmark.M3TAVERSAL_OUTCOME_SCENARIOS:
|
|
prompt_id = prompt["id"]
|
|
reply = CS_REPLIES[prompt_id]
|
|
results.append(
|
|
{
|
|
"prompt_id": prompt_id,
|
|
"marker": f"WL-M3TAVERSAL-SANDBOX-{prompt_id}-20260713",
|
|
"mode": "sandbox_expected_answer_fixture",
|
|
"runtime": prompt["runtime"],
|
|
"mutates_kb": False,
|
|
"evidence_tier": "sandbox_expected_answer_fixture",
|
|
"prompt": prompt["message"],
|
|
"expected_outcome": prompt["expected_outcome"],
|
|
"reply": reply,
|
|
"claim_ceiling": "Fixture answer for sandbox-first benchmark scoring; not a live Telegram reply.",
|
|
}
|
|
)
|
|
return results
|
|
|
|
|
|
def _build_dc_results() -> list[dict[str, Any]]:
|
|
results = []
|
|
for prompt in benchmark.M3TAVERSAL_DIRECT_CLAIM_FOLLOWUP_SCENARIOS:
|
|
prompt_id = prompt["id"]
|
|
reply = DC_REPLIES[prompt_id]
|
|
results.append(
|
|
{
|
|
"prompt_id": prompt_id,
|
|
"marker": f"WL-M3TAVERSAL-DIRECT-SANDBOX-{prompt_id}-20260713",
|
|
"mode": "sandbox_expected_answer_fixture",
|
|
"runtime": prompt["runtime"],
|
|
"mutates_kb": False,
|
|
"evidence_tier": "sandbox_expected_answer_fixture",
|
|
"prompt": prompt["message"],
|
|
"expected_answer": prompt["expected_answer"],
|
|
"expected_follow_up": prompt["expected_follow_up"],
|
|
"reply": reply,
|
|
"claim_ceiling": "Fixture answer for direct-claim follow-up benchmark scoring; not a live Telegram reply.",
|
|
}
|
|
)
|
|
return results
|
|
|
|
|
|
def build_results(live_suite_score: Path = DEFAULT_LIVE_SUITE_SCORE) -> dict[str, Any]:
|
|
live_results = _load_live_oe_results(live_suite_score)
|
|
cs_results = _build_cs_results()
|
|
dc_results = _build_dc_results()
|
|
results = live_results + cs_results + dc_results
|
|
score = benchmark.score_results(
|
|
results,
|
|
include_m3taversal_outcomes=True,
|
|
include_direct_claim_followups=True,
|
|
)
|
|
return {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"mode": "mixed_live_retained_and_sandbox_fixture",
|
|
"mutates_kb": False,
|
|
"live_suite_score": str(live_suite_score),
|
|
"benchmark_layers": benchmark.BENCHMARK_LAYERS,
|
|
"evidence_tiers": {
|
|
"OE-01..OE-05": "live_telegram_readonly_retained",
|
|
"CS-01..CS-09": "sandbox_expected_answer_fixture",
|
|
"DC-01..DC-06": "sandbox_expected_answer_fixture",
|
|
},
|
|
"claim_ceiling": (
|
|
"OE rows are retained live Telegram read-only evidence. CS rows are sandbox fixture answers for broad "
|
|
"m3taversal-style outcomes. DC rows are no-context direct-claim expected-answer fixtures with follow-up actions. "
|
|
"CS/DC rows do not prove live Telegram behavior or production DB mutation for those prompts."
|
|
),
|
|
"score": score,
|
|
"results": results,
|
|
}
|
|
|
|
|
|
def write_markdown(path: Path, report: dict[str, Any]) -> None:
|
|
lines = [
|
|
"# Working Leo m3taversal Outcome Sandbox Results",
|
|
"",
|
|
f"Generated UTC: `{report['generated_at_utc']}`",
|
|
f"Mode: `{report['mode']}`",
|
|
f"Mutates KB: `{report['mutates_kb']}`",
|
|
f"Overall full-catalog score: `{report['score']['pass']}`",
|
|
f"Prompts scored: `{report['score']['passes']}/{report['score']['expected_prompt_count']}`",
|
|
"",
|
|
"## Evidence Tiers",
|
|
"",
|
|
]
|
|
lines.extend(f"- `{key}`: `{value}`" for key, value in report["evidence_tiers"].items())
|
|
lines.extend(["", "## Scenario Scores", ""])
|
|
for item in report["score"]["scores"]:
|
|
lines.append(f"- `{item['prompt_id']}` / `{item['dimension']}`: `pass={item['pass']}`")
|
|
lines.extend(["", "## Claim Ceiling", "", report["claim_ceiling"], ""])
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--live-suite-score", type=Path, default=DEFAULT_LIVE_SUITE_SCORE)
|
|
parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
|
|
parser.add_argument(
|
|
"--markdown-out",
|
|
type=Path,
|
|
default=REPO_REPORT_DIR / "working-leo-m3taversal-outcome-sandbox-results-current.md",
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
report = build_results(args.live_suite_score)
|
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
|
args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
write_markdown(args.markdown_out, report)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"out": str(args.out),
|
|
"markdown_out": str(args.markdown_out),
|
|
"pass": report["score"]["pass"],
|
|
"passes": report["score"]["passes"],
|
|
"expected_prompt_count": report["score"]["expected_prompt_count"],
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return 0 if report["score"]["pass"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|