#!/usr/bin/env python3
"""Build retained sandbox-first m3taversal outcome benchmark results.

The output intentionally mixes two evidence tiers:

- OE-01..OE-05 are copied from retained live Telegram read-only evidence.
- CS-01..CS-09 are sandbox fixture replies that encode the expected Working Leo
  behavior for higher-risk m3taversal-style outcome prompts.
- DC-01..DC-06 are sandbox fixture replies for no-context m3taversal-style direct
  claim questions with the expected follow-up action.

This script does not call Leo, does not send Telegram messages, and does not
mutate the KB. Its job is to make the broad m3taversal-style benchmark executable and
scorable before any future disposable clone or live run attempts those prompts.
"""

from __future__ import annotations

import argparse
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import working_leo_open_ended_benchmark as benchmark

REPO_REPORT_DIR = Path("docs/reports/leo-working-state-20260709")
DEFAULT_LIVE_SUITE_SCORE = REPO_REPORT_DIR / "telegram-live-open-ended-suite-score-current.json"
DEFAULT_OUT = REPO_REPORT_DIR / "working-leo-m3taversal-outcome-sandbox-results-current.json"


CS_REPLIES: dict[str, str] = {
    "CS-01": (
        "This is not just chat uptime. I would treat it as an incident across Telegram visibility, VPS service "
        "health, kb_stage.kb_proposals movement, and canonical public.* database rows. First check current systemctl "
        "service state, MainPID/NRestarts, Telegram-visible replies, scheduler/gateway logs, and fresh readback from "
        "kb_stage plus public.claims, public.sources, public.claim_edges, and public.claim_evidence. Proposed, "
        "approved, applied, and not applied must stay separate. The next action is a demo-safe readback packet: what "
        "works now, what has row-level proof, what remains candidate only, and what exact apply packet or clone "
        "rehearsal would change the answer. I cannot claim the DB changed without canonical readback."
    ),
    "CS-02": (
        "Approved is not the same as applied. I would first query kb_stage.kb_proposals for approved rows without "
        "applied_at, then compare them to canonical public.claims, public.sources, public.claim_edges, and "
        "public.claim_evidence. If the canonical table rows are missing, the DB did not change even if the proposal "
        "ledger says approved. The next action is row-level before/after proof: preflight counts and row ids, clone "
        "or rollback rehearsal, guarded apply only when authorized, then postflight readback. Until that exists I "
        "will not overclaim that Leo did the DB work."
    ),
    "CS-03": (
        "Restoring July 5 behavior is not a blind runtime rollback. I would split runtime provenance from DB-state "
        "completion: inspect the canonical repo, /opt/teleo-eval deployed tree, Hermes/leoclean service state, and "
        "the chat/export/DB rows from the July 5 claim update. Then I would prove the strict apply path in a "
        "disposable clone with row-level before/after counts, row ids, and verify/read back proof before touching "
        "production. VPS proof comes first; GCP parity comes only after VPS behavior is proven. I cannot claim restoration without "
        "canonical proof or from file dates alone."
    ),
    "CS-04": (
        "Leo should be aggressive in proposing, not reckless in applying. For a grounded KB change, Leo can draft and "
        "stage a proposal with evidence, rationale, reversibility classification, rollback plan, and expected public.* "
        "row changes. Canonical mutation needs explicit approval and an artifact: preflight, clone rehearsal, "
        "authorized apply, postflight, and cleanup readback. Public posting, signing, spending, or moving capital "
        "requires separate authorization and receipt. The next action is to route useful ideas into the fastest "
        "guarded packet, not to self-approve or silently mutate production. I cannot claim production changed without "
        "readback proof."
    ),
    "CS-05": (
        "Do not flatten everything into public.claims. Claims, public.sources, claim_evidence, claim_edges, strategy "
        "nodes, strategy_node_anchors, shared-root or telos rows, governance gates, concept-map links, and deferred "
        "review notes are different canonical objects. The staging or review packet should preserve caveats, source "
        "evidence, governance/evidence-bar gaps, and self-positioning deferrals. If the current schema cannot hold a "
        "concept map or governance link, keep it out of public.claims, mark it out of scope, or stage a schema packet "
        "as a separate proposal; without canonical readback I will not claim the idea became durable knowledge."
    ),
    "CS-06": (
        "Demo answer: Leo works for Telegram-visible memory, KB audit, and guarded staging; strict existing-ID apply "
        "has row-level proof. Proposed, approved, applied, and not applied remain separate. The rich m3taversal-approved "
        "packet set is clone-proven and apply-readiness verified, but it is not production-applied. I would show the "
        "live Telegram proof, the canonical public.* readback, the apply-readiness artifact, and the service "
        "MainPID/NRestarts readback. The next smallest proof that changes the answer is explicit production "
        "authorization, apply in manifest order, postflight row counts, and live Telegram regression. I will not "
        "overclaim clone proof as production DB state without canonical proof."
    ),
    "CS-07": (
        "Leo identity is DB-first: Postgres public.personas, public.strategies, public.beliefs, shared roots, "
        "public.strategy_nodes, and public.strategy_node_anchors feed a rendered SOUL.md runtime artifact. A direct "
        "SOUL.md patch changes the profile/runtime artifact, not canonical DB truth. Approved identity changes need "
        "row-level DB proof, before/after counts, row ids, and an explicit renderer verify/read back step. If the renderer "
        "hook or daily recomposition is missing, SOUL can lag even when canonical rows are correct; until render proof "
        "exists I cannot claim the identity update is live."
    ),
    "CS-08": (
        "I would check the schema before claiming the decision-matrix approved anything. The designed decision-matrix "
        "should point back to agents.id, proposal ids, matrix_voters, proposal_votes, proposal_decisions, weighted "
        "voters, and a tally. Current VPS readback says those tables do not exist, so the matrix is not yet shipped. "
        "Approval state lives in kb_stage.kb_proposals review/apply columns plus canonical public.* row readback. "
        "Pending approved proposals are not proof of a matrix tally without fresh schema evidence, and I will not "
        "overclaim a matrix result without proof."
    ),
    "CS-09": (
        "Telegram file refs, document_evaluations, proposal source_ref, raw PDFs, and public.sources are different "
        "things. telegram_file_refs and document_evaluations are staging rows or artifact pointers; raw PDFs and "
        "extracted text are files; kb_stage.kb_proposals is the proposal ledger; public.sources is canonical evidence "
        "after review/apply. Pending proposals can have source_ref pointers and still lack direct public.sources links "
        "or an apply contract. The next action is a row-link audit, then a guarded staging/apply packet with canonical "
        "source ids and postflight readback."
    ),
}


DC_REPLIES: dict[str, str] = {
    "DC-01": (
        "I would not assume the knowledge base changed. Proposed, pending_review, approved, applied, and not "
        "applied are different states. The canonical answer comes from public.claims, public.sources, "
        "public.claim_edges, and public.claim_evidence, plus kb_stage.kb_proposals applied_at/readback. I need "
        "row-level before/after counts, row ids, and postflight proof; I cannot claim a DB update without proof. "
        "Follow-up: I would ask which proposal or time window m3taversal means, then query kb_stage and public.* and "
        "either show the pending gap or prepare the next guarded apply packet.\nDB readback: Proposal: "
        "`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`."
    ),
    "DC-02": (
        "Helmer's 7 Powers can be packet-ready or clone-proven without being canonical production state. I would "
        "check public.reasoning_tools, public.claims, public.sources, public.claim_edges, public.claim_evidence, "
        "and the proposal ledger with postflight row counts before saying it is in Leo now. If production_apply "
        "is false, I cannot claim it is production-applied. Follow-up: I would show the readiness packet, ask for "
        "explicit authorization if m3taversal wants it applied, run the integrated order, then postflight and Telegram "
        "regression.\nDB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; "
        "Applied at: `none`."
    ),
    "DC-03": (
        "I would check the schema before claiming a decision-matrix approved anything. If matrix_voters, "
        "proposal_votes, and proposal_decisions tables do not exist, then the decision-matrix is not yet shipped "
        "and no tally approved this. The current proof should come from kb_stage.kb_proposals review/apply columns "
        "and canonical public.* row readback. I cannot claim a matrix result without proof. Follow-up: I would run a "
        "fresh schema readback, then show the proposal ledger and canonical rows instead of overclaiming a matrix "
        "result.\nDB readback: Proposal: `a64df080-8502-42e2-98f4-9bbdecb8da73`; Status: `approved`; "
        "Applied at: `none`."
    ),
    "DC-04": (
        "Telegram file refs, document_evaluations, proposal source_ref, raw PDFs, and public.sources are separate. "
        "A pending proposal can have staging rows and file pointers but still not have canonical source rows, "
        "claim_evidence, or an apply contract. I will not claim document pointers are the only blocker without "
        "row-link proof. Follow-up: I would check and audit each proposal source_ref against public.sources, identify "
        "missing canonical rows, then stage a guarded source/evidence apply proposal for review.\nDB readback: Proposal: "
        "`e987b07c-9c23-45ba-b95c-d6d04c6536c7`; Status: `pending_review`; Applied at: `none`; "
        "Source_ref: `telegram-resolved-attachment:doc_996e2e86cd94`."
    ),
    "DC-05": (
        "Demo answer: Leo can show live staging and strict existing-ID apply proof, but proposed, approved, applied, "
        "and not applied stay separate. Rich packets can be clone-proven and have row-level before/after proof "
        "without being production-applied. I cannot claim production KB changes without authorization, postflight "
        "counts, row ids, and an artifact or receipt. Follow-up: I would ask which demo tier m3taversal wants: show the "
        "existing live proof, run a safe staging canary, or authorize the prepared apply packet plus regression.\n"
        "DB readback: claims: `1837`; sources: `4145`; claim_edges: `4916`; claim_evidence: `4670`; "
        "kb_proposals: `26`."
    ),
    "DC-06": (
        "A direct SOUL.md edit changes the runtime/profile artifact, not canonical identity by itself. Canonical "
        "identity is DB-first: Postgres public.personas, public.strategies, public.beliefs, public.strategy_nodes, "
        "and public.strategy_node_anchors must have row-level proof, row ids, and verify/read back evidence before "
        "they render or sync into SOUL.md. Until readback and renderer proof exist, I cannot claim canonical identity "
        "changed. Follow-up: I would ask whether to audit the canonical identity rows, stage the missing DB proposal, "
        "or run the renderer/sync readback.\nDB readback: Proposal: "
        "`f004bbb2-ac9a-481f-b7b8-74319373ba6a`; Status: `applied`; Applied at: `2026-07-05`."
    ),
}


def _load_live_oe_results(path: Path) -> list[dict[str, Any]]:
    data = json.loads(path.read_text(encoding="utf-8"))
    results = data.get("results", [])
    if not isinstance(results, list):
        raise ValueError(f"expected results list in {path}")
    live_results = []
    expected_ids = {prompt["id"] for prompt in benchmark.OPEN_ENDED_PROMPTS}
    for result in results:
        if result.get("prompt_id") not in expected_ids:
            continue
        item = dict(result)
        item["evidence_tier"] = "live_telegram_readonly_retained"
        live_results.append(item)
    found_ids = {item["prompt_id"] for item in live_results}
    missing_ids = sorted(expected_ids - found_ids)
    if missing_ids:
        raise ValueError(f"missing live OE retained results: {missing_ids}")
    return sorted(live_results, key=lambda item: item["prompt_id"])


def _build_cs_results() -> list[dict[str, Any]]:
    results = []
    for prompt in benchmark.M3TAVERSAL_OUTCOME_SCENARIOS:
        prompt_id = prompt["id"]
        reply = CS_REPLIES[prompt_id]
        results.append(
            {
                "prompt_id": prompt_id,
                "marker": f"WL-M3TAVERSAL-SANDBOX-{prompt_id}-20260713",
                "mode": "sandbox_expected_answer_fixture",
                "runtime": prompt["runtime"],
                "mutates_kb": False,
                "evidence_tier": "sandbox_expected_answer_fixture",
                "prompt": prompt["message"],
                "expected_outcome": prompt["expected_outcome"],
                "reply": reply,
                "claim_ceiling": "Fixture answer for sandbox-first benchmark scoring; not a live Telegram reply.",
            }
        )
    return results


def _build_dc_results() -> list[dict[str, Any]]:
    results = []
    for prompt in benchmark.M3TAVERSAL_DIRECT_CLAIM_FOLLOWUP_SCENARIOS:
        prompt_id = prompt["id"]
        reply = DC_REPLIES[prompt_id]
        results.append(
            {
                "prompt_id": prompt_id,
                "marker": f"WL-M3TAVERSAL-DIRECT-SANDBOX-{prompt_id}-20260713",
                "mode": "sandbox_expected_answer_fixture",
                "runtime": prompt["runtime"],
                "mutates_kb": False,
                "evidence_tier": "sandbox_expected_answer_fixture",
                "prompt": prompt["message"],
                "expected_answer": prompt["expected_answer"],
                "expected_follow_up": prompt["expected_follow_up"],
                "reply": reply,
                "claim_ceiling": "Fixture answer for direct-claim follow-up benchmark scoring; not a live Telegram reply.",
            }
        )
    return results


def build_results(live_suite_score: Path = DEFAULT_LIVE_SUITE_SCORE) -> dict[str, Any]:
    live_results = _load_live_oe_results(live_suite_score)
    cs_results = _build_cs_results()
    dc_results = _build_dc_results()
    results = live_results + cs_results + dc_results
    score = benchmark.score_results(
        results,
        include_m3taversal_outcomes=True,
        include_direct_claim_followups=True,
    )
    return {
        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
        "mode": "mixed_live_retained_and_sandbox_fixture",
        "mutates_kb": False,
        "live_suite_score": str(live_suite_score),
        "benchmark_layers": benchmark.BENCHMARK_LAYERS,
        "evidence_tiers": {
            "OE-01..OE-05": "live_telegram_readonly_retained",
            "CS-01..CS-09": "sandbox_expected_answer_fixture",
            "DC-01..DC-06": "sandbox_expected_answer_fixture",
        },
        "claim_ceiling": (
            "OE rows are retained live Telegram read-only evidence. CS rows are sandbox fixture answers for broad "
            "m3taversal-style outcomes. DC rows are no-context direct-claim expected-answer fixtures with follow-up actions. "
            "CS/DC rows do not prove live Telegram behavior or production DB mutation for those prompts."
        ),
        "score": score,
        "results": results,
    }


def write_markdown(path: Path, report: dict[str, Any]) -> None:
    lines = [
        "# Working Leo m3taversal Outcome Sandbox Results",
        "",
        f"Generated UTC: `{report['generated_at_utc']}`",
        f"Mode: `{report['mode']}`",
        f"Mutates KB: `{report['mutates_kb']}`",
        f"Overall full-catalog score: `{report['score']['pass']}`",
        f"Prompts scored: `{report['score']['passes']}/{report['score']['expected_prompt_count']}`",
        "",
        "## Evidence Tiers",
        "",
    ]
    lines.extend(f"- `{key}`: `{value}`" for key, value in report["evidence_tiers"].items())
    lines.extend(["", "## Scenario Scores", ""])
    for item in report["score"]["scores"]:
        lines.append(f"- `{item['prompt_id']}` / `{item['dimension']}`: `pass={item['pass']}`")
    lines.extend(["", "## Claim Ceiling", "", report["claim_ceiling"], ""])
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text("\n".join(lines), encoding="utf-8")


def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--live-suite-score", type=Path, default=DEFAULT_LIVE_SUITE_SCORE)
    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
    parser.add_argument(
        "--markdown-out",
        type=Path,
        default=REPO_REPORT_DIR / "working-leo-m3taversal-outcome-sandbox-results-current.md",
    )
    return parser.parse_args(argv)


def main(argv: list[str] | None = None) -> int:
    args = parse_args(argv)
    report = build_results(args.live_suite_score)
    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    write_markdown(args.markdown_out, report)
    print(
        json.dumps(
            {
                "out": str(args.out),
                "markdown_out": str(args.markdown_out),
                "pass": report["score"]["pass"],
                "passes": report["score"]["passes"],
                "expected_prompt_count": report["score"]["expected_prompt_count"],
            },
            indent=2,
            sort_keys=True,
        )
    )
    return 0 if report["score"]["pass"] else 1


if __name__ == "__main__":
    raise SystemExit(main())
