63 lines
2.4 KiB
Python
63 lines
2.4 KiB
Python
"""Tests for scripts/build_working_leo_m3taversal_outcome_sandbox.py."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
|
|
|
import build_working_leo_m3taversal_outcome_sandbox as builder # noqa: E402
|
|
import working_leo_open_ended_benchmark as benchmark # noqa: E402
|
|
|
|
|
|
def test_builder_combines_live_oe_and_sandbox_cs_results():
|
|
report = builder.build_results()
|
|
results = report["results"]
|
|
by_id = {item["prompt_id"]: item for item in results}
|
|
|
|
assert report["score"]["pass"] is True
|
|
assert report["score"]["passes"] == 20
|
|
assert report["score"]["expected_prompt_count"] == 20
|
|
assert report["mutates_kb"] is False
|
|
assert set(by_id) == {
|
|
prompt["id"]
|
|
for prompt in benchmark.prompt_catalog(
|
|
include_m3taversal_outcomes=True,
|
|
include_direct_claim_followups=True,
|
|
)
|
|
}
|
|
assert all(by_id[f"OE-0{idx}"]["evidence_tier"] == "live_telegram_readonly_retained" for idx in range(1, 6))
|
|
assert all(by_id[f"CS-0{idx}"]["evidence_tier"] == "sandbox_expected_answer_fixture" for idx in range(1, 10))
|
|
assert all(by_id[f"DC-0{idx}"]["evidence_tier"] == "sandbox_expected_answer_fixture" for idx in range(1, 7))
|
|
assert all(by_id[f"DC-0{idx}"]["expected_follow_up"] for idx in range(1, 7))
|
|
assert "CS/DC rows do not prove live Telegram behavior" in report["claim_ceiling"]
|
|
|
|
|
|
def test_builder_rejects_missing_live_oe_results(tmp_path: Path):
|
|
bad_source = tmp_path / "bad.json"
|
|
bad_source.write_text(json.dumps({"results": [{"prompt_id": "OE-01", "reply": "x"}]}))
|
|
|
|
try:
|
|
builder.build_results(bad_source)
|
|
except ValueError as exc:
|
|
assert "missing live OE retained results" in str(exc)
|
|
else:
|
|
raise AssertionError("expected missing live OE results to fail")
|
|
|
|
|
|
def test_builder_writes_markdown_claim_ceiling(tmp_path: Path):
|
|
report = builder.build_results()
|
|
out = tmp_path / "sandbox.md"
|
|
|
|
builder.write_markdown(out, report)
|
|
|
|
text = out.read_text()
|
|
assert "Working Leo m3taversal Outcome Sandbox Results" in text
|
|
assert "OE-01..OE-05" in text
|
|
assert "CS-01..CS-09" in text
|
|
assert "DC-01..DC-06" in text
|
|
assert "not a live Telegram reply" not in text
|
|
assert "CS/DC rows do not prove live Telegram behavior" in text
|