83 lines
4.4 KiB
Python
83 lines
4.4 KiB
Python
"""Tests for the open-ended Working Leo benchmark."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
|
|
|
import working_leo_open_ended_benchmark as bench # noqa: E402
|
|
|
|
|
|
def test_prompts_are_open_ended_and_not_id_led():
|
|
assert len(bench.OPEN_ENDED_PROMPTS) >= 5
|
|
joined = "\n".join(prompt["message"] for prompt in bench.OPEN_ENDED_PROMPTS)
|
|
assert "00957f6c-9883-4015-95a4-6b09367efb0e" not in joined
|
|
assert "c167933e-d513-4f43-9335-d5d8aeb259f2" not in joined
|
|
assert "same state as last night" in joined
|
|
|
|
|
|
def test_scores_good_cory_style_answer_as_pass():
|
|
prompt = bench.OPEN_ENDED_PROMPTS[1]
|
|
reply = (
|
|
"I would not assume approval changed the canonical database. I need to split proposed, pending_review, "
|
|
"approved, applied, and not applied. I would query kb_stage plus canonical public.claims, public.sources, "
|
|
"public.claim_edges, and public.claim_evidence, then give row-level before/after counts and row ids. "
|
|
"Until that readback exists I cannot claim it applied. The next action is a guarded staging or apply packet "
|
|
"with preflight, clone rehearsal, and postflight proof."
|
|
)
|
|
result = bench.score_reply(prompt, reply)
|
|
assert result["pass"] is True
|
|
|
|
|
|
def test_scores_live_style_open_ended_triage_answer_as_pass():
|
|
prompt = bench.OPEN_ENDED_PROMPTS[0]
|
|
reply = (
|
|
"Cory likely means the agents did not advance the graph state beyond yesterday: same claims, same edges, "
|
|
"same proposal queue. I would first check kb_stage.kb_proposals for new proposals since last night, then "
|
|
"check whether agent processes actually executed through gateway logs or scheduler logs. If proposals exist "
|
|
"but the canonical graph is unchanged, that is the approved-not-applied gap. What counts as fixed is new "
|
|
"canonical rows in public.claims, public.claim_edges, or public.claim_evidence with created_at after the "
|
|
"baseline and provenance back to a stage to approve to apply chain. A full-green run means agent fires, "
|
|
"proposal lands in kb_stage, reviewer approves, apply tool writes canonical rows, and readback confirms. "
|
|
"Proposed means pending_review in kb_stage only. Approved means approval is a human gate, not a write. "
|
|
"Applied means public.* rows exist; the only ground truth is whether the row is present in the canonical "
|
|
"table, and the canonical row is the authoritative test. Approved with zero canonical effect means the graph "
|
|
"is unchanged."
|
|
)
|
|
result = bench.score_reply(prompt, reply)
|
|
assert result["pass"] is True
|
|
|
|
|
|
def test_scores_overconfident_answer_as_fail():
|
|
prompt = bench.OPEN_ENDED_PROMPTS[1]
|
|
reply = "Yes, I already applied the changes to the database. We talked about it yesterday, so the KB is updated."
|
|
result = bench.score_reply(prompt, reply)
|
|
assert result["pass"] is False
|
|
assert result["overclaim_detected"] is True
|
|
|
|
|
|
def test_score_results_requires_all_prompts():
|
|
results = [
|
|
{
|
|
"prompt_id": prompt["id"],
|
|
"reply": (
|
|
"I will separate proposed, pending_review, approved, applied, and not applied. "
|
|
"Approved is not the same as applied. "
|
|
"I will query canonical public.claims, public.sources, public.claim_edges, and public.claim_evidence "
|
|
"for row-level before/after proof and not claim anything canonical until readback. "
|
|
"The next action is to stage or review a guarded apply packet. "
|
|
"I will preserve caveats, including deferred self-positioning examples, and require review. "
|
|
"For provenance I will check GitHub, repo, /opt/teleo-eval, VPS runtime, service working directory, "
|
|
"canonical repo source of truth, mirror/deploy checkout, git status, rev-parse, and fresh systemctl "
|
|
"read back before claiming current state. "
|
|
"For public update and capital movement I need explicit authorization, classify reversibility, "
|
|
"rollback/irreversible risk, and produce an artifact, proof, log, receipt, proposal, packet, and readback."
|
|
),
|
|
}
|
|
for prompt in bench.OPEN_ENDED_PROMPTS
|
|
]
|
|
score = bench.score_results(results)
|
|
assert score["pass"] is True
|