496 lines
18 KiB
Python
496 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import stat
|
|
import sys
|
|
import urllib.error
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
|
|
|
import prepare_kb_source_manifest as preparer # noqa: E402
|
|
|
|
SOURCE_TEXT = (
|
|
"Source preparation note.\n\n"
|
|
"A staged proposal remains non-canonical until review and guarded apply both succeed.\n\n"
|
|
"Every prepared claim must remain bound to an exact source quote.\n"
|
|
)
|
|
CLAIM_QUOTE = "Every prepared claim must remain bound to an exact source quote."
|
|
DUPLICATE_QUOTE = "A staged proposal remains non-canonical until review and guarded apply both succeed."
|
|
CANDIDATE_ID = "11111111-1111-4111-8111-111111111111"
|
|
|
|
|
|
def _args(tmp_path: Path, artifact: Path, context: Path) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
artifact=artifact,
|
|
text=None,
|
|
kb_context=context,
|
|
identity="document:source-preparation-test-v1",
|
|
source_key="source_preparation_test_v1",
|
|
source_type="article",
|
|
title="Source preparation test",
|
|
locator="artifact://source-preparation/test-v1",
|
|
output_dir=tmp_path / "private-output",
|
|
key_file=tmp_path / "openrouter-key",
|
|
provider="openrouter",
|
|
model="fixture/model",
|
|
openrouter_url="https://openrouter.invalid/v1/chat/completions",
|
|
timeout_seconds=30,
|
|
max_budget_usd=0.10,
|
|
)
|
|
|
|
|
|
def _inputs(tmp_path: Path) -> tuple[Path, Path]:
|
|
artifact = tmp_path / "source.md"
|
|
artifact.write_text(SOURCE_TEXT, encoding="utf-8")
|
|
context = tmp_path / "context.json"
|
|
context.write_text(
|
|
json.dumps(
|
|
{
|
|
"database_search_query": "staged proposal exact source quote",
|
|
"candidate_claims": [
|
|
{
|
|
"id": CANDIDATE_ID,
|
|
"text": "A staged proposal is not canonical before guarded apply.",
|
|
"score": 3,
|
|
"evidence_count": 2,
|
|
"edge_count": 1,
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "openrouter-key").write_text("fixture-secret", encoding="utf-8")
|
|
return artifact, context
|
|
|
|
|
|
def _model_output(*, claims: bool = True, bad_reference: bool = False) -> str:
|
|
claim_rows = []
|
|
if claims:
|
|
quote_ref = "S99999" if bad_reference else "S00003"
|
|
claim_rows = [
|
|
{
|
|
"claim_key": "exact_quotes_bind_prepared_claims",
|
|
"type": "structural",
|
|
"text": "Prepared claims remain auditable when their evidence is bound to exact source quotes.",
|
|
"quote_ref": quote_ref,
|
|
"confidence": 0.7,
|
|
"tags": ["provenance", "review"],
|
|
"evidence": [{"quote_ref": quote_ref, "role": "grounds"}],
|
|
}
|
|
]
|
|
return json.dumps(
|
|
{
|
|
"schema": preparer.MODEL_OUTPUT_SCHEMA,
|
|
"claims": claim_rows,
|
|
"duplicates": [
|
|
{
|
|
"claim_id": CANDIDATE_ID,
|
|
"source_quote_ref": "S00002",
|
|
"reason": "The canonical candidate already captures the review-before-apply argument.",
|
|
}
|
|
],
|
|
"extraction_notes": "One duplicate argument and one novel exact-quote provenance claim.",
|
|
}
|
|
)
|
|
|
|
|
|
def test_prepare_builds_private_compiler_validated_manifest_without_db_write(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
artifact, context = _inputs(tmp_path)
|
|
monkeypatch.setattr(
|
|
preparer,
|
|
"call_openrouter",
|
|
lambda *_args, **_kwargs: (_model_output(), {"prompt_tokens": 100, "completion_tokens": 50}),
|
|
)
|
|
|
|
receipt = preparer.prepare(_args(tmp_path, artifact, context))
|
|
|
|
assert receipt["status"] == "prepared"
|
|
assert receipt["database_write_performed"] is False
|
|
assert receipt["canonical_apply_performed"] is False
|
|
assert receipt["retrieval"]["canonical_candidate_ids"] == [CANDIDATE_ID]
|
|
assert receipt["extraction"]["claim_count"] == 1
|
|
assert receipt["compiler"]["validated"] is True
|
|
assert receipt["compiler"]["strict_child_proposal_id"]
|
|
output_dir = Path(receipt["outputs"]["output_dir"])
|
|
manifest = json.loads(Path(receipt["outputs"]["manifest"]).read_text(encoding="utf-8"))
|
|
assert manifest["schema"] == preparer.compiler.MANIFEST_SCHEMA_V2
|
|
assert manifest["extractor"]["version"] == preparer.EXTRACTOR_VERSION == "2"
|
|
assert manifest["dedupe"]["duplicates"][0]["claim_id"] == CANDIDATE_ID
|
|
assert manifest["claims"][0]["quote"] == CLAIM_QUOTE
|
|
assert manifest["dedupe"]["duplicates"][0]["source_quote"] == DUPLICATE_QUOTE
|
|
selected = receipt["extraction"]["selected_source_references"]
|
|
assert [(row["kind"], row["reference"]) for row in selected] == [
|
|
("claim", "S00003"),
|
|
("evidence", "S00003"),
|
|
("duplicate", "S00002"),
|
|
]
|
|
assert all(row["char_end"] > row["char_start"] for row in selected)
|
|
assert all(len(row["quote_sha256"]) == 64 for row in selected)
|
|
assert receipt["replay"]["exact_selected_locations_validated"] is True
|
|
assert receipt["extraction"]["evidence_count"] == 1
|
|
assert receipt["extraction"]["duplicate_judgment_count"] == 1
|
|
assert stat.S_IMODE(output_dir.stat().st_mode) == 0o700
|
|
for path in output_dir.iterdir():
|
|
assert stat.S_IMODE(path.stat().st_mode) == 0o600
|
|
|
|
|
|
def test_prepare_retries_once_when_model_invents_a_source_reference(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
artifact, context = _inputs(tmp_path)
|
|
outputs = iter((_model_output(bad_reference=True), _model_output()))
|
|
prompts: list[str] = []
|
|
|
|
def fake_call(prompt: str, **_kwargs):
|
|
prompts.append(prompt)
|
|
return next(outputs), {}
|
|
|
|
monkeypatch.setattr(preparer, "call_openrouter", fake_call)
|
|
|
|
receipt = preparer.prepare(_args(tmp_path, artifact, context))
|
|
|
|
assert receipt["status"] == "prepared"
|
|
assert len(receipt["extraction"]["attempts"]) == 2
|
|
assert "previous extraction failed deterministic validation" in prompts[1]
|
|
assert "must select one of the bracketed SOURCE fragment IDs" in prompts[1]
|
|
|
|
|
|
def test_source_fragments_preserve_exact_unicode_and_punctuation() -> None:
|
|
text = "Heading\n\nIdentity pins use an exact version - never a floating alias.\nCurly: \u201creviewed\u201d.\n"
|
|
|
|
fragments = preparer.build_source_fragments(text)
|
|
|
|
assert fragments == {
|
|
"S00001": "Heading",
|
|
"S00002": "Identity pins use an exact version - never a floating alias.",
|
|
"S00003": "Curly: \u201creviewed\u201d.",
|
|
}
|
|
assert all(value in text for value in fragments.values())
|
|
|
|
|
|
def test_source_fragment_ids_are_dense_across_blank_lines() -> None:
|
|
fragments = preparer.build_source_fragments("first\n\n\nsecond\n")
|
|
|
|
assert fragments == {"S00001": "first", "S00002": "second"}
|
|
|
|
|
|
def test_source_fragment_index_trims_indentation_but_preserves_exact_offsets() -> None:
|
|
text = " indented Unicode: \u201creviewed\u201d \nnext\n"
|
|
|
|
fragments, locations = preparer.build_source_fragment_index(text)
|
|
|
|
assert fragments["S00001"] == "indented Unicode: \u201creviewed\u201d"
|
|
selected = locations["S00001"]
|
|
assert selected["line"] == 1
|
|
assert text[selected["char_start"] : selected["char_end"]] == fragments["S00001"]
|
|
|
|
|
|
def test_bundle_location_validation_rejects_ambiguous_rebound() -> None:
|
|
text = "same line\nsame line\n"
|
|
fragments, locations = preparer.build_source_fragment_index(text)
|
|
extraction = preparer.resolve_model_output(
|
|
{
|
|
"claims": [
|
|
{
|
|
"claim_key": "second_occurrence",
|
|
"type": "structural",
|
|
"text": "The second occurrence was selected.",
|
|
"quote_ref": "S00002",
|
|
"confidence": 0.5,
|
|
"tags": [],
|
|
"evidence": [{"quote_ref": "S00002", "role": "grounds"}],
|
|
}
|
|
],
|
|
"duplicates": [],
|
|
"extraction_notes": "Select the second repeated line.",
|
|
},
|
|
fragments,
|
|
locations,
|
|
)
|
|
fake_bundle = {
|
|
"quote_bindings": [
|
|
{
|
|
"kind": "claim",
|
|
"claim_key": "second_occurrence",
|
|
"quote": "same line",
|
|
"first_start": 0,
|
|
"first_end": 9,
|
|
},
|
|
{
|
|
"kind": "evidence",
|
|
"claim_key": "second_occurrence",
|
|
"evidence_role": "grounds",
|
|
"quote": "same line",
|
|
"first_start": 0,
|
|
"first_end": 9,
|
|
},
|
|
]
|
|
}
|
|
|
|
with pytest.raises(preparer.PreparationError, match="ambiguous repeated quote"):
|
|
preparer.validate_bundle_source_locations(fake_bundle, extraction["selected_source_references"])
|
|
|
|
|
|
def test_prepare_returns_no_novel_claims_without_manifest_or_stageable_packet(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
artifact, context = _inputs(tmp_path)
|
|
monkeypatch.setattr(preparer, "call_openrouter", lambda *_args, **_kwargs: (_model_output(claims=False), {}))
|
|
|
|
receipt = preparer.prepare(_args(tmp_path, artifact, context))
|
|
|
|
assert receipt["status"] == "no_novel_claims"
|
|
assert receipt["outputs"]["manifest"] is None
|
|
assert receipt["compiler"]["validated"] is False
|
|
assert "do not stage an empty proposal" in receipt["next_action"]
|
|
|
|
|
|
def test_binary_artifact_requires_explicit_utf8_extraction(tmp_path: Path) -> None:
|
|
artifact = tmp_path / "source.pdf"
|
|
artifact.write_bytes(b"%PDF-1.7\x00binary")
|
|
|
|
with pytest.raises(preparer.PreparationError, match=r"supply --text for \.pdf"):
|
|
preparer.extract_utf8_text(artifact, None)
|
|
|
|
|
|
def test_repo_native_jsonl_transcript_is_accepted_as_strict_utf8(tmp_path: Path) -> None:
|
|
artifact = tmp_path / "telegram.jsonl"
|
|
artifact.write_text('{"chat_id":1,"message_id":2,"message":"hello"}\n', encoding="utf-8")
|
|
|
|
assert preparer.extract_utf8_text(artifact, None) == artifact.read_bytes()
|
|
|
|
|
|
def test_openrouter_key_cannot_be_redirected_to_an_unapproved_endpoint(tmp_path: Path) -> None:
|
|
with pytest.raises(preparer.PreparationError, match=r"must equal https://openrouter\.ai"):
|
|
preparer.call_openrouter(
|
|
"prompt",
|
|
model="fixture/model",
|
|
key_file=tmp_path / "private-file",
|
|
url="https://example.invalid/collect",
|
|
timeout_seconds=1,
|
|
)
|
|
|
|
|
|
def test_claude_cli_uses_stdin_without_tools_or_session_persistence_and_retains_cost(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
prompt = "private source prompt"
|
|
observed: dict[str, object] = {}
|
|
|
|
def fake_run(command, **kwargs):
|
|
observed["command"] = command
|
|
observed.update(kwargs)
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stderr="",
|
|
stdout=json.dumps(
|
|
{
|
|
"subtype": "success",
|
|
"is_error": False,
|
|
"result": _model_output(),
|
|
"total_cost_usd": 0.0123,
|
|
"usage": {"input_tokens": 321, "output_tokens": 123},
|
|
"modelUsage": {"claude-sonnet-test": {"costUSD": 0.0123}},
|
|
}
|
|
),
|
|
)
|
|
|
|
monkeypatch.setattr(preparer.shutil, "which", lambda _name: "/usr/local/bin/claude")
|
|
monkeypatch.setattr(preparer.subprocess, "run", fake_run)
|
|
|
|
content, usage = preparer.call_claude_cli(
|
|
prompt,
|
|
model="sonnet",
|
|
timeout_seconds=30,
|
|
max_budget_usd=0.10,
|
|
)
|
|
|
|
command = observed["command"]
|
|
assert prompt not in command
|
|
assert observed["input"] == prompt
|
|
assert "--no-session-persistence" in command
|
|
assert command[command.index("--tools") + 1] == ""
|
|
assert command[command.index("--max-budget-usd") + 1] == "0.1"
|
|
assert json.loads(content)["schema"] == preparer.MODEL_OUTPUT_SCHEMA
|
|
assert usage["total_cost_usd"] == 0.0123
|
|
assert usage["session_persistence"] is False
|
|
assert usage["tools_enabled"] is False
|
|
|
|
|
|
def test_prepare_routes_claude_cli_and_records_provider_cost(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
artifact, context = _inputs(tmp_path)
|
|
args = _args(tmp_path, artifact, context)
|
|
args.provider = "claude-cli"
|
|
args.model = "sonnet"
|
|
monkeypatch.setattr(
|
|
preparer,
|
|
"call_openrouter",
|
|
lambda *_args, **_kwargs: pytest.fail("OpenRouter must not be called for the Claude CLI provider"),
|
|
)
|
|
monkeypatch.setattr(
|
|
preparer,
|
|
"call_claude_cli",
|
|
lambda *_args, **_kwargs: (_model_output(), {"total_cost_usd": 0.0123}),
|
|
)
|
|
|
|
receipt = preparer.prepare(args)
|
|
|
|
assert receipt["extraction"]["provider"] == "claude-cli"
|
|
assert receipt["extraction"]["attempts"][0]["usage"]["total_cost_usd"] == 0.0123
|
|
assert receipt["extraction"]["cost_control"] == {
|
|
"max_total_budget_usd": 0.10,
|
|
"reported_total_cost_usd": 0.0123,
|
|
}
|
|
assert receipt["replay"]["extractor"]["name"] == "claude-cli:sonnet"
|
|
manifest = json.loads(Path(receipt["outputs"]["manifest"]).read_text(encoding="utf-8"))
|
|
assert manifest["extractor"]["name"] == "claude-cli:sonnet"
|
|
|
|
|
|
def test_claude_retry_budget_is_reduced_by_reported_first_attempt_cost(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
artifact, context = _inputs(tmp_path)
|
|
args = _args(tmp_path, artifact, context)
|
|
args.provider = "claude-cli"
|
|
args.model = "sonnet"
|
|
outputs = iter(((_model_output(bad_reference=True), 0.04), (_model_output(), 0.03)))
|
|
budgets: list[float] = []
|
|
|
|
def fake_claude(_prompt: str, *, model: str, timeout_seconds: int, max_budget_usd: float):
|
|
assert model == "sonnet"
|
|
assert timeout_seconds == 30
|
|
budgets.append(max_budget_usd)
|
|
output, cost = next(outputs)
|
|
return output, {"total_cost_usd": cost}
|
|
|
|
monkeypatch.setattr(preparer, "call_claude_cli", fake_claude)
|
|
|
|
receipt = preparer.prepare(args)
|
|
|
|
assert budgets == pytest.approx([0.10, 0.06])
|
|
assert receipt["extraction"]["cost_control"]["reported_total_cost_usd"] == pytest.approx(0.07)
|
|
|
|
|
|
def test_provider_errors_redact_secret_and_private_detail(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
key_file = tmp_path / "key"
|
|
key_file.write_text("fixture-secret", encoding="utf-8")
|
|
error = urllib.error.HTTPError(
|
|
preparer.DEFAULT_OPENROUTER_URL,
|
|
401,
|
|
"unauthorized",
|
|
{},
|
|
io.BytesIO(b"fixture-secret and private source quote"),
|
|
)
|
|
|
|
def fail_urlopen(*_args, **_kwargs):
|
|
raise error
|
|
|
|
monkeypatch.setattr(preparer.urllib.request, "urlopen", fail_urlopen)
|
|
|
|
with pytest.raises(preparer.PreparationError) as caught:
|
|
preparer.call_openrouter(
|
|
"private source quote",
|
|
model="fixture/model",
|
|
key_file=key_file,
|
|
url=preparer.DEFAULT_OPENROUTER_URL,
|
|
timeout_seconds=1,
|
|
)
|
|
|
|
rendered = str(caught.value)
|
|
assert "fixture-secret" not in rendered
|
|
assert "private source quote" not in rendered
|
|
assert "provider detail redacted" in rendered
|
|
|
|
|
|
def test_cli_emits_only_public_status_for_private_preparation_receipt(
|
|
tmp_path: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
sentinel = "PRIVATE-SOURCE-RECEIPT-SENTINEL"
|
|
private_path = tmp_path / sentinel / "preparation-receipt.json"
|
|
private_receipt = {
|
|
"status": "prepared",
|
|
"source": {"title": sentinel, "artifact_path": str(private_path)},
|
|
"retrieval": {"database_search_query": sentinel},
|
|
"outputs": {"receipt": str(private_path)},
|
|
}
|
|
monkeypatch.setattr(preparer, "prepare", lambda _args: private_receipt)
|
|
|
|
status = preparer.main(
|
|
[
|
|
"--artifact",
|
|
str(tmp_path / sentinel / "source.md"),
|
|
"--kb-context",
|
|
str(tmp_path / sentinel / "context.json"),
|
|
"--identity",
|
|
f"document:{sentinel}",
|
|
"--source-key",
|
|
"private_source",
|
|
"--source-type",
|
|
"article",
|
|
"--title",
|
|
sentinel,
|
|
"--locator",
|
|
f"artifact://{sentinel}",
|
|
"--output-dir",
|
|
str(private_path.parent),
|
|
]
|
|
)
|
|
|
|
stdout = capsys.readouterr().out
|
|
assert status == 0
|
|
assert json.loads(stdout) == {
|
|
"schema": preparer.STATUS_SCHEMA,
|
|
"status": "prepared",
|
|
"database_write_performed": False,
|
|
"canonical_apply_performed": False,
|
|
"private_outputs_written": True,
|
|
}
|
|
assert sentinel not in stdout
|
|
assert str(private_path) not in stdout
|
|
|
|
|
|
def test_cli_redacts_private_preparation_failure_from_stdout(
|
|
tmp_path: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
sentinel = "PRIVATE-PREPARATION-FAILURE-SENTINEL"
|
|
|
|
def fail(_args):
|
|
raise preparer.PreparationError(f"{sentinel}: {tmp_path / sentinel}")
|
|
|
|
monkeypatch.setattr(preparer, "prepare", fail)
|
|
status = preparer.main(
|
|
[
|
|
"--artifact",
|
|
str(tmp_path / sentinel / "source.md"),
|
|
"--kb-context",
|
|
str(tmp_path / sentinel / "context.json"),
|
|
"--identity",
|
|
f"document:{sentinel}",
|
|
"--source-key",
|
|
"private_source",
|
|
"--source-type",
|
|
"article",
|
|
"--title",
|
|
sentinel,
|
|
"--locator",
|
|
f"artifact://{sentinel}",
|
|
"--output-dir",
|
|
str(tmp_path / sentinel / "output"),
|
|
]
|
|
)
|
|
|
|
stdout = capsys.readouterr().out
|
|
public_status = json.loads(stdout)
|
|
assert status == 2
|
|
assert public_status["reason"] == "source_preparation_failed"
|
|
assert sentinel not in stdout
|
|
assert str(tmp_path) not in stdout
|