Preserve atomic claim text in source proposals
This commit is contained in:
parent
8f70e39bdd
commit
ee3ee1d998
6 changed files with 31 additions and 3 deletions
|
|
@ -149,7 +149,8 @@ new candidates with confidence at or below `0.75`, has the model select
|
||||||
densely numbered non-empty source fragments, resolves those IDs to exact source substrings, records
|
densely numbered non-empty source fragments, resolves those IDs to exact source substrings, records
|
||||||
duplicate judgments, and validates a v2 manifest through the source compiler.
|
duplicate judgments, and validates a v2 manifest through the source compiler.
|
||||||
Unknown line IDs are rejected rather than fuzzily matched. It writes private
|
Unknown line IDs are rejected rather than fuzzily matched. It writes private
|
||||||
files only. A separate
|
files only. The extracted atomic proposition is the proposed claim text; exact
|
||||||
|
source wording remains separately hash-bound as quote and evidence. A separate
|
||||||
`teleo-kb propose-source` call is required to create a `pending_review` row;
|
`teleo-kb propose-source` call is required to create a `pending_review` row;
|
||||||
neither command applies canonical rows.
|
neither command applies canonical rows.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -149,7 +149,9 @@ This performs a canonical claim retrieval before model extraction, allows at
|
||||||
most three single-source candidates, caps confidence at `0.75`, binds every
|
most three single-source candidates, caps confidence at `0.75`, binds every
|
||||||
claim/evidence/duplicate judgment to a dense numbered source fragment that is resolved
|
claim/evidence/duplicate judgment to a dense numbered source fragment that is resolved
|
||||||
deterministically to an exact substring, and writes private mode-`0600`
|
deterministically to an exact substring, and writes private mode-`0600`
|
||||||
outputs. It performs no database write. If status is
|
outputs. The concise extracted proposition becomes the candidate claim text;
|
||||||
|
the exact source wording remains its hash-bound quote and evidence. It performs
|
||||||
|
no database write. If status is
|
||||||
`no_novel_claims`, inspect the duplicate judgments and do not stage an empty
|
`no_novel_claims`, inspect the duplicate judgments and do not stage an empty
|
||||||
packet.
|
packet.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -390,6 +390,7 @@ def build_parent_proposal(
|
||||||
{
|
{
|
||||||
"claim_key": claim["claim_key"],
|
"claim_key": claim["claim_key"],
|
||||||
"type": claim["type"],
|
"type": claim["type"],
|
||||||
|
"headline": claim["text"],
|
||||||
"text": claim["text"],
|
"text": claim["text"],
|
||||||
"body": claim["quote"],
|
"body": claim["quote"],
|
||||||
"confidence": claim["confidence"],
|
"confidence": claim["confidence"],
|
||||||
|
|
@ -471,6 +472,8 @@ def _validate_compiled_child(
|
||||||
raise CompilerError("strict child claim count differs from the extraction manifest")
|
raise CompilerError("strict child claim count differs from the extraction manifest")
|
||||||
if len({row.get("id") for row in claims}) != len(claims):
|
if len({row.get("id") for row in claims}) != len(claims):
|
||||||
raise CompilerError("strict child contains duplicate claim ids")
|
raise CompilerError("strict child contains duplicate claim ids")
|
||||||
|
if [row.get("text") for row in claims] != [row["text"] for row in manifest["claims"]]:
|
||||||
|
raise CompilerError("strict child claim text differs from the extracted atomic propositions")
|
||||||
if len({row.get("id") for row in sources}) != len(sources):
|
if len({row.get("id") for row in sources}) != len(sources):
|
||||||
raise CompilerError("strict child contains duplicate source ids")
|
raise CompilerError("strict child contains duplicate source ids")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ RECEIPT_SCHEMA = "livingip.kbSourcePreparationReceipt.v1"
|
||||||
MODEL_OUTPUT_SCHEMA = "livingip.kbSourceModelExtraction.v2"
|
MODEL_OUTPUT_SCHEMA = "livingip.kbSourceModelExtraction.v2"
|
||||||
RESOLVED_OUTPUT_SCHEMA = "livingip.kbSourceResolvedExtraction.v1"
|
RESOLVED_OUTPUT_SCHEMA = "livingip.kbSourceResolvedExtraction.v1"
|
||||||
DEFAULT_MODEL = "anthropic/claude-sonnet-4.5"
|
DEFAULT_MODEL = "anthropic/claude-sonnet-4.5"
|
||||||
|
EXTRACTOR_VERSION = "2"
|
||||||
DEFAULT_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
DEFAULT_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
|
||||||
DEFAULT_KEY_FILE = Path("/opt/teleo-eval/secrets/openrouter-key")
|
DEFAULT_KEY_FILE = Path("/opt/teleo-eval/secrets/openrouter-key")
|
||||||
TEXT_SUFFIXES = {".csv", ".htm", ".html", ".json", ".md", ".rst", ".txt", ".xml"}
|
TEXT_SUFFIXES = {".csv", ".htm", ".html", ".json", ".md", ".rst", ".txt", ".xml"}
|
||||||
|
|
@ -373,7 +374,7 @@ def build_manifest(
|
||||||
"schema": compiler.MANIFEST_SCHEMA_V2,
|
"schema": compiler.MANIFEST_SCHEMA_V2,
|
||||||
"artifact_sha256": artifact_sha256,
|
"artifact_sha256": artifact_sha256,
|
||||||
"extracted_text_sha256": text_sha256,
|
"extracted_text_sha256": text_sha256,
|
||||||
"extractor": {"name": f"openrouter:{args.model}", "version": "1"},
|
"extractor": {"name": f"openrouter:{args.model}", "version": EXTRACTOR_VERSION},
|
||||||
"source": {
|
"source": {
|
||||||
"identity": args.identity,
|
"identity": args.identity,
|
||||||
"source_key": args.source_key,
|
"source_key": args.source_key,
|
||||||
|
|
|
||||||
|
|
@ -108,10 +108,14 @@ def test_compiles_deterministic_hash_bound_pending_review_bundle_without_db_writ
|
||||||
assert apply_payload["contract_version"] == 2
|
assert apply_payload["contract_version"] == 2
|
||||||
assert apply_payload["source_proposal_id"] == parent["id"]
|
assert apply_payload["source_proposal_id"] == parent["id"]
|
||||||
assert len(apply_payload["claims"]) == 1
|
assert len(apply_payload["claims"]) == 1
|
||||||
|
assert apply_payload["claims"][0]["text"] == _manifest()["claims"][0]["text"]
|
||||||
|
assert apply_payload["claims"][0]["text"] != CLAIM_QUOTE
|
||||||
assert len(apply_payload["sources"]) == 2
|
assert len(apply_payload["sources"]) == 2
|
||||||
assert len(apply_payload["evidence"]) == 2
|
assert len(apply_payload["evidence"]) == 2
|
||||||
assert first["hashes"]["artifact_sha256"] in {row["hash"] for row in apply_payload["sources"]}
|
assert first["hashes"]["artifact_sha256"] in {row["hash"] for row in apply_payload["sources"]}
|
||||||
assert first["compiler_contract"]["source_from_chat_labels"] is False
|
assert first["compiler_contract"]["source_from_chat_labels"] is False
|
||||||
|
assert parent["payload"]["claim_candidates"][0]["headline"] == _manifest()["claims"][0]["text"]
|
||||||
|
assert parent["payload"]["claim_candidates"][0]["body"] == CLAIM_QUOTE
|
||||||
|
|
||||||
without_bundle_hash = copy.deepcopy(first)
|
without_bundle_hash = copy.deepcopy(first)
|
||||||
bundle_hash = without_bundle_hash["hashes"].pop("bundle_content_sha256")
|
bundle_hash = without_bundle_hash["hashes"].pop("bundle_content_sha256")
|
||||||
|
|
@ -156,6 +160,22 @@ def test_v2_manifest_carries_canonical_dedupe_candidates_into_review_packet(tmp_
|
||||||
assert staged_assessment == assessment
|
assert staged_assessment == assessment
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejects_normalized_child_that_replaces_atomic_claim_text_with_quote(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
original = compiler.stage.prepare_normalized_child
|
||||||
|
|
||||||
|
def replace_claim_text(parent: dict) -> dict:
|
||||||
|
child = original(parent)
|
||||||
|
child["payload"]["apply_payload"]["claims"][0]["text"] = CLAIM_QUOTE
|
||||||
|
return child
|
||||||
|
|
||||||
|
monkeypatch.setattr(compiler.stage, "prepare_normalized_child", replace_claim_text)
|
||||||
|
|
||||||
|
with pytest.raises(compiler.CompilerError, match="claim text differs from the extracted atomic propositions"):
|
||||||
|
_compile(tmp_path)
|
||||||
|
|
||||||
|
|
||||||
def test_v2_manifest_rejects_unretrieved_or_unbound_duplicate_judgments(tmp_path: Path) -> None:
|
def test_v2_manifest_rejects_unretrieved_or_unbound_duplicate_judgments(tmp_path: Path) -> None:
|
||||||
manifest = _manifest()
|
manifest = _manifest()
|
||||||
manifest["schema"] = compiler.MANIFEST_SCHEMA_V2
|
manifest["schema"] = compiler.MANIFEST_SCHEMA_V2
|
||||||
|
|
|
||||||
|
|
@ -119,6 +119,7 @@ def test_prepare_builds_private_compiler_validated_manifest_without_db_write(
|
||||||
output_dir = Path(receipt["outputs"]["output_dir"])
|
output_dir = Path(receipt["outputs"]["output_dir"])
|
||||||
manifest = json.loads(Path(receipt["outputs"]["manifest"]).read_text(encoding="utf-8"))
|
manifest = json.loads(Path(receipt["outputs"]["manifest"]).read_text(encoding="utf-8"))
|
||||||
assert manifest["schema"] == preparer.compiler.MANIFEST_SCHEMA_V2
|
assert manifest["schema"] == preparer.compiler.MANIFEST_SCHEMA_V2
|
||||||
|
assert manifest["extractor"]["version"] == preparer.EXTRACTOR_VERSION == "2"
|
||||||
assert manifest["dedupe"]["duplicates"][0]["claim_id"] == CANDIDATE_ID
|
assert manifest["dedupe"]["duplicates"][0]["claim_id"] == CANDIDATE_ID
|
||||||
assert manifest["claims"][0]["quote"] == CLAIM_QUOTE
|
assert manifest["claims"][0]["quote"] == CLAIM_QUOTE
|
||||||
assert manifest["dedupe"]["duplicates"][0]["source_quote"] == DUPLICATE_QUOTE
|
assert manifest["dedupe"]["duplicates"][0]["source_quote"] == DUPLICATE_QUOTE
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue