diff --git a/docs/kb-rebuild-and-recompile.md b/docs/kb-rebuild-and-recompile.md index 540b5e3..96c5a87 100644 --- a/docs/kb-rebuild-and-recompile.md +++ b/docs/kb-rebuild-and-recompile.md @@ -145,9 +145,11 @@ teleo-kb prepare-source \ ``` It queries canonical claims before extraction, caps a single document at three -new candidates with confidence at or below `0.75`, requires exact source -substrings, records duplicate judgments, and validates a v2 manifest through -the source compiler. It writes private files only. A separate +new candidates with confidence at or below `0.75`, has the model select +numbered source lines, resolves those IDs to exact source substrings, records +duplicate judgments, and validates a v2 manifest through the source compiler. +Unknown line IDs are rejected rather than fuzzily matched. It writes private +files only. A separate `teleo-kb propose-source` call is required to create a `pending_review` row; neither command applies canonical rows. diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md index 1abf597..9ad0c14 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md @@ -68,8 +68,10 @@ Apply these gates before drafting the answer or proposing a next action: or `source_ref` is not source identity. `teleo-kb prepare-source` is shipped on the VPS for a text-like artifact under the private `state/kb-source-inbox` root or supplied strict UTF-8 - extraction. It retrieves canonical candidates and emits an exact-quote - private manifest without a DB write. `teleo-kb propose-source` can stage one + extraction. It retrieves canonical candidates, asks the model to select + numbered source lines, resolves those IDs to byte-exact text, and rejects + unknown IDs instead of fuzzy-matching. It emits an exact-quote private + manifest without a DB write. `teleo-kb propose-source` can stage one deterministic `pending_review` packet without apply authorization. Automatic Telegram attachment capture and arbitrary tweet fetching are not shipped. Current `approve_claim` cannot insert diff --git a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md index c525858..0c06dc1 100644 --- a/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md +++ b/hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md @@ -61,8 +61,10 @@ Apply these before drafting the answer or proposing a next action: source identity. `teleo-kb prepare-source` is shipped on the VPS for a text-like artifact under the private `state/kb-source-inbox` root or a supplied strict UTF-8 extraction in that root. It reads - nearby canonical claims before model extraction, rejects invented quotes, and - emits a private manifest without a DB write. `teleo-kb propose-source` can + nearby canonical claims before model extraction. The model selects numbered + source lines; the compiler resolves those IDs back to byte-exact text and + rejects unknown IDs rather than fuzzy-matching invented prose. It emits a + private manifest without a DB write. `teleo-kb propose-source` can then stage one deterministic `pending_review` packet. Automatic Telegram attachment capture and arbitrary tweet fetching are not shipped. Approval begins before guarded canonical apply. @@ -145,11 +147,19 @@ provide a separate strict UTF-8 extraction with `--text`: This performs a canonical claim retrieval before model extraction, allows at most three single-source candidates, caps confidence at `0.75`, binds every -claim/evidence/duplicate judgment to an exact source substring, and writes -private mode-`0600` outputs. It performs no database write. If status is +claim/evidence/duplicate judgment to a numbered source line that is resolved +deterministically to an exact substring, and writes private mode-`0600` +outputs. It performs no database write. If status is `no_novel_claims`, inspect the duplicate judgments and do not stage an empty packet. +For each newly supported document format, run one representative live +preparation before calling it working. Verify the receipt reports both database +and canonical apply as false, every emitted quote is an exact substring of the +retained extraction, and canonical counts remain unchanged. After staging, +verify only one `pending_review` proposal was added and an identical rerun is +idempotent. + After reviewing a `prepared` receipt, stage exactly one reviewable proposal: ```bash diff --git a/scripts/prepare_kb_source_manifest.py b/scripts/prepare_kb_source_manifest.py index 432a653..b4b0b3e 100644 --- a/scripts/prepare_kb_source_manifest.py +++ b/scripts/prepare_kb_source_manifest.py @@ -28,7 +28,8 @@ sys.path.insert(0, str(HERE)) import compile_kb_source_packet as compiler # noqa: E402 RECEIPT_SCHEMA = "livingip.kbSourcePreparationReceipt.v1" -MODEL_OUTPUT_SCHEMA = "livingip.kbSourceModelExtraction.v1" +MODEL_OUTPUT_SCHEMA = "livingip.kbSourceModelExtraction.v2" +RESOLVED_OUTPUT_SCHEMA = "livingip.kbSourceResolvedExtraction.v1" DEFAULT_MODEL = "anthropic/claude-sonnet-4.5" DEFAULT_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" DEFAULT_KEY_FILE = Path("/opt/teleo-eval/secrets/openrouter-key") @@ -38,6 +39,10 @@ MAX_CANDIDATES = 20 MAX_CLAIMS = 3 MAX_ATTEMPTS = 2 +MODEL_CLAIM_KEYS = {"claim_key", "type", "text", "quote_ref", "confidence", "tags", "evidence"} +MODEL_EVIDENCE_KEYS = {"quote_ref", "role"} +MODEL_DUPLICATE_KEYS = {"claim_id", "source_quote_ref", "reason"} + class PreparationError(RuntimeError): """Raised when a source cannot safely become a reviewable manifest.""" @@ -118,8 +123,23 @@ def _load_context(path: Path) -> dict[str, Any]: return {"database_search_query": query.strip(), "candidate_claims": normalized} -def build_prompt(text: str, context: dict[str, Any], repair_error: str | None = None) -> str: +def build_source_fragments(text: str) -> dict[str, str]: + """Label non-empty source lines so the model selects text instead of copying it.""" + fragments = { + f"L{line_number:05d}": line + for line_number, line in enumerate(text.splitlines(), start=1) + if line.strip() + } + if not fragments: + raise PreparationError("extracted text has no selectable source lines") + return fragments + + +def build_prompt( + fragments: dict[str, str], context: dict[str, Any], repair_error: str | None = None +) -> str: candidates = json.dumps(context["candidate_claims"], indent=2, ensure_ascii=True) + source = "\n".join(f"[{reference}] {line}" for reference, line in fragments.items()) repair = "" if repair_error: repair = ( @@ -136,7 +156,8 @@ Canonical claims retrieved before extraction: Rules: - Return 0 to {MAX_CLAIMS} genuinely novel, arguable claims. A fact, summary, heading, implementation detail, or restatement is not automatically a claim. - Prefer no new claim when a canonical candidate already makes the same argument. Record that in duplicates instead. -- Every claim quote, evidence quote, and duplicate source_quote must be copied byte-for-byte from SOURCE and be a contiguous exact substring. +- Select quote_ref and source_quote_ref only from the bracketed line IDs in SOURCE. Do not copy or rewrite source prose. +- Each selected line becomes the exact quote deterministically after your response. - A single source cannot justify confidence above 0.75. - claim type must be one of: {", ".join(sorted(compiler.ap.CLAIM_TYPES))}. - evidence role must be one of: {", ".join(sorted(compiler.ap.EVIDENCE_ROLES))}. @@ -152,20 +173,20 @@ Return exactly this JSON object and no prose: "claim_key": "stable_key", "type": "empirical|structural|normative|meta|concept", "text": "concise proposition", - "quote": "exact source substring", + "quote_ref": "L00001", "confidence": 0.0, "tags": ["tag"], - "evidence": [{{"quote": "exact source substring", "role": "grounds|illustrates|contradicts"}}] + "evidence": [{{"quote_ref": "L00001", "role": "grounds|illustrates|contradicts"}}] }} ], "duplicates": [ - {{"claim_id": "retrieved canonical UUID", "source_quote": "exact source substring", "reason": "same argument"}} + {{"claim_id": "retrieved canonical UUID", "source_quote_ref": "L00001", "reason": "same argument"}} ], "extraction_notes": "short reviewer-facing note" }} {repair} SOURCE START -{text} +{source} SOURCE END """ @@ -251,14 +272,96 @@ def parse_model_output(content: str) -> dict[str, Any]: for index, claim in enumerate(claims): if not isinstance(claim, dict): raise PreparationError(f"model extraction claims[{index}] must be an object") + if set(claim) != MODEL_CLAIM_KEYS: + raise PreparationError( + f"model extraction claims[{index}] keys must equal {sorted(MODEL_CLAIM_KEYS)}" + ) confidence = claim.get("confidence") if confidence is not None and ( isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not 0 <= confidence <= 0.75 ): raise PreparationError(f"model extraction claims[{index}].confidence must be between 0 and 0.75 or null") + evidence = claim.get("evidence") + if not isinstance(evidence, list) or not evidence: + raise PreparationError(f"model extraction claims[{index}].evidence must be a non-empty list") + for evidence_index, row in enumerate(evidence): + if not isinstance(row, dict) or set(row) != MODEL_EVIDENCE_KEYS: + raise PreparationError( + f"model extraction claims[{index}].evidence[{evidence_index}] keys must equal " + f"{sorted(MODEL_EVIDENCE_KEYS)}" + ) + for index, duplicate in enumerate(duplicates): + if not isinstance(duplicate, dict) or set(duplicate) != MODEL_DUPLICATE_KEYS: + raise PreparationError( + f"model extraction duplicates[{index}] keys must equal {sorted(MODEL_DUPLICATE_KEYS)}" + ) return value +def _resolve_reference(reference: Any, fragments: dict[str, str], label: str) -> str: + if not isinstance(reference, str) or reference not in fragments: + raise PreparationError(f"{label} must select one of the bracketed SOURCE line IDs") + return fragments[reference] + + +def resolve_model_output(selection: dict[str, Any], fragments: dict[str, str]) -> dict[str, Any]: + claims: list[dict[str, Any]] = [] + selected_references: list[dict[str, str]] = [] + for claim_index, claim in enumerate(selection["claims"]): + quote_ref = claim["quote_ref"] + quote = _resolve_reference(quote_ref, fragments, f"claims[{claim_index}].quote_ref") + selected_references.append({"kind": "claim", "reference": quote_ref}) + evidence: list[dict[str, str]] = [] + for evidence_index, row in enumerate(claim["evidence"]): + evidence_ref = row["quote_ref"] + evidence.append( + { + "quote": _resolve_reference( + evidence_ref, + fragments, + f"claims[{claim_index}].evidence[{evidence_index}].quote_ref", + ), + "role": row["role"], + } + ) + selected_references.append({"kind": "evidence", "reference": evidence_ref}) + claims.append( + { + "claim_key": claim["claim_key"], + "type": claim["type"], + "text": claim["text"], + "quote": quote, + "confidence": claim["confidence"], + "tags": claim["tags"], + "evidence": evidence, + } + ) + + duplicates: list[dict[str, str]] = [] + for duplicate_index, duplicate in enumerate(selection["duplicates"]): + source_quote_ref = duplicate["source_quote_ref"] + duplicates.append( + { + "claim_id": duplicate["claim_id"], + "source_quote": _resolve_reference( + source_quote_ref, + fragments, + f"duplicates[{duplicate_index}].source_quote_ref", + ), + "reason": duplicate["reason"], + } + ) + selected_references.append({"kind": "duplicate", "reference": source_quote_ref}) + + return { + "schema": RESOLVED_OUTPUT_SCHEMA, + "claims": claims, + "duplicates": duplicates, + "extraction_notes": selection["extraction_notes"], + "selected_source_references": selected_references, + } + + def build_manifest( *, args: argparse.Namespace, @@ -324,6 +427,7 @@ def prepare(args: argparse.Namespace) -> dict[str, Any]: artifact_bytes = _read_nonempty(args.artifact, "artifact") text_bytes = extract_utf8_text(args.artifact, args.text) text = text_bytes.decode("utf-8", errors="strict") + fragments = build_source_fragments(text) context = _load_context(args.kb_context) requested_output_dir = args.output_dir.expanduser() @@ -351,7 +455,7 @@ def prepare(args: argparse.Namespace) -> dict[str, Any]: manifest: dict[str, Any] | None = None for attempt in range(1, MAX_ATTEMPTS + 1): raw, usage = call_openrouter( - build_prompt(text, context, repair_error), + build_prompt(fragments, context, repair_error), model=args.model, key_file=args.key_file, url=args.openrouter_url, @@ -359,7 +463,8 @@ def prepare(args: argparse.Namespace) -> dict[str, Any]: ) attempts.append({"attempt": attempt, "response_sha256": sha256_bytes(raw.encode("utf-8")), "usage": usage}) try: - extraction = parse_model_output(raw) + selection = parse_model_output(raw) + extraction = resolve_model_output(selection, fragments) compiler._validate_dedupe( { "database_search_query": context["database_search_query"], @@ -440,6 +545,8 @@ def prepare(args: argparse.Namespace) -> dict[str, Any]: "model": args.model, "attempts": attempts, "claim_count": len(extraction["claims"]), + "selectable_source_line_count": len(fragments), + "selected_source_references": extraction["selected_source_references"], "notes": extraction["extraction_notes"], }, "outputs": { diff --git a/tests/test_prepare_kb_source_manifest.py b/tests/test_prepare_kb_source_manifest.py index 4627b05..5bc7732 100644 --- a/tests/test_prepare_kb_source_manifest.py +++ b/tests/test_prepare_kb_source_manifest.py @@ -66,19 +66,19 @@ def _inputs(tmp_path: Path) -> tuple[Path, Path]: return artifact, context -def _model_output(*, claims: bool = True, bad_quote: bool = False) -> str: +def _model_output(*, claims: bool = True, bad_reference: bool = False) -> str: claim_rows = [] if claims: - quote = "Invented quote." if bad_quote else CLAIM_QUOTE + quote_ref = "L99999" if bad_reference else "L00005" claim_rows = [ { "claim_key": "exact_quotes_bind_prepared_claims", "type": "structural", "text": "Prepared claims remain auditable when their evidence is bound to exact source quotes.", - "quote": quote, + "quote_ref": quote_ref, "confidence": 0.7, "tags": ["provenance", "review"], - "evidence": [{"quote": quote, "role": "grounds"}], + "evidence": [{"quote_ref": quote_ref, "role": "grounds"}], } ] return json.dumps( @@ -88,7 +88,7 @@ def _model_output(*, claims: bool = True, bad_quote: bool = False) -> str: "duplicates": [ { "claim_id": CANDIDATE_ID, - "source_quote": DUPLICATE_QUOTE, + "source_quote_ref": "L00003", "reason": "The canonical candidate already captures the review-before-apply argument.", } ], @@ -120,14 +120,23 @@ def test_prepare_builds_private_compiler_validated_manifest_without_db_write( manifest = json.loads(Path(receipt["outputs"]["manifest"]).read_text(encoding="utf-8")) assert manifest["schema"] == preparer.compiler.MANIFEST_SCHEMA_V2 assert manifest["dedupe"]["duplicates"][0]["claim_id"] == CANDIDATE_ID + assert manifest["claims"][0]["quote"] == CLAIM_QUOTE + assert manifest["dedupe"]["duplicates"][0]["source_quote"] == DUPLICATE_QUOTE + assert receipt["extraction"]["selected_source_references"] == [ + {"kind": "claim", "reference": "L00005"}, + {"kind": "evidence", "reference": "L00005"}, + {"kind": "duplicate", "reference": "L00003"}, + ] assert stat.S_IMODE(output_dir.stat().st_mode) == 0o700 for path in output_dir.iterdir(): assert stat.S_IMODE(path.stat().st_mode) == 0o600 -def test_prepare_retries_once_when_model_invents_a_quote(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_prepare_retries_once_when_model_invents_a_source_reference( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: artifact, context = _inputs(tmp_path) - outputs = iter((_model_output(bad_quote=True), _model_output())) + outputs = iter((_model_output(bad_reference=True), _model_output())) prompts: list[str] = [] def fake_call(prompt: str, **_kwargs): @@ -141,7 +150,20 @@ def test_prepare_retries_once_when_model_invents_a_quote(tmp_path: Path, monkeyp assert receipt["status"] == "prepared" assert len(receipt["extraction"]["attempts"]) == 2 assert "previous extraction failed deterministic validation" in prompts[1] - assert "not an exact substring" in prompts[1] + assert "must select one of the bracketed SOURCE line IDs" in prompts[1] + + +def test_source_fragments_preserve_exact_unicode_and_punctuation() -> None: + text = "Heading\n\nIdentity pins use an exact version - never a floating alias.\nCurly: \u201creviewed\u201d.\n" + + fragments = preparer.build_source_fragments(text) + + assert fragments == { + "L00001": "Heading", + "L00003": "Identity pins use an exact version - never a floating alias.", + "L00004": "Curly: \u201creviewed\u201d.", + } + assert all(value in text for value in fragments.values()) def test_prepare_returns_no_novel_claims_without_manifest_or_stageable_packet(