Merge pull request #129 from living-ip/codex/leo-dense-source-refs-20260713

Use dense source references for document extraction

docs/kb-rebuild-and-recompile.md

@@ -146,7 +146,7 @@ teleo-kb prepare-source \
 
 It queries canonical claims before extraction, caps a single document at three
 new candidates with confidence at or below `0.75`, has the model select
-numbered source lines, resolves those IDs to exact source substrings, records
+densely numbered non-empty source fragments, resolves those IDs to exact source substrings, records
 duplicate judgments, and validates a v2 manifest through the source compiler.
 Unknown line IDs are rejected rather than fuzzily matched. It writes private
 files only. A separate

hermes-agent/leoclean-skills/vps/teleo-kb-bridge/REFERENCE.md

@@ -69,7 +69,7 @@ Apply these gates before drafting the answer or proposing a next action:
    on the VPS for a text-like artifact under the private
    `state/kb-source-inbox` root or supplied strict UTF-8
    extraction. It retrieves canonical candidates, asks the model to select
-   numbered source lines, resolves those IDs to byte-exact text, and rejects
+   dense numbered non-empty source fragments, resolves those IDs to byte-exact text, and rejects
    unknown IDs instead of fuzzy-matching. It emits an exact-quote private
    manifest without a DB write. `teleo-kb propose-source` can stage one
    deterministic `pending_review` packet without apply authorization. Automatic

hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md

@@ -61,8 +61,8 @@ Apply these before drafting the answer or proposing a next action:
    source identity. `teleo-kb prepare-source` is shipped on the VPS for a
    text-like artifact under the private `state/kb-source-inbox` root or a
    supplied strict UTF-8 extraction in that root. It reads
-   nearby canonical claims before model extraction. The model selects numbered
-   source lines; the compiler resolves those IDs back to byte-exact text and
+   nearby canonical claims before model extraction. The model selects dense,
+   numbered non-empty source fragments; the compiler resolves those IDs back to byte-exact text and
    rejects unknown IDs rather than fuzzy-matching invented prose. It emits a
    private manifest without a DB write. `teleo-kb propose-source` can
    then stage one deterministic `pending_review` packet. Automatic Telegram
@@ -147,7 +147,7 @@ provide a separate strict UTF-8 extraction with `--text`:
 
 This performs a canonical claim retrieval before model extraction, allows at
 most three single-source candidates, caps confidence at `0.75`, binds every
-claim/evidence/duplicate judgment to a numbered source line that is resolved
+claim/evidence/duplicate judgment to a dense numbered source fragment that is resolved
 deterministically to an exact substring, and writes private mode-`0600`
 outputs. It performs no database write. If status is
 `no_novel_claims`, inspect the duplicate judgments and do not stage an empty

scripts/prepare_kb_source_manifest.py

@@ -125,11 +125,10 @@ def _load_context(path: Path) -> dict[str, Any]:
 
 def build_source_fragments(text: str) -> dict[str, str]:
     """Label non-empty source lines so the model selects text instead of copying it."""
-    fragments = {
-        f"L{line_number:05d}": line
-        for line_number, line in enumerate(text.splitlines(), start=1)
-        if line.strip()
-    }
+    fragments: dict[str, str] = {}
+    for line in text.splitlines():
+        if line.strip():
+            fragments[f"S{len(fragments) + 1:05d}"] = line
     if not fragments:
         raise PreparationError("extracted text has no selectable source lines")
     return fragments
@@ -156,7 +155,7 @@ Canonical claims retrieved before extraction:
 Rules:
 - Return 0 to {MAX_CLAIMS} genuinely novel, arguable claims. A fact, summary, heading, implementation detail, or restatement is not automatically a claim.
 - Prefer no new claim when a canonical candidate already makes the same argument. Record that in duplicates instead.
-- Select quote_ref and source_quote_ref only from the bracketed line IDs in SOURCE. Do not copy or rewrite source prose.
+- Select quote_ref and source_quote_ref only from the bracketed fragment IDs in SOURCE. Do not copy or rewrite source prose.
 - Each selected line becomes the exact quote deterministically after your response.
 - A single source cannot justify confidence above 0.75.
 - claim type must be one of: {", ".join(sorted(compiler.ap.CLAIM_TYPES))}.
@@ -173,14 +172,14 @@ Return exactly this JSON object and no prose:
       "claim_key": "stable_key",
       "type": "empirical|structural|normative|meta|concept",
       "text": "concise proposition",
-      "quote_ref": "L00001",
+      "quote_ref": "S00001",
       "confidence": 0.0,
       "tags": ["tag"],
-      "evidence": [{{"quote_ref": "L00001", "role": "grounds|illustrates|contradicts"}}]
+      "evidence": [{{"quote_ref": "S00001", "role": "grounds|illustrates|contradicts"}}]
     }}
   ],
   "duplicates": [
-    {{"claim_id": "retrieved canonical UUID", "source_quote_ref": "L00001", "reason": "same argument"}}
+    {{"claim_id": "retrieved canonical UUID", "source_quote_ref": "S00001", "reason": "same argument"}}
   ],
   "extraction_notes": "short reviewer-facing note"
 }}
@@ -300,7 +299,7 @@ def parse_model_output(content: str) -> dict[str, Any]:
 
 def _resolve_reference(reference: Any, fragments: dict[str, str], label: str) -> str:
     if not isinstance(reference, str) or reference not in fragments:
-        raise PreparationError(f"{label} must select one of the bracketed SOURCE line IDs")
+        raise PreparationError(f"{label} must select one of the bracketed SOURCE fragment IDs")
     return fragments[reference]
 
 

tests/test_prepare_kb_source_manifest.py

@@ -69,7 +69,7 @@ def _inputs(tmp_path: Path) -> tuple[Path, Path]:
 def _model_output(*, claims: bool = True, bad_reference: bool = False) -> str:
     claim_rows = []
     if claims:
-        quote_ref = "L99999" if bad_reference else "L00005"
+        quote_ref = "S99999" if bad_reference else "S00003"
         claim_rows = [
             {
                 "claim_key": "exact_quotes_bind_prepared_claims",
@@ -88,7 +88,7 @@ def _model_output(*, claims: bool = True, bad_reference: bool = False) -> str:
             "duplicates": [
                 {
                     "claim_id": CANDIDATE_ID,
-                    "source_quote_ref": "L00003",
+                    "source_quote_ref": "S00002",
                     "reason": "The canonical candidate already captures the review-before-apply argument.",
                 }
             ],
@@ -123,9 +123,9 @@ def test_prepare_builds_private_compiler_validated_manifest_without_db_write(
     assert manifest["claims"][0]["quote"] == CLAIM_QUOTE
     assert manifest["dedupe"]["duplicates"][0]["source_quote"] == DUPLICATE_QUOTE
     assert receipt["extraction"]["selected_source_references"] == [
-        {"kind": "claim", "reference": "L00005"},
-        {"kind": "evidence", "reference": "L00005"},
-        {"kind": "duplicate", "reference": "L00003"},
+        {"kind": "claim", "reference": "S00003"},
+        {"kind": "evidence", "reference": "S00003"},
+        {"kind": "duplicate", "reference": "S00002"},
     ]
     assert stat.S_IMODE(output_dir.stat().st_mode) == 0o700
     for path in output_dir.iterdir():
@@ -150,7 +150,7 @@ def test_prepare_retries_once_when_model_invents_a_source_reference(
     assert receipt["status"] == "prepared"
     assert len(receipt["extraction"]["attempts"]) == 2
     assert "previous extraction failed deterministic validation" in prompts[1]
-    assert "must select one of the bracketed SOURCE line IDs" in prompts[1]
+    assert "must select one of the bracketed SOURCE fragment IDs" in prompts[1]
 
 
 def test_source_fragments_preserve_exact_unicode_and_punctuation() -> None:
@@ -159,13 +159,19 @@ def test_source_fragments_preserve_exact_unicode_and_punctuation() -> None:
     fragments = preparer.build_source_fragments(text)
 
     assert fragments == {
-        "L00001": "Heading",
-        "L00003": "Identity pins use an exact version - never a floating alias.",
-        "L00004": "Curly: \u201creviewed\u201d.",
+        "S00001": "Heading",
+        "S00002": "Identity pins use an exact version - never a floating alias.",
+        "S00003": "Curly: \u201creviewed\u201d.",
     }
     assert all(value in text for value in fragments.values())
 
 
+def test_source_fragment_ids_are_dense_across_blank_lines() -> None:
+    fragments = preparer.build_source_fragments("first\n\n\nsecond\n")
+
+    assert fragments == {"S00001": "first", "S00002": "second"}
+
+
 def test_prepare_returns_no_novel_claims_without_manifest_or_stageable_packet(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None: