fix: 3 extraction bugs causing 0% approval rate

1. Remove hardcoded [[_map]] dead link from reconstruct_claim_content
2. Add source status check + archive move after extraction (prevents re-extraction loops)
3. Fix wiki link slug→space normalization (resolves hyphens to spaces before stripping)

Pentagon-Agent: Epimetheus <0144398e-4ed3-4fe2-95a3-3d72e1abf887>
This commit is contained in:
m3taversal 2026-04-01 14:57:38 +01:00
parent 2253f48993
commit eee2b5c78b
3 changed files with 49 additions and 6 deletions

View file

@ -100,6 +100,13 @@ for SOURCE in $SOURCES; do
BASENAME=$(basename "$SOURCE" .md)
BRANCH="extract/$BASENAME"
# Skip already-processed sources (status set by extraction script, merged to main)
if grep -q "^status: processed\|^status: enrichment\|^status: null-result" "$SOURCE" 2>/dev/null; then
echo "[$(date)] [$COUNT/$MAX] SKIP $BASENAME (already processed — status in frontmatter)" >> $LOG
SKIPPED=$((SKIPPED + 1))
continue
fi
# Skip conversation archives — valuable content enters through standalone sources,
# inline tags (SOURCE:/CLAIM:), and transcript review. Raw conversations produce
# low-quality claims with schema failures. (Epimetheus session 4)
@ -244,6 +251,28 @@ Pentagon-Agent: Epimetheus <3D35839A-7722-4740-B93D-51157F7D5E70>" >> $LOG 2>&1
SUCCESS=$((SUCCESS + 1))
echo " -> SUCCESS ($CHANGED files)" >> $LOG
# Move source from queue/ to archive/ on main worktree (prevents re-extraction)
DOMAIN=$(grep -m1 "^domain:" "$MAIN_REPO/inbox/queue/$BASENAME.md" 2>/dev/null | sed 's/^domain:\s*//')
ARCHIVE_DIR="$MAIN_REPO/inbox/archive/${DOMAIN:-uncategorized}"
mkdir -p "$ARCHIVE_DIR"
if [ -f "$MAIN_REPO/inbox/queue/$BASENAME.md" ]; then
mv "$MAIN_REPO/inbox/queue/$BASENAME.md" "$ARCHIVE_DIR/$BASENAME.md"
cd $MAIN_REPO
git add "inbox/queue/$BASENAME.md" "inbox/archive/" 2>/dev/null
git commit -m "pipeline: archive $BASENAME after extraction
Pentagon-Agent: Epimetheus <3D35839A-7722-4740-B93D-51157F7D5E70>" >> $LOG 2>&1
for attempt in 1 2 3; do
git pull --rebase origin main >> $LOG 2>&1
git push origin main >> $LOG 2>&1 && break
sleep 1
done
cd $REPO
git fetch origin main >> $LOG 2>&1
git reset --hard origin/main >> $LOG 2>&1
echo " -> Archived $BASENAME to $ARCHIVE_DIR" >> $LOG
fi
# Back to main
git checkout -f main >> $LOG 2>&1

View file

@ -163,15 +163,29 @@ def fix_frontmatter(content: str, domain: str, agent: str) -> tuple[str, list[st
def fix_wiki_links(content: str, existing_claims: set[str]) -> tuple[str, list[str]]:
"""Strip brackets from broken wiki links, keeping the text. Returns (fixed_content, fixes)."""
"""Fix or strip broken wiki links. Resolves slug→space mismatches before stripping.
The LLM often generates wiki links as slugs (hyphens) but KB filenames use spaces.
Try normalizing hyphensspaces before giving up and stripping brackets.
"""
fixes = []
# Build a lookup: normalized (lowercased, hyphens→spaces) → original stem
_normalized_lookup: dict[str, str] = {}
for stem in existing_claims:
_normalized_lookup[stem.lower().replace("-", " ")] = stem
def replace_broken(match):
link = match.group(1).strip()
if link not in existing_claims:
fixes.append(f"stripped_wiki_link:{link[:60]}")
return link # Keep text, remove brackets
return match.group(0)
if link in existing_claims:
return match.group(0) # Exact match — keep as-is
# Try normalizing slug to spaces
normalized = link.lower().replace("-", " ")
if normalized in _normalized_lookup:
resolved = _normalized_lookup[normalized]
fixes.append(f"resolved_wiki_link:{link[:40]}->{resolved[:40]}")
return f"[[{resolved}]]"
fixes.append(f"stripped_wiki_link:{link[:60]}")
return link # Keep text, remove brackets
fixed = WIKI_LINK_RE.sub(replace_broken, content)
return fixed, fixes

View file

@ -262,7 +262,7 @@ def reconstruct_claim_content(claim, domain, agent):
]
for r in related[:5]:
lines.append(f"- [[{r}]]")
lines.extend(["", "Topics:", "- [[_map]]", ""])
lines.extend(["", "Topics:", ""])
return "\n".join(lines)