From 716cc438909f9ac3021aac36a8141af0417bb5cc Mon Sep 17 00:00:00 2001 From: m3taversal Date: Thu, 16 Apr 2026 12:38:31 +0100 Subject: [PATCH] extraction quality: trust hierarchy + verified tagging + telegram review endpoint Three fixes for conversation-sourced claim quality: 1. Trust hierarchy in extraction prompt: bot-generated numbers are flagged as unverified context, not evidence. Directional claims are extractable but specific figures require external verification. Prevents laundering bot guesses into the KB as evidence. 2. Conversation-sourced claims tagged with verified: false and source_type: conversation in frontmatter. Downstream consumers (Leo, dashboard) can filter/flag these for verification. 3. GET /api/telegram-extractions endpoint for daily spot-checking. Shows recent Telegram-sourced PRs with claim titles, status, merge rate, and eval issues. Quick review surface. Co-Authored-By: Claude Opus 4.6 (1M context) --- diagnostics/dashboard_routes.py | 74 +++++++++++++++++++++++++++++++++ lib/extract.py | 7 +++- lib/extraction_prompt.py | 20 +++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/diagnostics/dashboard_routes.py b/diagnostics/dashboard_routes.py index 4b912c8..7e95309 100644 --- a/diagnostics/dashboard_routes.py +++ b/diagnostics/dashboard_routes.py @@ -1109,6 +1109,79 @@ async def handle_pr_lifecycle(request): conn.close() +# ─── GET /api/telegram-extractions ─────────────────────────────────────── + +async def handle_telegram_extractions(request): + """Review surface for Telegram conversation extractions. + + Shows recent PRs sourced from Telegram conversations with claim titles, + status, and source info. Designed for quick daily spot-checking. + + Query params: + days (int): lookback window (default 7, max 90) + """ + conn = request.app["_get_conn"]() + try: + days = min(int(request.query.get("days", "7")), 90) + day_filter = f"-{days}" + + # Find PRs from Telegram sources (source_path contains 'telegram' or submitted_by is @m3taversal via bot) + rows = conn.execute( + """SELECT p.number, p.agent, p.domain, p.tier, p.status, + p.created_at, p.merged_at, p.description, p.source_path, + p.submitted_by, p.branch, p.eval_issues, p.leo_verdict + FROM prs p + WHERE (p.source_path LIKE '%telegram%' OR p.source_path LIKE '%futardio%') + AND p.created_at > datetime('now', ? || ' days') + ORDER BY p.number DESC""", + (day_filter,), + ).fetchall() + + prs = [] + for r in rows: + desc = r["description"] or "" + claim_titles = [t.strip() for t in desc.split("|") if t.strip()] if desc.strip() else [] + + issues = None + if r["eval_issues"]: + try: + issues = json.loads(r["eval_issues"]) if isinstance(r["eval_issues"], str) else r["eval_issues"] + except (json.JSONDecodeError, TypeError): + pass + + prs.append({ + "number": r["number"], + "agent": r["agent"], + "domain": r["domain"], + "tier": r["tier"], + "status": r["status"], + "created_at": r["created_at"], + "merged_at": r["merged_at"], + "claim_titles": claim_titles, + "source_path": r["source_path"], + "submitted_by": r["submitted_by"], + "eval_issues": issues, + "leo_verdict": r["leo_verdict"], + }) + + # Summary stats + merged = sum(1 for p in prs if p["status"] == "merged") + closed = sum(1 for p in prs if p["status"] == "closed") + open_prs = sum(1 for p in prs if p["status"] == "open") + + return web.json_response({ + "days": days, + "total": len(prs), + "merged": merged, + "closed": closed, + "open": open_prs, + "merge_rate": round(merged / len(prs) * 100, 1) if prs else 0, + "prs": prs, + }) + finally: + conn.close() + + # ─── Registration ────────────────────────────────────────────────────────── def register_dashboard_routes(app: web.Application, get_conn): @@ -1125,3 +1198,4 @@ def register_dashboard_routes(app: web.Application, get_conn): app.router.add_get("/api/trace/{trace_id}", handle_trace) app.router.add_get("/api/growth", handle_growth) app.router.add_get("/api/pr-lifecycle", handle_pr_lifecycle) + app.router.add_get("/api/telegram-extractions", handle_telegram_extractions) diff --git a/lib/extract.py b/lib/extract.py index ae32e63..d82de83 100644 --- a/lib/extract.py +++ b/lib/extract.py @@ -215,7 +215,7 @@ def _parse_extraction_json(text: str) -> dict | None: return None -def _build_claim_content(claim: dict, agent: str) -> str: +def _build_claim_content(claim: dict, agent: str, source_format: str | None = None) -> str: """Build claim markdown file content from extraction JSON.""" today = date.today().isoformat() domain = claim.get("domain", "") @@ -265,6 +265,9 @@ def _build_claim_content(claim: dict, agent: str) -> str: lines.append(f"scope: {scope}") if sourcer: lines.append(f'sourcer: "{sourcer}"') + if source_format and source_format.lower() == "conversation": + lines.append("verified: false") + lines.append("source_type: conversation") lines.extend(edge_lines) lines.append("---") lines.append("") @@ -401,7 +404,7 @@ async def _extract_one_source( filename = Path(filename).name # Strip directory components — LLM output may contain path traversal if not filename.endswith(".md"): filename += ".md" - content = _build_claim_content(c, agent_lower) + content = _build_claim_content(c, agent_lower, source_format=source_format) claim_files.append({"filename": filename, "domain": c.get("domain", domain), "content": content}) # Build entity file contents diff --git a/lib/extraction_prompt.py b/lib/extraction_prompt.py index ae633f9..8502f10 100644 --- a/lib/extraction_prompt.py +++ b/lib/extraction_prompt.py @@ -178,6 +178,26 @@ casual or too specific to the conversation context). When the AI agent drops its confidence score after a correction, that CONFIRMS the human was right. Low confidence (0.3-0.5) after pushback = strong signal the correction is valid. +### Trust hierarchy for numbers and specifics + +**CRITICAL:** Neither the human NOR the AI agent should be treated as authoritative sources +for specific numbers, dates, dollar amounts, or statistics UNLESS they cite a verifiable +external source (on-chain data, official announcements, published reports). + +- **Bot-generated numbers are ALWAYS unverified.** When the AI agent says "$25.6M committed + capital" or "15x oversubscription" — these are the bot's best guess, NOT verified data. + NEVER extract bot-generated numbers as evidence in a claim. +- **Human-asserted numbers are ALSO unverified** unless they cite a source. "It raised $11.4M" + from the human is a claim about a number, not proof of the number. +- **Extract the DIRECTIONAL insight, not the specific figures.** "Curated launches attracted + significantly more committed capital than permissionless launches" is extractable. + "$25.6M vs $11.4M" is not — unless the conversation cites where those numbers come from. +- **If specific figures are important to the claim, flag them.** Add a note in the claim body: + "Note: specific figures cited in conversation require verification against on-chain data." + +The goal: capture WHAT the human is asserting (the mechanism, the direction, the pattern) +without laundering unverified numbers into the knowledge base as if they were evidence. + ### Anti-circularity rule If the AI agent is simply reflecting the human's thesis back (restating what the human said