teleo-codex/ops/diagnostics/dashboard_health.py
m3taversal 05d74d5e32 sync: import all VPS pipeline + diagnostics code as baseline
Imports 67 files from VPS (/opt/teleo-eval/) into repo as the single source
of truth. Previously only 8 of 67 files existed in repo — the rest were
deployed directly to VPS via SCP, causing massive drift.

Includes:
- pipeline/lib/: 33 Python modules (daemon core, extraction, evaluation, merge, cascade, cross-domain, costs, attribution, etc.)
- pipeline/: main daemon (teleo-pipeline.py), reweave.py, batch-extract-50.sh
- diagnostics/: 19 files (4-page dashboard, alerting, daily digest, review queue, tier1 metrics)
- agent-state/: bootstrap, lib-state, cascade inbox processor, schema
- systemd/: service unit files for reference
- deploy.sh: rsync-based deploy with --dry-run, syntax checks, dirty-tree gate
- research-session.sh: updated with Step 8.5 digest + cascade inbox processing

No new code written — all files are exact copies from VPS as of 2026-04-06.
From this point forward: edit in repo, commit, then deploy.sh.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-07 00:00:00 +01:00

223 lines
8.9 KiB
Python

"""Page 2: Knowledge Health — "What do we know and how good is it?"
Renders: claims by domain, Herfindahl index, evidence freshness,
orphan ratio, link density, confidence distribution, extraction yield.
Data sources: /api/vital-signs, /api/herfindahl, /api/extraction-yield-by-domain,
/api/domains, claim-index (cached).
"""
import json
from datetime import datetime
from shared_ui import render_page
def render_health_page(vital_signs: dict, domain_breakdown: dict, now: datetime) -> str:
"""Render the Knowledge Health page."""
# --- Vital signs data ---
vs_orphan = vital_signs.get("orphan_ratio", {})
orphan_ratio_val = vs_orphan.get("ratio")
orphan_color = {"healthy": "green", "warning": "yellow", "critical": "red"}.get(vs_orphan.get("status", ""), "")
orphan_display = f"{orphan_ratio_val:.1%}" if orphan_ratio_val is not None else ""
vs_linkage = vital_signs.get("linkage_density") or {}
linkage_display = f'{vs_linkage.get("avg_outgoing_links", "")}'
cross_domain_ratio = vs_linkage.get("cross_domain_ratio")
cross_domain_color = "green" if cross_domain_ratio and cross_domain_ratio >= 0.15 else (
"yellow" if cross_domain_ratio and cross_domain_ratio >= 0.05 else "red"
) if cross_domain_ratio is not None else ""
vs_fresh = vital_signs.get("evidence_freshness") or {}
fresh_display = f'{vs_fresh.get("median_age_days", "")}' if vs_fresh.get("median_age_days") else ""
fresh_pct = vs_fresh.get("fresh_30d_pct", 0)
vs_conf = vital_signs.get("confidence_distribution", {})
# Domain activity
stagnant = vital_signs.get("domain_activity", {}).get("stagnant", [])
active_domains = vital_signs.get("domain_activity", {}).get("active", [])
claim_status = vital_signs.get("claim_index_status", "unavailable")
# Domain breakdown table
domain_rows = ""
for domain, stats in sorted(domain_breakdown.items(), key=lambda x: x[1].get("knowledge_prs", 0), reverse=True):
if stats.get("knowledge_prs", 0) > 0:
top_contribs = ", ".join(f'{c["handle"]} ({c["claims"]})' for c in stats.get("contributors", [])[:3])
domain_rows += f"""<tr>
<td style="color:#58a6ff">{domain}</td>
<td>{stats["knowledge_prs"]}</td>
<td>{stats["total_prs"]}</td>
<td style="font-size:12px;color:#8b949e">{top_contribs}</td>
</tr>"""
body = f"""
<!-- Vital Signs Cards -->
<div class="grid">
<div class="card">
<div class="label">Orphan Ratio</div>
<div class="value {orphan_color}">{orphan_display}</div>
<div class="detail">{vs_orphan.get("count", "?")} / {vs_orphan.get("total", "?")} claims &middot; target &lt;15%</div>
</div>
<div class="card">
<div class="label">Avg Links/Claim</div>
<div class="value">{linkage_display}</div>
<div class="detail">cross-domain: <span class="{cross_domain_color}">{f"{cross_domain_ratio:.1%}" if cross_domain_ratio is not None else ""}</span> &middot; target 15-30%</div>
</div>
<div class="card">
<div class="label">Evidence Freshness</div>
<div class="value">{fresh_display}<span style="font-size:14px;color:#8b949e">d median</span></div>
<div class="detail">{vs_fresh.get("fresh_30d_count", "?")} claims &lt;30d old &middot; {fresh_pct:.0f}% fresh</div>
</div>
<div class="card">
<div class="label">Confidence Spread</div>
<div class="value" style="font-size:16px">{" / ".join(f"{vs_conf.get(k, 0)}" for k in ["proven", "likely", "experimental", "speculative"])}</div>
<div class="detail">proven / likely / experimental / speculative</div>
</div>
<div class="card">
<div class="label">Claim Index</div>
<div class="value {'green' if claim_status == 'live' else 'red'}">{claim_status}</div>
<div class="detail">{vs_orphan.get("total", "?")} claims indexed</div>
</div>
</div>
<!-- Herfindahl + Domain Yield (loaded via JS) -->
<div class="row">
<div class="section">
<div class="section-title">Domain Concentration</div>
<div id="herfindahl-container" class="card" style="text-align:center;padding:24px">
<div class="label">Loading...</div>
</div>
</div>
<div class="section">
<div class="section-title">Extraction Yield by Domain</div>
<div id="yield-domain-container" class="card">
<div style="color:#8b949e;text-align:center;padding:16px">Loading...</div>
</div>
</div>
</div>
<!-- Charts -->
<div class="row">
<div class="chart-container">
<h2>Claims by Domain</h2>
<canvas id="domainChart"></canvas>
</div>
<div class="chart-container">
<h2>Confidence Distribution</h2>
<canvas id="confidenceChart"></canvas>
</div>
</div>
<!-- Domain Breakdown Table -->
<div class="section">
<div class="section-title">Contributions by Domain</div>
<div class="card">
<table>
<tr><th>Domain</th><th>Knowledge PRs</th><th>Total PRs</th><th>Top Contributors</th></tr>
{domain_rows if domain_rows else "<tr><td colspan='4' style='color:#8b949e'>No domain data</td></tr>"}
</table>
</div>
</div>
<!-- Stagnation Alerts -->
{"" if not stagnant else f'''
<div class="section">
<div class="section-title" style="color:#d29922">Stagnation Alerts</div>
<div class="card">
<p style="color:#d29922">Domains with no PR activity in 7 days: <strong>{", ".join(stagnant)}</strong></p>
</div>
</div>
'''}
"""
scripts = f"""<script>
// --- Herfindahl index ---
fetch('/api/herfindahl?days=30')
.then(r => r.json())
.then(data => {{
const container = document.getElementById('herfindahl-container');
const statusColor = data.status === 'diverse' ? 'green' : data.status === 'moderate' ? 'yellow' : 'red';
let domainsHtml = data.domains.map(d =>
'<div style="display:flex;justify-content:space-between;padding:4px 0;border-bottom:1px solid #21262d">' +
'<span>' + esc(d.domain) + '</span>' +
'<span style="color:#8b949e">' + d.count + ' (' + (d.share * 100).toFixed(1) + '%)</span></div>'
).join('');
container.innerHTML =
'<div class="value ' + statusColor + '">' + data.hhi.toFixed(4) + '</div>' +
'<div class="detail">' + data.status + ' &middot; ' + data.total_merged + ' merged (30d)</div>' +
'<div style="margin-top:12px;text-align:left">' + domainsHtml + '</div>';
}}).catch(() => {{}});
// --- Extraction yield by domain ---
fetch('/api/extraction-yield-by-domain?days=30')
.then(r => r.json())
.then(data => {{
const container = document.getElementById('yield-domain-container');
if (!data.domains || data.domains.length === 0) {{
container.innerHTML = '<div style="color:#8b949e;text-align:center;padding:16px">No yield data</div>';
return;
}}
let html = '<table><tr><th>Domain</th><th>PRs</th><th>Merged</th><th>Yield</th></tr>';
data.domains.forEach(d => {{
const yieldColor = d.yield >= 0.5 ? 'green' : d.yield >= 0.3 ? 'yellow' : 'red';
html += '<tr><td>' + esc(d.domain) + '</td><td>' + d.total_prs + '</td>' +
'<td>' + d.merged + '</td><td class="' + yieldColor + '">' + (d.yield * 100).toFixed(1) + '%</td></tr>';
}});
html += '</table>';
container.innerHTML = html;
}}).catch(() => {{}});
// --- Domain distribution chart ---
const domainData = {json.dumps({d: s.get("knowledge_prs", 0) for d, s in domain_breakdown.items() if s.get("knowledge_prs", 0) > 0})};
const domainLabels = Object.keys(domainData);
const domainValues = Object.values(domainData);
if (domainLabels.length > 0) {{
const colors = ['#58a6ff', '#3fb950', '#d29922', '#f0883e', '#bc8cff', '#f85149', '#8b949e', '#ec4899'];
new Chart(document.getElementById('domainChart'), {{
type: 'doughnut',
data: {{
labels: domainLabels,
datasets: [{{ data: domainValues, backgroundColor: domainLabels.map((_, i) => colors[i % colors.length]), borderColor: '#161b22', borderWidth: 2 }}],
}},
options: {{
responsive: true,
plugins: {{ legend: {{ position: 'right', labels: {{ boxWidth: 12, font: {{ size: 11 }} }} }} }},
}},
}});
}}
// --- Confidence distribution chart ---
const confData = {json.dumps(vs_conf)};
const confLabels = Object.keys(confData);
const confValues = Object.values(confData);
if (confLabels.length > 0) {{
const confColors = {{ 'proven': '#3fb950', 'likely': '#58a6ff', 'experimental': '#d29922', 'speculative': '#f85149', 'unknown': '#8b949e' }};
new Chart(document.getElementById('confidenceChart'), {{
type: 'bar',
data: {{
labels: confLabels,
datasets: [{{ data: confValues, backgroundColor: confLabels.map(l => confColors[l] || '#8b949e') }}],
}},
options: {{
responsive: true,
plugins: {{ legend: {{ display: false }} }},
scales: {{
y: {{ title: {{ display: true, text: 'Claims' }}, min: 0 }},
x: {{ grid: {{ display: false }} }},
}},
}},
}});
}}
</script>"""
return render_page(
title="Knowledge Health",
subtitle="What do we know and how good is it?",
active_path="/health",
body_html=body,
scripts=scripts,
timestamp=now.strftime("%Y-%m-%d %H:%M UTC"),
)