teleo-codex/ops/diagnostics/dashboard_ops.py

"""Page 1: Pipeline Operations — "Is the machine running?"

Renders: queue depth, throughput, error rate, stage flow, breakers,
funnel, rejection reasons, fix cycle, time-series charts.

All data comes from existing endpoints: /api/metrics, /api/snapshots,
/api/stage-times, /api/alerts, /api/fix-rates.
"""

import json
from datetime import datetime, timezone

from shared_ui import render_page


def render_ops_page(metrics: dict, snapshots: list, changes: list,
                    vital_signs: dict, now: datetime) -> str:
    """Render the Pipeline Operations page."""

    # --- Prepare chart data ---
    timestamps = [s["ts"] for s in snapshots]
    throughput_data = [s.get("throughput_1h", 0) for s in snapshots]
    approval_data = [(s.get("approval_rate") or 0) * 100 for s in snapshots]
    open_prs_data = [s.get("open_prs", 0) for s in snapshots]
    merged_data = [s.get("merged_total", 0) for s in snapshots]

    rej_wiki = [s.get("rejection_broken_wiki_links", 0) for s in snapshots]
    rej_schema = [s.get("rejection_frontmatter_schema", 0) for s in snapshots]
    rej_dup = [s.get("rejection_near_duplicate", 0) for s in snapshots]
    rej_conf = [s.get("rejection_confidence", 0) for s in snapshots]
    rej_other = [s.get("rejection_other", 0) for s in snapshots]

    # origin_agent/origin_human removed — replaced by /api/growth chart

    annotations_js = json.dumps([
        {
            "type": "line", "xMin": c["ts"], "xMax": c["ts"],
            "borderColor": "#d29922" if c["type"] == "prompt" else "#58a6ff",
            "borderWidth": 1, "borderDash": [4, 4],
            "label": {"display": True, "content": f"{c['type']}: {c.get('to', '?')}",
                      "position": "start", "backgroundColor": "#161b22",
                      "color": "#8b949e", "font": {"size": 10}},
        }
        for c in changes
    ])

    # --- Status helpers ---
    sm = metrics["status_map"]
    ar = metrics["approval_rate"]
    ar_color = "green" if ar > 0.5 else ("yellow" if ar > 0.2 else "red")
    fr_color = "green" if metrics["fix_rate"] > 0.3 else ("yellow" if metrics["fix_rate"] > 0.1 else "red")

    vs_review = vital_signs["review_throughput"]
    vs_status_color = {"healthy": "green", "warning": "yellow", "critical": "red"}.get(vs_review["status"], "yellow")

    # --- Rejection reasons table ---
    reason_rows = "".join(
        f'<tr><td><code>{r["tag"]}</code></td><td>{r["unique_prs"]}</td>'
        f'<td style="color:#8b949e">{r["count"]}</td></tr>'
        for r in metrics["rejection_reasons"]
    )

    # --- Breaker rows ---
    breaker_rows = ""
    for name, info in metrics["breakers"].items():
        state = info["state"]
        color = "green" if state == "closed" else ("red" if state == "open" else "yellow")
        age = f'{info.get("age_s", "?")}s ago' if "age_s" in info else "-"
        breaker_rows += f'<tr><td>{name}</td><td class="{color}">{state}</td><td>{info["failures"]}</td><td>{age}</td></tr>'

    # --- Funnel ---
    funnel = vital_signs["funnel"]

    # --- Queue staleness ---
    qs = vital_signs.get("queue_staleness", {})
    stale_count = qs.get("stale_count", 0)
    stale_status = qs.get("status", "healthy")
    stale_color = {"healthy": "green", "warning": "yellow", "critical": "red"}.get(stale_status, "")

    body = f"""
<!-- Hero Cards -->
<div class="grid">
  <div class="card">
    <div class="label">Throughput</div>
    <div class="value">{metrics["throughput_1h"]}<span style="font-size:14px;color:#8b949e">/hr</span></div>
    <div class="detail">merged last hour</div>
  </div>
  <div class="card">
    <div class="label">Approval Rate (24h)</div>
    <div class="value {ar_color}">{ar:.1%}</div>
    <div class="detail">{metrics["approved_24h"]}/{metrics["evaluated_24h"]} evaluated</div>
  </div>
  <div class="card">
    <div class="label">Review Backlog</div>
    <div class="value {vs_status_color}">{vs_review["backlog"]}</div>
    <div class="detail">{vs_review["open_prs"]} open + {vs_review["reviewing_prs"]} reviewing + {vs_review["approved_waiting"]} approved</div>
  </div>
  <div class="card">
    <div class="label">Merged Total</div>
    <div class="value green">{sm.get("merged", 0)}</div>
    <div class="detail">{sm.get("closed", 0)} closed</div>
  </div>
  <div class="card">
    <div class="label">Fix Success</div>
    <div class="value {fr_color}">{metrics["fix_rate"]:.1%}</div>
    <div class="detail">{metrics["fix_succeeded"]}/{metrics["fix_attempted"]} fixed</div>
  </div>
  <div class="card">
    <div class="label">Time to Merge</div>
    <div class="value">{f"{metrics['median_ttm_minutes']:.0f}" if metrics["median_ttm_minutes"] else "—"}<span style="font-size:14px;color:#8b949e">min</span></div>
    <div class="detail">median (24h)</div>
  </div>
</div>

<!-- Alert Banner (loaded via JS) -->
<div id="alert-banner"></div>

<!-- Pipeline Funnel -->
<div class="section">
  <div class="section-title">Pipeline Funnel</div>
  <div class="funnel">
    <div class="funnel-step"><div class="num">{funnel["sources_total"]}</div><div class="lbl">Sources</div></div>
    <div class="funnel-arrow">&rarr;</div>
    <div class="funnel-step"><div class="num" style="color:#f0883e">{funnel["sources_queued"]}</div><div class="lbl">In Queue</div></div>
    <div class="funnel-arrow">&rarr;</div>
    <div class="funnel-step"><div class="num">{funnel["sources_extracted"]}</div><div class="lbl">Extracted</div></div>
    <div class="funnel-arrow">&rarr;</div>
    <div class="funnel-step"><div class="num">{funnel["prs_total"]}</div><div class="lbl">PRs Created</div></div>
    <div class="funnel-arrow">&rarr;</div>
    <div class="funnel-step"><div class="num green">{funnel["prs_merged"]}</div><div class="lbl">Merged</div></div>
    <div class="funnel-arrow">&rarr;</div>
    <div class="funnel-step"><div class="num blue">{funnel["conversion_rate"]:.1%}</div><div class="lbl">Conversion</div></div>
  </div>
  <div style="margin-top:8px;font-size:12px;color:#8b949e">
    Queue staleness: <span class="{stale_color}">{stale_count} stale</span>
    {f'(oldest: {qs.get("oldest_age_days", "?")}d)' if stale_count > 0 else ""}
  </div>
</div>

<!-- Stage Dwell Times (loaded via JS) -->
<div class="section">
  <div class="section-title">Stage Dwell Times</div>
  <div id="stage-times-container" class="grid"></div>
</div>

<!-- Charts -->
<div id="no-chart-data" class="card" style="text-align:center;padding:40px;margin:16px 0;display:none">
  <p style="color:#8b949e">No time-series data yet.</p>
</div>
<div id="chart-section">
<div class="row">
  <div class="chart-container">
    <h2>Throughput &amp; Approval Rate</h2>
    <canvas id="throughputChart"></canvas>
  </div>
  <div class="chart-container">
    <h2>Rejection Reasons Over Time</h2>
    <canvas id="rejectionChart"></canvas>
  </div>
</div>
<div class="row">
  <div class="chart-container">
    <h2>PR Backlog</h2>
    <canvas id="backlogChart"></canvas>
  </div>
  <div class="chart-container">
    <h2>Cumulative Growth</h2>
    <canvas id="growthChart"></canvas>
  </div>
</div>
</div>

<!-- PR Trace Lookup -->
<div class="section">
  <div class="section-title">PR Trace Lookup</div>
  <div class="card">
    <div style="display:flex;gap:8px;align-items:center">
      <input id="trace-pr-input" type="number" placeholder="Enter PR number"
        style="background:#0d1117;border:1px solid #30363d;color:#c9d1d9;padding:8px 12px;border-radius:6px;width:180px;font-size:14px">
      <button onclick="loadTrace()" style="background:#238636;color:#fff;border:none;padding:8px 16px;border-radius:6px;cursor:pointer;font-size:13px;font-weight:600">Trace</button>
    </div>
    <div id="trace-result" style="margin-top:12px"></div>
  </div>
</div>

<!-- Tables -->
<div class="row">
  <div class="section">
    <div class="section-title">Top Rejection Reasons (24h)</div>
    <div class="card">
      <table>
        <tr><th>Issue</th><th>PRs</th><th style="color:#8b949e">Events</th></tr>
        {reason_rows if reason_rows else "<tr><td colspan='3' style='color:#8b949e'>No rejections in 24h</td></tr>"}
      </table>
    </div>
  </div>
  <div class="section">
    <div class="section-title">Circuit Breakers</div>
    <div class="card">
      <table>
        <tr><th>Stage</th><th>State</th><th>Failures</th><th>Last Success</th></tr>
        {breaker_rows if breaker_rows else "<tr><td colspan='4' style='color:#8b949e'>No breaker data</td></tr>"}
      </table>
    </div>
  </div>
</div>
"""

    scripts = f"""<script>
const timestamps = {json.dumps(timestamps)};

// --- Alerts banner ---
fetch('/api/alerts')
.then(r => r.json())
.then(data => {{
  if (data.alerts && data.alerts.length > 0) {{
    const critical = data.alerts.filter(a => a.severity === 'critical');
    const warning = data.alerts.filter(a => a.severity === 'warning');
    let html = '';
    if (critical.length > 0) {{
      html += '<div class="alert-banner alert-critical">' +
        critical.map(a => '!! ' + esc(a.title)).join('<br>') + '</div>';
    }}
    if (warning.length > 0) {{
      html += '<div class="alert-banner alert-warning">' +
        warning.map(a => '! ' + esc(a.title)).join('<br>') + '</div>';
    }}
    document.getElementById('alert-banner').innerHTML = html;
  }}
}}).catch(() => {{}});

// --- Stage dwell times ---
fetch('/api/stage-times?hours=24')
.then(r => r.json())
.then(data => {{
  const container = document.getElementById('stage-times-container');
  const stages = data.stages || {{}};
  if (Object.keys(stages).length === 0) {{
    container.innerHTML = '<div class="card" style="grid-column:1/-1;text-align:center;color:#8b949e">No stage timing data yet</div>';
    return;
  }}
  let html = '';
  for (const [label, info] of Object.entries(stages)) {{
    const color = info.median_minutes < 5 ? 'green' : info.median_minutes < 30 ? 'yellow' : 'red';
    html += '<div class="card"><div class="label">' + esc(label) + '</div>' +
      '<div class="value ' + color + '">' + info.median_minutes.toFixed(1) + '<span style="font-size:14px;color:#8b949e">min</span></div>' +
      '<div class="detail">median (' + info.count + ' PRs)' +
      (info.p90_minutes ? ' &middot; p90: ' + info.p90_minutes.toFixed(1) + 'min' : '') +
      '</div></div>';
  }}
  container.innerHTML = html;
}}).catch(() => {{}});

// --- Time-series charts ---
if (timestamps.length === 0) {{
  document.getElementById('chart-section').style.display = 'none';
  document.getElementById('no-chart-data').style.display = 'block';
}} else {{

const throughputData = {json.dumps(throughput_data)};
const approvalData = {json.dumps(approval_data)};
const openPrsData = {json.dumps(open_prs_data)};
const mergedData = {json.dumps(merged_data)};
const rejWiki = {json.dumps(rej_wiki)};
const rejSchema = {json.dumps(rej_schema)};
const rejDup = {json.dumps(rej_dup)};
const rejConf = {json.dumps(rej_conf)};
const rejOther = {json.dumps(rej_other)};
const annotations = {annotations_js};

new Chart(document.getElementById('throughputChart'), {{
  type: 'line',
  data: {{
    labels: timestamps,
    datasets: [
      {{ label: 'Throughput/hr', data: throughputData, borderColor: '#58a6ff', backgroundColor: 'rgba(88,166,255,0.1)', fill: true, tension: 0.3, yAxisID: 'y', pointRadius: 1 }},
      {{ label: 'Approval %', data: approvalData, borderColor: '#3fb950', borderDash: [4,2], tension: 0.3, yAxisID: 'y1', pointRadius: 1 }},
    ],
  }},
  options: {{
    responsive: true,
    interaction: {{ mode: 'index', intersect: false }},
    scales: {{
      x: {{ type: 'time', time: {{ unit: 'hour', displayFormats: {{ hour: 'MMM d HH:mm' }} }}, grid: {{ display: false }} }},
      y: {{ position: 'left', title: {{ display: true, text: 'PRs/hr' }}, min: 0 }},
      y1: {{ position: 'right', title: {{ display: true, text: 'Approval %' }}, min: 0, max: 100, grid: {{ drawOnChartArea: false }} }},
    }},
    plugins: {{ annotation: {{ annotations }}, legend: {{ labels: {{ boxWidth: 12 }} }} }},
  }},
}});

new Chart(document.getElementById('rejectionChart'), {{
  type: 'line',
  data: {{
    labels: timestamps,
    datasets: [
      {{ label: 'Wiki Links', data: rejWiki, borderColor: '#f85149', backgroundColor: 'rgba(248,81,73,0.2)', fill: true, tension: 0.3, pointRadius: 0 }},
      {{ label: 'Schema', data: rejSchema, borderColor: '#d29922', backgroundColor: 'rgba(210,153,34,0.2)', fill: true, tension: 0.3, pointRadius: 0 }},
      {{ label: 'Duplicate', data: rejDup, borderColor: '#8b949e', backgroundColor: 'rgba(139,148,158,0.2)', fill: true, tension: 0.3, pointRadius: 0 }},
      {{ label: 'Confidence', data: rejConf, borderColor: '#bc8cff', backgroundColor: 'rgba(188,140,255,0.2)', fill: true, tension: 0.3, pointRadius: 0 }},
      {{ label: 'Other', data: rejOther, borderColor: '#6e7681', backgroundColor: 'rgba(110,118,129,0.15)', fill: true, tension: 0.3, pointRadius: 0 }},
    ],
  }},
  options: {{
    responsive: true,
    scales: {{
      x: {{ type: 'time', time: {{ unit: 'hour', displayFormats: {{ hour: 'MMM d HH:mm' }} }}, grid: {{ display: false }} }},
      y: {{ stacked: true, min: 0, title: {{ display: true, text: 'Count (24h)' }} }},
    }},
    plugins: {{ annotation: {{ annotations }}, legend: {{ labels: {{ boxWidth: 12 }} }} }},
  }},
}});

new Chart(document.getElementById('backlogChart'), {{
  type: 'line',
  data: {{
    labels: timestamps,
    datasets: [
      {{ label: 'Open PRs', data: openPrsData, borderColor: '#d29922', backgroundColor: 'rgba(210,153,34,0.15)', fill: true, tension: 0.3, pointRadius: 1 }},
      {{ label: 'Merged (total)', data: mergedData, borderColor: '#3fb950', tension: 0.3, pointRadius: 1 }},
    ],
  }},
  options: {{
    responsive: true,
    scales: {{
      x: {{ type: 'time', time: {{ unit: 'hour', displayFormats: {{ hour: 'MMM d HH:mm' }} }}, grid: {{ display: false }} }},
      y: {{ min: 0, title: {{ display: true, text: 'PRs' }} }},
    }},
    plugins: {{ legend: {{ labels: {{ boxWidth: 12 }} }} }},
  }},
}});

}} // end if timestamps

// Growth chart loaded async from /api/growth (independent of snapshots)
fetch('/api/growth?days=90')
.then(r => r.json())
.then(data => {{
  if (!data.dates || data.dates.length === 0) return;
  new Chart(document.getElementById('growthChart'), {{
    type: 'line',
    data: {{
      labels: data.dates,
      datasets: [
        {{ label: 'Sources', data: data.sources, borderColor: '#58a6ff', backgroundColor: 'rgba(88,166,255,0.1)', fill: true, tension: 0.3, pointRadius: 1 }},
        {{ label: 'PRs Created', data: data.prs, borderColor: '#d29922', backgroundColor: 'rgba(210,153,34,0.1)', fill: false, tension: 0.3, pointRadius: 1 }},
        {{ label: 'Merged', data: data.merged, borderColor: '#3fb950', backgroundColor: 'rgba(63,185,80,0.1)', fill: false, tension: 0.3, pointRadius: 1 }},
      ],
    }},
    options: {{
      responsive: true,
      interaction: {{ mode: 'index', intersect: false }},
      scales: {{
        x: {{ type: 'time', time: {{ unit: 'day', displayFormats: {{ day: 'MMM d' }} }}, grid: {{ display: false }} }},
        y: {{ min: 0, title: {{ display: true, text: 'Cumulative Count' }} }},
      }},
      plugins: {{ legend: {{ labels: {{ boxWidth: 12 }} }} }},
    }},
  }});
}}).catch(() => {{}});

// --- PR Trace Lookup ---
document.getElementById('trace-pr-input').addEventListener('keydown', e => {{ if (e.key === 'Enter') loadTrace(); }});

function loadTrace() {{
  const pr = document.getElementById('trace-pr-input').value.trim();
  const container = document.getElementById('trace-result');
  if (!pr) {{ container.innerHTML = '<p style="color:#8b949e">Enter a PR number</p>'; return; }}
  container.innerHTML = '<p style="color:#8b949e">Loading...</p>';

  fetch('/api/trace/' + encodeURIComponent(pr))
  .then(r => r.json())
  .then(data => {{
    if (!data.pr && data.timeline.length === 0) {{
      container.innerHTML = '<p style="color:#8b949e">No trace found for PR ' + esc(pr) + '</p>';
      return;
    }}

    const stageColors = {{
      ingest: '#58a6ff', validate: '#d29922', evaluate: '#f0883e',
      merge: '#3fb950', cascade: '#bc8cff', cross_domain: '#79c0ff'
    }};

    let html = '';

    // PR summary
    if (data.pr) {{
      const p = data.pr;
      html += '<div style="margin-bottom:12px;padding:8px 12px;background:#21262d;border-radius:6px;font-size:13px">' +
        '<strong>PR #' + esc(String(p.number)) + '</strong> &middot; ' +
        '<span style="color:' + (p.status === 'merged' ? '#3fb950' : '#d29922') + '">' + esc(p.status) + '</span>' +
        ' &middot; ' + esc(p.domain || 'general') +
        ' &middot; ' + esc(p.agent || '?') +
        ' &middot; ' + esc(p.tier || '?') +
        ' &middot; created ' + esc(p.created_at || '') +
        (p.merged_at ? ' &middot; merged ' + esc(p.merged_at) : '') +
        '</div>';
    }}

    // Timeline
    if (data.timeline.length > 0) {{
      html += '<div style="font-size:12px;font-weight:600;color:#8b949e;margin-bottom:6px;text-transform:uppercase">Timeline</div>';
      html += '<table style="font-size:12px"><tr><th>Time</th><th>Stage</th><th>Event</th><th>Details</th></tr>';
      for (const evt of data.timeline) {{
        const sc = stageColors[evt.stage] || '#8b949e';
        const detail = evt.detail || {{}};
        // Show key fields inline, expandable full JSON
        const keyFields = [];
        if (detail.issues) keyFields.push('issues: ' + detail.issues.join(', '));
        if (detail.agent) keyFields.push('agent: ' + detail.agent);
        if (detail.tier) keyFields.push('tier: ' + detail.tier);
        if (detail.leo) keyFields.push('leo: ' + detail.leo);
        if (detail.domain) keyFields.push('domain: ' + detail.domain);
        if (detail.pass != null) keyFields.push('pass: ' + detail.pass);
        if (detail.attempt) keyFields.push('attempt: ' + detail.attempt);
        const summary = keyFields.length > 0 ? esc(keyFields.join(' | ')) : '';
        const fullJson = JSON.stringify(detail, null, 2);
        const detailId = 'trace-detail-' + Math.random().toString(36).substr(2, 6);

        html += '<tr>' +
          '<td style="white-space:nowrap;color:#8b949e">' + esc(evt.timestamp) + '</td>' +
          '<td><span style="color:' + sc + ';font-weight:600">' + esc(evt.stage) + '</span></td>' +
          '<td>' + esc(evt.event) + '</td>' +
          '<td>' + summary +
            (Object.keys(detail).length > 0
              ? ' <a style="color:#58a6ff;cursor:pointer;font-size:11px" onclick="document.getElementById(\\\'' + detailId + '\\\').style.display=document.getElementById(\\\'' + detailId + '\\\').style.display===\\\'none\\\'?\\\'block\\\':\\\'none\\\'">[json]</a>' +
                '<pre id="' + detailId + '" style="display:none;margin-top:4px;background:#0d1117;padding:6px;border-radius:4px;font-size:11px;overflow-x:auto;max-width:500px">' + esc(fullJson) + '</pre>'
              : '') +
          '</td></tr>';
      }}
      html += '</table>';
    }}

    // Reviews
    if (data.reviews && data.reviews.length > 0) {{
      html += '<div style="font-size:12px;font-weight:600;color:#8b949e;margin:12px 0 6px;text-transform:uppercase">Reviews</div>';
      html += '<table style="font-size:12px"><tr><th>Claim</th><th>Outcome</th><th>Reviewer</th><th>Reason</th></tr>';
      for (const rv of data.reviews) {{
        const outColor = rv.outcome === 'approved' ? '#3fb950' : rv.outcome === 'rejected' ? '#f85149' : '#d29922';
        html += '<tr>' +
          '<td style="max-width:250px;overflow:hidden;text-overflow:ellipsis">' + esc(rv.claim_path || '-') + '</td>' +
          '<td><span class="badge" style="background:' + outColor + '33;color:' + outColor + '">' + esc(rv.outcome || '-') + '</span></td>' +
          '<td>' + esc(rv.reviewer || '-') + '</td>' +
          '<td>' + esc(rv.rejection_reason || '') + '</td></tr>';
      }}
      html += '</table>';
    }}

    container.innerHTML = html;
  }})
  .catch(err => {{
    container.innerHTML = '<p style="color:#f85149">Error: ' + esc(err.message) + '</p>';
  }});
}}
</script>"""

    return render_page(
        title="Pipeline Operations",
        subtitle="Is the machine running?",
        active_path="/ops",
        body_html=body,
        scripts=scripts,
        timestamp=now.strftime("%Y-%m-%d %H:%M UTC"),
    )