"""Tests for structured rejection feedback system.""" import json import pytest from lib.feedback import ( QUALITY_GATES, format_rejection_comment, get_agent_error_patterns, parse_rejection_comment, ) # ─── Quality gate coverage ───────────────────────────────────────────────── class TestQualityGates: def test_all_eval_tags_have_gates(self): """Every issue tag used by evaluate.py should have a quality gate entry.""" eval_tags = { "broken_wiki_links", "frontmatter_schema", "title_overclaims", "confidence_miscalibration", "date_errors", "factual_discrepancy", "near_duplicate", "scope_error", } for tag in eval_tags: assert tag in QUALITY_GATES, f"Missing quality gate for eval tag: {tag}" def test_post_extract_tags_have_gates(self): """Issue tags from post_extract.py should also have quality gate entries.""" post_extract_tags = { "opsec_internal_deal_terms", "body_too_thin", "title_too_few_words", "title_not_proposition", } for tag in post_extract_tags: assert tag in QUALITY_GATES, f"Missing quality gate for post_extract tag: {tag}" def test_every_gate_has_required_fields(self): for tag, gate in QUALITY_GATES.items(): assert "gate" in gate, f"{tag} missing 'gate'" assert "description" in gate, f"{tag} missing 'description'" assert "fix" in gate, f"{tag} missing 'fix'" assert "severity" in gate, f"{tag} missing 'severity'" assert gate["severity"] in ("blocking", "warning"), f"{tag} invalid severity" # ─── format_rejection_comment ────────────────────────────────────────────── class TestFormatRejectionComment: def test_single_blocking_issue(self): comment = format_rejection_comment(["frontmatter_schema"]) assert "\n\nSome text' data = parse_rejection_comment(body) assert data["issues"] == ["scope_error"] def test_parse_no_rejection(self): assert parse_rejection_comment("Just a normal comment") is None def test_parse_malformed_json(self): assert parse_rejection_comment("") is None # ─── get_agent_error_patterns ────────────────────────────────────────────── class TestAgentErrorPatterns: def test_empty_agent(self, conn): result = get_agent_error_patterns(conn, "rio") assert result["total_prs"] == 0 assert result["trend"] == "no_data" def test_agent_with_rejections(self, conn): # Insert some test PRs conn.execute( """INSERT INTO prs (number, branch, status, agent, eval_issues, last_attempt, domain) VALUES (1, 'rio/test-1', 'closed', 'rio', '["frontmatter_schema", "confidence_miscalibration"]', datetime('now'), 'internet-finance')""" ) conn.execute( """INSERT INTO prs (number, branch, status, agent, eval_issues, last_attempt, domain) VALUES (2, 'rio/test-2', 'merged', 'rio', '[]', datetime('now'), 'internet-finance')""" ) conn.execute( """INSERT INTO prs (number, branch, status, agent, eval_issues, last_attempt, domain) VALUES (3, 'rio/test-3', 'closed', 'rio', '["frontmatter_schema"]', datetime('now'), 'internet-finance')""" ) result = get_agent_error_patterns(conn, "rio") assert result["total_prs"] == 3 assert result["rejected_prs"] == 2 assert result["approval_rate"] == round(1/3, 3) # frontmatter_schema should be top issue (appears in 2 PRs) top = result["top_issues"] assert len(top) > 0 assert top[0]["tag"] == "frontmatter_schema" assert top[0]["count"] == 2 assert "fix" in top[0] # Guidance included def test_agent_with_all_approvals(self, conn): conn.execute( """INSERT INTO prs (number, branch, status, agent, eval_issues, last_attempt, domain) VALUES (1, 'clay/test-1', 'merged', 'clay', '[]', datetime('now'), 'entertainment')""" ) result = get_agent_error_patterns(conn, "clay") assert result["total_prs"] == 1 assert result["rejected_prs"] == 0 assert result["approval_rate"] == 1.0