552 lines
18 KiB
Python
552 lines
18 KiB
Python
"""Unit tests for scripts/apply_worker.py.
|
|
|
|
Pure tests: query building, candidate parsing, render-hook expansion, and the
|
|
report-only enablement gate -- no live database. Runnable under pytest or
|
|
standalone (`python3 tests/test_apply_worker.py`).
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import stat
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
try:
|
|
import pytest
|
|
except ImportError: # standalone fallback so the file runs without pytest installed
|
|
import contextlib
|
|
import types
|
|
|
|
pytest = types.SimpleNamespace()
|
|
|
|
@contextlib.contextmanager
|
|
def _raises(exc):
|
|
try:
|
|
yield
|
|
except exc:
|
|
return
|
|
raise AssertionError(f"expected {exc.__name__} to be raised")
|
|
|
|
def _fixture(*_args, **_kwargs):
|
|
def decorator(func):
|
|
return func
|
|
|
|
return decorator
|
|
|
|
pytest.raises = _raises
|
|
pytest.fixture = _fixture
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
|
|
|
import apply_worker as w # noqa: E402
|
|
|
|
|
|
def _dependency_restore():
|
|
original_load_password = w.ap.load_password
|
|
original_load_failure_state = w.load_failure_state
|
|
original_fetch_candidates = w.fetch_candidates
|
|
original_apply_one = w.apply_one
|
|
original_render_one = w.render_one
|
|
original_subprocess_run = w.subprocess.run
|
|
yield
|
|
w.ap.load_password = original_load_password
|
|
w.load_failure_state = original_load_failure_state
|
|
w.fetch_candidates = original_fetch_candidates
|
|
w.apply_one = original_apply_one
|
|
w.render_one = original_render_one
|
|
w.subprocess.run = original_subprocess_run
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def restore_apply_worker_dependencies():
|
|
restore = _dependency_restore()
|
|
next(restore)
|
|
try:
|
|
yield
|
|
finally:
|
|
try:
|
|
next(restore)
|
|
except StopIteration:
|
|
pass
|
|
|
|
|
|
# --- candidate query ------------------------------------------------------ #
|
|
def test_candidate_query_filters_on_approved():
|
|
sql = w.build_candidate_query()
|
|
# The structural guarantee: worker can only ever act on approved proposals.
|
|
assert "status = 'approved'" in sql
|
|
# Never selects pending_review (no auto-approve possible).
|
|
assert "pending_review" not in sql
|
|
|
|
|
|
def test_candidate_query_requires_apply_payload():
|
|
sql = w.build_candidate_query()
|
|
assert "payload ? 'apply_payload'" in sql
|
|
|
|
|
|
def test_candidate_query_lists_worker_types_only():
|
|
sql = w.build_candidate_query()
|
|
for t in w.WORKER_TYPES:
|
|
assert f"'{t}'" in sql
|
|
# a non-applyable type must not appear
|
|
assert "reject_claim" not in sql
|
|
|
|
|
|
def test_candidate_query_can_exclude_poisoned_ids_before_limit():
|
|
sql = w.build_candidate_query(excluded_ids=("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",))
|
|
assert "id::text not in" in sql.lower()
|
|
assert "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" in sql
|
|
assert sql.lower().index("id::text not in") < sql.lower().index("limit 20")
|
|
|
|
|
|
def test_worker_includes_approved_claim_bundles_without_unproven_render_hook():
|
|
assert "approve_claim" in w.WORKER_TYPES
|
|
assert "approve_claim" not in w.RENDER_TYPES
|
|
sql = w.build_candidate_query()
|
|
assert "'approve_claim'" in sql
|
|
|
|
|
|
# --- candidate parsing ---------------------------------------------------- #
|
|
def test_parse_candidates_skips_blank_lines():
|
|
out = '{"id": "a", "proposal_type": "add_edge", "agent_id": null}\n\n'
|
|
rows = w.parse_candidates(out)
|
|
assert len(rows) == 1
|
|
assert rows[0]["id"] == "a"
|
|
|
|
|
|
# --- render hook ---------------------------------------------------------- #
|
|
def test_render_command_none_when_unset():
|
|
assert w.build_render_command("", "agent-1") is None
|
|
|
|
|
|
def test_render_command_none_without_agent():
|
|
assert w.build_render_command("python3 render_soul.py --agent-id {agent_id}", None) is None
|
|
|
|
|
|
def test_render_command_expands_agent_id():
|
|
cmd = w.build_render_command("python3 render_soul.py --agent-id {agent_id}", "abc")
|
|
assert cmd == ["python3", "render_soul.py", "--agent-id", "abc"]
|
|
|
|
|
|
# --- enablement gate ------------------------------------------------------ #
|
|
def _args(**over):
|
|
base = dict(
|
|
enable=False,
|
|
limit=20,
|
|
max_per_tick=1,
|
|
max_attempts=3,
|
|
failure_state_file=None,
|
|
applied_by="kb-apply",
|
|
apply_script="apply_proposal.py",
|
|
render_cmd="",
|
|
receipt_dir="/private/tmp/kb-apply-worker-test-receipts",
|
|
secrets_file="x",
|
|
container="teleo-pg",
|
|
db="teleo",
|
|
host="127.0.0.1",
|
|
role="kb_apply",
|
|
)
|
|
base.update(over)
|
|
return argparse.Namespace(**base)
|
|
|
|
|
|
def _write_valid_private_receipt(path: Path, proposal_id: str) -> None:
|
|
from_claim = "11111111-1111-4111-8111-111111111111"
|
|
to_claim = "22222222-2222-4222-8222-222222222222"
|
|
proposal = {
|
|
"id": proposal_id,
|
|
"proposal_type": "add_edge",
|
|
"status": "applied",
|
|
"payload": {
|
|
"apply_payload": {
|
|
"from_claim": from_claim,
|
|
"to_claim": to_claim,
|
|
"edge_type": "supports",
|
|
"weight": 0.75,
|
|
}
|
|
},
|
|
"reviewed_by_handle": "reviewer",
|
|
"reviewed_by_agent_id": None,
|
|
"reviewed_at": "2026-07-15T10:00:00+00:00",
|
|
"review_note": "Approved for worker receipt testing.",
|
|
"applied_by_handle": "kb-apply",
|
|
"applied_by_agent_id": None,
|
|
"applied_at": "2026-07-15T10:01:00+00:00",
|
|
}
|
|
receipt = w.ap.replay_receipt.build_replay_receipt(
|
|
proposal,
|
|
{
|
|
"claim_edges": [
|
|
{
|
|
"id": "33333333-3333-4333-8333-333333333333",
|
|
"from_claim": from_claim,
|
|
"to_claim": to_claim,
|
|
"edge_type": "supports",
|
|
"weight": 0.75,
|
|
"created_by": None,
|
|
}
|
|
]
|
|
},
|
|
apply_sql_sha256="a" * 64,
|
|
apply_sql_source="exact_executed_sql",
|
|
exported_at_utc="2026-07-15T10:02:00+00:00",
|
|
)
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
path.write_text(json.dumps(receipt), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
|
|
|
|
def _load_private_receipt(path: Path) -> dict:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def _write_private_receipt(path: Path, receipt: dict) -> None:
|
|
path.write_text(json.dumps(receipt), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
|
|
|
|
def _completed(returncode: int, *, stdout: str = "", stderr: str = ""):
|
|
return w.subprocess.CompletedProcess([], returncode, stdout=stdout, stderr=stderr)
|
|
|
|
|
|
def _postcommit_receipt_failure(proposal_id: str, detail: str = "receipt write failed") -> str:
|
|
return (
|
|
f"proposal {proposal_id} {w.POSTCOMMIT_RECEIPT_FAILURE_MARKER}: {detail}"
|
|
f"{w.POSTCOMMIT_RECEIPT_RECOVERY_MARKER}"
|
|
f"apply_proposal.py {proposal_id} --receipt-only"
|
|
)
|
|
|
|
|
|
# --- private receipt and post-COMMIT recovery ---------------------------- #
|
|
def test_private_receipt_requires_complete_canonical_rows_and_hashes():
|
|
proposal_id = "99999999-9999-4999-8999-999999999999"
|
|
receipt_path = Path(tempfile.mkdtemp()) / "receipt.json"
|
|
_write_valid_private_receipt(receipt_path, proposal_id)
|
|
|
|
missing_rows = _load_private_receipt(receipt_path)
|
|
missing_rows.pop("canonical_rows")
|
|
_write_private_receipt(receipt_path, missing_rows)
|
|
with pytest.raises(RuntimeError, match="row-level contract is invalid"):
|
|
w.assert_private_replay_receipt(receipt_path, proposal_id)
|
|
|
|
_write_valid_private_receipt(receipt_path, proposal_id)
|
|
missing_hashes = _load_private_receipt(receipt_path)
|
|
missing_hashes.pop("hashes")
|
|
_write_private_receipt(receipt_path, missing_hashes)
|
|
with pytest.raises(RuntimeError, match="row-level contract is invalid"):
|
|
w.assert_private_replay_receipt(receipt_path, proposal_id)
|
|
|
|
|
|
def test_private_receipt_rejects_canonical_row_tampering_without_recomputed_hashes():
|
|
proposal_id = "88888888-8888-4888-8888-888888888888"
|
|
receipt_path = Path(tempfile.mkdtemp()) / "receipt.json"
|
|
_write_valid_private_receipt(receipt_path, proposal_id)
|
|
tampered = _load_private_receipt(receipt_path)
|
|
tampered["canonical_rows"]["claim_edges"][0]["weight"] = 0.5
|
|
_write_private_receipt(receipt_path, tampered)
|
|
|
|
with pytest.raises(RuntimeError, match="row-level contract is invalid|internally inconsistent"):
|
|
w.assert_private_replay_receipt(receipt_path, proposal_id)
|
|
|
|
|
|
def test_apply_one_supplies_deterministic_private_receipt_path():
|
|
proposal_id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
|
|
receipt_dir = Path(tempfile.mkdtemp()) / "receipts"
|
|
calls = []
|
|
|
|
def _run(cmd, **_kwargs):
|
|
calls.append(cmd)
|
|
receipt_path = Path(cmd[cmd.index("--receipt-out") + 1])
|
|
_write_valid_private_receipt(receipt_path, proposal_id)
|
|
return _completed(0, stdout='{"applied": true}')
|
|
|
|
w.subprocess.run = _run
|
|
w.apply_one(_args(receipt_dir=str(receipt_dir)), proposal_id)
|
|
|
|
expected = receipt_dir.resolve() / f"{proposal_id}.json"
|
|
assert len(calls) == 1
|
|
assert calls[0][calls[0].index("--receipt-dir") + 1] == str(expected.parent)
|
|
assert calls[0][calls[0].index("--receipt-out") + 1] == str(expected)
|
|
assert stat.S_IMODE(expected.stat().st_mode) == 0o600
|
|
|
|
|
|
def test_apply_one_recovers_postcommit_receipt_failure_without_reapplying():
|
|
proposal_id = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"
|
|
receipt_dir = Path(tempfile.mkdtemp()) / "receipts"
|
|
calls = []
|
|
|
|
def _run(cmd, **_kwargs):
|
|
calls.append(cmd)
|
|
if len(calls) == 1:
|
|
return _completed(
|
|
1,
|
|
stderr=_postcommit_receipt_failure(proposal_id),
|
|
)
|
|
assert "--receipt-only" in cmd
|
|
receipt_path = Path(cmd[cmd.index("--receipt-out") + 1])
|
|
_write_valid_private_receipt(receipt_path, proposal_id)
|
|
return _completed(0, stdout='{"replay_ready": true}')
|
|
|
|
w.subprocess.run = _run
|
|
w.apply_one(_args(receipt_dir=str(receipt_dir)), proposal_id)
|
|
|
|
assert len(calls) == 2
|
|
assert "--receipt-only" not in calls[0]
|
|
assert "--receipt-only" in calls[1]
|
|
assert calls[0][calls[0].index("--receipt-out") + 1] == calls[1][calls[1].index("--receipt-out") + 1]
|
|
|
|
|
|
def test_apply_one_does_not_recover_a_precommit_apply_failure():
|
|
calls = []
|
|
|
|
def _run(cmd, **_kwargs):
|
|
calls.append(cmd)
|
|
return _completed(1, stderr="psql transaction failed before commit")
|
|
|
|
w.subprocess.run = _run
|
|
try:
|
|
w.apply_one(_args(receipt_dir=tempfile.mkdtemp()), "p1")
|
|
except w.PostCommitReceiptError as exc:
|
|
raise AssertionError("pre-COMMIT failure was misclassified as committed") from exc
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
raise AssertionError("expected pre-COMMIT apply failure")
|
|
|
|
assert len(calls) == 1
|
|
|
|
|
|
def test_apply_one_classifies_unrecovered_receipt_as_postcommit():
|
|
calls = []
|
|
|
|
def _run(cmd, **_kwargs):
|
|
calls.append(cmd)
|
|
if len(calls) == 1:
|
|
return _completed(1, stderr=_postcommit_receipt_failure("p1"))
|
|
return _completed(1, stderr="receipt directory remains unavailable")
|
|
|
|
w.subprocess.run = _run
|
|
with pytest.raises(w.PostCommitReceiptError):
|
|
w.apply_one(_args(receipt_dir=tempfile.mkdtemp()), "p1")
|
|
|
|
assert len(calls) == 2
|
|
assert "--receipt-only" in calls[1]
|
|
|
|
|
|
def test_forged_marker_in_unknown_field_traceback_is_a_normal_apply_failure():
|
|
proposal_id = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"
|
|
calls = []
|
|
|
|
def _run(cmd, **_kwargs):
|
|
calls.append(cmd)
|
|
return _completed(
|
|
1,
|
|
stderr=(
|
|
"Traceback (most recent call last):\n"
|
|
"ValueError: approve_claim apply_payload contains unknown fields: "
|
|
f"['{w.POSTCOMMIT_RECEIPT_FAILURE_MARKER}']"
|
|
),
|
|
)
|
|
|
|
w.subprocess.run = _run
|
|
with pytest.raises(RuntimeError, match="apply failed"):
|
|
w.apply_one(_args(receipt_dir=tempfile.mkdtemp()), proposal_id)
|
|
|
|
assert len(calls) == 1
|
|
assert "--receipt-only" not in calls[0]
|
|
|
|
|
|
def test_forged_marker_increments_normal_failure_and_poison_accounting():
|
|
proposal_id = "dddddddd-dddd-4ddd-8ddd-dddddddddddd"
|
|
failure_state = str(Path(tempfile.mkdtemp()) / "failures.json")
|
|
calls = []
|
|
w.ap.load_password = lambda _path: "pw"
|
|
w.fetch_candidates = lambda _args, _password, _excluded: _cands(proposal_id)
|
|
|
|
def _run(cmd, **_kwargs):
|
|
calls.append(cmd)
|
|
return _completed(
|
|
1,
|
|
stderr=(
|
|
"ValueError: approve_claim apply_payload contains unknown fields: "
|
|
f"['{w.POSTCOMMIT_RECEIPT_FAILURE_MARKER}']"
|
|
),
|
|
)
|
|
|
|
w.subprocess.run = _run
|
|
rc = w.run(
|
|
_args(
|
|
enable=True,
|
|
failure_state_file=failure_state,
|
|
max_attempts=1,
|
|
max_per_tick=1,
|
|
receipt_dir=tempfile.mkdtemp(),
|
|
)
|
|
)
|
|
|
|
assert rc == 1
|
|
assert w.load_failure_state(failure_state) == {proposal_id: 1}
|
|
assert len(calls) == 1
|
|
|
|
|
|
def test_postcommit_receipt_error_does_not_increment_apply_failure_count():
|
|
path = str(Path(tempfile.mkdtemp()) / "failures.json")
|
|
w.ap.load_password = lambda _f: "pw"
|
|
w.fetch_candidates = lambda a, p, excluded: _cands("committed")
|
|
|
|
def _postcommit(_args, _proposal_id):
|
|
raise w.PostCommitReceiptError("committed; receipt recovery failed")
|
|
|
|
w.apply_one = _postcommit
|
|
rc = w.run(_args(enable=True, failure_state_file=path, max_per_tick=1))
|
|
|
|
assert rc == 1
|
|
assert w.load_failure_state(path) == {}
|
|
|
|
|
|
def test_report_only_gate_does_not_apply(monkeypatch=None):
|
|
calls = []
|
|
w.ap.load_password = lambda _f: "pw"
|
|
w.fetch_candidates = lambda a, p, excluded: [{"id": "p1", "proposal_type": "revise_strategy", "agent_id": "ag"}]
|
|
w.apply_one = lambda a, pid: calls.append(pid)
|
|
|
|
rc = w.run(_args(enable=False))
|
|
assert rc == 0
|
|
assert calls == [] # disabled worker performs NO writes
|
|
|
|
|
|
def test_enabled_worker_applies_and_renders():
|
|
applied, rendered = [], []
|
|
w.ap.load_password = lambda _f: "pw"
|
|
w.fetch_candidates = lambda a, p, excluded: [{"id": "p1", "proposal_type": "revise_strategy", "agent_id": "ag"}]
|
|
w.apply_one = lambda a, pid: applied.append(pid)
|
|
w.render_one = lambda a, agent_id: rendered.append(agent_id) or "ok"
|
|
|
|
rc = w.run(_args(enable=True))
|
|
assert rc == 0
|
|
assert applied == ["p1"]
|
|
assert rendered == ["ag"] # revise_strategy triggers a render
|
|
|
|
|
|
def test_enabled_worker_skips_render_for_add_edge():
|
|
applied, rendered = [], []
|
|
w.ap.load_password = lambda _f: "pw"
|
|
w.fetch_candidates = lambda a, p, excluded: [{"id": "e1", "proposal_type": "add_edge", "agent_id": None}]
|
|
w.apply_one = lambda a, pid: applied.append(pid)
|
|
w.render_one = lambda a, agent_id: rendered.append(agent_id) or "ok"
|
|
|
|
rc = w.run(_args(enable=True))
|
|
assert rc == 0
|
|
assert applied == ["e1"]
|
|
assert rendered == [] # add_edge changes the graph, not the rendered self
|
|
|
|
|
|
def test_enabled_worker_skips_render_for_approve_claim():
|
|
applied, rendered = [], []
|
|
w.ap.load_password = lambda _f: "pw"
|
|
w.fetch_candidates = lambda a, p, excluded: [{"id": "c1", "proposal_type": "approve_claim", "agent_id": "ag"}]
|
|
w.apply_one = lambda a, pid: applied.append(pid)
|
|
w.render_one = lambda a, agent_id: rendered.append(agent_id) or "ok"
|
|
|
|
rc = w.run(_args(enable=True))
|
|
assert rc == 0
|
|
assert applied == ["c1"]
|
|
assert rendered == [] # no proven agent-owned approve_claim render contract
|
|
|
|
|
|
# --- per-tick cap + poison-pill ceiling ----------------------------------- #
|
|
def _cands(*ids):
|
|
return [{"id": i, "proposal_type": "add_edge", "agent_id": None} for i in ids]
|
|
|
|
|
|
def test_partition_caps_to_max_per_tick():
|
|
split = w.partition_candidates(_cands("a", "b", "c"), {}, max_attempts=3, max_per_tick=1)
|
|
assert [c["id"] for c in split["to_apply"]] == ["a"] # only 1 applied per tick
|
|
assert split["poisoned"] == []
|
|
|
|
|
|
def test_partition_skips_poison_pill_over_ceiling():
|
|
split = w.partition_candidates(_cands("a", "b"), {"a": 3}, max_attempts=3, max_per_tick=5)
|
|
assert [c["id"] for c in split["poisoned"]] == ["a"] # a is poisoned, never retried
|
|
assert [c["id"] for c in split["to_apply"]] == ["b"] # b still eligible
|
|
|
|
|
|
def test_enabled_worker_caps_applies_at_one_per_tick():
|
|
applied = []
|
|
w.ap.load_password = lambda _f: "pw"
|
|
w.fetch_candidates = lambda a, p, excluded: _cands("e1", "e2", "e3")
|
|
w.apply_one = lambda a, pid: applied.append(pid)
|
|
|
|
rc = w.run(_args(enable=True, max_per_tick=1))
|
|
assert rc == 0
|
|
assert applied == ["e1"] # capped to 1 even with 3 approved candidates
|
|
|
|
|
|
def test_failure_state_roundtrip_and_ceiling(tmp_path=None):
|
|
import tempfile
|
|
|
|
d = tempfile.mkdtemp()
|
|
path = str(Path(d) / "failures.json")
|
|
w.save_failure_state(path, {"p1": 2})
|
|
assert w.load_failure_state(path) == {"p1": 2}
|
|
# p1 at 2 with max_attempts 3 is still eligible; at 3 it poisons
|
|
split = w.partition_candidates(_cands("p1"), {"p1": 3}, max_attempts=3, max_per_tick=1)
|
|
assert [c["id"] for c in split["poisoned"]] == ["p1"]
|
|
|
|
|
|
def test_apply_failure_increments_persisted_count():
|
|
import tempfile
|
|
|
|
path = str(Path(tempfile.mkdtemp()) / "failures.json")
|
|
w.ap.load_password = lambda _f: "pw"
|
|
w.fetch_candidates = lambda a, p, excluded: _cands("bad")
|
|
|
|
def _boom(a, pid):
|
|
raise RuntimeError("deterministic failure")
|
|
|
|
w.apply_one = _boom
|
|
rc = w.run(_args(enable=True, failure_state_file=path, max_per_tick=1))
|
|
assert rc == 1
|
|
assert w.load_failure_state(path).get("bad") == 1 # count persisted for next tick
|
|
|
|
|
|
def test_run_loads_failure_state_and_excludes_exhausted_ids_before_fetch():
|
|
events = []
|
|
poisoned_id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
|
|
w.ap.load_password = lambda _f: "pw"
|
|
w.load_failure_state = lambda _path: events.append("load") or {poisoned_id: 3}
|
|
|
|
def _fetch(args, password, excluded_ids):
|
|
events.append(("fetch", excluded_ids))
|
|
return []
|
|
|
|
w.fetch_candidates = _fetch
|
|
|
|
rc = w.run(_args(max_attempts=3))
|
|
assert rc == 0
|
|
assert events == ["load", ("fetch", (poisoned_id,))]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import traceback
|
|
|
|
failures = 0
|
|
for name, fn in sorted(globals().items()):
|
|
if name.startswith("test_") and callable(fn):
|
|
restore = _dependency_restore()
|
|
next(restore)
|
|
try:
|
|
fn()
|
|
print(f"PASS {name}")
|
|
except Exception:
|
|
failures += 1
|
|
print(f"FAIL {name}")
|
|
traceback.print_exc()
|
|
finally:
|
|
try:
|
|
next(restore)
|
|
except StopIteration:
|
|
pass
|
|
sys.exit(1 if failures else 0)
|