15c30f1fbe
Some checks are pending
CI / lint-and-test (push) Waiting to run
* make Leo skill onboarding clean-context reproducible * harden Leo clean-context skill acceptance
285 lines
11 KiB
Python
285 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REPORT_DIR = ROOT / "docs" / "reports" / "leo-working-state-20260709"
|
|
MANIFEST = REPORT_DIR / "skill-pack-manifest.json"
|
|
|
|
|
|
def _skill(name: str) -> str:
|
|
return (ROOT / ".agents" / "skills" / name / "SKILL.md").read_text(encoding="utf-8")
|
|
|
|
|
|
def test_manifest_indexes_valid_repo_native_skills() -> None:
|
|
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
|
|
assert manifest["status"] == "repo_native_path_validated"
|
|
assert manifest["repo_skill_root"] == ".agents/skills"
|
|
assert manifest["validation"] == "tests/test_repo_skill_pack.py"
|
|
assert "37/37" in manifest["claim_ceiling"]
|
|
assert "PRs #146 and #147 are merged" in manifest["claim_ceiling"]
|
|
assert "PR #148 is open candidate evidence only" in manifest["claim_ceiling"]
|
|
|
|
for entry in manifest["skills"]:
|
|
path = ROOT / entry["path"]
|
|
text = path.read_text(encoding="utf-8")
|
|
assert path.is_file()
|
|
assert text.startswith("---\n")
|
|
assert f"name: {entry['name']}\n" in text.split("---\n", 2)[1]
|
|
assert "description:" in text.split("---\n", 2)[1]
|
|
|
|
for relative in manifest["reference_files"]:
|
|
assert (ROOT / relative).is_file(), relative
|
|
|
|
|
|
def test_manifest_covers_the_required_onboarding_surface() -> None:
|
|
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
expected = {
|
|
"company_product_context",
|
|
"vps",
|
|
"gcp",
|
|
"hermes_runtime",
|
|
"database_provenance",
|
|
"ingestion",
|
|
"proposal_apply",
|
|
"reconstruction",
|
|
"identity",
|
|
"testing",
|
|
"security",
|
|
"secrets",
|
|
"incident_recovery",
|
|
}
|
|
assert set(manifest["required_coverage"]) == expected
|
|
skill_names = {entry["name"] for entry in manifest["skills"]}
|
|
repo_skill_names = {path.parent.name for path in (ROOT / ".agents" / "skills").glob("*/SKILL.md")}
|
|
assert skill_names == repo_skill_names
|
|
for owners in manifest["required_coverage"].values():
|
|
assert owners
|
|
assert set(owners) <= skill_names
|
|
|
|
|
|
def test_path_validator_and_clean_installer() -> None:
|
|
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
validator = ROOT / manifest["validator"]
|
|
result = subprocess.run(
|
|
[str(validator), "--root", str(ROOT)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = json.loads(result.stdout)
|
|
assert payload["status"] == "pass"
|
|
assert payload["skill_count"] == len(manifest["skills"])
|
|
assert payload["coverage_area_count"] == len(manifest["required_coverage"])
|
|
assert payload["problems"] == []
|
|
|
|
|
|
def test_installer_copies_only_manifest_skills(tmp_path: Path) -> None:
|
|
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
installer = ROOT / manifest["installer"]
|
|
target = tmp_path / "skills"
|
|
result = subprocess.run(
|
|
[str(installer), "--root", str(ROOT), "--target", str(target)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = json.loads(result.stdout)
|
|
assert payload["status"] == "pass"
|
|
assert payload["installed_skill_count"] == len(manifest["skills"])
|
|
assert {path.name for path in target.iterdir()} == {entry["name"] for entry in manifest["skills"]}
|
|
assert all((target / entry["name"] / "SKILL.md").is_file() for entry in manifest["skills"])
|
|
|
|
|
|
def test_installer_rejects_manifest_path_traversal(tmp_path: Path) -> None:
|
|
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
manifest["skills"][0]["name"] = "../escape"
|
|
malicious_manifest = tmp_path / "manifest.json"
|
|
malicious_manifest.write_text(json.dumps(manifest), encoding="utf-8")
|
|
target = tmp_path / "target" / "skills"
|
|
|
|
result = subprocess.run(
|
|
[
|
|
str(ROOT / manifest["installer"]),
|
|
"--root",
|
|
str(ROOT),
|
|
"--manifest",
|
|
str(malicious_manifest),
|
|
"--target",
|
|
str(target),
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert "invalid skill name" in result.stdout
|
|
assert not (tmp_path / "target" / "escape").exists()
|
|
assert not target.exists()
|
|
|
|
|
|
def test_isolated_install_canary_reaches_t2_without_claiming_agent_reasoning(tmp_path: Path) -> None:
|
|
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
canary = ROOT / manifest["isolated_install_canary"]
|
|
output = tmp_path / "receipt.json"
|
|
result = subprocess.run(
|
|
[str(canary), "--root", str(ROOT), "--output", str(output)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = json.loads(output.read_text(encoding="utf-8"))
|
|
assert payload["status"] == "pass"
|
|
assert payload["required_tier"] == "T2_runtime"
|
|
assert payload["current_tier"] == "T2_runtime"
|
|
assert payload["proof_scope"] == "deterministic_isolated_install_and_route_validation"
|
|
assert payload["fresh_temporary_skill_root"] is True
|
|
assert payload["ambient_skill_root_used"] is False
|
|
assert payload["agent_reasoning_exercised"] is False
|
|
assert payload["installed_skill_count"] == len(manifest["skills"])
|
|
assert all(payload["semantic_checks"].values())
|
|
assert payload["canary_exit_status"] == 0
|
|
assert payload["cleanup_readback"]["temporary_skill_root_removed"] is True
|
|
assert payload["missing_to_reach_required_tier"] == []
|
|
|
|
|
|
def test_fresh_agent_receipt_rejects_stale_routes() -> None:
|
|
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
receipt = json.loads((ROOT / manifest["fresh_agent_receipt"]).read_text(encoding="utf-8"))
|
|
|
|
assert receipt["status"] == "pass"
|
|
assert receipt["evidence_origin"]["fork_turns"] == "none"
|
|
assert receipt["evidence_origin"]["memory_used"] is False
|
|
assert all(receipt["stale_claims_rejected"].values())
|
|
assert receipt["validator"]["status"] == "pass"
|
|
assert receipt["validator"]["skill_count"] == 16
|
|
assert receipt["repository_readback"]["unchanged"] is True
|
|
assert receipt["cleanup_readback"]["temporary_skill_root_removed"] is True
|
|
assert receipt["external_runtime_contacted"] is False
|
|
assert receipt["production_mutation"] is False
|
|
assert receipt["contains_secrets"] is False
|
|
assert receipt["problems"] == []
|
|
|
|
|
|
def test_db_change_skill_tracks_guarded_v2_runtime_proof() -> None:
|
|
text = _skill("teleo-kb-db-change-workflow")
|
|
squashed = " ".join(text.split())
|
|
|
|
for required in (
|
|
"`approve_claim` v2",
|
|
"kb_review",
|
|
"kb_apply",
|
|
"kb_gate_owner",
|
|
"NOINHERIT",
|
|
"37/37",
|
|
"exact table deltas",
|
|
"source hashes",
|
|
"trusted operator-only canonical writer",
|
|
"run_approve_claim_isolated_container_canary.sh",
|
|
"all `13` checks",
|
|
"`34/34`",
|
|
"PR #86",
|
|
):
|
|
assert required in text
|
|
|
|
assert "Production was not applied" in squashed
|
|
assert "Arbitrary production document/tweet ingestion is not proven" in squashed
|
|
assert "three approved-but-unapplied legacy packets" in squashed
|
|
|
|
|
|
def test_gcp_skill_uses_current_operator_access_and_parity_truth() -> None:
|
|
text = _skill("teleo-gcp-parity-ops")
|
|
squashed = " ".join(text.split())
|
|
|
|
for required in (
|
|
"ssh teleo-gcp-staging",
|
|
"teleo-pgvector-standby",
|
|
"PostgreSQL 16.14",
|
|
"public IP disabled",
|
|
"39/39",
|
|
"52,167/52,167",
|
|
"`148735`, `NRestarts=0`",
|
|
"NRestarts=0",
|
|
"state.db",
|
|
"leoclean-cloudsql-memory-sync.service",
|
|
"gcp-db-first-restore-current.json",
|
|
"gcp-db-first-parity-current.json",
|
|
"gcp-db-first-blind-claim-current.json",
|
|
"gcp-db-first-cleanup-current.json",
|
|
"gcp-db-first-live-deploy-restart-current.json",
|
|
"18/18",
|
|
"6/6",
|
|
"PID `304036`",
|
|
"fail closed",
|
|
"invalid_target",
|
|
"older staging copy",
|
|
):
|
|
assert required in text
|
|
|
|
assert "does not store a Google password" in squashed
|
|
assert "firewall-source-dependent" in text
|
|
assert "Do not call this path working until a live `status` run passes" in text
|
|
assert "Do not describe a passing Hermes memory sync as canonical KB parity" in text
|
|
assert "Do not call control-plane inventory" in text
|
|
|
|
|
|
def test_vps_onboarding_and_m3taversal_skills_point_to_current_proof() -> None:
|
|
vps = _skill("teleo-vps-runtime-ops")
|
|
provenance = _skill("teleo-infra-provenance")
|
|
onboarding = _skill("teleo-leo-onboarding")
|
|
outcomes = _skill("working-leo-m3taversal-outcomes")
|
|
|
|
assert "reasoning tools `17`" in vps
|
|
assert "current source hashes" in vps
|
|
assert "`teleo-auto-deploy.timer`" in vps
|
|
assert "separate state changes" in vps
|
|
assert "/opt/teleo-eval/.last-deploy-sha" in provenance
|
|
assert "does not prove a gateway restart" in " ".join(provenance.split())
|
|
assert "gcp-cloud-sql-t3-live-readonly-current.md" in onboarding
|
|
assert "gcp-db-first-working-leo-20260714.md" in onboarding
|
|
assert "gcp-db-first-parity-current.json" in onboarding
|
|
assert "gcp-db-first-blind-claim-current.json" in onboarding
|
|
assert "gcp-db-first-cleanup-current.json" in onboarding
|
|
assert "ssh -o BatchMode=yes teleo-gcp-staging true" in onboarding
|
|
assert "37/37" in outcomes
|
|
assert "Approved is not the same as applied" in outcomes
|
|
assert "Address `@m3taversal` only as `m3taversal`, exactly" in outcomes
|
|
assert "Next proof-changing follow-up:" in outcomes
|
|
assert "working-leo-current-proof-20260712.md" in outcomes
|
|
assert "Cory" not in outcomes
|
|
|
|
|
|
def test_reconstruction_and_pr_state_are_routed_without_candidate_overclaim() -> None:
|
|
reconstruction = _skill("teleo-reconstruction-recovery")
|
|
onboarding = _skill("teleo-leo-onboarding")
|
|
gcp = _skill("teleo-gcp-parity-ops")
|
|
current_truth = (REPORT_DIR / "current-truth-index.md").read_text(encoding="utf-8")
|
|
|
|
assert "ops/run_local_genesis_ledger_rebuild.py" in reconstruction
|
|
assert "scripts/verify_leo_unseen_reasoning_chain.py" in reconstruction
|
|
assert "Semantic source-to-blank-database recompilation remains incomplete" in reconstruction
|
|
assert "PRs #146 and #147 are merged repository truth" in onboarding
|
|
assert "PR #148 remains an open GCP" in onboarding
|
|
assert "candidate evidence only" in gcp
|
|
assert "PRs #146 and #147 are merged into canonical `main`" in current_truth
|
|
assert "PR #148 remains open candidate evidence" in current_truth
|
|
|
|
|
|
def test_active_onboarding_prose_uses_exact_m3taversal_identity() -> None:
|
|
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
active_files = [
|
|
*[ROOT / relative for relative in manifest["scan_files"]],
|
|
*[ROOT / entry["path"] for entry in manifest["skills"]],
|
|
]
|
|
for path in active_files:
|
|
text = path.read_text(encoding="utf-8")
|
|
prose = re.sub(r"`[^`]*\/[^`]+\.(?:md|json|svg)`", "", text)
|
|
assert re.search(r"\bcory(?:-style)?\b", prose, flags=re.IGNORECASE) is None, path
|