Harden staging preflight receipt publication

docs/gcp-leoclean-nosend-preflight.md

@@ -33,15 +33,25 @@ Run from the exact clean, tracked source revision with an already authenticated
 Cloud account that has the reviewed IAP/OS Login read path:
 
 ```bash
+receipt_dir="$(mktemp -d "${TMPDIR:-/tmp}/livingip-leoclean-preflight.XXXXXX")"
+chmod 0700 "$receipt_dir"
 python3 ops/check_gcp_leoclean_nosend_preflight.py \
-  --output /tmp/leoclean-nosend-gcp-preflight.json
+  --output "$receipt_dir/preflight.json"
 ```
 
-The output file is created with mode `0600`. Authentication, permission, and
-connection failures emit only a bounded category; command stderr is not copied
-into the receipt. The checker rejects an alternate repository root, an output
-path inside the source tree, untracked critical inputs, and source drift during
-the live observation.
+The output parent must already exist, be owned by the operator, and have no
+group/other write permission. The output name must not exist. The checker
+creates it once with no-clobber/no-symlink semantics, keeps the same file and
+directory identity open through publication, writes mode `0600`, and fsyncs
+the file and directory. It never removes or overwrites the output pathname;
+an ambiguous publication failure can therefore leave a private residue for
+operator inspection.
+
+Authentication, permission, connection, and receipt-publication failures emit
+only a bounded category; command stderr and filesystem exception details are
+not copied into the receipt or stdout. The checker rejects an alternate
+repository root, an output path inside the source tree, untracked critical
+inputs, and source drift during the live observation.
 
 ## Claim ceiling and rollback
 
@@ -52,7 +62,8 @@ secret access, database authentication, scoped PostgreSQL permissions, image
 publication, deployment, restart, rollback, model behavior, parity, traffic,
 or production readiness.
 
-This slice changes no repository-independent service state. Its local rollback
-boundary is deletion of the branch/worktree and any generated receipt. Any
-time-bounded SSH key created by the access path expires after at most five
-minutes.
+This slice changes no repository-independent service state beyond the stated
+possibility of a time-bounded SSH key. Its local rollback boundary is deletion
+of the branch/worktree; generated receipts and ambiguous private residues are
+operator-owned evidence and are not deleted automatically. Any time-bounded
+SSH key created by the access path expires after at most five minutes.

docs/workstreams/gcp-leoclean/goals.md

@@ -71,7 +71,22 @@ cut over safely and retire the VPS after soak.
   #210 head. It checks exact control-plane and host prerequisites without
   reading secrets, logging into PostgreSQL, deploying, or restarting. Its
   bounded IAP/SSH path uses a fixed remote command and a maximum five-minute
-  key lifetime, but no live run is currently authorized.
+  key lifetime. Receipt publication now requires a pre-existing private
+  operator-owned directory, creates one mode-`0600` no-clobber/no-symlink
+  file through a held directory descriptor, verifies stable identity, fsyncs
+  both file and directory, and preserves ambiguous residues. No live run is
+  currently authorized.
+- Artifact Registry cleanup policy is not an additional hard gate for the
+  package: immutable tags prevent deletion of the retained tagged candidate.
+  The post-push/finalization path must still re-prove the exact retained
+  candidate tag-to-digest binding immediately before deployment.
+- Current offline evidence for the preflight extraction: 243 focused
+  preflight/package/OCI tests pass; Ruff, Python compilation, formatting, and
+  diff checks pass. The repository suite produced 2,497 passes and 5 expected
+  skips inside the desktop sandbox; all 18 sandbox-denied Docker, loopback, and
+  Swift-cache cases passed in a separately authorized 59-test local rerun,
+  reconciling the suite to 2,515 passes and 5 skips. No cloud or registry
+  endpoint was contacted.
 - Historical only: the superseded preflight branch records an earlier live
   observation of `e2-standard-4` on `teleo-staging-europe-west6`
   (`10.60.0.0/20`) with an external IPv4 attachment. That observation is not

ops/check_gcp_leoclean_nosend_preflight.py

@@ -11,6 +11,7 @@ import json
 import os
 import re
 import shlex
+import stat
 import subprocess
 import sys
 from collections.abc import Callable
@@ -467,7 +468,10 @@ def _host_readback(runner: Runner, private_ip: str) -> dict[str, Any]:
     _require(value["project"] == package.PROJECT, "host metadata project drifted")
     _require(value["instance"] == package.INSTANCE, "host metadata instance drifted")
     _require(value["service_account"] == package.SERVICE_ACCOUNT, "host metadata service account drifted")
-    _require(version is not None and int(version.group(1)) >= MINIMUM_DOCKER_MAJOR, "Docker server is below the reviewed minimum")
+    _require(
+        version is not None and int(version.group(1)) >= MINIMUM_DOCKER_MAJOR,
+        "Docker server is below the reviewed minimum",
+    )
     _require(value["docker_server_os"] == "linux", "Docker server OS is not Linux")
     _require(value["docker_server_arch"] == "amd64", "Docker server architecture is not amd64")
     try:
@@ -553,6 +557,151 @@ def _validated_repo_root(repo_root: Path) -> Path:
     return resolved
 
 
+def _normalized_output_path(path: Path) -> Path:
+    expanded = path.expanduser()
+    if not expanded.is_absolute():
+        expanded = Path.cwd() / expanded
+    if expanded.name in {"", ".", ".."}:
+        raise PreflightError("invalid_output_path", "preflight receipt output path is invalid")
+    try:
+        parent = expanded.parent.resolve(strict=True)
+    except OSError as exc:
+        raise PreflightError(
+            "invalid_output_path",
+            "preflight receipt parent must already exist",
+        ) from exc
+    return parent / expanded.name
+
+
+def _assert_private_output_parent(parent_descriptor: int, parent: Path) -> os.stat_result:
+    try:
+        observed = os.fstat(parent_descriptor)
+        named = os.stat(parent, follow_symlinks=False)
+    except OSError as exc:
+        raise PreflightError(
+            "invalid_output_path",
+            "preflight receipt parent cannot be identified safely",
+        ) from exc
+    if not (
+        stat.S_ISDIR(observed.st_mode)
+        and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino)
+        and observed.st_uid == os.geteuid()
+        and observed.st_mode & 0o022 == 0
+    ):
+        raise PreflightError(
+            "invalid_output_path",
+            "preflight receipt parent must be operator-owned and not group/world writable",
+        )
+    return observed
+
+
+def _output_entry_stat(parent_descriptor: int, name: str) -> os.stat_result | None:
+    try:
+        return os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False)
+    except FileNotFoundError:
+        return None
+    except OSError as exc:
+        raise PreflightError(
+            "invalid_output_path",
+            "preflight receipt output cannot be identified safely",
+        ) from exc
+
+
+def _open_private_output_parent(output_path: Path) -> int:
+    flags = os.O_RDONLY
+    if hasattr(os, "O_DIRECTORY"):
+        flags |= os.O_DIRECTORY
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(output_path.parent, flags)
+    except OSError as exc:
+        raise PreflightError(
+            "invalid_output_path",
+            "preflight receipt parent is missing or unsafe",
+        ) from exc
+    try:
+        _assert_private_output_parent(descriptor, output_path.parent)
+        if _output_entry_stat(descriptor, output_path.name) is not None:
+            raise PreflightError(
+                "invalid_output_path",
+                "preflight receipt output already exists",
+            )
+    except (OSError, PreflightError):
+        os.close(descriptor)
+        raise
+    return descriptor
+
+
+def _write_all(descriptor: int, payload: bytes) -> None:
+    offset = 0
+    while offset < len(payload):
+        written = os.write(descriptor, payload[offset:])
+        if written <= 0:
+            raise OSError("preflight receipt write made no progress")
+        offset += written
+
+
+def _publish_private_receipt(
+    output_path: Path,
+    parent_descriptor: int,
+    payload: dict[str, Any],
+) -> None:
+    try:
+        encoded = (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8")
+    except (TypeError, ValueError) as exc:
+        raise PreflightError(
+            "receipt_publication_failed",
+            "preflight receipt could not be encoded",
+        ) from exc
+
+    descriptor: int | None = None
+    try:
+        _assert_private_output_parent(parent_descriptor, output_path.parent)
+        if _output_entry_stat(parent_descriptor, output_path.name) is not None:
+            raise PreflightError(
+                "receipt_publication_failed",
+                "preflight receipt output was created before publication",
+            )
+        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+        descriptor = os.open(output_path.name, flags, 0o600, dir_fd=parent_descriptor)
+        os.fchmod(descriptor, 0o600)
+        before = os.fstat(descriptor)
+        _write_all(descriptor, encoded)
+        os.fsync(descriptor)
+        after = os.fstat(descriptor)
+        named = os.stat(output_path.name, dir_fd=parent_descriptor, follow_symlinks=False)
+        through_path = os.stat(output_path, follow_symlinks=False)
+        if not (
+            stat.S_ISREG(after.st_mode)
+            and (before.st_dev, before.st_ino) == (after.st_dev, after.st_ino)
+            and (after.st_dev, after.st_ino) == (named.st_dev, named.st_ino)
+            and (after.st_dev, after.st_ino) == (through_path.st_dev, through_path.st_ino)
+            and after.st_uid == os.geteuid()
+            and stat.S_IMODE(after.st_mode) == 0o600
+            and after.st_nlink == 1
+            and after.st_size == len(encoded)
+        ):
+            raise PreflightError(
+                "receipt_publication_failed",
+                "published preflight receipt identity or posture drifted",
+            )
+        _assert_private_output_parent(parent_descriptor, output_path.parent)
+        os.fsync(parent_descriptor)
+    except PreflightError:
+        raise
+    except OSError as exc:
+        raise PreflightError(
+            "receipt_publication_failed",
+            "preflight receipt publication failed",
+        ) from exc
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+
+
 def _source_revision(
     repo_root: Path,
     *,
@@ -591,34 +740,38 @@ def _source_revision(
     return source_revision
 
 
-def main() -> int:
+def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[1])
     parser.add_argument("--output", type=Path)
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     output_path: Path | None = None
+    output_parent_descriptor: int | None = None
     try:
         repo_root = _validated_repo_root(args.repo_root)
         if args.output:
-            output_path = args.output.expanduser().resolve()
+            output_path = _normalized_output_path(args.output)
             try:
                 output_path.relative_to(repo_root)
             except ValueError:
                 pass
             else:
                 raise PreflightError("invalid_output_path", "preflight receipt must be written outside the source tree")
+            output_parent_descriptor = _open_private_output_parent(output_path)
         source_revision = _source_revision(repo_root)
         payload = build_preflight(source_revision=source_revision)
         if _source_revision(repo_root) != source_revision:
             raise PreflightError("source_revision_changed", "source revision changed during preflight")
+        if output_path is not None and output_parent_descriptor is not None:
+            _publish_private_receipt(output_path, output_parent_descriptor, payload)
     except (PreflightError, subprocess.SubprocessError, OSError) as exc:
         category = exc.category if isinstance(exc, PreflightError) else "local_preflight_unavailable"
         payload = {"schema": SCHEMA, "status": "blocked", "category": category}
         print(json.dumps(payload, separators=(",", ":"), sort_keys=True))
         return 69 if category in {"authentication_required", "permission_denied", "connection_unavailable"} else 1
-    if output_path:
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-        package.write_json(output_path, payload, mode=0o600)
+    finally:
+        if output_parent_descriptor is not None:
+            os.close(output_parent_descriptor)
     print(json.dumps(payload, indent=2, sort_keys=True))
     return 0
 

tests/test_gcp_leoclean_nosend_preflight.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 import copy
 import hashlib
 import json
+import os
+import stat
 import subprocess
 import sys
 from collections.abc import Callable
@@ -30,9 +32,7 @@ def live_fixtures() -> dict[str, object]:
                 "machineTypes/e2-standard-4"
             ),
             "cpuPlatform": "Intel Broadwell",
-            "serviceAccounts": [
-                {"email": "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com"}
-            ],
+            "serviceAccounts": [{"email": "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com"}],
             "disks": [
                 {
                     "boot": True,
@@ -169,9 +169,10 @@ def test_exact_read_only_preflight_passes_without_exposing_private_ip() -> None:
         "ssh_key_maximum_lifetime": "5m",
     }
     assert payload["host"]["cloudsql_private_tcp_5432"] == "reachable"
-    assert payload["control_plane"]["cloudsql"]["private_ip_sha256"] == hashlib.sha256(
-        PRIVATE_IP.encode("ascii")
-    ).hexdigest()
+    assert (
+        payload["control_plane"]["cloudsql"]["private_ip_sha256"]
+        == hashlib.sha256(PRIVATE_IP.encode("ascii")).hexdigest()
+    )
     assert PRIVATE_IP not in json.dumps(payload)
     assert len(calls) == 7
     assert calls[:-1] == [
@@ -446,7 +447,12 @@ def test_preflight_rejects_an_unrelated_clean_repository(tmp_path: Path) -> None
 
 def test_direct_cli_imports_then_rejects_an_unrelated_repository(tmp_path: Path) -> None:
     completed = subprocess.run(
-        [sys.executable, str(preflight.SOURCE_ROOT / "ops" / "check_gcp_leoclean_nosend_preflight.py"), "--repo-root", str(tmp_path)],
+        [
+            sys.executable,
+            str(preflight.SOURCE_ROOT / "ops" / "check_gcp_leoclean_nosend_preflight.py"),
+            "--repo-root",
+            str(tmp_path),
+        ],
         text=True,
         capture_output=True,
         check=False,
@@ -457,3 +463,154 @@ def test_direct_cli_imports_then_rejects_an_unrelated_repository(tmp_path: Path)
         "schema": "livingip.leocleanNoSendGcpPreflight.v1",
         "status": "blocked",
     }
+
+
+def _private_output_directory(tmp_path: Path) -> Path:
+    directory = tmp_path / "private-receipts"
+    directory.mkdir(mode=0o700)
+    directory.chmod(0o700)
+    return directory
+
+
+def _stub_successful_cli(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:
+    payload: dict[str, object] = {
+        "schema": preflight.SCHEMA,
+        "status": "pass",
+        "source_revision": REVISION,
+    }
+    monkeypatch.setattr(preflight, "_source_revision", lambda *_args, **_kwargs: REVISION)
+    monkeypatch.setattr(preflight, "build_preflight", lambda **_kwargs: payload)
+    return payload
+
+
+def test_cli_publishes_one_private_no_clobber_receipt(
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    payload = _stub_successful_cli(monkeypatch)
+    output = _private_output_directory(tmp_path) / "preflight.json"
+
+    assert preflight.main(["--output", str(output)]) == 0
+
+    assert json.loads(capsys.readouterr().out) == payload
+    assert json.loads(output.read_text(encoding="utf-8")) == payload
+    observed = output.stat(follow_symlinks=False)
+    assert stat.S_ISREG(observed.st_mode)
+    assert stat.S_IMODE(observed.st_mode) == 0o600
+    assert observed.st_uid == os.geteuid()
+    assert observed.st_nlink == 1
+
+
+@pytest.mark.parametrize("entry_kind", ["file", "directory", "symlink"])
+def test_cli_rejects_every_preexisting_output_before_live_observation(
+    entry_kind: str,
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    output = _private_output_directory(tmp_path) / "preflight.json"
+    if entry_kind == "file":
+        output.write_text("existing\n", encoding="utf-8")
+    elif entry_kind == "directory":
+        output.mkdir()
+    else:
+        target = output.with_name("target.json")
+        target.write_text("target\n", encoding="utf-8")
+        output.symlink_to(target)
+    called = False
+
+    def unexpected_live_observation(**_kwargs: object) -> dict[str, object]:
+        nonlocal called
+        called = True
+        raise AssertionError("live observation must not run")
+
+    monkeypatch.setattr(preflight, "build_preflight", unexpected_live_observation)
+
+    assert preflight.main(["--output", str(output)]) == 1
+
+    assert called is False
+    assert json.loads(capsys.readouterr().out) == {
+        "category": "invalid_output_path",
+        "schema": preflight.SCHEMA,
+        "status": "blocked",
+    }
+
+
+@pytest.mark.parametrize("unsafe_mode", [0o720, 0o702, 0o777])
+def test_cli_rejects_group_or_world_writable_output_parent(
+    unsafe_mode: int,
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    parent = _private_output_directory(tmp_path)
+    parent.chmod(unsafe_mode)
+    called = False
+
+    def unexpected_live_observation(**_kwargs: object) -> dict[str, object]:
+        nonlocal called
+        called = True
+        raise AssertionError("live observation must not run")
+
+    monkeypatch.setattr(preflight, "build_preflight", unexpected_live_observation)
+
+    assert preflight.main(["--output", str(parent / "preflight.json")]) == 1
+
+    assert called is False
+    assert json.loads(capsys.readouterr().out)["category"] == "invalid_output_path"
+
+
+def test_publication_race_fails_without_overwriting_the_created_entry(
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    output = _private_output_directory(tmp_path) / "preflight.json"
+    monkeypatch.setattr(preflight, "_source_revision", lambda *_args, **_kwargs: REVISION)
+
+    def race(**_kwargs: object) -> dict[str, object]:
+        output.write_text("replacement-sentinel\n", encoding="utf-8")
+        return {"schema": preflight.SCHEMA, "status": "pass", "source_revision": REVISION}
+
+    monkeypatch.setattr(preflight, "build_preflight", race)
+
+    assert preflight.main(["--output", str(output)]) == 1
+
+    assert output.read_text(encoding="utf-8") == "replacement-sentinel\n"
+    assert json.loads(capsys.readouterr().out) == {
+        "category": "receipt_publication_failed",
+        "schema": preflight.SCHEMA,
+        "status": "blocked",
+    }
+
+
+def test_publication_failure_is_bounded_and_preserves_created_residue(
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    _stub_successful_cli(monkeypatch)
+    output = _private_output_directory(tmp_path) / "preflight.json"
+    real_fsync = preflight.os.fsync
+    sentinel = "private-publication-fault-sentinel"
+
+    def fail_file_fsync(descriptor: int) -> None:
+        if stat.S_ISREG(os.fstat(descriptor).st_mode):
+            raise OSError(sentinel)
+        real_fsync(descriptor)
+
+    monkeypatch.setattr(preflight.os, "fsync", fail_file_fsync)
+
+    assert preflight.main(["--output", str(output)]) == 1
+
+    stdout = capsys.readouterr().out
+    assert json.loads(stdout) == {
+        "category": "receipt_publication_failed",
+        "schema": preflight.SCHEMA,
+        "status": "blocked",
+    }
+    assert sentinel not in stdout
+    assert output.is_file()
+    assert stat.S_IMODE(output.stat().st_mode) == 0o600
+    assert json.loads(output.read_text(encoding="utf-8"))["status"] == "pass"