From b9081a537caac82dfc0b0205dc71ade843738a38 Mon Sep 17 00:00:00 2001 From: fwazb Date: Wed, 22 Jul 2026 18:24:17 -0700 Subject: [PATCH] Add fail-closed GCP no-send IAP runner --- docs/workstreams/gcp-leoclean/goals.md | 100 +- ops/gcp_leoclean_nosend_iap_remote.py | 1065 +++++++++++++++ ops/install_gcp_leoclean_nosend_release.py | 838 +++++++++++- ops/run_gcp_leoclean_nosend_iap.py | 1279 ++++++++++++++++++ tests/test_gcp_leoclean_nosend_iap_runner.py | 779 +++++++++++ tests/test_gcp_leoclean_nosend_installer.py | 345 +++++ 6 files changed, 4313 insertions(+), 93 deletions(-) create mode 100644 ops/gcp_leoclean_nosend_iap_remote.py create mode 100644 ops/run_gcp_leoclean_nosend_iap.py create mode 100644 tests/test_gcp_leoclean_nosend_iap_runner.py diff --git a/docs/workstreams/gcp-leoclean/goals.md b/docs/workstreams/gcp-leoclean/goals.md index bb8f92d..c69e093 100644 --- a/docs/workstreams/gcp-leoclean/goals.md +++ b/docs/workstreams/gcp-leoclean/goals.md @@ -24,38 +24,46 @@ until the GCP service has passed soak and restore testing. ## Current state -- Current main/base: `3f09bb2b6c8ac3113fc1743529a03c0e9b6ea587`. -- PRs #220, #221, #226, #227, #228, #229, and #230 are merged. Main therefore contains the - source-first V3 rebuild foundation, the PostgreSQL large-object boundary, - bounded Observatory prerequisite preservation, Cloud SQL provider-database - residual handling, and the manual exact-revision no-send publication - workflow plus the composable prerequisite/runtime ACL boundary and normalized - copied-artifact directory contract. -- The exact-main GCP preflight passed for `3f09bb2`: the staging VM and private - PostgreSQL 16 Cloud SQL endpoint were reachable, the Artifact Registry - repository was immutable, and no public database attachment was present. -- Manual publication run #3 passed for that exact revision. Artifact Registry - independently reports immutable manifest digest - `sha256:c326154acd0b9ae468449abe1c5fd398b6757f375a1a86fdb40e982c515a19a4` +- Current main/base: `3d35a7fafb41b61a94cb75de9c231afdc08b7f2b`. +- PRs #220, #221, #226, #227, #228, #229, #230, and #231 are merged. Main + therefore contains the source-first V3 rebuild foundation, the PostgreSQL + least-privilege boundary, the exact-revision publication workflow, the + hardened no-send installer with automatic transaction rollback, and the + container-bound `verify-database` command. +- The exact-main GCP preflight passed for `3d35a7f`: `teleo-staging-1` and the + private PostgreSQL 16 Cloud SQL endpoint were reachable, Artifact Registry was + immutable, and neither the VM nor Cloud SQL had a public attachment. +- Manual publication run #4 passed for that exact revision. The retained GitHub + artifact archive hashes to + `sha256:dbe7a34f2c03e2240de54c40afe0aefae00013dff51e17913140fa38f6e119ea`. + Its self-hashed publication journal records `fresh_push`, not reuse or + reconciliation, and binds release hash + `3e7b00d5ac12308cfc35e3b507258b659e6d95926f3c6241f67240d44105b72f`. +- Artifact Registry independently reports immutable manifest digest + `sha256:c69781400eecd5d2a932b9c7c988ed460365b8bc3f592fe95e51e4c1edf7b230` and config digest - `sha256:5c4d5e1bcfc8cd112e5c999217fce064b5be718409708a154e9d58f007a521ad`, - with the expected revision and package labels. This is a valid intermediate - candidate, not an approved deployment candidate. -- The original run artifact still needs to be retained and checked for its - non-reconstructible publication-outcome journal. Final registry state alone - cannot prove whether the run freshly pushed, reused, or reconciled a candidate. -- The published image cannot yet perform a container-bound database proof. A - narrow follow-up is adding a hash-bound `verify-database` command that uses the - attached staging service-account metadata identity, the scoped secret, and the - existing least-privilege SQL verifier from inside the running container. The - next deployable image must be published from the merge revision containing - that verifier. -- The GCP VM, private Cloud SQL instance, and scoped Secret Manager secret are - known to exist, but the scoped VM secret binding and runtime role have not been - activated for this service. -- Live Cloud SQL provisioning of `leoclean_kb_runtime`, VM access to the scoped - secret, installation of the no-send service, effective running-process - identity, restart behavior, and rollback remain unproven. + `sha256:c5fc9e715bcbee7d4fcc512b0c38aa4acee589836fbc11dbe9c6c34194959438`, + with the expected revision, input, artifact, identity, runtime, and target + labels. The release descriptor and build/push receipt validate against current + package code. +- A read-only IAM check confirms the scoped runtime secret exists with one + enabled version, grants `roles/secretmanager.secretAccessor` only to + `sa-teleo-staging-vm`, and the administrator secret does not grant that VM + access. The VM also has repository-level Artifact Registry reader and no + detected project-level role. +- The native PostgreSQL `leoclean_kb_runtime` role and password match remain + unproven. Cloud SQL control-plane user listing is not authoritative for native + PostgreSQL roles; the container-bound permission verifier is the intended + proof. +- Branch `feat/leoclean-iap-installer-runner` now contains the local review + candidate for the fixed-target IAP transfer/pull/install/restart/verify + runner and receipt-bound post-success rollback. It remains unmerged and has + not been used against GCP. The proportional suite passes 308 tests; the full + suite passes 3,013 tests with 3 skips when the repository's Python 3.11 + environment is first on `PATH`. +- Installation of the no-send service, effective running-process identity, + database permissions, restart behavior, and post-success rollback remain + unproven. - The source-first V3 reconstruction code is on main, but no canonical GCP promotion is implied by that merge. - Telegram still does not route to GCP. The VPS, production traffic, and @@ -63,18 +71,22 @@ until the GCP service has passed soak and restore testing. ## Execution order -1. Review and merge the container-bound database verifier. Run its disposable - PostgreSQL and OCI fail-closed tests, then publish one exact-main replacement - candidate and retain the complete publication artifact. -2. Add the narrow IAP transfer/install/restart/rollback runner that consumes the - finalized release bundle, pulls only its immutable digest, invokes the existing - installer, and binds the live container proof to systemd/container identity. -3. Provision `leoclean_kb_runtime`, grant only the staging VM access to the - scoped secret, and prove the Cloud SQL role's allowed reads and - function-only proposal staging plus denied writes and escalation. -4. Install the exact immutable image as a separate no-send GCP service. Prove - the running process uses the scoped identity without administrator fallback, - then prove restart and rollback. +1. Completed: merge the container-bound database verifier, run its disposable + PostgreSQL and OCI fail-closed tests, publish the exact-main candidate, retain + the complete artifact, and cross-check its receipts and registry digests. +2. Local implementation complete; review and merge the narrow fixed-target IAP + transfer/install/restart/verify runner plus receipt-bound post-success + rollback. It consumes only the finalized release artifact and expected + hashes, pulls only the immutable digest, preserves ephemeral credential + cleanup, and emits sanitized self-hashed receipts. +3. Prove or provision `leoclean_kb_runtime`. The VM already has access only to + the scoped secret; verify that secret matches the native role, allowed reads, + transaction-rolled-back function-only proposal staging, and denied writes or + escalation. +4. Install the exact immutable image as a separate no-send GCP service. Bind + systemd, PID/invocation, full container ID, image/config digests, and the + container database receipt; restart once and then execute the reviewed + post-success rollback proof. 5. Reconstruct or reconcile the V3 knowledge database into a separately identified canonical candidate and prove row/hash/source receipts before any promotion. @@ -88,7 +100,7 @@ until the GCP service has passed soak and restore testing. ## Authority and rollback boundaries -- Database role provisioning can be rolled back independently by refencing +- Database role provisioning can be rolled back independently by revoking the runtime role and removing its scoped secret access. - Service deployment can be rolled back independently to the prior immutable image or stopped without changing canonical data. diff --git a/ops/gcp_leoclean_nosend_iap_remote.py b/ops/gcp_leoclean_nosend_iap_remote.py new file mode 100644 index 0000000..9dd53dd --- /dev/null +++ b/ops/gcp_leoclean_nosend_iap_remote.py @@ -0,0 +1,1065 @@ +#!/usr/bin/python3 -I +"""Execute one root-owned, fixed-target leoclean no-send IAP request.""" + +from __future__ import annotations + +import fcntl +import hashlib +import io +import json +import os +import pwd +import re +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any + +SCHEMA = "livingip.leocleanNoSendIapRequest.v1" +RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v1" +PROJECT = "teleo-501523" +ZONE = "europe-west6-a" +INSTANCE = "teleo-staging-1" +SERVICE = "leoclean-gcp-nosend.service" +SERVICE_ACCOUNT = "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com" +IMAGE_REPOSITORY = "europe-west6-docker.pkg.dev/teleo-501523/teleo/leoclean-nosend-staging" +DOCKER = "/usr/bin/docker" +DOCKER_HOST = "unix:///var/run/docker.sock" +SERVICE_DOCKER_CONFIG = "/var/lib/livingip/leoclean-nosend/docker-config" +METADATA = "http://metadata.google.internal/computeMetadata/v1" +RUN_ROOT = Path("/var/lib/livingip/leoclean-nosend-iap/requests") +LOCK_PATH = Path("/run/leoclean-gcp-nosend-iap.lock") +UPLOAD_ROOT = Path("/tmp") +STAGE_ROOT = Path("/run/livingip-leoclean-nosend-iap-bootstrap") +REQUEST_ID = re.compile(r"^iap-[a-z0-9]{12,32}$") +HEX_40 = re.compile(r"^[0-9a-f]{40}$") +HEX_64 = re.compile(r"^[0-9a-f]{64}$") +IMAGE_REFERENCE = re.compile(rf"^{re.escape(IMAGE_REPOSITORY)}@sha256:([0-9a-f]{{64}})$") +OPERATIONS = ("install", "verify", "rollback") +MAX_ARCHIVE_BYTES = 8 * 1024 * 1024 +MAX_MEMBER_BYTES = 2 * 1024 * 1024 +MAX_MANIFEST_BYTES = 128 * 1024 +MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024 +MAX_TOKEN_BYTES = 64 * 1024 +EXPECTED_MEMBERS = { + "request-manifest.json", + "release-artifact.zip", + "release/release.json", + f"release/{SERVICE}", + "ops/gcp_leoclean_nosend_package.py", + "ops/install_gcp_leoclean_nosend_release.py", + "ops/leoclean_nosend_service_control.py", + "ops/gcp_leoclean_nosend_iap_remote.py", + "ops/run_gcp_leoclean_nosend_iap.py", + "scripts/leo_behavior_manifest.py", + "scripts/leo_identity_manifest.py", +} +RELEASE_ARTIFACT_MEMBERS = { + "evidence/oci-smoke.json", + "evidence/publish-outcome.json", + "evidence/runtime-receipt.json", + f"final/release-v3/{SERVICE}", + "final/release-v3/release.json", + "receipts/build-push-receipt.json", +} +SAFE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +Runner = Callable[..., subprocess.CompletedProcess[bytes]] + + +class RemoteError(RuntimeError): + """A remote request failed closed without exposing command output.""" + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise RemoteError(message) + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _canonical_sha256(value: Mapping[str, Any]) -> str: + try: + encoded = json.dumps( + value, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise RemoteError("request manifest could not be hashed") from exc + return _sha256_bytes(encoded) + + +def _strict_json(raw: bytes, label: str, max_bytes: int) -> dict[str, Any]: + _require(len(raw) <= max_bytes, f"{label} exceeded the safety limit") + + def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate key") + value[key] = item + return value + + try: + value = json.loads( + raw.decode("utf-8", "strict"), + object_pairs_hook=reject_duplicates, + parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("constant")), + ) + except (UnicodeError, ValueError, json.JSONDecodeError) as exc: + raise RemoteError(f"{label} was invalid JSON") from exc + _require(isinstance(value, dict), f"{label} was not a JSON object") + return value + + +def _run( + argv: list[str], + *, + runner: Runner, + label: str, + input_bytes: bytes | None = None, + timeout: int = 120, + require_success: bool = True, +) -> subprocess.CompletedProcess[bytes]: + try: + completed = runner( + argv, + check=False, + stdin=subprocess.DEVNULL if input_bytes is None else None, + input=input_bytes, + capture_output=True, + timeout=timeout, + env={"PATH": SAFE_PATH}, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise RemoteError(f"{label} failed") from exc + _require( + len(completed.stdout) <= MAX_COMMAND_OUTPUT_BYTES and len(completed.stderr) <= MAX_COMMAND_OUTPUT_BYTES, + f"{label} output exceeded the safety limit", + ) + if require_success: + _require(completed.returncode == 0, f"{label} failed") + return completed + + +def _read_stable_stage(path: Path, expected_sha256: str, max_bytes: int) -> bytes: + _require(path.parent.parent == STAGE_ROOT, "staged request path escaped its fixed root") + flags = os.O_RDONLY + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise RemoteError("request upload was missing or unsafe") from exc + try: + before = os.fstat(descriptor) + _require( + stat.S_ISREG(before.st_mode) + and before.st_uid == 0 + and stat.S_IMODE(before.st_mode) == 0o600 + and before.st_nlink == 1 + and before.st_size <= max_bytes, + "request upload posture was unsafe", + ) + chunks: list[bytes] = [] + remaining = max_bytes + 1 + while remaining: + chunk = os.read(descriptor, min(65536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + content = b"".join(chunks) + after = os.fstat(descriptor) + named = path.stat(follow_symlinks=False) + _require( + len(content) == after.st_size + and (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns) + == (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns) + and (after.st_dev, after.st_ino, after.st_mode, after.st_nlink) + == (named.st_dev, named.st_ino, named.st_mode, named.st_nlink) + and _sha256_bytes(content) == expected_sha256, + "request upload changed or differed from its hash", + ) + return content + except OSError as exc: + raise RemoteError("request upload could not be read") from exc + finally: + os.close(descriptor) + + +def _unlink_exact(path: Path, identity: tuple[int, int]) -> None: + observed = path.stat(follow_symlinks=False) + _require((observed.st_dev, observed.st_ino) == identity and stat.S_ISREG(observed.st_mode), "upload was replaced") + path.unlink() + + +def _ensure_root_directory( + path: Path, + mode: int, + *, + expected_uid: int = 0, + trusted_parent: Path = Path("/var/lib"), +) -> None: + _require(path.is_absolute() and path.is_relative_to(trusted_parent), "request directory escaped its fixed root") + missing: list[Path] = [] + cursor = path + while not cursor.exists(): + _require(not cursor.is_symlink(), "request directory path was unsafe") + missing.append(cursor) + cursor = cursor.parent + observed = cursor.stat(follow_symlinks=False) + _require(stat.S_ISDIR(observed.st_mode), "request directory parent was unsafe") + for directory in reversed(missing): + directory.mkdir(mode=mode if directory == path else 0o755) + os.chown(directory, expected_uid, os.getgid() if expected_uid != 0 else 0) + observed = path.stat(follow_symlinks=False) + _require( + stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0, + "request directory was unsafe", + ) + if path in missing: + path.chmod(mode) + cursor = path + while True: + observed = cursor.stat(follow_symlinks=False) + _require( + stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0, + "request directory ancestor was unsafe", + ) + if cursor == trusted_parent: + break + cursor = cursor.parent + + +def _acquire_lock(path: Path = LOCK_PATH, *, expected_uid: int = 0) -> int: + flags = os.O_RDWR | os.O_CREAT + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags, 0o600) + observed = os.fstat(descriptor) + named = path.stat(follow_symlinks=False) + _require( + stat.S_ISREG(observed.st_mode) + and observed.st_uid == expected_uid + and stat.S_IMODE(observed.st_mode) == 0o600 + and observed.st_nlink == 1 + and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino), + "request lock was unsafe", + ) + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + return descriptor + except (OSError, RemoteError) as exc: + if "descriptor" in locals(): + os.close(descriptor) + raise RemoteError("another request is active or the request lock was unsafe") from exc + + +def _copy_into_request(path: Path, content: bytes, mode: int = 0o600) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags, mode) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + except OSError as exc: + raise RemoteError("request file could not be secured") from exc + + +def _claim_request_directory( + run_root: Path, + request_id: str, + *, + expected_uid: int = 0, + trusted_parent: Path = Path("/var/lib"), +) -> Path: + _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") + _ensure_root_directory(run_root, 0o700, expected_uid=expected_uid, trusted_parent=trusted_parent) + request_dir = run_root / request_id + _require(not request_dir.exists() and not request_dir.is_symlink(), "request ID was already used") + request_dir.mkdir(mode=0o700) + os.chown(request_dir, expected_uid, os.getgid() if expected_uid != 0 else 0) + return request_dir + + +def _cleanup_upload_if_exact(path: Path, identity: tuple[int, int]) -> None: + try: + observed = path.stat(follow_symlinks=False) + if stat.S_ISREG(observed.st_mode) and (observed.st_dev, observed.st_ino) == identity: + path.unlink() + except OSError: + pass + + +def _validate_member_name(name: str) -> None: + path = Path(name) + _require( + name + and not name.startswith("/") + and not name.endswith("/") + and all(part not in {"", ".", ".."} for part in path.parts), + "archive member path was unsafe", + ) + + +def extract_archive(archive_path: Path, destination: Path) -> dict[str, Any]: + _require(archive_path.stat().st_size <= MAX_ARCHIVE_BYTES, "request archive exceeded the safety limit") + try: + with tarfile.open(archive_path, mode="r:gz") as archive: + return _extract_open_archive(archive, destination) + except (OSError, tarfile.TarError) as exc: + raise RemoteError("request archive was invalid") from exc + + +def _extract_open_archive(archive: tarfile.TarFile, destination: Path) -> dict[str, Any]: + members = archive.getmembers() + names = [member.name for member in members] + _require(len(names) == len(set(names)) and set(names) == EXPECTED_MEMBERS, "archive member set was not exact") + for member in members: + _validate_member_name(member.name) + _require( + member.isfile() + and member.size <= MAX_MEMBER_BYTES + and member.uid == 0 + and member.gid == 0 + and member.uname == "root" + and member.gname == "root" + and member.mtime == 0 + and stat.S_IMODE(member.mode) in {0o600, 0o644, 0o755}, + "archive member metadata was unsafe", + ) + manifest_member = archive.getmember("request-manifest.json") + _require(manifest_member.size <= MAX_MANIFEST_BYTES, "request manifest exceeded the safety limit") + manifest_stream = archive.extractfile(manifest_member) + _require(manifest_stream is not None, "request manifest was absent") + manifest = _strict_json(manifest_stream.read(), "request manifest", MAX_MANIFEST_BYTES) + files = manifest.get("files") + _require(isinstance(files, list), "request manifest file list was invalid") + file_map: dict[str, dict[str, Any]] = {} + for item in files: + _require( + isinstance(item, dict) and set(item) == {"path", "sha256", "size", "mode"}, + "request manifest file record was invalid", + ) + relative = item.get("path") + _require(isinstance(relative, str) and relative not in file_map, "request manifest file path was invalid") + file_map[relative] = item + _require(set(file_map) == EXPECTED_MEMBERS - {"request-manifest.json"}, "request manifest files were not exact") + for relative in sorted(file_map): + member = archive.getmember(relative) + stream = archive.extractfile(member) + _require(stream is not None, "request member was absent") + content = stream.read(MAX_MEMBER_BYTES + 1) + record = file_map[relative] + _require( + len(content) == member.size == record.get("size") + and stat.S_IMODE(member.mode) == record.get("mode") + and _sha256_bytes(content) == record.get("sha256"), + "request member differed from its manifest", + ) + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + _require(target.resolve().is_relative_to(destination.resolve()), "request member escaped extraction root") + _copy_into_request(target, content, stat.S_IMODE(member.mode)) + return manifest + + +def validate_manifest( + manifest: Mapping[str, Any], + *, + operation: str, + request_id: str, + runner_revision: str, + release_revision: str, +) -> None: + _require( + set(manifest) + == { + "schema", + "operation", + "request_id", + "runner_revision", + "release_revision", + "target", + "release", + "installer_source", + "release_artifact", + "files", + "manifest_sha256", + }, + "request manifest fields were not exact", + ) + _require( + manifest.get("schema") == SCHEMA + and manifest.get("operation") == operation + and manifest.get("request_id") == request_id + and manifest.get("runner_revision") == runner_revision + and manifest.get("release_revision") == release_revision, + "request manifest context differed", + ) + _require( + manifest.get("target") + == { + "project": PROJECT, + "zone": ZONE, + "instance": INSTANCE, + "service": SERVICE, + "service_account": SERVICE_ACCOUNT, + }, + "request target differed", + ) + stable = {key: value for key, value in manifest.items() if key != "manifest_sha256"} + _require(manifest.get("manifest_sha256") == _canonical_sha256(stable), "request manifest hash differed") + release = manifest.get("release") + _require( + isinstance(release, dict) + and set(release) == {"release_sha256", "image_reference", "image_digest", "image_config_digest"} + and HEX_64.fullmatch(str(release.get("release_sha256"))) is not None + and IMAGE_REFERENCE.fullmatch(str(release.get("image_reference"))) is not None + and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_digest"))) is not None + and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_config_digest"))) is not None, + "request release binding was invalid", + ) + _require( + release["image_reference"] == f"{IMAGE_REPOSITORY}@{release['image_digest']}", + "request image digest binding differed", + ) + artifact = manifest.get("release_artifact") + _require( + isinstance(artifact, dict) + and set(artifact) == {"archive_sha256", "archive_size", "members"} + and HEX_64.fullmatch(str(artifact.get("archive_sha256"))) is not None + and isinstance(artifact.get("archive_size"), int) + and 0 < artifact["archive_size"] <= MAX_ARCHIVE_BYTES + and isinstance(artifact.get("members"), list), + "release artifact binding was invalid", + ) + + +def validate_original_release_artifact( + artifact_path: Path, + *, + manifest: Mapping[str, Any], + payload_root: Path, +) -> None: + raw = artifact_path.read_bytes() + binding = manifest["release_artifact"] + _require( + len(raw) == binding["archive_size"] and _sha256_bytes(raw) == binding["archive_sha256"], + "original release artifact archive differed", + ) + try: + archive = zipfile.ZipFile(io.BytesIO(raw), mode="r") + except (OSError, zipfile.BadZipFile) as exc: + raise RemoteError("original release artifact archive was invalid") from exc + with archive: + infos = archive.infolist() + names = [info.filename for info in infos] + _require( + len(names) == len(set(names)) and set(names) == RELEASE_ARTIFACT_MEMBERS, + "release artifact members differed", + ) + observed: list[dict[str, Any]] = [] + payloads: dict[str, bytes] = {} + for info in infos: + unix_mode = (info.external_attr >> 16) & 0o177777 + _require( + not info.is_dir() + and info.file_size <= MAX_MEMBER_BYTES + and (unix_mode == 0 or stat.S_ISREG(unix_mode)), + "release artifact member was unsafe", + ) + content = archive.read(info) + payloads[info.filename] = content + observed.append({"path": info.filename, "sha256": _sha256_bytes(content), "size": len(content)}) + _require( + sorted(observed, key=lambda item: str(item["path"])) == binding["members"], + "release artifact member receipt differed", + ) + _require( + payloads["final/release-v3/release.json"] == (payload_root / "release/release.json").read_bytes() + and payloads[f"final/release-v3/{SERVICE}"] == (payload_root / f"release/{SERVICE}").read_bytes(), + "release artifact differed from extracted release", + ) + + +def _metadata(path: str, *, runner: Runner) -> bytes: + return _run( + [ + "/usr/bin/curl", + "--fail", + "--silent", + "--show-error", + "--max-time", + "5", + "--header", + "Metadata-Flavor: Google", + f"{METADATA}/{path}", + ], + runner=runner, + label="metadata read", + timeout=10, + ).stdout + + +def verify_service_account(*, runner: Runner) -> None: + try: + email = _metadata("instance/service-accounts/default/email", runner=runner).decode("ascii", "strict").strip() + project = _metadata("project/project-id", runner=runner).decode("ascii", "strict").strip() + instance = _metadata("instance/name", runner=runner).decode("ascii", "strict").strip() + zone = _metadata("instance/zone", runner=runner).decode("ascii", "strict").strip() + except UnicodeError as exc: + raise RemoteError("VM target identity was invalid") from exc + _require( + email == SERVICE_ACCOUNT and project == PROJECT and instance == INSTANCE and zone.endswith(f"/zones/{ZONE}"), + "VM target or service account identity differed", + ) + + +def _docker_prefix(config: Path) -> list[str]: + return [DOCKER, f"--host={DOCKER_HOST}", f"--config={config}"] + + +def pull_immutable_image(reference: str, request_dir: Path, *, runner: Runner) -> None: + _require(IMAGE_REFERENCE.fullmatch(reference) is not None, "image reference was not immutable") + token_receipt = _strict_json( + _metadata("instance/service-accounts/default/token", runner=runner), + "metadata token", + MAX_TOKEN_BYTES, + ) + token = token_receipt.get("access_token") + _require( + isinstance(token, str) + and 16 <= len(token) <= 8192 + and "\n" not in token + and token_receipt.get("token_type") == "Bearer", + "metadata token response was invalid", + ) + config = Path(tempfile.mkdtemp(prefix="docker-auth-", dir=request_dir)) + config.chmod(0o700) + try: + _run( + [ + *_docker_prefix(config), + "login", + "--username=oauth2accesstoken", + "--password-stdin", + "https://europe-west6-docker.pkg.dev", + ], + runner=runner, + label="registry login", + input_bytes=(token + "\n").encode("utf-8"), + ) + _run( + [*_docker_prefix(config), "image", "pull", reference], + runner=runner, + label="immutable image pull", + timeout=300, + ) + finally: + try: + _run( + [*_docker_prefix(config), "logout", "https://europe-west6-docker.pkg.dev"], + runner=runner, + label="registry logout", + ) + except RemoteError: + pass + shutil.rmtree(config, ignore_errors=True) + _require(not config.exists(), "ephemeral Docker credentials were not removed") + + +def _load_contract_modules(payload_root: Path) -> tuple[Any, Any]: + sys.path.insert(0, str(payload_root)) + try: + from ops import gcp_leoclean_nosend_package as package + from ops import install_gcp_leoclean_nosend_release as installer + except Exception as exc: + raise RemoteError("reviewed installer sources could not be loaded") from exc + _require( + package.PROJECT == PROJECT + and package.ZONE == ZONE + and package.INSTANCE == INSTANCE + and package.SERVICE == SERVICE + and package.SERVICE_ACCOUNT == SERVICE_ACCOUNT, + "reviewed installer target differed", + ) + return package, installer + + +def _verify_database( + *, + container_id: str, + run_id: str, + receipt_name: str, + request_dir: Path, + package_module: Any, + runner: Runner, +) -> dict[str, Any]: + completed = _run( + [ + DOCKER, + f"--host={DOCKER_HOST}", + f"--config={SERVICE_DOCKER_CONFIG}", + "container", + "exec", + "--user=0", + container_id, + "/opt/livingip/leoclean-nosend/venv/bin/python", + "/opt/livingip/leoclean-nosend/entrypoint.py", + "verify-database", + run_id, + ], + runner=runner, + label="container database verification", + timeout=300, + ) + receipt = _strict_json(completed.stdout, "database verification receipt", MAX_COMMAND_OUTPUT_BYTES) + _require( + receipt.get("schema") == "livingip.leocleanNoSendContainerDatabaseVerification.v1" + and receipt.get("status") == "pass" + and receipt.get("run_id") == run_id, + "database verification receipt did not pass", + ) + stable = {key: value for key, value in receipt.items() if key != "receipt_sha256"} + _require(receipt.get("receipt_sha256") == package_module.canonical_sha256(stable), "database receipt hash differed") + _require(re.fullmatch(r"[a-z][a-z0-9-]{1,62}\.json", receipt_name) is not None, "receipt name was invalid") + receipt_path = request_dir / receipt_name + _copy_into_request( + receipt_path, + (json.dumps(receipt, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"), + 0o600, + ) + return { + "schema": receipt["schema"], + "status": "pass", + "receipt_sha256": receipt["receipt_sha256"], + "receipt_path": str(receipt_path), + "proof": { + "proposal_staging": "function_only", + "proposal_transaction": "rolled_back", + "durable_proposal_created": False, + "canonical_direct_writes": "denied", + "stage_direct_writes": "denied", + "role_escalation": "denied", + }, + } + + +def _execute_explicit_rollback( + *, + installer_module: Any, + release_sha256: str, + paths: Any, + source_binding: Mapping[str, Any], + runner: Runner, +) -> dict[str, Any]: + """Call only the receipt-bound explicit rollback API; never model rollback as another install.""" + + rollback = getattr(installer_module, "rollback_release", None) + _require(callable(rollback), "reviewed installer lacked the explicit rollback interface") + result = rollback( + release_sha256, + paths=paths, + source_binding=source_binding, + runner=lambda argv: _run( + argv, + runner=runner, + label="rollback command", + require_success=False, + ), + execute_rollback=True, + ) + _require(isinstance(result, dict) and result.get("status") == "pass", "explicit rollback did not pass") + return result + + +def _rollback_result( + *, + manifest: Mapping[str, Any], + release: Mapping[str, Any], + rollback: Mapping[str, Any], + archive_sha256: str, + helper_sha256: str, +) -> dict[str, Any]: + posture = rollback.get("restored_service_posture") + restored_release = rollback.get("restored_release_sha256") + verification = rollback.get("verification") + _require( + rollback.get("schema") == "livingip.leocleanNoSendRollbackResult.v1" + and rollback.get("status") == "pass" + and rollback.get("mode") == "executed" + and rollback.get("rolled_back_release_sha256") == release["release_sha256"] + and posture in {"active", "inactive", "absent"}, + "explicit rollback result was invalid", + ) + if posture == "active": + _require( + isinstance(restored_release, str) + and HEX_64.fullmatch(restored_release) is not None + and isinstance(verification, dict) + and verification.get("active_state") == "active" + and verification.get("sub_state") == "running" + and isinstance(verification.get("container_id"), str) + and HEX_64.fullmatch(verification["container_id"]) is not None, + "active rollback result lacked exact restored verification", + ) + elif posture == "inactive": + _require( + isinstance(restored_release, str) + and HEX_64.fullmatch(restored_release) is not None + and verification is None, + "inactive rollback result was invalid", + ) + else: + _require(restored_release is None and verification is None, "absent rollback result was invalid") + return { + "schema": RESULT_SCHEMA, + "status": "pass", + "operation": "rollback", + "request_id": manifest["request_id"], + "runner_revision": manifest["runner_revision"], + "release_revision": manifest["release_revision"], + "release_sha256": release["release_sha256"], + "image_digest": release["image"]["digest"], + "image_config_digest": release["image"]["config_digest"], + "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"], + "request_binding": { + "archive_sha256": archive_sha256, + "helper_sha256": helper_sha256, + "manifest_sha256": manifest["manifest_sha256"], + }, + "service": { + "name": SERVICE, + "restored_posture": posture, + "restored_release_sha256": restored_release, + "verification": verification, + }, + "database": {"status": "not_run_after_explicit_rollback"}, + "authority": { + "sending_enabled": False, + "production_traffic_changed": False, + "vps_touched": False, + "canonical_proposal_approved": False, + }, + } + + +def execute_request( + manifest: Mapping[str, Any], + payload_root: Path, + request_dir: Path, + archive_sha256: str, + helper_sha256: str, + *, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + operation = str(manifest["operation"]) + release_binding = manifest["release"] + validate_original_release_artifact( + payload_root / "release-artifact.zip", + manifest=manifest, + payload_root=payload_root, + ) + package_module, installer_module = _load_contract_modules(payload_root) + bundle = payload_root / "release" + release, unit = installer_module.validate_release_bundle(bundle) + _require( + release["release_sha256"] == release_binding["release_sha256"] + and release["image"]["reference"] == release_binding["image_reference"] + and release["image"]["digest"] == release_binding["image_digest"] + and release["image"]["config_digest"] == release_binding["image_config_digest"] + and unit == package_module.render_systemd_unit(release).encode("utf-8"), + "extracted release differed from the request", + ) + source_binding = manifest["installer_source"] + installer_module._validate_source_binding(source_binding) + _require( + source_binding.get("revision") == manifest["runner_revision"], + "installer source revision differed from runner revision", + ) + _require( + release["image_input"]["runtime"]["teleo_git_head"] == manifest["release_revision"], + "release source revision differed from release revision", + ) + paths = installer_module.InstallPaths.under(Path("/")) + if operation == "install": + pull_immutable_image(release["image"]["reference"], request_dir, runner=runner) + install_receipt = installer_module.install_release( + bundle, + paths=paths, + source_binding=source_binding, + runner=lambda argv: _run( + argv, + runner=runner, + label="installer command", + require_success=False, + ), + execute_restart=True, + ) + _require(install_receipt.get("status") == "pass", "installation did not pass") + elif operation == "rollback": + rollback = _execute_explicit_rollback( + installer_module=installer_module, + release_sha256=release["release_sha256"], + paths=paths, + source_binding=source_binding, + runner=runner, + ) + return _rollback_result( + manifest=manifest, + release=release, + rollback=rollback, + archive_sha256=archive_sha256, + helper_sha256=helper_sha256, + ) + + def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]: + return _run( + argv, + runner=runner, + label="release verification command", + require_success=False, + ) + + if operation == "install": + try: + before = installer_module.verify_running_release( + release, + runner=contract_runner, + paths=paths, + ) + before_database = _verify_database( + container_id=before["container_id"], + run_id=f"{manifest['request_id']}-pre", + receipt_name="database-before-restart.json", + request_dir=request_dir, + package_module=package_module, + runner=runner, + ) + _run( + [installer_module.SYSTEMCTL, "restart", SERVICE], + runner=runner, + label="fixed service restart", + ) + after = installer_module.verify_running_release( + release, + runner=contract_runner, + paths=paths, + ) + _require( + before["main_pid"] != after["main_pid"] + and before["invocation_id"] != after["invocation_id"] + and before["container_id"] != after["container_id"] + and after["n_restarts"] >= before["n_restarts"], + "post-install restart did not create a new exact service/container identity", + ) + after_database = _verify_database( + container_id=after["container_id"], + run_id=f"{manifest['request_id']}-post", + receipt_name="database-after-restart.json", + request_dir=request_dir, + package_module=package_module, + runner=runner, + ) + except Exception as exc: + try: + recovery = _execute_explicit_rollback( + installer_module=installer_module, + release_sha256=release["release_sha256"], + paths=paths, + source_binding=source_binding, + runner=runner, + ) + _rollback_result( + manifest=manifest, + release=release, + rollback=recovery, + archive_sha256=archive_sha256, + helper_sha256=helper_sha256, + ) + except Exception as rollback_exc: + raise RemoteError("install_failed_rollback_incomplete") from rollback_exc + raise RemoteError("install_failed_rolled_back") from exc + service = { + "name": SERVICE, + "before_restart": before, + "after_restart": after, + } + database = { + "status": "pass", + "before_restart": before_database, + "after_restart": after_database, + } + else: + verification = installer_module.verify_running_release( + release, + runner=contract_runner, + paths=paths, + ) + database = _verify_database( + container_id=verification["container_id"], + run_id=f"{manifest['request_id']}-verify", + receipt_name="database-verify.json", + request_dir=request_dir, + package_module=package_module, + runner=runner, + ) + service = { + "name": SERVICE, + "verification": verification, + } + return { + "schema": RESULT_SCHEMA, + "status": "pass", + "operation": operation, + "request_id": manifest["request_id"], + "runner_revision": manifest["runner_revision"], + "release_revision": manifest["release_revision"], + "release_sha256": release["release_sha256"], + "image_digest": release["image"]["digest"], + "image_config_digest": release["image"]["config_digest"], + "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"], + "request_binding": { + "archive_sha256": archive_sha256, + "helper_sha256": helper_sha256, + "manifest_sha256": manifest["manifest_sha256"], + }, + "service": service, + "database": database, + "authority": { + "sending_enabled": False, + "production_traffic_changed": False, + "vps_touched": False, + "canonical_proposal_approved": False, + }, + } + + +def receive_request( + operation: str, + request_id: str, + runner_revision: str, + release_revision: str, + archive_sha256: str, + helper_sha256: str, + *, + runner: Runner = subprocess.run, + run_root: Path = RUN_ROOT, + lock_path: Path = LOCK_PATH, +) -> dict[str, Any]: + _require(os.geteuid() == 0, "remote execution required root") + _require(operation in OPERATIONS, "operation was not allowlisted") + _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") + _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid") + _require(HEX_40.fullmatch(release_revision) is not None, "release revision was invalid") + _require(HEX_64.fullmatch(archive_sha256) is not None, "archive hash was invalid") + _require(HEX_64.fullmatch(helper_sha256) is not None, "helper hash was invalid") + caller = os.environ.get("SUDO_USER", "") + _require(caller and caller != "root", "remote execution lacked a distinct OS Login caller") + try: + pwd.getpwnam(caller) + except KeyError as exc: + raise RemoteError("OS Login caller did not resolve") from exc + stage_dir = STAGE_ROOT / request_id + archive_upload = stage_dir / "request.tar.gz" + helper_upload = stage_dir / "remote-helper.py" + archive_stat = archive_upload.stat(follow_symlinks=False) + helper_stat = helper_upload.stat(follow_symlinks=False) + archive_identity = (archive_stat.st_dev, archive_stat.st_ino) + helper_identity = (helper_stat.st_dev, helper_stat.st_ino) + try: + stage_state = stage_dir.stat(follow_symlinks=False) + _require( + stat.S_ISDIR(stage_state.st_mode) + and stage_state.st_uid == 0 + and stage_state.st_gid == 0 + and stat.S_IMODE(stage_state.st_mode) == 0o700, + "root-owned bootstrap stage was unsafe", + ) + helper_content = _read_stable_stage(helper_upload, helper_sha256, MAX_MEMBER_BYTES) + archive_content = _read_stable_stage(archive_upload, archive_sha256, MAX_ARCHIVE_BYTES) + _require(_sha256_bytes(Path(__file__).read_bytes()) == helper_sha256, "executing helper differed from request") + lock_descriptor = _acquire_lock(lock_path) + try: + request_dir = _claim_request_directory(run_root, request_id) + secured_archive = request_dir / "request.tar.gz" + secured_helper = request_dir / "remote-helper.py" + _copy_into_request(secured_archive, archive_content) + _copy_into_request(secured_helper, helper_content) + _unlink_exact(archive_upload, archive_identity) + _unlink_exact(helper_upload, helper_identity) + stage_dir.rmdir() + payload_root = request_dir / "payload" + payload_root.mkdir(mode=0o700) + manifest = extract_archive(secured_archive, payload_root) + validate_manifest( + manifest, + operation=operation, + request_id=request_id, + runner_revision=runner_revision, + release_revision=release_revision, + ) + verify_service_account(runner=runner) + result = execute_request( + manifest, + payload_root, + request_dir, + archive_sha256, + helper_sha256, + runner=runner, + ) + result_path = request_dir / "result.json" + _copy_into_request( + result_path, + (json.dumps(result, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"), + 0o600, + ) + return result + finally: + os.close(lock_descriptor) + finally: + _cleanup_upload_if_exact(archive_upload, archive_identity) + _cleanup_upload_if_exact(helper_upload, helper_identity) + + +def main(argv: list[str] | None = None) -> int: + arguments = argv if argv is not None else sys.argv[1:] + try: + os.umask(0o077) + _require(len(arguments) == 6, "remote arguments were not exact") + result = receive_request(*arguments) + sys.stdout.write(json.dumps(result, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n") + return 0 + except RemoteError as exc: + error = str(exc) + if error not in {"install_failed_rolled_back", "install_failed_rollback_incomplete"}: + error = "remote_request_failed" + sys.stderr.write(json.dumps({"error": error, "status": "fail"}, separators=(",", ":")) + "\n") + return 65 + except Exception: + sys.stderr.write('{"error":"remote_request_failed","status":"fail"}\n') + return 65 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/install_gcp_leoclean_nosend_release.py b/ops/install_gcp_leoclean_nosend_release.py index 0faab6e..01bed06 100644 --- a/ops/install_gcp_leoclean_nosend_release.py +++ b/ops/install_gcp_leoclean_nosend_release.py @@ -4,6 +4,8 @@ from __future__ import annotations import argparse +import base64 +import binascii import contextlib import fcntl import hashlib @@ -27,12 +29,15 @@ if str(CONTRACT_ROOT) not in sys.path: from ops import gcp_leoclean_nosend_package as package # noqa: E402 -INSTALL_RECEIPT_SCHEMA = "livingip.leocleanNoSendInstallReceipt.v1" +INSTALL_RECEIPT_SCHEMA = "livingip.leocleanNoSendInstallReceipt.v2" +ROLLBACK_STATE_SCHEMA = "livingip.leocleanNoSendRollbackState.v1" +ROLLBACK_RESULT_SCHEMA = "livingip.leocleanNoSendRollbackResult.v1" SERVICE = package.SERVICE SYSTEMCTL = "/usr/bin/systemctl" MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024 MAX_RELEASE_BYTES = 8 * 1024 * 1024 MAX_SOURCE_BYTES = 8 * 1024 * 1024 +MAX_ROLLBACK_STATE_BYTES = 32 * 1024 * 1024 HEX_64 = re.compile(r"^[0-9a-f]{64}$") HEX_40 = re.compile(r"^[0-9a-f]{40}$") HEX_32 = re.compile(r"^[0-9a-f]{32}$") @@ -41,6 +46,7 @@ SOURCE_PATHS = ( "ops/install_gcp_leoclean_nosend_release.py", "ops/leoclean_nosend_service_control.py", ) +MANAGED_FILE_ROLES = frozenset({"unit", "helper", "release", "receipt", "rollback_state"}) SERVICE_EXEC_PROPERTIES = ("ExecStartPre", "ExecStart", "ExecStop") SERVICE_SECURITY_PROPERTIES = ( "RuntimeDirectory", @@ -134,6 +140,7 @@ class InstallPaths: docker_config: Path release: Path receipt: Path + rollback_state: Path lock: Path proc_root: Path dropin_directories: tuple[Path, ...] @@ -154,6 +161,7 @@ class InstallPaths: docker_config=state / "docker-config", release=state / "release.json", receipt=state / "install-receipt.json", + rollback_state=state / "rollback-state.json", lock=rooted("/run/leoclean-gcp-nosend-installer.lock"), proc_root=proc_root or rooted("/proc"), dropin_directories=tuple( @@ -173,6 +181,7 @@ class FileSnapshot: existed: bool content: bytes = b"" mode: int = 0 + identity: tuple[int, ...] | None = None @dataclass(frozen=True) @@ -465,12 +474,74 @@ def _snapshot(path: Path) -> FileSnapshot: return FileSnapshot(False) except OSError as exc: raise InstallError("installed file could not be identified") from exc - _require(stat.S_ISREG(observed.st_mode), "installed path was not a regular file") + _require( + stat.S_ISREG(observed.st_mode) and observed.st_nlink == 1, + "installed path was not a regular file", + ) + content = _read_stable_file( + path, + label="installed file", + max_bytes=MAX_ROLLBACK_STATE_BYTES, + expected_mode=stat.S_IMODE(observed.st_mode), + ) + named = path.stat(follow_symlinks=False) + _require( + (named.st_dev, named.st_ino, named.st_mode, named.st_nlink) + == (observed.st_dev, observed.st_ino, observed.st_mode, observed.st_nlink), + "installed file changed while it was snapshotted", + ) + return FileSnapshot( + True, + content, + stat.S_IMODE(observed.st_mode), + ( + observed.st_dev, + observed.st_ino, + observed.st_mode, + observed.st_nlink, + observed.st_size, + observed.st_mtime_ns, + observed.st_ctime_ns, + ), + ) + + +def _snapshot_to_json(snapshot: FileSnapshot) -> dict[str, Any]: + if not snapshot.existed: + return {"existed": False} + _require(snapshot.identity is not None, "rollback snapshot identity was missing") + return { + "existed": True, + "mode": snapshot.mode, + "sha256": _sha256_bytes(snapshot.content), + "content_base64": base64.b64encode(snapshot.content).decode("ascii"), + } + + +def _snapshot_from_json(value: Any, *, label: str) -> FileSnapshot: + _require(isinstance(value, dict), f"{label} was invalid") + if value == {"existed": False}: + return FileSnapshot(False) + _require( + set(value) == {"existed", "mode", "sha256", "content_base64"} + and value.get("existed") is True + and isinstance(value.get("mode"), int) + and not isinstance(value.get("mode"), bool) + and 0 <= value["mode"] <= 0o777 + and isinstance(value.get("sha256"), str) + and HEX_64.fullmatch(value["sha256"]) is not None + and isinstance(value.get("content_base64"), str), + f"{label} was invalid", + ) try: - content = path.read_bytes() - except OSError as exc: - raise InstallError("installed file could not be read") from exc - return FileSnapshot(True, content, stat.S_IMODE(observed.st_mode)) + content = base64.b64decode(value["content_base64"], validate=True) + except (ValueError, binascii.Error) as exc: + raise InstallError(f"{label} was invalid") from exc + _require( + len(content) <= MAX_ROLLBACK_STATE_BYTES and _sha256_bytes(content) == value["sha256"], + f"{label} was invalid", + ) + return FileSnapshot(True, content, value["mode"]) def _ensure_directory(path: Path, mode: int, created: list[Path]) -> None: @@ -573,19 +644,34 @@ def _publish_managed_file( raise -def _restore_snapshot(path: Path, snapshot: FileSnapshot, installed_identity: tuple[int, int] | None) -> None: +def _restore_snapshot( + path: Path, + snapshot: FileSnapshot, + installed_identity: tuple[int, ...] | None, +) -> tuple[int, int] | None: if installed_identity is not None: observed = path.stat(follow_symlinks=False) + observed_identity = ( + observed.st_dev, + observed.st_ino, + observed.st_mode, + observed.st_nlink, + observed.st_size, + observed.st_mtime_ns, + observed.st_ctime_ns, + ) _require( - stat.S_ISREG(observed.st_mode) and (observed.st_dev, observed.st_ino) == installed_identity, + stat.S_ISREG(observed.st_mode) + and (observed.st_dev, observed.st_ino) == installed_identity[:2] + and (len(installed_identity) == 2 or observed_identity == installed_identity), "rollback refused a replaced installed file", ) if snapshot.existed: - _atomic_write(path, snapshot.content, snapshot.mode) - return + return _atomic_write(path, snapshot.content, snapshot.mode) if path.exists() or path.is_symlink(): path.unlink() _fsync_directory(path.parent) + return None def _dropins_absent(paths: InstallPaths) -> None: @@ -979,27 +1065,174 @@ def _read_managed_release(paths: InstallPaths) -> dict[str, Any] | None: if not paths.release.exists() and not paths.release.is_symlink(): return None try: - value = package.load_json(paths.release, "installed release descriptor") + raw = _read_stable_file( + paths.release, + label="installed release descriptor", + max_bytes=MAX_RELEASE_BYTES, + expected_mode=0o600, + ) + value = _strict_json_bytes(raw, "installed release descriptor") + _require(isinstance(value, dict), "installed release descriptor was invalid") package.validate_release_descriptor(value) - except (OSError, package.PackageError) as exc: + except (OSError, InstallError, package.PackageError) as exc: raise InstallError("installed release descriptor was invalid") from exc return value +def _managed_files(paths: InstallPaths) -> dict[str, tuple[Path, int]]: + return { + "unit": (paths.unit, 0o644), + "helper": (paths.helper, 0o755), + "release": (paths.release, 0o600), + "receipt": (paths.receipt, 0o600), + "rollback_state": (paths.rollback_state, 0o600), + } + + +def _read_strict_json_file(path: Path, *, label: str, max_bytes: int, mode: int) -> tuple[dict[str, Any], bytes]: + raw = _read_stable_file(path, label=label, max_bytes=max_bytes, expected_mode=mode) + value = _strict_json_bytes(raw, label) + _require(isinstance(value, dict), f"{label} was invalid") + return value, raw + + +def _validate_rollback_state(value: Mapping[str, Any], *, current_release_sha256: str) -> None: + _require( + set(value) + == { + "schema", + "current_release_sha256", + "previous_files", + "previous_directories", + "previous_service", + "sha256", + } + and value.get("schema") == ROLLBACK_STATE_SCHEMA + and value.get("current_release_sha256") == current_release_sha256, + "rollback state was invalid", + ) + previous_files = value.get("previous_files") + _require( + isinstance(previous_files, dict) and set(previous_files) == MANAGED_FILE_ROLES, + "rollback state was invalid", + ) + decoded = { + role: _snapshot_from_json(previous_files[role], label=f"rollback {role} snapshot") + for role in sorted(previous_files) + } + prior_presence = {role: snapshot.existed for role, snapshot in decoded.items()} + _require( + not any(prior_presence.values()) or all(prior_presence.values()), + "rollback prior managed file set was partial", + ) + previous_directories = value.get("previous_directories") + _require( + isinstance(previous_directories, dict) + and set(previous_directories) == {"state_dir_existed", "docker_config_existed"} + and all(isinstance(item, bool) for item in previous_directories.values()), + "rollback state was invalid", + ) + previous_service = value.get("previous_service") + _require( + isinstance(previous_service, dict) + and set(previous_service) == {"posture", "active_state", "sub_state", "n_restarts"} + and previous_service.get("posture") in {"absent", "inactive", "active"} + and previous_service.get("active_state") in {"inactive", "active"} + and previous_service.get("sub_state") in {"dead", "running"} + and isinstance(previous_service.get("n_restarts"), int) + and not isinstance(previous_service.get("n_restarts"), bool) + and previous_service["n_restarts"] >= 0, + "rollback state was invalid", + ) + expected_state = { + "absent": ("inactive", "dead"), + "inactive": ("inactive", "dead"), + "active": ("active", "running"), + }[previous_service["posture"]] + _require( + (previous_service["active_state"], previous_service["sub_state"]) == expected_state + and ((previous_service["posture"] == "absent") == (not any(prior_presence.values()))), + "rollback state was invalid", + ) + stable = {key: item for key, item in value.items() if key != "sha256"} + _require(value.get("sha256") == package.canonical_sha256(stable), "rollback state self-hash drifted") + + +def _read_rollback_state(paths: InstallPaths, release_sha256: str) -> tuple[dict[str, Any], bytes]: + value, raw = _read_strict_json_file( + paths.rollback_state, + label="rollback state", + max_bytes=MAX_ROLLBACK_STATE_BYTES, + mode=0o600, + ) + _validate_rollback_state(value, current_release_sha256=release_sha256) + return value, raw + + +def _validate_install_receipt( + receipt: Mapping[str, Any], + *, + release: Mapping[str, Any], + paths: InstallPaths, + rollback_state_raw: bytes, +) -> None: + installed_files = receipt.get("installed_files") + rollback = receipt.get("rollback") + _require( + receipt.get("schema") == INSTALL_RECEIPT_SCHEMA + and receipt.get("status") == "pass" + and receipt.get("release_sha256") == release["release_sha256"] + and isinstance(installed_files, dict) + and set(installed_files) + == {"unit_sha256", "helper_sha256", "release_descriptor_sha256", "rollback_state_sha256"} + and installed_files.get("unit_sha256") + == _sha256_bytes( + _read_stable_file( + paths.unit, label="installed service unit", max_bytes=MAX_RELEASE_BYTES, expected_mode=0o644 + ) + ) + and installed_files.get("helper_sha256") + == _sha256_bytes( + _read_stable_file( + paths.helper, label="installed service helper", max_bytes=MAX_SOURCE_BYTES, expected_mode=0o755 + ) + ) + and installed_files.get("release_descriptor_sha256") + == _sha256_bytes( + _read_stable_file( + paths.release, + label="installed release descriptor", + max_bytes=MAX_RELEASE_BYTES, + expected_mode=0o600, + ) + ) + and installed_files.get("rollback_state_sha256") == _sha256_bytes(rollback_state_raw) + and isinstance(rollback, dict) + and rollback.get("state_sha256") == _sha256_bytes(rollback_state_raw) + and rollback.get("state_path") == str(paths.rollback_state), + "installed receipt was invalid", + ) + _require(isinstance(receipt.get("installer_source"), dict), "installed receipt source binding was invalid") + _validate_source_binding(receipt["installer_source"]) + _require( + receipt.get("image") + == { + "reference": release["image"]["reference"], + "digest": release["image"]["digest"], + "config_digest": release["image"]["config_digest"], + "input_sha256": release["image"]["input_sha256"], + }, + "installed receipt image binding was invalid", + ) + + def _validate_managed_prior_state(paths: InstallPaths, helper_source: bytes) -> dict[str, Any] | None: - unit_exists = paths.unit.exists() or paths.unit.is_symlink() - helper_exists = paths.helper.exists() or paths.helper.is_symlink() - release_exists = paths.release.exists() or paths.release.is_symlink() - receipt_exists = paths.receipt.exists() or paths.receipt.is_symlink() - if not any((unit_exists, helper_exists, release_exists, receipt_exists)): + managed = _managed_files(paths) + presence = {role: path.exists() or path.is_symlink() for role, (path, _mode) in managed.items()} + if not any(presence.values()): return None - _require(all((unit_exists, helper_exists, release_exists, receipt_exists)), "managed install state was partial") - for path, mode in ( - (paths.unit, 0o644), - (paths.helper, 0o755), - (paths.release, 0o600), - (paths.receipt, 0o600), - ): + _require(all(presence.values()), "managed install state was partial") + for path, mode in managed.values(): observed = path.stat(follow_symlinks=False) _require( stat.S_ISREG(observed.st_mode) @@ -1021,21 +1254,19 @@ def _validate_managed_prior_state(paths: InstallPaths, helper_source: bytes) -> _require(release is not None, "managed release state was absent") _require(paths.unit.read_bytes() == package.render_systemd_unit(release).encode(), "installed unit drifted") _require(paths.helper.is_file() and not paths.helper.is_symlink(), "installed helper was unsafe") - try: - receipt = package.load_json(paths.receipt, "install receipt") - except (OSError, package.PackageError) as exc: - raise InstallError("installed receipt was invalid") from exc - _require( - receipt.get("schema") == INSTALL_RECEIPT_SCHEMA - and receipt.get("status") == "pass" - and receipt.get("release_sha256") == release["release_sha256"], - "installed receipt was invalid", + rollback_state, rollback_state_raw = _read_rollback_state(paths, release["release_sha256"]) + del rollback_state + receipt, _receipt_raw = _read_strict_json_file( + paths.receipt, + label="install receipt", + max_bytes=MAX_RELEASE_BYTES, + mode=0o600, ) - _require( - isinstance(receipt.get("installed_files"), dict) - and receipt["installed_files"].get("unit_sha256") == _sha256_bytes(paths.unit.read_bytes()) - and receipt["installed_files"].get("helper_sha256") == _sha256_bytes(paths.helper.read_bytes()), - "installed file receipt drifted", + _validate_install_receipt( + receipt, + release=release, + paths=paths, + rollback_state_raw=rollback_state_raw, ) _require(bool(helper_source), "service-control helper source was empty") return release @@ -1062,6 +1293,65 @@ def _cleanup_created_directories(created: list[Path]) -> bool: return complete +def _directory_existed_safely(path: Path, mode: int) -> bool: + if not path.exists() and not path.is_symlink(): + return False + observed = path.stat(follow_symlinks=False) + _require( + stat.S_ISDIR(observed.st_mode) and observed.st_uid == os.geteuid() and stat.S_IMODE(observed.st_mode) == mode, + "prior install directory posture was unsafe", + ) + return True + + +def _prior_directory_state(paths: InstallPaths) -> dict[str, bool]: + state_existed = _directory_existed_safely(paths.state_dir, 0o700) + docker_existed = _directory_existed_safely(paths.docker_config, 0o700) + if docker_existed: + _require(state_existed and not any(paths.docker_config.iterdir()), "prior Docker config posture was unsafe") + if state_existed: + allowed = {paths.docker_config.name} if docker_existed else set() + observed = {path.name for path in paths.state_dir.iterdir()} + managed_names = {paths.release.name, paths.receipt.name, paths.rollback_state.name} + _require( + observed <= allowed | managed_names, + "prior install state directory contained an unmanaged entry", + ) + return { + "state_dir_existed": state_existed, + "docker_config_existed": docker_existed, + } + + +def _build_rollback_state( + release_sha256: str, + snapshots: Mapping[Path, FileSnapshot], + paths: InstallPaths, + previous_directories: Mapping[str, bool], + prior_posture: ServicePosture, +) -> dict[str, Any]: + path_to_role = {path: role for role, (path, _mode) in _managed_files(paths).items()} + _require(set(snapshots) == set(path_to_role), "rollback snapshot set was not exact") + stable = { + "schema": ROLLBACK_STATE_SCHEMA, + "current_release_sha256": release_sha256, + "previous_files": { + path_to_role[path]: _snapshot_to_json(snapshots[path]) + for path in sorted(snapshots, key=lambda item: path_to_role[item]) + }, + "previous_directories": dict(previous_directories), + "previous_service": { + "posture": prior_posture.kind, + "active_state": prior_posture.state["ActiveState"], + "sub_state": prior_posture.state["SubState"], + "n_restarts": prior_posture.state["NRestarts"], + }, + } + value = {**stable, "sha256": package.canonical_sha256(stable)} + _validate_rollback_state(value, current_release_sha256=release_sha256) + return value + + @contextlib.contextmanager def _install_lock(paths: InstallPaths): if paths.root == Path("/"): @@ -1129,6 +1419,7 @@ def _install_release_transaction( ) before = prior_posture.state prior_active = prior_posture.kind == "active" + previous_directories = _prior_directory_state(paths) created: list[Path] = [] try: @@ -1149,6 +1440,7 @@ def _install_release_transaction( paths.helper: _snapshot(paths.helper), paths.release: _snapshot(paths.release), paths.receipt: _snapshot(paths.receipt), + paths.rollback_state: _snapshot(paths.rollback_state), } installed_identities: dict[Path, tuple[int, int]] = {} attempted_paths: set[Path] = set() @@ -1198,6 +1490,20 @@ def _install_release_transaction( release_bytes = _canonical_json(release) attempted_paths.add(paths.release) _publish_managed_file(paths.release, release_bytes, 0o600, installed_identities) + rollback_state = _build_rollback_state( + release["release_sha256"], + snapshots, + paths, + previous_directories, + prior_posture, + ) + rollback_state_bytes = _canonical_json(rollback_state) + _require( + len(rollback_state_bytes) <= MAX_ROLLBACK_STATE_BYTES, + "rollback state exceeded the safety limit", + ) + attempted_paths.add(paths.rollback_state) + _publish_managed_file(paths.rollback_state, rollback_state_bytes, 0o600, installed_identities) receipt = { "schema": INSTALL_RECEIPT_SCHEMA, "status": "pass", @@ -1214,6 +1520,7 @@ def _install_release_transaction( "unit_sha256": _sha256_bytes(unit_content), "helper_sha256": _sha256_bytes(helper_content), "release_descriptor_sha256": _sha256_bytes(release_bytes), + "rollback_state_sha256": _sha256_bytes(rollback_state_bytes), }, "service_before": { "load_state": before["LoadState"], @@ -1228,6 +1535,8 @@ def _install_release_transaction( "previous_service_active": prior_active, "previous_service_posture": prior_posture.kind, "automatic_on_failure": True, + "state_path": str(paths.rollback_state), + "state_sha256": _sha256_bytes(rollback_state_bytes), }, "authority": { "image_pulled": False, @@ -1246,7 +1555,7 @@ def _install_release_transaction( _systemctl("stop", runner) except InstallError: rollback_errors.append("candidate_stop") - for path in (paths.receipt, paths.release, paths.unit, paths.helper): + for path in (paths.receipt, paths.rollback_state, paths.release, paths.unit, paths.helper): if path not in attempted_paths: continue try: @@ -1294,6 +1603,415 @@ def _install_release_transaction( raise InstallError("installation failed and was rolled back") from exc +def _json_from_snapshot(snapshot: FileSnapshot, *, label: str) -> dict[str, Any]: + _require(snapshot.existed, f"{label} was absent") + value = _strict_json_bytes(snapshot.content, label) + _require(isinstance(value, dict), f"{label} was invalid") + return value + + +def _validate_managed_snapshots( + snapshots: Mapping[str, FileSnapshot], + *, + paths: InstallPaths, +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + expected_modes = {role: mode for role, (_path, mode) in _managed_files(paths).items()} + _require(set(snapshots) == set(expected_modes), "managed snapshot set was not exact") + _require(all(snapshot.existed for snapshot in snapshots.values()), "managed snapshot set was partial") + _require( + all(snapshots[role].mode == expected_modes[role] for role in expected_modes), + "managed snapshot mode drifted", + ) + release = _json_from_snapshot(snapshots["release"], label="snapshot release descriptor") + try: + package.validate_release_descriptor(release) + except package.PackageError as exc: + raise InstallError("snapshot release descriptor was invalid") from exc + release_sha256 = release["release_sha256"] + _require( + snapshots["unit"].content == package.render_systemd_unit(release).encode("utf-8") + and snapshots["helper"].content.startswith(b"#!/usr/bin/python3 -I\n"), + "snapshot installed files differed from the release", + ) + rollback_state = _json_from_snapshot(snapshots["rollback_state"], label="snapshot rollback state") + _validate_rollback_state(rollback_state, current_release_sha256=release_sha256) + receipt = _json_from_snapshot(snapshots["receipt"], label="snapshot install receipt") + installed_files = receipt.get("installed_files") + rollback = receipt.get("rollback") + _require( + receipt.get("schema") == INSTALL_RECEIPT_SCHEMA + and receipt.get("status") == "pass" + and receipt.get("release_sha256") == release_sha256 + and isinstance(receipt.get("installer_source"), dict) + and isinstance(installed_files, dict) + and installed_files.get("unit_sha256") == _sha256_bytes(snapshots["unit"].content) + and installed_files.get("helper_sha256") == _sha256_bytes(snapshots["helper"].content) + and installed_files.get("release_descriptor_sha256") == _sha256_bytes(snapshots["release"].content) + and installed_files.get("rollback_state_sha256") == _sha256_bytes(snapshots["rollback_state"].content) + and isinstance(rollback, dict) + and rollback.get("state_path") == str(paths.rollback_state) + and rollback.get("state_sha256") == _sha256_bytes(snapshots["rollback_state"].content), + "snapshot install receipt was invalid", + ) + _validate_source_binding(receipt["installer_source"]) + return release, receipt, rollback_state + + +def _current_managed_snapshots(paths: InstallPaths) -> dict[str, FileSnapshot]: + return {role: _snapshot(path) for role, (path, _mode) in _managed_files(paths).items()} + + +def _previous_snapshots(rollback_state: Mapping[str, Any]) -> dict[str, FileSnapshot]: + previous_files = rollback_state["previous_files"] + _require(isinstance(previous_files, dict), "rollback prior file set was invalid") + return { + role: _snapshot_from_json(previous_files[role], label=f"rollback {role} snapshot") + for role in sorted(previous_files) + } + + +def _prepare_explicit_rollback( + expected_release_sha256: str, + *, + paths: InstallPaths, + source_binding: Mapping[str, Any], + runner: Runner, + sleeper: Sleeper, +) -> dict[str, Any]: + _require(HEX_64.fullmatch(expected_release_sha256) is not None, "expected release hash was invalid") + _bound_source_files(source_binding) + managed_release = _validate_managed_prior_state(paths, b"explicit rollback") + _require(managed_release is not None, "explicit rollback found no managed release") + current_snapshots = _current_managed_snapshots(paths) + current_release, current_receipt, rollback_state = _validate_managed_snapshots( + current_snapshots, + paths=paths, + ) + _require( + current_release == managed_release and current_release["release_sha256"] == expected_release_sha256, + "installed release did not match the expected release hash", + ) + previous_snapshots = _previous_snapshots(rollback_state) + previous_presence = [snapshot.existed for snapshot in previous_snapshots.values()] + previous_release: dict[str, Any] | None + if any(previous_presence): + _require(all(previous_presence), "rollback prior managed file set was partial") + previous_release, _previous_receipt, _previous_rollback = _validate_managed_snapshots( + previous_snapshots, + paths=paths, + ) + else: + previous_release = None + current_posture = _stable_service_posture( + current_release, + runner=runner, + paths=paths, + sleeper=sleeper, + ) + current_verification: dict[str, Any] | None = None + if current_posture.kind == "active": + current_verification = verify_running_release( + current_release, + runner=runner, + paths=paths, + sleeper=sleeper, + ) + _require( + current_verification["main_pid"] == current_posture.state["MainPID"] + and current_verification["n_restarts"] == current_posture.state["NRestarts"] + and current_verification["invocation_id"] == current_posture.state["InvocationID"], + "installed service changed during rollback preparation", + ) + previous_service = rollback_state["previous_service"] + _require( + (previous_release is None) == (previous_service["posture"] == "absent"), + "rollback prior release differed from its recorded posture", + ) + return { + "current_release": current_release, + "current_receipt": current_receipt, + "current_snapshots": current_snapshots, + "current_posture": current_posture, + "current_verification": current_verification, + "previous_snapshots": previous_snapshots, + "previous_release": previous_release, + "previous_directories": rollback_state["previous_directories"], + "previous_service": previous_service, + "rollback_state_sha256": _sha256_bytes(current_snapshots["rollback_state"].content), + } + + +def _build_rollback_plan( + expected_release_sha256: str, + prepared: Mapping[str, Any], + *, + paths: InstallPaths, + source_binding: Mapping[str, Any], +) -> dict[str, Any]: + previous_release = prepared["previous_release"] + return { + "schema": ROLLBACK_RESULT_SCHEMA, + "status": "pass", + "mode": "dry_run", + "installer_source": dict(source_binding), + "expected_release_sha256": expected_release_sha256, + "installed_release_sha256": prepared["current_release"]["release_sha256"], + "installed_receipt_release_sha256": prepared["current_receipt"]["release_sha256"], + "rollback_state_sha256": prepared["rollback_state_sha256"], + "current_service_posture": prepared["current_posture"].kind, + "target_service_posture": prepared["previous_service"]["posture"], + "target_release_sha256": previous_release["release_sha256"] if previous_release is not None else None, + "target": { + "service": SERVICE, + "unit": str(paths.unit), + "helper": str(paths.helper), + "state_directory": str(paths.state_dir), + }, + "authority": { + "execute_requires_root_and_execute_rollback": True, + "image_pulled": False, + "cloud_or_database_authority_changed": False, + "transport_authority_changed": False, + }, + } + + +def _remove_rollback_owned_directories(paths: InstallPaths, previous_directories: Mapping[str, bool]) -> None: + _require(not any(paths.docker_config.iterdir()), "rollback Docker config directory was not empty") + if not previous_directories["docker_config_existed"]: + paths.docker_config.rmdir() + _fsync_directory(paths.state_dir) + if not previous_directories["state_dir_existed"]: + _require(not any(paths.state_dir.iterdir()), "rollback state directory was not empty") + paths.state_dir.rmdir() + _fsync_directory(paths.state_dir.parent) + + +def _assert_snapshot_identities( + snapshots: Mapping[str, FileSnapshot], + *, + paths: InstallPaths, +) -> None: + for role, (path, expected_mode) in _managed_files(paths).items(): + snapshot = snapshots[role] + _require(snapshot.existed and snapshot.identity is not None, "installed rollback file identity was missing") + try: + observed = path.stat(follow_symlinks=False) + except OSError as exc: + raise InstallError("installed rollback file identity drifted") from exc + observed_identity = ( + observed.st_dev, + observed.st_ino, + observed.st_mode, + observed.st_nlink, + observed.st_size, + observed.st_mtime_ns, + observed.st_ctime_ns, + ) + _require( + stat.S_ISREG(observed.st_mode) + and observed.st_nlink == 1 + and stat.S_IMODE(observed.st_mode) == expected_mode + and observed_identity == snapshot.identity, + "installed rollback file identity drifted", + ) + + +def _execute_explicit_rollback( + expected_release_sha256: str, + prepared: Mapping[str, Any], + *, + paths: InstallPaths, + runner: Runner, + sleeper: Sleeper, +) -> dict[str, Any]: + current_release = prepared["current_release"] + current_snapshots = prepared["current_snapshots"] + current_posture: ServicePosture = prepared["current_posture"] + previous_snapshots = prepared["previous_snapshots"] + previous_release = prepared["previous_release"] + previous_service = prepared["previous_service"] + role_paths = {role: path for role, (path, _mode) in _managed_files(paths).items()} + restored_identities: dict[Path, tuple[int, int] | None] = {} + mutated_paths: set[Path] = set() + files_mutated = False + service_stopped = False + target_may_be_active = False + try: + _assert_snapshot_identities(current_snapshots, paths=paths) + if current_posture.kind == "active": + fresh_verification = verify_running_release( + current_release, + runner=runner, + paths=paths, + sleeper=sleeper, + ) + _require( + fresh_verification == prepared["current_verification"], + "service or container drifted before rollback stop", + ) + before_stop = _service_state(runner) + _require( + before_stop["MainPID"] == current_posture.state["MainPID"] + and before_stop["NRestarts"] == current_posture.state["NRestarts"] + and before_stop["InvocationID"] == current_posture.state["InvocationID"] + and before_stop["ActiveState"] == "active" + and before_stop["SubState"] == "running", + "service PID or invocation drifted before rollback stop", + ) + _systemctl("stop", runner) + service_stopped = True + stopped = _stable_service_posture( + current_release, + runner=runner, + paths=paths, + sleeper=sleeper, + ) + _require(stopped.kind == "inactive", "installed service did not stop into a trusted posture") + for role in ("receipt", "rollback_state", "release", "unit", "helper"): + path = role_paths[role] + files_mutated = True + mutated_paths.add(path) + restored_identities[path] = _restore_snapshot( + path, + previous_snapshots[role], + current_snapshots[role].identity, + ) + _systemctl("daemon-reload", runner) + if previous_service["posture"] == "active": + _require(previous_release is not None, "rollback active target release was missing") + target_may_be_active = True + _systemctl("restart", runner) + final_verification = verify_running_release( + previous_release, + runner=runner, + paths=paths, + sleeper=sleeper, + ) + _require( + final_verification["n_restarts"] == previous_service["n_restarts"], + "rollback did not restore the recorded active posture", + ) + final_posture = "active" + elif previous_service["posture"] == "inactive": + _require(previous_release is not None, "rollback inactive target release was missing") + final = _stable_service_posture( + previous_release, + runner=runner, + paths=paths, + sleeper=sleeper, + ) + _require( + final.kind == "inactive" and final.state["NRestarts"] == previous_service["n_restarts"], + "rollback did not restore the recorded inactive posture", + ) + final_verification = None + final_posture = "inactive" + else: + _require(previous_release is None, "rollback absent target unexpectedly had a release") + _systemctl("reset-failed", runner) + final = _stable_service_posture( + None, + runner=runner, + paths=paths, + sleeper=sleeper, + ) + _require(final.kind == "absent", "rollback did not restore the absent posture") + _remove_rollback_owned_directories(paths, prepared["previous_directories"]) + final_verification = None + final_posture = "absent" + return { + "schema": ROLLBACK_RESULT_SCHEMA, + "status": "pass", + "mode": "executed", + "rolled_back_release_sha256": expected_release_sha256, + "restored_release_sha256": previous_release["release_sha256"] if previous_release is not None else None, + "restored_service_posture": final_posture, + "verification": final_verification, + "authority": { + "image_pulled": False, + "cloud_or_database_authority_changed": False, + "transport_authority_changed": False, + }, + } + except Exception as exc: + recovery_errors: list[str] = [] + if target_may_be_active: + try: + _systemctl("stop", runner) + except InstallError: + recovery_errors.append("target_stop") + if files_mutated: + for role in ("receipt", "rollback_state", "release", "unit", "helper"): + path = role_paths[role] + if path not in mutated_paths: + continue + try: + if restored_identities.get(path) is None: + _require( + not path.exists() and not path.is_symlink(), + "rollback recovery refused a replaced pathname", + ) + _restore_snapshot(path, current_snapshots[role], restored_identities.get(path)) + except (OSError, InstallError): + recovery_errors.append("file_restore") + try: + _systemctl("daemon-reload", runner) + except InstallError: + recovery_errors.append("daemon_reload") + if service_stopped and current_posture.kind == "active": + try: + _systemctl("restart", runner) + verify_running_release(current_release, runner=runner, paths=paths, sleeper=sleeper) + except InstallError: + recovery_errors.append("current_restart") + if recovery_errors: + raise InstallError("explicit rollback failed and recovery was incomplete") from exc + if isinstance(exc, InstallError): + raise + raise InstallError("explicit rollback failed without changing the installed release") from exc + + +def rollback_release( + expected_release_sha256: str, + *, + paths: InstallPaths, + source_binding: Mapping[str, Any], + runner: Runner = _default_runner, + sleeper: Sleeper = time.sleep, + execute_rollback: bool = False, +) -> dict[str, Any]: + if not execute_rollback: + prepared = _prepare_explicit_rollback( + expected_release_sha256, + paths=paths, + source_binding=source_binding, + runner=runner, + sleeper=sleeper, + ) + return _build_rollback_plan( + expected_release_sha256, + prepared, + paths=paths, + source_binding=source_binding, + ) + with _install_lock(paths): + prepared = _prepare_explicit_rollback( + expected_release_sha256, + paths=paths, + source_binding=source_binding, + runner=runner, + sleeper=sleeper, + ) + return _execute_explicit_rollback( + expected_release_sha256, + prepared, + paths=paths, + runner=runner, + sleeper=sleeper, + ) + + def install_release( bundle: Path, *, @@ -1330,23 +2048,45 @@ class _Parser(argparse.ArgumentParser): def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = _Parser(description=__doc__, add_help=False) - parser.add_argument("--bundle", required=True, type=Path) + parser.add_argument("--bundle", type=Path) parser.add_argument("--execute-restart", action="store_true") - return parser.parse_args(argv) + parser.add_argument("--rollback-release") + parser.add_argument("--execute-rollback", action="store_true") + args = parser.parse_args(argv) + if args.rollback_release is None: + _require(args.bundle is not None and not args.execute_rollback, "invalid arguments") + else: + _require( + args.bundle is None and not args.execute_restart and HEX_64.fullmatch(args.rollback_release) is not None, + "invalid arguments", + ) + _require(not (args.execute_restart and args.execute_rollback), "invalid arguments") + return args def main(argv: list[str] | None = None) -> int: try: args = parse_args(argv) root = Path("/") - if args.execute_restart: - _require(os.geteuid() == 0, "live installation requires root") - result = install_release( - args.bundle.absolute(), - paths=InstallPaths.under(root), - source_binding=reviewed_source_binding(), - execute_restart=args.execute_restart, - ) + if args.execute_restart or args.execute_rollback: + _require(os.geteuid() == 0, "live installation or rollback requires root") + paths = InstallPaths.under(root) + binding = reviewed_source_binding() + if args.rollback_release is not None: + result = rollback_release( + args.rollback_release, + paths=paths, + source_binding=binding, + execute_rollback=args.execute_rollback, + ) + else: + _require(args.bundle is not None, "invalid arguments") + result = install_release( + args.bundle.absolute(), + paths=paths, + source_binding=binding, + execute_restart=args.execute_restart, + ) sys.stdout.write(json.dumps(result, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n") return 0 except (InstallError, package.PackageError) as exc: diff --git a/ops/run_gcp_leoclean_nosend_iap.py b/ops/run_gcp_leoclean_nosend_iap.py new file mode 100644 index 0000000..617acbc --- /dev/null +++ b/ops/run_gcp_leoclean_nosend_iap.py @@ -0,0 +1,1279 @@ +#!/usr/bin/env python3 +"""Build and optionally dispatch one fixed-target leoclean no-send IAP request.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +import re +import shlex +import stat +import subprocess +import sys +import tarfile +import tempfile +import textwrap +import zipfile +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any + +CONTRACT_ROOT = Path(__file__).resolve().parents[1] +if str(CONTRACT_ROOT) not in sys.path: + sys.path.insert(0, str(CONTRACT_ROOT)) + +from ops import gcp_leoclean_nosend_package as package # noqa: E402 +from ops import install_gcp_leoclean_nosend_release as installer # noqa: E402 + +SCHEMA = "livingip.leocleanNoSendIapRequest.v1" +PLAN_SCHEMA = "livingip.leocleanNoSendIapPlan.v1" +REMOTE_RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v1" +RESULT_RECEIPT_SCHEMA = "livingip.leocleanNoSendIapResultReceipt.v1" +EXECUTION_SCHEMA = "livingip.leocleanNoSendIapExecution.v1" +PROJECT = package.PROJECT +ZONE = package.ZONE +INSTANCE = package.INSTANCE +SERVICE = package.SERVICE +SERVICE_ACCOUNT = package.SERVICE_ACCOUNT +GCLOUD = "/opt/homebrew/bin/gcloud" +ORIGIN_URL = "git@github.com:living-ip/teleo-infrastructure.git" +REMOTE_UPLOAD_ROOT = "/tmp" +REMOTE_HELPER_SOURCE = "ops/gcp_leoclean_nosend_iap_remote.py" +KEY_LIFETIME = "5m" +MAX_ARCHIVE_BYTES = 8 * 1024 * 1024 +MAX_MEMBER_BYTES = 2 * 1024 * 1024 +MAX_COMMAND_OUTPUT_BYTES = 256 * 1024 +REQUEST_ID = re.compile(r"^iap-[a-z0-9]{12,32}$") +HEX_40 = re.compile(r"^[0-9a-f]{40}$") +HEX_64 = re.compile(r"^[0-9a-f]{64}$") +OPERATIONS = ("plan", "install", "verify", "rollback") +SOURCE_MEMBERS = ( + "ops/gcp_leoclean_nosend_package.py", + "ops/install_gcp_leoclean_nosend_release.py", + "ops/leoclean_nosend_service_control.py", + REMOTE_HELPER_SOURCE, + "ops/run_gcp_leoclean_nosend_iap.py", + "scripts/leo_behavior_manifest.py", + "scripts/leo_identity_manifest.py", +) +RELEASE_ARTIFACT_MEMBERS = ( + "evidence/oci-smoke.json", + "evidence/publish-outcome.json", + "evidence/runtime-receipt.json", + f"final/release-v3/{SERVICE}", + "final/release-v3/release.json", + "receipts/build-push-receipt.json", +) +FORBIDDEN_OUTPUT_MARKERS = ( + "authorization:", + "access_token", + "password", + "private_key", + "telegram", + "77.42.65.182", + "teleo-prod-1", +) +BOOTSTRAP_CODE = textwrap.dedent( + r""" + import fcntl, hashlib, io, json, os, pwd, re, stat, sys, tarfile + mode, operation, request_id, runner_revision, release_revision, archive_hash, helper_hash = sys.argv[1:] + fail = lambda: (_ for _ in ()).throw(RuntimeError("bootstrap_failed")) + if mode not in {"start", "cleanup"} or operation not in {"install", "verify", "rollback"}: fail() + if not re.fullmatch(r"iap-[a-z0-9]{12,32}", request_id): fail() + if not re.fullmatch(r"[0-9a-f]{40}", runner_revision) or not re.fullmatch(r"[0-9a-f]{40}", release_revision): fail() + if not re.fullmatch(r"[0-9a-f]{64}", archive_hash) or not re.fullmatch(r"[0-9a-f]{64}", helper_hash): fail() + caller = os.environ.get("SUDO_USER", "") + if not caller or caller == "root": fail() + caller_uid = pwd.getpwnam(caller).pw_uid + upload = f"/tmp/livingip-leoclean-nosend-{request_id}.tar.gz" + stage_root = "/run/livingip-leoclean-nosend-iap-bootstrap" + stage = f"{stage_root}/{request_id}" + lock_path = "/run/leoclean-gcp-nosend-iap-bootstrap.lock" + def stable(path, uid, expected, limit): + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)) + try: + before = os.fstat(fd); data = b"" + if not stat.S_ISREG(before.st_mode) or before.st_uid != uid or stat.S_IMODE(before.st_mode) != 0o600 or before.st_nlink != 1 or before.st_size > limit: fail() + while len(data) <= limit: + chunk = os.read(fd, min(65536, limit + 1 - len(data))) + if not chunk: break + data += chunk + after = os.fstat(fd); named = os.stat(path, follow_symlinks=False) + if len(data) != after.st_size or (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns) != (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns): fail() + if (after.st_dev, after.st_ino, after.st_mode, after.st_nlink) != (named.st_dev, named.st_ino, named.st_mode, named.st_nlink): fail() + if hashlib.sha256(data).hexdigest() != expected: fail() + return data, (after.st_dev, after.st_ino) + finally: os.close(fd) + def unlink_exact(path, identity): + observed = os.stat(path, follow_symlinks=False) + if not stat.S_ISREG(observed.st_mode) or (observed.st_dev, observed.st_ino) != identity: fail() + os.unlink(path) + def safe_identity(path, uid, limit): + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)) + try: + observed = os.fstat(fd); named = os.stat(path, follow_symlinks=False) + if not stat.S_ISREG(observed.st_mode) or observed.st_uid != uid or stat.S_IMODE(observed.st_mode) != 0o600 or observed.st_nlink != 1 or observed.st_size > limit: fail() + if (observed.st_dev, observed.st_ino, observed.st_mode, observed.st_nlink) != (named.st_dev, named.st_ino, named.st_mode, named.st_nlink): fail() + return (observed.st_dev, observed.st_ino) + finally: os.close(fd) + def write_exact(path, data): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(fd, "wb") as handle: handle.write(data); handle.flush(); os.fsync(handle.fileno()) + lock_fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) + try: + lock_state = os.fstat(lock_fd) + if lock_state.st_uid != 0 or stat.S_IMODE(lock_state.st_mode) != 0o600 or not stat.S_ISREG(lock_state.st_mode): fail() + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + if mode == "cleanup": + try: + identity = safe_identity(upload, caller_uid, 8 * 1024 * 1024) + unlink_exact(upload, identity) + except FileNotFoundError: pass + if os.path.isdir(stage) and not os.path.islink(stage): + state = os.stat(stage, follow_symlinks=False) + if state.st_uid != 0 or stat.S_IMODE(state.st_mode) != 0o700: fail() + for name in os.listdir(stage): + path = f"{stage}/{name}" + expected = archive_hash if name == "request.tar.gz" else helper_hash if name == "remote-helper.py" else None + if expected is None: fail() + _, identity = stable(path, 0, expected, 8 * 1024 * 1024) + unlink_exact(path, identity) + os.rmdir(stage) + print(json.dumps({"schema":"livingip.leocleanNoSendIapCleanup.v1","status":"pass","request_id":request_id}, sort_keys=True)) + raise SystemExit(0) + archive, upload_identity = stable(upload, caller_uid, archive_hash, 8 * 1024 * 1024) + if not os.path.isdir(stage_root): + os.mkdir(stage_root, 0o700); os.chown(stage_root, 0, 0) + root_state = os.stat(stage_root, follow_symlinks=False) + if root_state.st_uid != 0 or stat.S_IMODE(root_state.st_mode) != 0o700 or not stat.S_ISDIR(root_state.st_mode): fail() + os.mkdir(stage, 0o700); os.chown(stage, 0, 0) + write_exact(f"{stage}/request.tar.gz", archive) + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as request: + matches = [member for member in request.getmembers() if member.name == "ops/gcp_leoclean_nosend_iap_remote.py"] + if len(matches) != 1 or not matches[0].isfile() or matches[0].size > 2 * 1024 * 1024: fail() + stream = request.extractfile(matches[0]); helper = stream.read() if stream else b"" + if hashlib.sha256(helper).hexdigest() != helper_hash: fail() + write_exact(f"{stage}/remote-helper.py", helper) + unlink_exact(upload, upload_identity) + os.execve("/usr/bin/python3", ["/usr/bin/python3", "-I", f"{stage}/remote-helper.py", operation, request_id, runner_revision, release_revision, archive_hash, helper_hash], {"PATH":"/usr/bin:/bin:/usr/sbin:/sbin", "SUDO_USER":caller}) + finally: os.close(lock_fd) + """ +).strip() + +Runner = Callable[..., subprocess.CompletedProcess[bytes]] + + +class IapRequestError(RuntimeError): + """The request could not be proven safe enough to dispatch.""" + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise IapRequestError(message) + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _canonical_json(value: Mapping[str, Any]) -> bytes: + try: + return (json.dumps(value, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8") + except (TypeError, ValueError) as exc: + raise IapRequestError("request manifest could not be encoded") from exc + + +def _strict_json(raw: bytes, label: str) -> dict[str, Any]: + def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate key") + value[key] = item + return value + + try: + value = json.loads( + raw.decode("utf-8", "strict"), + object_pairs_hook=reject_duplicates, + parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("constant")), + ) + except (UnicodeError, ValueError, json.JSONDecodeError) as exc: + raise IapRequestError(f"{label} was invalid JSON") from exc + _require(isinstance(value, dict), f"{label} was not a JSON object") + return value + + +def _run_bounded( + argv: list[str], + *, + runner: Runner, + label: str, + input_bytes: bytes | None = None, + environment: Mapping[str, str] | None = None, +) -> subprocess.CompletedProcess[bytes]: + try: + completed = runner( + argv, + check=False, + stdin=subprocess.DEVNULL if input_bytes is None else None, + input=input_bytes, + capture_output=True, + timeout=300, + env=dict(environment or {"PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin"}), + ) + except (OSError, subprocess.SubprocessError) as exc: + raise IapRequestError(f"{label} failed") from exc + _require( + len(completed.stdout) <= MAX_COMMAND_OUTPUT_BYTES and len(completed.stderr) <= MAX_COMMAND_OUTPUT_BYTES, + f"{label} output exceeded the safety limit", + ) + _require(completed.returncode == 0, f"{label} failed") + return completed + + +def _validate_owned_directory(path: Path, label: str) -> Path: + _require(path.is_absolute() and not path.is_symlink(), f"{label} was unsafe") + resolved = path.resolve() + observed = resolved.stat(follow_symlinks=False) + _require( + stat.S_ISDIR(observed.st_mode) and observed.st_uid == os.geteuid() and observed.st_mode & 0o022 == 0, + f"{label} was unsafe", + ) + return resolved + + +def gcloud_environment(*, probe_runner: Runner = subprocess.run) -> dict[str, str]: + home_value = os.environ.get("HOME", "") + _require(home_value, "HOME was not configured") + home = _validate_owned_directory(Path(home_value), "HOME") + config_value = os.environ.get("CLOUDSDK_CONFIG") or str(home / ".config/gcloud") + config = _validate_owned_directory(Path(config_value), "Cloud SDK configuration") + candidates = [os.environ.get("CLOUDSDK_PYTHON"), str(CONTRACT_ROOT / ".venv/bin/python3")] + candidates.append(sys.executable) + selected: Path | None = None + for value in candidates: + if not value or not Path(value).is_absolute(): + continue + try: + candidate = Path(value).resolve(strict=True) + observed = candidate.stat(follow_symlinks=False) + except OSError: + continue + if not stat.S_ISREG(observed.st_mode) or not os.access(candidate, os.X_OK) or observed.st_mode & 0o022: + continue + try: + probe = probe_runner( + [str(candidate), "--version"], + check=False, + stdin=subprocess.DEVNULL, + capture_output=True, + timeout=10, + env={"PATH": "/usr/bin:/bin"}, + ) + except (OSError, subprocess.SubprocessError): + continue + output = (probe.stdout + probe.stderr).decode("ascii", "ignore") + match = re.search(r"Python ([0-9]+)\.([0-9]+)(?:\.[0-9]+)?", output) + if probe.returncode == 0 and match and (int(match.group(1)), int(match.group(2))) >= (3, 10): + selected = candidate + break + _require(selected is not None, "no compatible absolute Cloud SDK Python was available") + return { + "PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin", + "HOME": str(home), + "CLOUDSDK_CONFIG": str(config), + "CLOUDSDK_PYTHON": str(selected), + } + + +def _git_output(arguments: list[str], *, runner: Runner) -> bytes: + completed = _run_bounded(["/usr/bin/git", "-C", str(CONTRACT_ROOT), *arguments], runner=runner, label="Git check") + return completed.stdout + + +def validate_clean_revision(runner_revision: str, *, runner: Runner = subprocess.run) -> None: + _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision must be one full lowercase Git commit") + top = _git_output(["rev-parse", "--show-toplevel"], runner=runner).decode("utf-8", "strict").strip() + head = _git_output(["rev-parse", "--verify", "HEAD^{commit}"], runner=runner).decode("ascii", "strict").strip() + dirty = _git_output(["status", "--porcelain=v1", "-z", "--untracked-files=all"], runner=runner) + origin_main = ( + _git_output( + ["rev-parse", "--verify", "refs/remotes/origin/main^{commit}"], + runner=runner, + ) + .decode("ascii", "strict") + .strip() + ) + _require(Path(top).resolve() == CONTRACT_ROOT, "source checkout was not the contract repository") + _require(head == runner_revision, "source checkout differed from the requested runner revision") + _require(origin_main == runner_revision, "runner revision was not exact local origin/main") + _require(not dirty, "source checkout was dirty") + + +def validate_remote_main(runner_revision: str, *, runner: Runner = subprocess.run) -> None: + _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid") + origin = _git_output(["remote", "get-url", "origin"], runner=runner).decode("utf-8", "strict").strip() + _require(origin == ORIGIN_URL, "origin remote was not the fixed LivingIP SSH repository") + readback = _git_output(["ls-remote", "--exit-code", "origin", "refs/heads/main"], runner=runner) + try: + line = readback.decode("ascii", "strict").strip() + except UnicodeError as exc: + raise IapRequestError("remote main readback was invalid") from exc + _require(line == f"{runner_revision}\trefs/heads/main", "runner revision was not exact live origin/main") + + +def _read_stable_file(path: Path, *, root: Path, max_bytes: int = MAX_MEMBER_BYTES) -> tuple[bytes, int]: + _require(path.is_absolute(), "request member path was not absolute") + try: + relative = path.relative_to(root) + except ValueError as exc: + raise IapRequestError("request member escaped its fixed root") from exc + _require(not any(part in {"", ".", ".."} for part in relative.parts), "request member path was unsafe") + flags = os.O_RDONLY + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise IapRequestError("request member was missing or unsafe") from exc + try: + before = os.fstat(descriptor) + _require( + stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size <= max_bytes, + "request member was not a bounded regular file", + ) + content = b"" + while len(content) <= max_bytes: + chunk = os.read(descriptor, min(65536, max_bytes + 1 - len(content))) + if not chunk: + break + content += chunk + after = os.fstat(descriptor) + named = path.stat(follow_symlinks=False) + identity = (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns) + _require( + len(content) == after.st_size + and identity + == (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns) + and (after.st_dev, after.st_ino, after.st_mode, after.st_nlink) + == (named.st_dev, named.st_ino, named.st_mode, named.st_nlink), + "request member changed while it was read", + ) + return content, stat.S_IMODE(after.st_mode) + except OSError as exc: + raise IapRequestError("request member could not be read") from exc + finally: + os.close(descriptor) + + +def _validate_expected_bindings( + *, + expected_artifact_sha256: str, + expected_release_sha256: str, + expected_image_digest: str, + expected_image_config_digest: str, +) -> dict[str, str]: + _require(HEX_64.fullmatch(expected_artifact_sha256) is not None, "expected artifact hash was invalid") + _require(HEX_64.fullmatch(expected_release_sha256) is not None, "expected release hash was invalid") + _require( + re.fullmatch(r"sha256:[0-9a-f]{64}", expected_image_digest) is not None, + "expected image manifest digest was invalid", + ) + _require( + re.fullmatch(r"sha256:[0-9a-f]{64}", expected_image_config_digest) is not None, + "expected image config digest was invalid", + ) + return { + "release_artifact_sha256": expected_artifact_sha256, + "release_sha256": expected_release_sha256, + "image_digest": expected_image_digest, + "image_config_digest": expected_image_config_digest, + } + + +def validate_release( + bundle: Path, + release_revision: str, + *, + expected_release_sha256: str, + expected_image_digest: str, + expected_image_config_digest: str, +) -> tuple[dict[str, Any], bytes]: + try: + release, unit = installer.validate_release_bundle(bundle) + except (installer.InstallError, package.PackageError) as exc: + raise IapRequestError("finalized release bundle did not validate") from exc + image = release.get("image") + runtime = release.get("image_input", {}).get("runtime") + _require( + isinstance(image, dict) and package.IMAGE_REFERENCE.fullmatch(str(image.get("reference"))), + "image was not immutable", + ) + _require( + isinstance(runtime, dict) and runtime.get("teleo_git_head") == release_revision, + "release differed from release revision", + ) + _require( + release.get("release_sha256") == expected_release_sha256 + and image.get("digest") == expected_image_digest + and image.get("config_digest") == expected_image_config_digest + and image.get("reference") == f"{package.IMAGE_REPOSITORY}@{expected_image_digest}", + "release differed from the independently supplied expected binding", + ) + return release, unit + + +def validate_release_artifact( + artifact: Path, + *, + bundle: Path, + release: Mapping[str, Any], + expected_artifact_sha256: str, +) -> tuple[dict[str, Any], bytes]: + raw, mode = _read_stable_file(artifact.absolute(), root=artifact.absolute().parent, max_bytes=MAX_ARCHIVE_BYTES) + _require(mode & 0o022 == 0, "release artifact archive was writable by group or other") + _require( + _sha256_bytes(raw) == expected_artifact_sha256, + "release artifact differed from the independently supplied expected hash", + ) + try: + archive = zipfile.ZipFile(io.BytesIO(raw), mode="r") + except (OSError, zipfile.BadZipFile) as exc: + raise IapRequestError("release artifact archive was invalid") from exc + with archive: + infos = archive.infolist() + names = [info.filename for info in infos] + _require( + names == list(RELEASE_ARTIFACT_MEMBERS) or sorted(names) == sorted(RELEASE_ARTIFACT_MEMBERS), + "release artifact member allowlist was not exact", + ) + _require(len(names) == len(set(names)), "release artifact contained duplicate members") + members: list[dict[str, Any]] = [] + payloads: dict[str, bytes] = {} + for info in infos: + path = Path(info.filename) + unix_mode = (info.external_attr >> 16) & 0o177777 + _require( + info.filename + and not info.filename.startswith("/") + and all(part not in {"", ".", ".."} for part in path.parts) + and not info.is_dir() + and info.file_size <= MAX_MEMBER_BYTES + and (unix_mode == 0 or stat.S_ISREG(unix_mode)), + "release artifact contained an unsafe member", + ) + content = archive.read(info) + _require(len(content) == info.file_size, "release artifact member size differed") + payloads[info.filename] = content + members.append( + { + "path": info.filename, + "sha256": _sha256_bytes(content), + "size": len(content), + } + ) + bundle_root = bundle.resolve() + release_bytes, _release_mode = _read_stable_file(bundle_root / "release.json", root=bundle_root) + unit_bytes, _unit_mode = _read_stable_file(bundle_root / SERVICE, root=bundle_root) + _require( + payloads["final/release-v3/release.json"] == release_bytes + and payloads[f"final/release-v3/{SERVICE}"] == unit_bytes, + "release bundle differed from its original artifact archive", + ) + build_receipt = _strict_json(payloads["receipts/build-push-receipt.json"], "artifact build/push receipt") + publish_outcome = _strict_json(payloads["evidence/publish-outcome.json"], "artifact publish outcome") + _require(build_receipt == release["build_push_receipt"], "artifact build/push receipt differed from release") + try: + package.validate_publish_outcome(publish_outcome, image_input=dict(release["image_input"])) + except package.PackageError as exc: + raise IapRequestError("artifact publish outcome did not validate") from exc + _require(publish_outcome.get("status") == "pass", "artifact publish outcome did not pass") + return ( + { + "archive_sha256": _sha256_bytes(raw), + "archive_size": len(raw), + "members": sorted(members, key=lambda item: str(item["path"])), + }, + raw, + ) + + +def _member_payloads(bundle: Path, unit: bytes, release_artifact: bytes) -> dict[str, tuple[bytes, int]]: + payloads: dict[str, tuple[bytes, int]] = {} + for relative in SOURCE_MEMBERS: + payloads[relative] = _read_stable_file(CONTRACT_ROOT / relative, root=CONTRACT_ROOT) + release_bytes, release_mode = _read_stable_file(bundle.resolve() / "release.json", root=bundle.resolve()) + _unit_bytes, unit_mode = _read_stable_file(bundle.resolve() / SERVICE, root=bundle.resolve()) + _require(_unit_bytes == unit, "release unit changed after validation") + payloads["release/release.json"] = (release_bytes, release_mode) + payloads[f"release/{SERVICE}"] = (unit, unit_mode) + payloads["release-artifact.zip"] = (release_artifact, 0o600) + return payloads + + +def build_manifest( + *, + operation: str, + request_id: str, + runner_revision: str, + release_revision: str, + release: Mapping[str, Any], + payloads: Mapping[str, tuple[bytes, int]], + source_binding: Mapping[str, Any], + release_artifact: Mapping[str, Any], +) -> dict[str, Any]: + _require(operation in OPERATIONS, "operation was not allowlisted") + _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") + files = [ + { + "path": relative, + "sha256": _sha256_bytes(content), + "size": len(content), + "mode": mode, + } + for relative, (content, mode) in sorted(payloads.items()) + ] + stable = { + "schema": SCHEMA, + "operation": operation, + "request_id": request_id, + "runner_revision": runner_revision, + "release_revision": release_revision, + "target": { + "project": PROJECT, + "zone": ZONE, + "instance": INSTANCE, + "service": SERVICE, + "service_account": SERVICE_ACCOUNT, + }, + "release": { + "release_sha256": release["release_sha256"], + "image_reference": release["image"]["reference"], + "image_digest": release["image"]["digest"], + "image_config_digest": release["image"]["config_digest"], + }, + "installer_source": dict(source_binding), + "release_artifact": dict(release_artifact), + "files": files, + } + return {**stable, "manifest_sha256": package.canonical_sha256(stable)} + + +def _tar_info(name: str, *, size: int, mode: int) -> tarfile.TarInfo: + _require(name and not name.startswith("/") and ".." not in Path(name).parts, "archive member path was unsafe") + info = tarfile.TarInfo(name) + info.size = size + info.mode = mode + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + info.mtime = 0 + info.type = tarfile.REGTYPE + return info + + +def build_archive( + output: Path, + *, + manifest: Mapping[str, Any], + payloads: Mapping[str, tuple[bytes, int]], +) -> str: + _require(output.is_absolute(), "archive output path was not absolute") + parent = output.parent + parent_state = parent.stat(follow_symlinks=False) + _require( + stat.S_ISDIR(parent_state.st_mode) + and parent_state.st_uid == os.geteuid() + and parent_state.st_mode & 0o077 == 0, + "archive output directory was not private", + ) + _require(not output.exists() and not output.is_symlink(), "archive output already existed") + manifest_bytes = _canonical_json(manifest) + members = {"request-manifest.json": (manifest_bytes, 0o600), **payloads} + temporary: Path | None = None + try: + descriptor, temporary_name = tempfile.mkstemp(prefix=".iap-request-", dir=parent) + temporary = Path(temporary_name) + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb", closefd=True) as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive: + for relative, (content, mode) in sorted(members.items()): + archive.addfile(_tar_info(relative, size=len(content), mode=mode), io.BytesIO(content)) + raw.flush() + os.fsync(raw.fileno()) + _require(temporary.stat().st_size <= MAX_ARCHIVE_BYTES, "request archive exceeded the safety limit") + os.link(temporary, output, follow_symlinks=False) + temporary.unlink() + temporary = None + directory = os.open(parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + return package.sha256_file(output) + except OSError as exc: + raise IapRequestError("request archive could not be published") from exc + finally: + if temporary is not None: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def remote_names(request_id: str) -> tuple[str, str]: + _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") + return ( + f"livingip-leoclean-nosend-{request_id}.tar.gz", + f"livingip-leoclean-nosend-{request_id}-ssh-key", + ) + + +def build_gcloud_argv( + *, + operation: str, + request_id: str, + runner_revision: str, + release_revision: str, + archive: Path, + archive_sha256: str, + ssh_key: Path, + helper_sha256: str, +) -> tuple[list[str], list[str], list[str]]: + _require(operation in {"install", "verify", "rollback"}, "plan is local-only") + _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") + _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid") + _require(HEX_40.fullmatch(release_revision) is not None, "release revision was invalid") + _require(HEX_64.fullmatch(archive_sha256) is not None, "archive hash was invalid") + _require(HEX_64.fullmatch(helper_sha256) is not None, "helper hash was invalid") + archive_name, key_name = remote_names(request_id) + _require(archive.name == archive_name and ssh_key.name == key_name, "request filenames were not exact") + key_flag = f"--ssh-key-file={ssh_key}" + scp = [ + GCLOUD, + "compute", + "scp", + f"--project={PROJECT}", + f"--zone={ZONE}", + "--tunnel-through-iap", + f"--ssh-key-expire-after={KEY_LIFETIME}", + key_flag, + "--quiet", + str(archive), + f"{INSTANCE}:{REMOTE_UPLOAD_ROOT}/", + ] + remote = [ + "sudo", + "--", + "/usr/bin/python3", + "-I", + "-c", + BOOTSTRAP_CODE, + "start", + operation, + request_id, + runner_revision, + release_revision, + archive_sha256, + helper_sha256, + ] + cleanup_remote = [*remote[:6], "cleanup", *remote[7:]] + ssh = [ + GCLOUD, + "compute", + "ssh", + INSTANCE, + f"--project={PROJECT}", + f"--zone={ZONE}", + "--tunnel-through-iap", + f"--ssh-key-expire-after={KEY_LIFETIME}", + key_flag, + "--quiet", + f"--command={shlex.join(remote)}", + ] + cleanup = [*ssh[:-1], f"--command={shlex.join(cleanup_remote)}"] + return scp, ssh, cleanup + + +def _action_receipts(commands: tuple[list[str], list[str], list[str]]) -> list[dict[str, str]]: + labels = ("upload_request", "execute_remote_request", "cleanup_failed_transfer") + return [ + { + "label": label, + "argv_sha256": package.canonical_sha256({"argv": argv}), + } + for label, argv in zip(labels, commands, strict=True) + ] + + +def prepare_request( + operation: str, + request_id: str, + runner_revision: str, + release_revision: str, + bundle: Path, + release_artifact_path: Path, + output_directory: Path, + expected_artifact_sha256: str, + expected_release_sha256: str, + expected_image_digest: str, + expected_image_config_digest: str, + *, + git_runner: Runner = subprocess.run, +) -> dict[str, Any]: + _require(operation in OPERATIONS, "operation was not allowlisted") + _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") + output_directory = output_directory.absolute() + output_state = output_directory.stat(follow_symlinks=False) + _require( + stat.S_ISDIR(output_state.st_mode) + and output_state.st_uid == os.geteuid() + and output_state.st_mode & 0o077 == 0, + "request output directory was not private", + ) + expected = _validate_expected_bindings( + expected_artifact_sha256=expected_artifact_sha256, + expected_release_sha256=expected_release_sha256, + expected_image_digest=expected_image_digest, + expected_image_config_digest=expected_image_config_digest, + ) + validate_clean_revision(runner_revision, runner=git_runner) + release, unit = validate_release( + bundle.absolute(), + release_revision, + expected_release_sha256=expected_release_sha256, + expected_image_digest=expected_image_digest, + expected_image_config_digest=expected_image_config_digest, + ) + release_artifact, release_artifact_bytes = validate_release_artifact( + release_artifact_path.absolute(), + bundle=bundle.absolute(), + release=release, + expected_artifact_sha256=expected_artifact_sha256, + ) + source_binding = installer.reviewed_source_binding() + _require( + source_binding.get("revision") == runner_revision, "installer source binding differed from runner revision" + ) + payloads = _member_payloads(bundle.absolute(), unit, release_artifact_bytes) + manifest = build_manifest( + operation=operation, + request_id=request_id, + runner_revision=runner_revision, + release_revision=release_revision, + release=release, + payloads=payloads, + source_binding=source_binding, + release_artifact=release_artifact, + ) + archive_name, key_name = remote_names(request_id) + archive = output_directory / archive_name + ssh_key = output_directory / key_name + _require( + not ssh_key.exists() + and not ssh_key.is_symlink() + and not Path(f"{ssh_key}.pub").exists() + and not Path(f"{ssh_key}.pub").is_symlink(), + "request-local SSH key path already existed", + ) + helper_sha256 = _sha256_bytes(payloads[REMOTE_HELPER_SOURCE][0]) + try: + archive_sha256 = build_archive(archive, manifest=manifest, payloads=payloads) + except Exception: + for created in (archive,): + try: + observed = created.stat(follow_symlinks=False) + if stat.S_ISREG(observed.st_mode) and observed.st_uid == os.geteuid() and observed.st_nlink == 1: + created.unlink() + except FileNotFoundError: + pass + raise + if operation == "plan": + actions: list[dict[str, str]] = [] + else: + commands = build_gcloud_argv( + operation=operation, + request_id=request_id, + runner_revision=runner_revision, + release_revision=release_revision, + archive=archive, + archive_sha256=archive_sha256, + ssh_key=ssh_key, + helper_sha256=helper_sha256, + ) + actions = _action_receipts(commands) + return { + "schema": PLAN_SCHEMA, + "status": "pass", + "mode": "dry_run", + "operation": operation, + "request_id": request_id, + "runner_revision": runner_revision, + "release_revision": release_revision, + "target": manifest["target"], + "release": manifest["release"], + "release_artifact": manifest["release_artifact"], + "expected": expected, + "request": { + "archive": str(archive), + "archive_sha256": archive_sha256, + "helper_sha256": helper_sha256, + "manifest_sha256": manifest["manifest_sha256"], + "ssh_key": str(ssh_key), + }, + "actions": actions, + "authority": { + "gcp_mutation": operation in {"install", "rollback"}, + "service_restart": operation in {"install", "rollback"}, + "database_or_secret_provisioning": False, + "production_or_transport": False, + "vps": False, + }, + } + + +def _cleanup_local_ssh_key(path: Path) -> None: + for candidate in (path, Path(f"{path}.pub")): + try: + observed = candidate.stat(follow_symlinks=False) + except FileNotFoundError: + continue + _require( + stat.S_ISREG(observed.st_mode) and observed.st_uid == os.geteuid() and observed.st_nlink == 1, + "request-local SSH key cleanup refused an unsafe path", + ) + candidate.unlink() + + +def _validate_verification(value: Any, label: str) -> None: + _require( + isinstance(value, dict) + and set(value) + == { + "active_state", + "sub_state", + "main_pid", + "n_restarts", + "invocation_id", + "supervising_process", + "container_id", + "container_pid", + "health", + "validated_process_environment_fields", + "dropins", + "environment_files", + } + and value.get("active_state") == "active" + and value.get("sub_state") == "running" + and isinstance(value.get("main_pid"), int) + and value["main_pid"] > 1 + and isinstance(value.get("n_restarts"), int) + and value["n_restarts"] >= 0 + and re.fullmatch(r"[0-9a-f]{32}", str(value.get("invocation_id"))) is not None + and HEX_64.fullmatch(str(value.get("container_id"))) is not None + and isinstance(value.get("container_pid"), int) + and value["container_pid"] > 1 + and value.get("health") == "healthy" + and value.get("dropins") == "absent" + and value.get("environment_files") == "absent", + f"{label} was invalid", + ) + + +def _validate_database_summary(value: Any, request_id: str, expected_name: str) -> None: + expected_path = f"/var/lib/livingip/leoclean-nosend-iap/requests/{request_id}/{expected_name}" + _require( + isinstance(value, dict) + and set(value) == {"schema", "status", "receipt_sha256", "receipt_path", "proof"} + and value.get("schema") == "livingip.leocleanNoSendContainerDatabaseVerification.v1" + and value.get("status") == "pass" + and HEX_64.fullmatch(str(value.get("receipt_sha256"))) is not None + and value.get("receipt_path") == expected_path, + "database verification summary was invalid", + ) + _require( + value.get("proof") + == { + "proposal_staging": "function_only", + "proposal_transaction": "rolled_back", + "durable_proposal_created": False, + "canonical_direct_writes": "denied", + "stage_direct_writes": "denied", + "role_escalation": "denied", + }, + "database verification proof summary was invalid", + ) + + +def validate_remote_result(plan: Mapping[str, Any], result: Mapping[str, Any]) -> None: + _require( + set(result) + == { + "schema", + "status", + "operation", + "request_id", + "runner_revision", + "release_revision", + "release_sha256", + "image_digest", + "image_config_digest", + "release_artifact_sha256", + "request_binding", + "service", + "database", + "authority", + }, + "remote result fields were not exact", + ) + operation = plan["operation"] + request_id = plan["request_id"] + request = plan["request"] + release = plan["release"] + _require( + result.get("schema") == REMOTE_RESULT_SCHEMA + and result.get("status") == "pass" + and result.get("operation") == operation + and result.get("request_id") == request_id + and result.get("runner_revision") == plan["runner_revision"] + and result.get("release_revision") == plan["release_revision"] + and result.get("release_sha256") == release["release_sha256"] + and result.get("image_digest") == release["image_digest"] + and result.get("image_config_digest") == release["image_config_digest"] + and result.get("release_artifact_sha256") == plan["release_artifact"]["archive_sha256"] + and result.get("request_binding") + == { + "archive_sha256": request["archive_sha256"], + "helper_sha256": request["helper_sha256"], + "manifest_sha256": request["manifest_sha256"], + } + and result.get("authority") + == { + "sending_enabled": False, + "production_traffic_changed": False, + "vps_touched": False, + "canonical_proposal_approved": False, + }, + "remote result did not match the request", + ) + service = result["service"] + database = result["database"] + if operation == "install": + _require( + isinstance(service, dict) and set(service) == {"name", "before_restart", "after_restart"}, + "install service result was invalid", + ) + _require(service["name"] == SERVICE, "install service target differed") + _validate_verification(service["before_restart"], "pre-restart verification") + _validate_verification(service["after_restart"], "post-restart verification") + before = service["before_restart"] + after = service["after_restart"] + _require( + before["main_pid"] != after["main_pid"] + and before["invocation_id"] != after["invocation_id"] + and before["container_id"] != after["container_id"], + "restart identities did not change", + ) + _require( + isinstance(database, dict) + and set(database) == {"status", "before_restart", "after_restart"} + and database.get("status") == "pass", + "install database result was invalid", + ) + _validate_database_summary(database["before_restart"], request_id, "database-before-restart.json") + _validate_database_summary(database["after_restart"], request_id, "database-after-restart.json") + elif operation == "verify": + _require( + isinstance(service, dict) and set(service) == {"name", "verification"} and service.get("name") == SERVICE, + "verify service result was invalid", + ) + _validate_verification(service["verification"], "verify running release") + _validate_database_summary(database, request_id, "database-verify.json") + else: + _require( + isinstance(service, dict) + and set(service) == {"name", "restored_posture", "restored_release_sha256", "verification"} + and service.get("name") == SERVICE + and service.get("restored_posture") in {"active", "inactive", "absent"} + and database == {"status": "not_run_after_explicit_rollback"}, + "rollback result was invalid", + ) + if service["restored_posture"] == "active": + _require( + HEX_64.fullmatch(str(service["restored_release_sha256"])) is not None, "restored release was invalid" + ) + _validate_verification(service["verification"], "restored active verification") + elif service["restored_posture"] == "inactive": + _require( + HEX_64.fullmatch(str(service["restored_release_sha256"])) is not None + and service["verification"] is None, + "restored inactive result was invalid", + ) + else: + _require( + service["restored_release_sha256"] is None and service["verification"] is None, + "restored absent result was invalid", + ) + + +def _write_result_receipt(plan: Mapping[str, Any], result: Mapping[str, Any]) -> tuple[Path, dict[str, Any]]: + archive = Path(str(plan["request"]["archive"])) + output = archive.parent / f"livingip-leoclean-nosend-result-{plan['request_id']}.json" + _require(not output.exists() and not output.is_symlink(), "local result receipt already existed") + stable = { + "schema": RESULT_RECEIPT_SCHEMA, + "status": "pass", + "operation": plan["operation"], + "request_id": plan["request_id"], + "runner_revision": plan["runner_revision"], + "release_revision": plan["release_revision"], + "request": { + "archive_sha256": plan["request"]["archive_sha256"], + "helper_sha256": plan["request"]["helper_sha256"], + "manifest_sha256": plan["request"]["manifest_sha256"], + }, + "release": dict(plan["release"]), + "release_artifact": dict(plan["release_artifact"]), + "expected": dict(plan["expected"]), + "remote_result": dict(result), + } + receipt = {**stable, "receipt_sha256": package.canonical_sha256(stable)} + content = _canonical_json(receipt) + descriptor, temporary_name = tempfile.mkstemp(prefix=".iap-result-", dir=output.parent) + temporary: Path | None = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.link(temporary, output, follow_symlinks=False) + temporary.unlink() + temporary = None + directory = os.open(output.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + except OSError as exc: + raise IapRequestError("local result receipt could not be published") from exc + finally: + try: + if temporary is not None: + temporary.unlink() + except FileNotFoundError: + pass + return output, receipt + + +def execute_plan( + plan: Mapping[str, Any], + *, + runner: Runner = subprocess.run, + git_runner: Runner = subprocess.run, + gcloud_env: Mapping[str, str] | None = None, +) -> dict[str, Any]: + _require( + set(plan) + == { + "schema", + "status", + "mode", + "operation", + "request_id", + "runner_revision", + "release_revision", + "target", + "release", + "release_artifact", + "expected", + "request", + "actions", + "authority", + } + and plan.get("schema") == PLAN_SCHEMA + and plan.get("status") == "pass" + and plan.get("mode") == "dry_run", + "request plan fields were not exact", + ) + operation = plan.get("operation") + _require(operation in {"install", "verify", "rollback"}, "operation was not executable") + request_id = plan.get("request_id") + runner_revision = plan.get("runner_revision") + release_revision = plan.get("release_revision") + _require(isinstance(request_id, str) and REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") + _require( + isinstance(runner_revision, str) and HEX_40.fullmatch(runner_revision) is not None, + "runner revision was invalid", + ) + _require( + isinstance(release_revision, str) and HEX_40.fullmatch(release_revision) is not None, + "release revision was invalid", + ) + _require( + plan.get("target") + == { + "project": PROJECT, + "zone": ZONE, + "instance": INSTANCE, + "service": SERVICE, + "service_account": SERVICE_ACCOUNT, + }, + "request target was not exact", + ) + expected = plan.get("expected") + _require( + isinstance(expected, dict) + and expected + == _validate_expected_bindings( + expected_artifact_sha256=str(expected.get("release_artifact_sha256")), + expected_release_sha256=str(expected.get("release_sha256")), + expected_image_digest=str(expected.get("image_digest")), + expected_image_config_digest=str(expected.get("image_config_digest")), + ) + and plan.get("release") + == { + "release_sha256": expected["release_sha256"], + "image_reference": f"{package.IMAGE_REPOSITORY}@{expected['image_digest']}", + "image_digest": expected["image_digest"], + "image_config_digest": expected["image_config_digest"], + } + and isinstance(plan.get("release_artifact"), dict) + and plan["release_artifact"].get("archive_sha256") == expected["release_artifact_sha256"], + "request expected bindings were not exact", + ) + request = plan.get("request") + _require( + isinstance(request, dict) + and set(request) == {"archive", "archive_sha256", "helper_sha256", "manifest_sha256", "ssh_key"}, + "request files were invalid", + ) + archive = Path(str(request["archive"])) + ssh_key = Path(str(request["ssh_key"])) + _require(archive.is_absolute() and ssh_key.is_absolute(), "request file path was not absolute") + archive_content, archive_mode = _read_stable_file(archive, root=archive.parent, max_bytes=MAX_ARCHIVE_BYTES) + _require( + archive_mode == 0o600 + and _sha256_bytes(archive_content) == request.get("archive_sha256") + and HEX_64.fullmatch(str(request.get("helper_sha256"))) is not None + and HEX_64.fullmatch(str(request.get("manifest_sha256"))) is not None, + "request files differed from the dry-run plan", + ) + _require( + not ssh_key.exists() + and not ssh_key.is_symlink() + and not Path(f"{ssh_key}.pub").exists() + and not Path(f"{ssh_key}.pub").is_symlink(), + "request-local SSH key path was already occupied", + ) + expected_commands = build_gcloud_argv( + operation=str(operation), + request_id=request_id, + runner_revision=runner_revision, + release_revision=release_revision, + archive=archive, + archive_sha256=str(request["archive_sha256"]), + ssh_key=ssh_key, + helper_sha256=str(request["helper_sha256"]), + ) + _require( + plan.get("actions") == _action_receipts(expected_commands), "request actions differed from the fixed IAP plan" + ) + scp, ssh, cleanup = expected_commands + validate_remote_main(runner_revision, runner=git_runner) + environment = dict(gcloud_env or gcloud_environment()) + try: + try: + _run_bounded(list(scp), runner=runner, label="IAP upload", environment=environment) + completed = _run_bounded(list(ssh), runner=runner, label="IAP remote operation", environment=environment) + except IapRequestError as exc: + try: + _run_bounded( + list(cleanup), + runner=runner, + label="IAP failed-transfer cleanup", + environment=environment, + ) + except IapRequestError as cleanup_exc: + raise IapRequestError("IAP operation failed and remote upload cleanup was unproven") from cleanup_exc + raise exc + finally: + _cleanup_local_ssh_key(ssh_key) + result = _strict_json(completed.stdout, "remote result") + validate_remote_result(plan, result) + encoded = json.dumps(result, allow_nan=False, sort_keys=True).casefold() + _require(not any(marker in encoded for marker in FORBIDDEN_OUTPUT_MARKERS), "remote result was not sanitized") + receipt_path, receipt = _write_result_receipt(plan, result) + return { + "schema": EXECUTION_SCHEMA, + "status": "pass", + "operation": operation, + "request_id": request_id, + "runner_revision": runner_revision, + "release_revision": release_revision, + **expected, + "receipt_path": str(receipt_path), + "receipt_sha256": receipt["receipt_sha256"], + } + + +class _Parser(argparse.ArgumentParser): + def error(self, _message: str) -> None: + raise IapRequestError("invalid arguments") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = _Parser(description=__doc__, add_help=False) + parser.add_argument("--operation", required=True, choices=OPERATIONS) + parser.add_argument("--request-id", required=True) + parser.add_argument("--runner-revision", required=True) + parser.add_argument("--release-revision", required=True) + parser.add_argument("--bundle", required=True, type=Path) + parser.add_argument("--release-artifact", required=True, type=Path) + parser.add_argument("--output-directory", required=True, type=Path) + parser.add_argument("--expected-artifact-sha256", required=True) + parser.add_argument("--expected-release-sha256", required=True) + parser.add_argument("--expected-image-digest", required=True) + parser.add_argument("--expected-image-config-digest", required=True) + parser.add_argument("--execute", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + plan = prepare_request( + args.operation, + args.request_id, + args.runner_revision, + args.release_revision, + args.bundle, + args.release_artifact, + args.output_directory, + args.expected_artifact_sha256, + args.expected_release_sha256, + args.expected_image_digest, + args.expected_image_config_digest, + ) + result = execute_plan(plan) if args.execute else plan + sys.stdout.write(json.dumps(result, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n") + return 0 + except (IapRequestError, installer.InstallError, package.PackageError): + sys.stderr.write('{"error":"iap_request_failed","status":"fail"}\n') + return 65 + except Exception: + sys.stderr.write('{"error":"iap_request_failed","status":"fail"}\n') + return 65 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_gcp_leoclean_nosend_iap_runner.py b/tests/test_gcp_leoclean_nosend_iap_runner.py new file mode 100644 index 0000000..c02fc3e --- /dev/null +++ b/tests/test_gcp_leoclean_nosend_iap_runner.py @@ -0,0 +1,779 @@ +from __future__ import annotations + +import io +import json +import os +import stat +import subprocess +import tarfile +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from ops import gcp_leoclean_nosend_iap_remote as remote +from ops import run_gcp_leoclean_nosend_iap as local + + +def completed( + argv: list[str], + *, + returncode: int = 0, + stdout: bytes = b"", + stderr: bytes = b"", +) -> subprocess.CompletedProcess[bytes]: + return subprocess.CompletedProcess(argv, returncode, stdout, stderr) + + +def private_directory(path: Path) -> Path: + path.mkdir(mode=0o700) + path.chmod(0o700) + return path + + +def fixed_commands(tmp_path: Path, operation: str = "install") -> tuple[list[str], list[str], list[str]]: + request_id = "iap-abcdefghijkl" + archive_name, key_name = local.remote_names(request_id) + archive = tmp_path / archive_name + ssh_key = tmp_path / key_name + archive.write_bytes(b"archive") + archive.chmod(0o600) + return local.build_gcloud_argv( + operation=operation, + request_id=request_id, + runner_revision="a" * 40, + release_revision="b" * 40, + archive=archive, + archive_sha256="c" * 64, + ssh_key=ssh_key, + helper_sha256="d" * 64, + ) + + +def executable_plan(tmp_path: Path, operation: str) -> dict[str, Any]: + request_id = "iap-abcdefghijkl" + archive_name, key_name = local.remote_names(request_id) + archive = tmp_path / archive_name + ssh_key = tmp_path / key_name + archive.write_bytes(b"archive") + archive.chmod(0o600) + archive_sha256 = local._sha256_bytes(archive.read_bytes()) + helper_sha256 = "c" * 64 + commands = local.build_gcloud_argv( + operation=operation, + request_id=request_id, + runner_revision="a" * 40, + release_revision="b" * 40, + archive=archive, + archive_sha256=archive_sha256, + ssh_key=ssh_key, + helper_sha256=helper_sha256, + ) + expected = { + "release_artifact_sha256": "d" * 64, + "release_sha256": "e" * 64, + "image_digest": f"sha256:{'f' * 64}", + "image_config_digest": f"sha256:{'1' * 64}", + } + return { + "schema": local.PLAN_SCHEMA, + "status": "pass", + "mode": "dry_run", + "operation": operation, + "request_id": "iap-abcdefghijkl", + "runner_revision": "a" * 40, + "release_revision": "b" * 40, + "target": { + "project": local.PROJECT, + "zone": local.ZONE, + "instance": local.INSTANCE, + "service": local.SERVICE, + "service_account": local.SERVICE_ACCOUNT, + }, + "release": { + "release_sha256": expected["release_sha256"], + "image_reference": f"{local.package.IMAGE_REPOSITORY}@{expected['image_digest']}", + "image_digest": expected["image_digest"], + "image_config_digest": expected["image_config_digest"], + }, + "release_artifact": { + "archive_sha256": expected["release_artifact_sha256"], + "archive_size": 1, + "members": [], + }, + "expected": expected, + "request": { + "archive": str(archive), + "archive_sha256": archive_sha256, + "helper_sha256": helper_sha256, + "manifest_sha256": "d" * 64, + "ssh_key": str(ssh_key), + }, + "actions": local._action_receipts(commands), + "authority": { + "gcp_mutation": operation in {"install", "rollback"}, + "service_restart": operation in {"install", "rollback"}, + "database_or_secret_provisioning": False, + "production_or_transport": False, + "vps": False, + }, + } + + +def test_gcloud_commands_are_fixed_iap_only_and_expire_keys(tmp_path: Path) -> None: + scp, ssh, cleanup = fixed_commands(tmp_path) + + assert scp[:3] == [local.GCLOUD, "compute", "scp"] + assert ssh[:4] == [local.GCLOUD, "compute", "ssh", local.INSTANCE] + for command in (scp, ssh, cleanup): + assert f"--project={local.PROJECT}" in command + assert f"--zone={local.ZONE}" in command + assert "--tunnel-through-iap" in command + assert "--ssh-key-expire-after=5m" in command + encoded = json.dumps({"scp": scp, "ssh": ssh, "cleanup": cleanup}) + assert local.INSTANCE in encoded + assert local.SERVICE not in encoded + assert "teleo-prod-1" not in encoded + assert "77.42.65.182" not in encoded + assert "telegram" not in encoded.casefold() + assert "sudo -- /usr/bin/python3 -I -c" in ssh[-1] + assert "/tmp/livingip-leoclean-nosend-remote" not in ssh[-1] + + +def test_embedded_root_bootstrap_compiles() -> None: + compile(local.BOOTSTRAP_CODE, "", "exec") + + +def test_live_origin_main_readback_is_exact() -> None: + revision = "a" * 40 + responses = iter( + [ + f"{local.ORIGIN_URL}\n".encode(), + f"{revision}\trefs/heads/main\n".encode(), + ] + ) + + def runner(argv: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + return completed(argv, stdout=next(responses)) + + local.validate_remote_main(revision, runner=runner) + + +@pytest.mark.parametrize( + "responses", + [ + [b"git@github.com:someone-else/teleo-infrastructure.git\n"], + [ + f"{local.ORIGIN_URL}\n".encode(), + f"{'b' * 40}\trefs/heads/main\n".encode(), + ], + ], +) +def test_live_origin_main_readback_rejects_different_authority(responses: list[bytes]) -> None: + output = iter(responses) + + def runner(argv: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + return completed(argv, stdout=next(output)) + + with pytest.raises(local.IapRequestError): + local.validate_remote_main("a" * 40, runner=runner) + + +@pytest.mark.parametrize("value", ["main", "a" * 39, "A" * 40, "a" * 41, "a" * 39 + ";"]) +def test_revision_input_is_exact(value: str, tmp_path: Path) -> None: + archive_name, key_name = local.remote_names("iap-abcdefghijkl") + with pytest.raises(local.IapRequestError, match="revision"): + local.build_gcloud_argv( + operation="install", + request_id="iap-abcdefghijkl", + runner_revision=value, + release_revision="b" * 40, + archive=tmp_path / archive_name, + archive_sha256="b" * 64, + ssh_key=tmp_path / key_name, + helper_sha256="c" * 64, + ) + + +@pytest.mark.parametrize("value", ["iap-short", "iap-abcdefghijkl;id", "../iap-abcdefghijkl", "iap-ABCDEFGHIJKL"]) +def test_request_id_rejects_shell_and_path_fragments(value: str) -> None: + with pytest.raises(local.IapRequestError, match="request ID"): + local.remote_names(value) + + +def test_mutable_release_reference_is_rejected(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + release = { + "image": {"reference": f"{local.package.IMAGE_REPOSITORY}:latest"}, + "image_input": {"runtime": {"teleo_git_head": "a" * 40}}, + } + monkeypatch.setattr(local.installer, "validate_release_bundle", lambda _bundle: (release, b"unit")) + + with pytest.raises(local.IapRequestError, match="immutable"): + local.validate_release( + tmp_path, + "a" * 40, + expected_release_sha256="b" * 64, + expected_image_digest=f"sha256:{'c' * 64}", + expected_image_config_digest=f"sha256:{'d' * 64}", + ) + + +def test_stable_reader_rejects_symlink(tmp_path: Path) -> None: + root = private_directory(tmp_path / "root") + target = root / "target" + target.write_text("safe", encoding="utf-8") + alias = root / "alias" + alias.symlink_to(target) + + with pytest.raises(local.IapRequestError, match="unsafe"): + local._read_stable_file(alias.absolute(), root=root.absolute()) + + +def archive_manifest(payloads: dict[str, tuple[bytes, int]]) -> dict[str, Any]: + files = [ + {"path": path, "sha256": local._sha256_bytes(content), "size": len(content), "mode": mode} + for path, (content, mode) in sorted(payloads.items()) + ] + stable = { + "schema": remote.SCHEMA, + "operation": "verify", + "request_id": "iap-abcdefghijkl", + "runner_revision": "a" * 40, + "release_revision": "b" * 40, + "target": { + "project": remote.PROJECT, + "zone": remote.ZONE, + "instance": remote.INSTANCE, + "service": remote.SERVICE, + "service_account": remote.SERVICE_ACCOUNT, + }, + "release": { + "release_sha256": "b" * 64, + "image_reference": f"{remote.IMAGE_REPOSITORY}@sha256:{'c' * 64}", + "image_digest": f"sha256:{'c' * 64}", + "image_config_digest": f"sha256:{'d' * 64}", + }, + "installer_source": {"revision": "a" * 40, "files": [], "sha256": "e" * 64}, + "release_artifact": {"archive_sha256": "f" * 64, "archive_size": 1, "members": []}, + "files": files, + } + return {**stable, "manifest_sha256": remote._canonical_sha256(stable)} + + +def exact_payloads() -> dict[str, tuple[bytes, int]]: + return {name: (f"payload:{name}\n".encode(), 0o644) for name in remote.EXPECTED_MEMBERS - {"request-manifest.json"}} + + +def test_archive_is_deterministic_and_extracts_exact_members(tmp_path: Path) -> None: + first_root = private_directory(tmp_path / "first") + second_root = private_directory(tmp_path / "second") + extracted = private_directory(tmp_path / "extracted") + payloads = exact_payloads() + manifest = archive_manifest(payloads) + + first_hash = local.build_archive(first_root / "request.tar.gz", manifest=manifest, payloads=payloads) + second_hash = local.build_archive(second_root / "request.tar.gz", manifest=manifest, payloads=payloads) + observed = remote.extract_archive(first_root / "request.tar.gz", extracted) + + assert first_hash == second_hash + assert (first_root / "request.tar.gz").read_bytes() == (second_root / "request.tar.gz").read_bytes() + assert observed == manifest + assert {str(path.relative_to(extracted)) for path in extracted.rglob("*") if path.is_file()} == ( + remote.EXPECTED_MEMBERS - {"request-manifest.json"} + ) + + +def write_unsafe_archive(path: Path, *, kind: str) -> None: + with tarfile.open(path, "w:gz") as archive: + for index, name in enumerate(sorted(remote.EXPECTED_MEMBERS)): + info = tarfile.TarInfo(name) + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + info.mtime = 0 + info.mode = 0o600 + content = b"{}" + if kind == "link" and index == 0: + info.type = tarfile.SYMTYPE + info.linkname = "/etc/passwd" + info.size = 0 + archive.addfile(info) + continue + if kind == "oversize" and index == 0: + content = b"x" * (remote.MAX_MEMBER_BYTES + 1) + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + if kind == "duplicate": + info = tarfile.TarInfo("request-manifest.json") + info.size = 2 + info.mode = 0o600 + info.uid = info.gid = info.mtime = 0 + info.uname = info.gname = "root" + archive.addfile(info, io.BytesIO(b"{}")) + + +@pytest.mark.parametrize("kind", ["link", "oversize", "duplicate"]) +def test_archive_rejects_links_oversize_and_duplicate_members(tmp_path: Path, kind: str) -> None: + path = tmp_path / f"{kind}.tar.gz" + destination = private_directory(tmp_path / kind) + write_unsafe_archive(path, kind=kind) + + with pytest.raises(remote.RemoteError): + remote.extract_archive(path, destination) + + +def test_archive_output_requires_private_directory(tmp_path: Path) -> None: + output = tmp_path / "public" + output.mkdir(mode=0o755) + output.chmod(0o755) + with pytest.raises(local.IapRequestError, match="private"): + local.build_archive(output / "request.tar.gz", manifest={}, payloads={}) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("expected_artifact_sha256", "not-a-hash"), + ("expected_release_sha256", "a" * 63), + ("expected_image_digest", "latest"), + ("expected_image_config_digest", "sha256:" + "A" * 64), + ], +) +def test_expected_release_inputs_are_exact(field: str, value: str) -> None: + arguments = { + "expected_artifact_sha256": "a" * 64, + "expected_release_sha256": "b" * 64, + "expected_image_digest": f"sha256:{'c' * 64}", + "expected_image_config_digest": f"sha256:{'d' * 64}", + } + arguments[field] = value + with pytest.raises(local.IapRequestError, match="expected"): + local._validate_expected_bindings(**arguments) + + +def test_release_and_original_artifact_require_independent_expected_values( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + release = { + "release_sha256": "a" * 64, + "image": { + "reference": f"{local.package.IMAGE_REPOSITORY}@sha256:{'b' * 64}", + "digest": f"sha256:{'b' * 64}", + "config_digest": f"sha256:{'c' * 64}", + }, + "image_input": {"runtime": {"teleo_git_head": "d" * 40}}, + } + monkeypatch.setattr(local.installer, "validate_release_bundle", lambda _bundle: (release, b"unit")) + with pytest.raises(local.IapRequestError, match="independently supplied"): + local.validate_release( + tmp_path, + "d" * 40, + expected_release_sha256="e" * 64, + expected_image_digest=f"sha256:{'b' * 64}", + expected_image_config_digest=f"sha256:{'c' * 64}", + ) + + artifact = tmp_path / "artifact.zip" + artifact.write_bytes(b"not-the-approved-artifact") + artifact.chmod(0o600) + with pytest.raises(local.IapRequestError, match="independently supplied"): + local.validate_release_artifact( + artifact, + bundle=tmp_path, + release=release, + expected_artifact_sha256="f" * 64, + ) + + +def test_dry_run_plan_records_only_bounded_action_hashes(tmp_path: Path) -> None: + plan = executable_plan(tmp_path, "install") + encoded = json.dumps(plan, sort_keys=True) + + assert [item["label"] for item in plan["actions"]] == [ + "upload_request", + "execute_remote_request", + "cleanup_failed_transfer", + ] + assert all(set(item) == {"label", "argv_sha256"} for item in plan["actions"]) + assert local.BOOTSTRAP_CODE not in encoded + assert "--command=" not in encoded + assert "sudo --" not in encoded + + +def test_gcloud_environment_selects_absolute_compatible_python( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + home = private_directory(tmp_path / "home") + config = private_directory(home / "gcloud") + python = tmp_path / "python311" + python.write_text("placeholder", encoding="utf-8") + python.chmod(0o700) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("CLOUDSDK_CONFIG", str(config)) + monkeypatch.setenv("CLOUDSDK_PYTHON", str(python)) + + environment = local.gcloud_environment( + probe_runner=lambda argv, **_kwargs: completed(argv, stdout=b"Python 3.11.9\n") + ) + + assert environment["CLOUDSDK_PYTHON"] == str(python.resolve()) + assert environment["CLOUDSDK_CONFIG"] == str(config.resolve()) + + +def test_local_result_receipt_is_private_and_self_hashed(tmp_path: Path) -> None: + plan = executable_plan(tmp_path, "verify") + result = {"bounded": "remote-result"} + + path, receipt = local._write_result_receipt(plan, result) + stable = {key: value for key, value in receipt.items() if key != "receipt_sha256"} + + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert receipt["receipt_sha256"] == local.package.canonical_sha256(stable) + assert json.loads(path.read_text(encoding="utf-8")) == receipt + + +def test_execute_failure_does_not_forward_command_output(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + plan = executable_plan(tmp_path, "install") + monkeypatch.setattr(local, "validate_remote_main", lambda *_args, **_kwargs: None) + + def runner(argv: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + return completed(argv, returncode=1, stderr=b"password=extremely-secret") + + with pytest.raises(local.IapRequestError) as caught: + local.execute_plan(plan, runner=runner, gcloud_env={"PATH": "/usr/bin"}) + assert "extremely-secret" not in str(caught.value) + assert "password" not in str(caught.value).casefold() + + +def test_execute_rejects_unsanitized_remote_result(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + plan = executable_plan(tmp_path, "verify") + monkeypatch.setattr(local, "validate_remote_main", lambda *_args, **_kwargs: None) + calls = 0 + + def runner(argv: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + nonlocal calls + calls += 1 + if calls == 1: + return completed(argv) + payload = { + "schema": local.REMOTE_RESULT_SCHEMA, + "status": "pass", + "operation": "verify", + "request_id": "iap-abcdefghijkl", + "access_token": "not-allowed", + } + return completed(argv, stdout=json.dumps(payload).encode()) + + with pytest.raises(local.IapRequestError) as caught: + local.execute_plan(plan, runner=runner, gcloud_env={"PATH": "/usr/bin"}) + assert "not-allowed" not in str(caught.value) + + +def test_execute_rederives_commands_and_rejects_plan_tampering(tmp_path: Path) -> None: + plan = executable_plan(tmp_path, "verify") + plan["actions"][1]["argv_sha256"] = "0" * 64 + + with pytest.raises(local.IapRequestError, match="fixed IAP plan"): + local.execute_plan(plan, runner=lambda argv, **_kwargs: completed(argv)) + + +@pytest.mark.parametrize("fail_pull", [False, True]) +def test_ephemeral_registry_credentials_are_always_removed(tmp_path: Path, fail_pull: bool) -> None: + calls: list[tuple[list[str], bytes | None]] = [] + + def runner(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[bytes]: + input_bytes = kwargs.get("input") + calls.append((argv, input_bytes)) + if argv[0] == "/usr/bin/curl": + return completed(argv, stdout=b'{"access_token":"abcdefghijklmnop","token_type":"Bearer"}') + if "pull" in argv and fail_pull: + return completed(argv, returncode=1, stderr=b"registry said secret") + return completed(argv) + + reference = f"{remote.IMAGE_REPOSITORY}@sha256:{'a' * 64}" + if fail_pull: + with pytest.raises(remote.RemoteError, match="immutable image pull"): + remote.pull_immutable_image(reference, tmp_path, runner=runner) + else: + remote.pull_immutable_image(reference, tmp_path, runner=runner) + assert not any(path.name.startswith("docker-auth-") for path in tmp_path.iterdir()) + assert all("abcdefghijklmnop" not in " ".join(argv) for argv, _input in calls) + assert any(input_bytes == b"abcdefghijklmnop\n" for _argv, input_bytes in calls) + assert any("logout" in argv for argv, _input in calls) + + +def test_metadata_target_binds_project_zone_instance_and_service_account() -> None: + values = { + "instance/service-accounts/default/email": remote.SERVICE_ACCOUNT, + "project/project-id": remote.PROJECT, + "instance/name": remote.INSTANCE, + "instance/zone": f"projects/123/zones/{remote.ZONE}", + } + + def runner(argv: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + path = argv[-1].removeprefix(f"{remote.METADATA}/") + return completed(argv, stdout=values[path].encode()) + + remote.verify_service_account(runner=runner) + + values["instance/name"] = "wrong-instance" + with pytest.raises(remote.RemoteError, match="target or service account"): + remote.verify_service_account(runner=runner) + + +def test_outer_lock_and_reused_request_id_fail_closed(tmp_path: Path) -> None: + lock = tmp_path / "runner.lock" + first = remote._acquire_lock(lock, expected_uid=os.geteuid()) + try: + with pytest.raises(remote.RemoteError, match="another request"): + remote._acquire_lock(lock, expected_uid=os.geteuid()) + finally: + os.close(first) + + root = tmp_path / "requests" + claimed = remote._claim_request_directory( + root, + "iap-abcdefghijkl", + expected_uid=os.geteuid(), + trusted_parent=tmp_path, + ) + assert stat.S_IMODE(claimed.stat().st_mode) == 0o700 + with pytest.raises(remote.RemoteError, match="already used"): + remote._claim_request_directory( + root, + "iap-abcdefghijkl", + expected_uid=os.geteuid(), + trusted_parent=tmp_path, + ) + + +def test_rollback_uses_only_explicit_receipt_bound_interface(tmp_path: Path) -> None: + observed: dict[str, Any] = {} + + def rollback(release_sha256: str, **kwargs: Any) -> dict[str, Any]: + observed.update({"release_sha256": release_sha256, **kwargs}) + return {"schema": "rollback", "status": "pass"} + + installer = SimpleNamespace(rollback_release=rollback) + result = remote._execute_explicit_rollback( + installer_module=installer, + release_sha256="a" * 64, + paths=object(), + source_binding={"revision": "b" * 40}, + runner=lambda argv, **_kwargs: completed(argv), + ) + + assert result["status"] == "pass" + assert observed["release_sha256"] == "a" * 64 + assert observed["execute_rollback"] is True + assert "install_release" not in observed + + +def test_rollback_fails_when_explicit_interface_is_absent(tmp_path: Path) -> None: + with pytest.raises(remote.RemoteError, match="lacked the explicit rollback"): + remote._execute_explicit_rollback( + installer_module=SimpleNamespace(), + release_sha256="a" * 64, + paths=object(), + source_binding={}, + runner=lambda argv, **_kwargs: completed(argv), + ) + + +@pytest.mark.parametrize("failure_point", ["first_database", "restart", "second_database"]) +def test_post_install_proof_failure_executes_explicit_rollback( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + failure_point: str, +) -> None: + release = { + "release_sha256": "a" * 64, + "image": { + "reference": f"{remote.IMAGE_REPOSITORY}@sha256:{'b' * 64}", + "digest": f"sha256:{'b' * 64}", + "config_digest": f"sha256:{'c' * 64}", + }, + "image_input": {"runtime": {"teleo_git_head": "d" * 40}}, + } + manifest = { + "operation": "install", + "request_id": "iap-abcdefghijkl", + "runner_revision": "e" * 40, + "release_revision": "d" * 40, + "manifest_sha256": "f" * 64, + "release": { + "release_sha256": release["release_sha256"], + "image_reference": release["image"]["reference"], + "image_digest": release["image"]["digest"], + "image_config_digest": release["image"]["config_digest"], + }, + "release_artifact": {"archive_sha256": "1" * 64}, + "installer_source": {"revision": "e" * 40}, + } + verifications = iter( + [ + { + "main_pid": 10, + "invocation_id": "2" * 32, + "container_id": "3" * 64, + "n_restarts": 0, + }, + { + "main_pid": 11, + "invocation_id": "4" * 32, + "container_id": "5" * 64, + "n_restarts": 0, + }, + ] + ) + rollbacks: list[str] = [] + + class Paths: + @staticmethod + def under(_root: Path) -> object: + return object() + + installer_module = SimpleNamespace( + SYSTEMCTL="/usr/bin/systemctl", + InstallPaths=Paths, + validate_release_bundle=lambda _bundle: (release, b"unit"), + _validate_source_binding=lambda _binding: None, + install_release=lambda *_args, **_kwargs: {"status": "pass"}, + verify_running_release=lambda *_args, **_kwargs: next(verifications), + rollback_release=lambda release_sha256, **_kwargs: ( + rollbacks.append(release_sha256) + or { + "schema": "livingip.leocleanNoSendRollbackResult.v1", + "status": "pass", + "mode": "executed", + "rolled_back_release_sha256": release_sha256, + "restored_release_sha256": None, + "restored_service_posture": "absent", + "verification": None, + } + ), + ) + package_module = SimpleNamespace( + render_systemd_unit=lambda _release: "unit", + canonical_sha256=local.package.canonical_sha256, + ) + monkeypatch.setattr(remote, "validate_original_release_artifact", lambda *_args, **_kwargs: None) + monkeypatch.setattr(remote, "_load_contract_modules", lambda _root: (package_module, installer_module)) + monkeypatch.setattr(remote, "pull_immutable_image", lambda *_args, **_kwargs: None) + database_calls = 0 + + def verify_database(**_kwargs: Any) -> dict[str, Any]: + nonlocal database_calls + database_calls += 1 + if failure_point == "first_database" and database_calls == 1: + raise remote.RemoteError("database failed") + if failure_point == "second_database" and database_calls == 2: + raise remote.RemoteError("database failed") + return {"status": "pass"} + + monkeypatch.setattr(remote, "_verify_database", verify_database) + + def runner(argv: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + if failure_point == "restart" and argv[:2] == ["/usr/bin/systemctl", "restart"]: + return completed(argv, returncode=1) + return completed(argv) + + with pytest.raises(remote.RemoteError, match="install_failed_rolled_back"): + remote.execute_request( + manifest, + tmp_path, + tmp_path, + "6" * 64, + "7" * 64, + runner=runner, + ) + assert rollbacks == [release["release_sha256"]] + + +@pytest.mark.parametrize( + ("posture", "restored_release", "verification"), + [ + ( + "active", + "e" * 64, + { + "active_state": "active", + "sub_state": "running", + "container_id": "f" * 64, + }, + ), + ("inactive", "e" * 64, None), + ("absent", None, None), + ], +) +def test_rollback_result_accepts_exact_recorded_postures( + posture: str, + restored_release: str | None, + verification: dict[str, Any] | None, +) -> None: + manifest = { + "request_id": "iap-abcdefghijkl", + "runner_revision": "a" * 40, + "release_revision": "d" * 40, + "manifest_sha256": "1" * 64, + "release_artifact": {"archive_sha256": "2" * 64}, + } + release = { + "release_sha256": "b" * 64, + "image": {"digest": f"sha256:{'c' * 64}", "config_digest": f"sha256:{'d' * 64}"}, + } + rollback = { + "schema": "livingip.leocleanNoSendRollbackResult.v1", + "status": "pass", + "mode": "executed", + "rolled_back_release_sha256": "b" * 64, + "restored_release_sha256": restored_release, + "restored_service_posture": posture, + "verification": verification, + } + + result = remote._rollback_result( + manifest=manifest, + release=release, + rollback=rollback, + archive_sha256="3" * 64, + helper_sha256="4" * 64, + ) + + assert result["service"]["restored_posture"] == posture + assert result["database"] == {"status": "not_run_after_explicit_rollback"} + + +def test_absent_rollback_rejects_invented_database_or_runtime_verification() -> None: + manifest = { + "request_id": "iap-abcdefghijkl", + "runner_revision": "a" * 40, + "release_revision": "d" * 40, + "manifest_sha256": "1" * 64, + "release_artifact": {"archive_sha256": "2" * 64}, + } + release = { + "release_sha256": "b" * 64, + "image": {"digest": f"sha256:{'c' * 64}", "config_digest": f"sha256:{'d' * 64}"}, + } + rollback = { + "schema": "livingip.leocleanNoSendRollbackResult.v1", + "status": "pass", + "mode": "executed", + "rolled_back_release_sha256": "b" * 64, + "restored_release_sha256": None, + "restored_service_posture": "absent", + "verification": {"active_state": "active"}, + } + with pytest.raises(remote.RemoteError, match="absent rollback"): + remote._rollback_result( + manifest=manifest, + release=release, + rollback=rollback, + archive_sha256="3" * 64, + helper_sha256="4" * 64, + ) diff --git a/tests/test_gcp_leoclean_nosend_installer.py b/tests/test_gcp_leoclean_nosend_installer.py index 28fa409..ce43976 100644 --- a/tests/test_gcp_leoclean_nosend_installer.py +++ b/tests/test_gcp_leoclean_nosend_installer.py @@ -337,6 +337,20 @@ def test_alternate_repo_root_cannot_bind_the_executing_installer(tmp_path: Path) installer.parse_args(["--bundle", str(tmp_path), "--repo-root", str(tmp_path)]) +def test_explicit_rollback_cli_is_dry_run_unless_execute_flag_is_separate() -> None: + release_sha256 = "a" * 64 + + dry_run = installer.parse_args(["--rollback-release", release_sha256]) + execute = installer.parse_args(["--rollback-release", release_sha256, "--execute-rollback"]) + + assert dry_run.execute_rollback is False + assert execute.execute_rollback is True + with pytest.raises(installer.InstallError, match="invalid arguments"): + installer.parse_args(["--execute-rollback"]) + with pytest.raises(installer.InstallError, match="invalid arguments"): + installer.parse_args(["--bundle", "/tmp/release", "--rollback-release", release_sha256]) + + def test_rehashed_alternate_source_bytes_cannot_bypass_actual_source_binding(tmp_path: Path) -> None: bundle, _release, _image = release_bundle(tmp_path, "2") paths = install_paths(tmp_path) @@ -372,6 +386,7 @@ def test_disposable_root_install_is_digest_bound_idempotent_and_exact(tmp_path: ) assert receipt["status"] == "pass" + assert receipt["schema"] == "livingip.leocleanNoSendInstallReceipt.v2" assert receipt["release_sha256"] == release["release_sha256"] assert host.restart_calls == 1 assert paths.unit.read_text(encoding="utf-8") == package.render_systemd_unit(release) @@ -379,6 +394,11 @@ def test_disposable_root_install_is_digest_bound_idempotent_and_exact(tmp_path: assert paths.helper.stat().st_mode & 0o777 == 0o755 assert paths.release.stat().st_mode & 0o777 == 0o600 assert paths.receipt.stat().st_mode & 0o777 == 0o600 + assert paths.rollback_state.stat().st_mode & 0o777 == 0o600 + rollback_state = json.loads(paths.rollback_state.read_text(encoding="utf-8")) + assert rollback_state["schema"] == installer.ROLLBACK_STATE_SCHEMA + assert rollback_state["current_release_sha256"] == release["release_sha256"] + assert receipt["rollback"]["state_sha256"] == package.sha256_file(paths.rollback_state) assert not list(paths.docker_config.iterdir()) assert not list(paths.unit.parent.glob(f".{paths.unit.name}.*.tmp")) @@ -962,6 +982,331 @@ def test_effective_process_environment_override_is_rejected_and_rolled_back(tmp_ assert not paths.state_dir.exists() +def test_explicit_rollback_is_dry_run_by_default_and_removes_first_install(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + before_calls = list(host.calls) + before_stops = host.stop_calls + before_restarts = host.restart_calls + + plan = installer.rollback_release( + str(release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + ) + + assert plan["mode"] == "dry_run" + assert plan["installed_release_sha256"] == release["release_sha256"] + assert plan["target_release_sha256"] is None + assert plan["target_service_posture"] == "absent" + assert host.stop_calls == before_stops + assert host.restart_calls == before_restarts + assert host.calls != before_calls + assert host.active + + with pytest.raises(installer.InstallError, match="did not match the expected release hash"): + installer.rollback_release( + "f" * 64, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_rollback=True, + ) + assert host.stop_calls == before_stops + + result = installer.rollback_release( + str(release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_rollback=True, + ) + + assert result == { + "schema": installer.ROLLBACK_RESULT_SCHEMA, + "status": "pass", + "mode": "executed", + "rolled_back_release_sha256": release["release_sha256"], + "restored_release_sha256": None, + "restored_service_posture": "absent", + "verification": None, + "authority": { + "image_pulled": False, + "cloud_or_database_authority_changed": False, + "transport_authority_changed": False, + }, + } + assert not host.active + assert host.loaded_release is None + assert not paths.unit.exists() + assert not paths.helper.exists() + assert not paths.state_dir.exists() + + +def test_explicit_rollback_restores_exact_previous_active_release_and_history(tmp_path: Path) -> None: + first_bundle, first_release, first_image = release_bundle(tmp_path, "2") + second_bundle, second_release, second_image = release_bundle(tmp_path, "4") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(first_release, first_image) + host.add_release(second_release, second_image) + installer.install_release( + first_bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + first_files = {role: path.read_bytes() for role, (path, _mode) in installer._managed_files(paths).items()} + installer.install_release( + second_bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + + result = installer.rollback_release( + str(second_release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_rollback=True, + ) + + assert result["restored_release_sha256"] == first_release["release_sha256"] + assert result["restored_service_posture"] == "active" + assert host.current_release == first_release + assert {role: path.read_bytes() for role, (path, _mode) in installer._managed_files(paths).items()} == first_files + + nested = installer.rollback_release( + str(first_release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_rollback=True, + ) + assert nested["restored_service_posture"] == "absent" + assert not paths.state_dir.exists() + + +def test_explicit_rollback_restores_previous_inactive_posture(tmp_path: Path) -> None: + first_bundle, first_release, first_image = release_bundle(tmp_path, "2") + second_bundle, second_release, second_image = release_bundle(tmp_path, "4") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(first_release, first_image) + host.add_release(second_release, second_image) + installer.install_release( + first_bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + host._stop() + installer.install_release( + second_bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + restarts = host.restart_calls + + result = installer.rollback_release( + str(second_release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_rollback=True, + ) + + assert result["restored_release_sha256"] == first_release["release_sha256"] + assert result["restored_service_posture"] == "inactive" + assert not host.active + assert host.loaded_release == first_release + assert host.restart_calls == restarts + + +def test_failed_explicit_rollback_recovers_current_release(tmp_path: Path) -> None: + first_bundle, first_release, first_image = release_bundle(tmp_path, "2") + second_bundle, second_release, second_image = release_bundle(tmp_path, "4") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(first_release, first_image) + host.add_release(second_release, second_image) + installer.install_release( + first_bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + installer.install_release( + second_bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + second_files = {role: path.read_bytes() for role, (path, _mode) in installer._managed_files(paths).items()} + fail_next_reload = True + + def runner(arguments: list[str]) -> subprocess.CompletedProcess[bytes]: + nonlocal fail_next_reload + if arguments[:2] == [installer.SYSTEMCTL, "daemon-reload"] and fail_next_reload: + fail_next_reload = False + return subprocess.CompletedProcess(arguments, 1, stdout=b"", stderr=b"injected") + return host(arguments) + + with pytest.raises(installer.InstallError, match="systemd daemon-reload failed"): + installer.rollback_release( + str(second_release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=runner, + sleeper=lambda _seconds: None, + execute_rollback=True, + ) + + assert host.active + assert host.current_release == second_release + assert {role: path.read_bytes() for role, (path, _mode) in installer._managed_files(paths).items()} == second_files + + +def test_explicit_rollback_rejects_receipt_drift_before_service_stop(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + receipt = json.loads(paths.receipt.read_text(encoding="utf-8")) + receipt["release_sha256"] = "f" * 64 + paths.receipt.write_text(json.dumps(receipt), encoding="utf-8") + paths.receipt.chmod(0o600) + stop_calls = host.stop_calls + + with pytest.raises(installer.InstallError, match=r"(installed|snapshot install) receipt was invalid"): + installer.rollback_release( + str(release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_rollback=True, + ) + + assert host.stop_calls == stop_calls + assert host.active + + +def test_explicit_rollback_rejects_path_inode_drift_before_service_stop(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + prepared = installer._prepare_explicit_rollback( + str(release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + ) + unit = paths.unit.read_bytes() + paths.unit.unlink() + paths.unit.write_bytes(unit) + paths.unit.chmod(0o644) + stop_calls = host.stop_calls + + with pytest.raises(installer.InstallError, match="file identity drifted"): + installer._execute_explicit_rollback( + str(release["release_sha256"]), + prepared, + paths=paths, + runner=host, + sleeper=lambda _seconds: None, + ) + + assert host.stop_calls == stop_calls + assert host.active + + +def test_explicit_rollback_rejects_pid_drift_before_service_stop(tmp_path: Path) -> None: + bundle, release, image = release_bundle(tmp_path, "2") + paths = install_paths(tmp_path) + host = FakeHost(paths) + host.add_release(release, image) + installer.install_release( + bundle, + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + execute_restart=True, + ) + prepared = installer._prepare_explicit_rollback( + str(release["release_sha256"]), + paths=paths, + source_binding=source_binding(), + runner=host, + sleeper=lambda _seconds: None, + ) + host.main_pid += 1 + host._write_process_environment() + stop_calls = host.stop_calls + + with pytest.raises(installer.InstallError, match=r"changed during verification|drifted before rollback stop"): + installer._execute_explicit_rollback( + str(release["release_sha256"]), + prepared, + paths=paths, + runner=host, + sleeper=lambda _seconds: None, + ) + + assert host.stop_calls == stop_calls + assert host.active + + def test_control_helper_stops_only_trusted_full_container_id(tmp_path: Path) -> None: cid_parent = tmp_path / "runtime" cid_parent.mkdir(mode=0o700)