#!/usr/bin/env python3 """Build and optionally dispatch one fixed-target leoclean no-send IAP request.""" from __future__ import annotations import argparse import gzip import hashlib import io import json import os import re import shlex import stat import subprocess import sys import tarfile import tempfile import textwrap import zipfile from collections.abc import Callable, Mapping from pathlib import Path from typing import Any CONTRACT_ROOT = Path(__file__).resolve().parents[1] if str(CONTRACT_ROOT) not in sys.path: sys.path.insert(0, str(CONTRACT_ROOT)) from ops import gcp_leoclean_nosend_package as package # noqa: E402 from ops import install_gcp_leoclean_nosend_release as installer # noqa: E402 SCHEMA = "livingip.leocleanNoSendIapRequest.v1" PLAN_SCHEMA = "livingip.leocleanNoSendIapPlan.v1" REMOTE_RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v1" RESULT_RECEIPT_SCHEMA = "livingip.leocleanNoSendIapResultReceipt.v1" EXECUTION_SCHEMA = "livingip.leocleanNoSendIapExecution.v1" PROJECT = package.PROJECT ZONE = package.ZONE INSTANCE = package.INSTANCE SERVICE = package.SERVICE SERVICE_ACCOUNT = package.SERVICE_ACCOUNT GCLOUD = "/opt/homebrew/bin/gcloud" ORIGIN_URL = "git@github.com:living-ip/teleo-infrastructure.git" REMOTE_UPLOAD_ROOT = "/tmp" REMOTE_HELPER_SOURCE = "ops/gcp_leoclean_nosend_iap_remote.py" KEY_LIFETIME = "5m" MAX_ARCHIVE_BYTES = 8 * 1024 * 1024 MAX_MEMBER_BYTES = 2 * 1024 * 1024 MAX_COMMAND_OUTPUT_BYTES = 256 * 1024 REQUEST_ID = re.compile(r"^iap-[a-z0-9]{12,32}$") HEX_40 = re.compile(r"^[0-9a-f]{40}$") HEX_64 = re.compile(r"^[0-9a-f]{64}$") OPERATIONS = ("plan", "install", "verify", "rollback") SOURCE_MEMBERS = ( "ops/gcp_leoclean_nosend_package.py", "ops/install_gcp_leoclean_nosend_release.py", "ops/leoclean_nosend_service_control.py", REMOTE_HELPER_SOURCE, "ops/run_gcp_leoclean_nosend_iap.py", "scripts/leo_behavior_manifest.py", "scripts/leo_identity_manifest.py", ) RELEASE_ARTIFACT_MEMBERS = ( "evidence/oci-smoke.json", "evidence/publish-outcome.json", "evidence/runtime-receipt.json", f"final/release-v3/{SERVICE}", "final/release-v3/release.json", "receipts/build-push-receipt.json", ) FORBIDDEN_OUTPUT_MARKERS = ( "authorization:", "access_token", "password", "private_key", "telegram", "77.42.65.182", "teleo-prod-1", ) BOOTSTRAP_CODE = textwrap.dedent( r""" import fcntl, hashlib, io, json, os, pwd, re, stat, sys, tarfile mode, operation, request_id, runner_revision, release_revision, archive_hash, helper_hash = sys.argv[1:] fail = lambda: (_ for _ in ()).throw(RuntimeError("bootstrap_failed")) if mode not in {"start", "cleanup"} or operation not in {"install", "verify", "rollback"}: fail() if not re.fullmatch(r"iap-[a-z0-9]{12,32}", request_id): fail() if not re.fullmatch(r"[0-9a-f]{40}", runner_revision) or not re.fullmatch(r"[0-9a-f]{40}", release_revision): fail() if not re.fullmatch(r"[0-9a-f]{64}", archive_hash) or not re.fullmatch(r"[0-9a-f]{64}", helper_hash): fail() caller = os.environ.get("SUDO_USER", "") if not caller or caller == "root": fail() caller_uid = pwd.getpwnam(caller).pw_uid upload = f"/tmp/livingip-leoclean-nosend-{request_id}.tar.gz" stage_root = "/run/livingip-leoclean-nosend-iap-bootstrap" stage = f"{stage_root}/{request_id}" lock_path = "/run/leoclean-gcp-nosend-iap-bootstrap.lock" def stable(path, uid, expected, limit): fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)) try: before = os.fstat(fd); data = b"" if not stat.S_ISREG(before.st_mode) or before.st_uid != uid or stat.S_IMODE(before.st_mode) != 0o600 or before.st_nlink != 1 or before.st_size > limit: fail() while len(data) <= limit: chunk = os.read(fd, min(65536, limit + 1 - len(data))) if not chunk: break data += chunk after = os.fstat(fd); named = os.stat(path, follow_symlinks=False) if len(data) != after.st_size or (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns) != (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns): fail() if (after.st_dev, after.st_ino, after.st_mode, after.st_nlink) != (named.st_dev, named.st_ino, named.st_mode, named.st_nlink): fail() if hashlib.sha256(data).hexdigest() != expected: fail() return data, (after.st_dev, after.st_ino) finally: os.close(fd) def unlink_exact(path, identity): observed = os.stat(path, follow_symlinks=False) if not stat.S_ISREG(observed.st_mode) or (observed.st_dev, observed.st_ino) != identity: fail() os.unlink(path) def safe_identity(path, uid, limit): fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)) try: observed = os.fstat(fd); named = os.stat(path, follow_symlinks=False) if not stat.S_ISREG(observed.st_mode) or observed.st_uid != uid or stat.S_IMODE(observed.st_mode) != 0o600 or observed.st_nlink != 1 or observed.st_size > limit: fail() if (observed.st_dev, observed.st_ino, observed.st_mode, observed.st_nlink) != (named.st_dev, named.st_ino, named.st_mode, named.st_nlink): fail() return (observed.st_dev, observed.st_ino) finally: os.close(fd) def write_exact(path, data): fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600) with os.fdopen(fd, "wb") as handle: handle.write(data); handle.flush(); os.fsync(handle.fileno()) lock_fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) try: lock_state = os.fstat(lock_fd) if lock_state.st_uid != 0 or stat.S_IMODE(lock_state.st_mode) != 0o600 or not stat.S_ISREG(lock_state.st_mode): fail() fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) if mode == "cleanup": try: identity = safe_identity(upload, caller_uid, 8 * 1024 * 1024) unlink_exact(upload, identity) except FileNotFoundError: pass if os.path.isdir(stage) and not os.path.islink(stage): state = os.stat(stage, follow_symlinks=False) if state.st_uid != 0 or stat.S_IMODE(state.st_mode) != 0o700: fail() for name in os.listdir(stage): path = f"{stage}/{name}" expected = archive_hash if name == "request.tar.gz" else helper_hash if name == "remote-helper.py" else None if expected is None: fail() _, identity = stable(path, 0, expected, 8 * 1024 * 1024) unlink_exact(path, identity) os.rmdir(stage) print(json.dumps({"schema":"livingip.leocleanNoSendIapCleanup.v1","status":"pass","request_id":request_id}, sort_keys=True)) raise SystemExit(0) archive, upload_identity = stable(upload, caller_uid, archive_hash, 8 * 1024 * 1024) if not os.path.isdir(stage_root): os.mkdir(stage_root, 0o700); os.chown(stage_root, 0, 0) root_state = os.stat(stage_root, follow_symlinks=False) if root_state.st_uid != 0 or stat.S_IMODE(root_state.st_mode) != 0o700 or not stat.S_ISDIR(root_state.st_mode): fail() os.mkdir(stage, 0o700); os.chown(stage, 0, 0) write_exact(f"{stage}/request.tar.gz", archive) with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as request: matches = [member for member in request.getmembers() if member.name == "ops/gcp_leoclean_nosend_iap_remote.py"] if len(matches) != 1 or not matches[0].isfile() or matches[0].size > 2 * 1024 * 1024: fail() stream = request.extractfile(matches[0]); helper = stream.read() if stream else b"" if hashlib.sha256(helper).hexdigest() != helper_hash: fail() write_exact(f"{stage}/remote-helper.py", helper) unlink_exact(upload, upload_identity) os.execve("/usr/bin/python3", ["/usr/bin/python3", "-I", f"{stage}/remote-helper.py", operation, request_id, runner_revision, release_revision, archive_hash, helper_hash], {"PATH":"/usr/bin:/bin:/usr/sbin:/sbin", "SUDO_USER":caller}) finally: os.close(lock_fd) """ ).strip() Runner = Callable[..., subprocess.CompletedProcess[bytes]] class IapRequestError(RuntimeError): """The request could not be proven safe enough to dispatch.""" def _require(condition: bool, message: str) -> None: if not condition: raise IapRequestError(message) def _sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() def _canonical_json(value: Mapping[str, Any]) -> bytes: try: return (json.dumps(value, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8") except (TypeError, ValueError) as exc: raise IapRequestError("request manifest could not be encoded") from exc def _strict_json(raw: bytes, label: str) -> dict[str, Any]: def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: value: dict[str, Any] = {} for key, item in pairs: if key in value: raise ValueError("duplicate key") value[key] = item return value try: value = json.loads( raw.decode("utf-8", "strict"), object_pairs_hook=reject_duplicates, parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("constant")), ) except (UnicodeError, ValueError, json.JSONDecodeError) as exc: raise IapRequestError(f"{label} was invalid JSON") from exc _require(isinstance(value, dict), f"{label} was not a JSON object") return value def _run_bounded( argv: list[str], *, runner: Runner, label: str, input_bytes: bytes | None = None, environment: Mapping[str, str] | None = None, ) -> subprocess.CompletedProcess[bytes]: try: completed = runner( argv, check=False, stdin=subprocess.DEVNULL if input_bytes is None else None, input=input_bytes, capture_output=True, timeout=300, env=dict(environment or {"PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin"}), ) except (OSError, subprocess.SubprocessError) as exc: raise IapRequestError(f"{label} failed") from exc _require( len(completed.stdout) <= MAX_COMMAND_OUTPUT_BYTES and len(completed.stderr) <= MAX_COMMAND_OUTPUT_BYTES, f"{label} output exceeded the safety limit", ) _require(completed.returncode == 0, f"{label} failed") return completed def _validate_owned_directory(path: Path, label: str) -> Path: _require(path.is_absolute() and not path.is_symlink(), f"{label} was unsafe") resolved = path.resolve() observed = resolved.stat(follow_symlinks=False) _require( stat.S_ISDIR(observed.st_mode) and observed.st_uid == os.geteuid() and observed.st_mode & 0o022 == 0, f"{label} was unsafe", ) return resolved def gcloud_environment(*, probe_runner: Runner = subprocess.run) -> dict[str, str]: home_value = os.environ.get("HOME", "") _require(home_value, "HOME was not configured") home = _validate_owned_directory(Path(home_value), "HOME") config_value = os.environ.get("CLOUDSDK_CONFIG") or str(home / ".config/gcloud") config = _validate_owned_directory(Path(config_value), "Cloud SDK configuration") candidates = [os.environ.get("CLOUDSDK_PYTHON"), str(CONTRACT_ROOT / ".venv/bin/python3")] candidates.append(sys.executable) selected: Path | None = None for value in candidates: if not value or not Path(value).is_absolute(): continue try: candidate = Path(value).resolve(strict=True) observed = candidate.stat(follow_symlinks=False) except OSError: continue if not stat.S_ISREG(observed.st_mode) or not os.access(candidate, os.X_OK) or observed.st_mode & 0o022: continue try: probe = probe_runner( [str(candidate), "--version"], check=False, stdin=subprocess.DEVNULL, capture_output=True, timeout=10, env={"PATH": "/usr/bin:/bin"}, ) except (OSError, subprocess.SubprocessError): continue output = (probe.stdout + probe.stderr).decode("ascii", "ignore") match = re.search(r"Python ([0-9]+)\.([0-9]+)(?:\.[0-9]+)?", output) if probe.returncode == 0 and match and (int(match.group(1)), int(match.group(2))) >= (3, 10): selected = candidate break _require(selected is not None, "no compatible absolute Cloud SDK Python was available") return { "PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin", "HOME": str(home), "CLOUDSDK_CONFIG": str(config), "CLOUDSDK_PYTHON": str(selected), } def _git_output(arguments: list[str], *, runner: Runner) -> bytes: completed = _run_bounded(["/usr/bin/git", "-C", str(CONTRACT_ROOT), *arguments], runner=runner, label="Git check") return completed.stdout def validate_clean_revision(runner_revision: str, *, runner: Runner = subprocess.run) -> None: _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision must be one full lowercase Git commit") top = _git_output(["rev-parse", "--show-toplevel"], runner=runner).decode("utf-8", "strict").strip() head = _git_output(["rev-parse", "--verify", "HEAD^{commit}"], runner=runner).decode("ascii", "strict").strip() dirty = _git_output(["status", "--porcelain=v1", "-z", "--untracked-files=all"], runner=runner) origin_main = ( _git_output( ["rev-parse", "--verify", "refs/remotes/origin/main^{commit}"], runner=runner, ) .decode("ascii", "strict") .strip() ) _require(Path(top).resolve() == CONTRACT_ROOT, "source checkout was not the contract repository") _require(head == runner_revision, "source checkout differed from the requested runner revision") _require(origin_main == runner_revision, "runner revision was not exact local origin/main") _require(not dirty, "source checkout was dirty") def validate_remote_main(runner_revision: str, *, runner: Runner = subprocess.run) -> None: _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid") origin = _git_output(["remote", "get-url", "origin"], runner=runner).decode("utf-8", "strict").strip() _require(origin == ORIGIN_URL, "origin remote was not the fixed LivingIP SSH repository") readback = _git_output(["ls-remote", "--exit-code", "origin", "refs/heads/main"], runner=runner) try: line = readback.decode("ascii", "strict").strip() except UnicodeError as exc: raise IapRequestError("remote main readback was invalid") from exc _require(line == f"{runner_revision}\trefs/heads/main", "runner revision was not exact live origin/main") def _read_stable_file(path: Path, *, root: Path, max_bytes: int = MAX_MEMBER_BYTES) -> tuple[bytes, int]: _require(path.is_absolute(), "request member path was not absolute") try: relative = path.relative_to(root) except ValueError as exc: raise IapRequestError("request member escaped its fixed root") from exc _require(not any(part in {"", ".", ".."} for part in relative.parts), "request member path was unsafe") flags = os.O_RDONLY if hasattr(os, "O_CLOEXEC"): flags |= os.O_CLOEXEC if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: descriptor = os.open(path, flags) except OSError as exc: raise IapRequestError("request member was missing or unsafe") from exc try: before = os.fstat(descriptor) _require( stat.S_ISREG(before.st_mode) and before.st_nlink == 1 and before.st_size <= max_bytes, "request member was not a bounded regular file", ) content = b"" while len(content) <= max_bytes: chunk = os.read(descriptor, min(65536, max_bytes + 1 - len(content))) if not chunk: break content += chunk after = os.fstat(descriptor) named = path.stat(follow_symlinks=False) identity = (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns) _require( len(content) == after.st_size and identity == (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns) and (after.st_dev, after.st_ino, after.st_mode, after.st_nlink) == (named.st_dev, named.st_ino, named.st_mode, named.st_nlink), "request member changed while it was read", ) return content, stat.S_IMODE(after.st_mode) except OSError as exc: raise IapRequestError("request member could not be read") from exc finally: os.close(descriptor) def _validate_expected_bindings( *, expected_artifact_sha256: str, expected_release_sha256: str, expected_image_digest: str, expected_image_config_digest: str, ) -> dict[str, str]: _require(HEX_64.fullmatch(expected_artifact_sha256) is not None, "expected artifact hash was invalid") _require(HEX_64.fullmatch(expected_release_sha256) is not None, "expected release hash was invalid") _require( re.fullmatch(r"sha256:[0-9a-f]{64}", expected_image_digest) is not None, "expected image manifest digest was invalid", ) _require( re.fullmatch(r"sha256:[0-9a-f]{64}", expected_image_config_digest) is not None, "expected image config digest was invalid", ) return { "release_artifact_sha256": expected_artifact_sha256, "release_sha256": expected_release_sha256, "image_digest": expected_image_digest, "image_config_digest": expected_image_config_digest, } def validate_release( bundle: Path, release_revision: str, *, expected_release_sha256: str, expected_image_digest: str, expected_image_config_digest: str, ) -> tuple[dict[str, Any], bytes]: try: release, unit = installer.validate_release_bundle(bundle) except (installer.InstallError, package.PackageError) as exc: raise IapRequestError("finalized release bundle did not validate") from exc image = release.get("image") runtime = release.get("image_input", {}).get("runtime") _require( isinstance(image, dict) and package.IMAGE_REFERENCE.fullmatch(str(image.get("reference"))), "image was not immutable", ) _require( isinstance(runtime, dict) and runtime.get("teleo_git_head") == release_revision, "release differed from release revision", ) _require( release.get("release_sha256") == expected_release_sha256 and image.get("digest") == expected_image_digest and image.get("config_digest") == expected_image_config_digest and image.get("reference") == f"{package.IMAGE_REPOSITORY}@{expected_image_digest}", "release differed from the independently supplied expected binding", ) return release, unit def validate_release_artifact( artifact: Path, *, bundle: Path, release: Mapping[str, Any], expected_artifact_sha256: str, ) -> tuple[dict[str, Any], bytes]: raw, mode = _read_stable_file(artifact.absolute(), root=artifact.absolute().parent, max_bytes=MAX_ARCHIVE_BYTES) _require(mode & 0o022 == 0, "release artifact archive was writable by group or other") _require( _sha256_bytes(raw) == expected_artifact_sha256, "release artifact differed from the independently supplied expected hash", ) try: archive = zipfile.ZipFile(io.BytesIO(raw), mode="r") except (OSError, zipfile.BadZipFile) as exc: raise IapRequestError("release artifact archive was invalid") from exc with archive: infos = archive.infolist() names = [info.filename for info in infos] _require( names == list(RELEASE_ARTIFACT_MEMBERS) or sorted(names) == sorted(RELEASE_ARTIFACT_MEMBERS), "release artifact member allowlist was not exact", ) _require(len(names) == len(set(names)), "release artifact contained duplicate members") members: list[dict[str, Any]] = [] payloads: dict[str, bytes] = {} for info in infos: path = Path(info.filename) unix_mode = (info.external_attr >> 16) & 0o177777 _require( info.filename and not info.filename.startswith("/") and all(part not in {"", ".", ".."} for part in path.parts) and not info.is_dir() and info.file_size <= MAX_MEMBER_BYTES and (unix_mode == 0 or stat.S_ISREG(unix_mode)), "release artifact contained an unsafe member", ) content = archive.read(info) _require(len(content) == info.file_size, "release artifact member size differed") payloads[info.filename] = content members.append( { "path": info.filename, "sha256": _sha256_bytes(content), "size": len(content), } ) bundle_root = bundle.resolve() release_bytes, _release_mode = _read_stable_file(bundle_root / "release.json", root=bundle_root) unit_bytes, _unit_mode = _read_stable_file(bundle_root / SERVICE, root=bundle_root) _require( payloads["final/release-v3/release.json"] == release_bytes and payloads[f"final/release-v3/{SERVICE}"] == unit_bytes, "release bundle differed from its original artifact archive", ) build_receipt = _strict_json(payloads["receipts/build-push-receipt.json"], "artifact build/push receipt") publish_outcome = _strict_json(payloads["evidence/publish-outcome.json"], "artifact publish outcome") _require(build_receipt == release["build_push_receipt"], "artifact build/push receipt differed from release") try: package.validate_publish_outcome(publish_outcome, image_input=dict(release["image_input"])) except package.PackageError as exc: raise IapRequestError("artifact publish outcome did not validate") from exc _require(publish_outcome.get("status") == "pass", "artifact publish outcome did not pass") return ( { "archive_sha256": _sha256_bytes(raw), "archive_size": len(raw), "members": sorted(members, key=lambda item: str(item["path"])), }, raw, ) def _member_payloads(bundle: Path, unit: bytes, release_artifact: bytes) -> dict[str, tuple[bytes, int]]: payloads: dict[str, tuple[bytes, int]] = {} for relative in SOURCE_MEMBERS: payloads[relative] = _read_stable_file(CONTRACT_ROOT / relative, root=CONTRACT_ROOT) release_bytes, release_mode = _read_stable_file(bundle.resolve() / "release.json", root=bundle.resolve()) _unit_bytes, unit_mode = _read_stable_file(bundle.resolve() / SERVICE, root=bundle.resolve()) _require(_unit_bytes == unit, "release unit changed after validation") payloads["release/release.json"] = (release_bytes, release_mode) payloads[f"release/{SERVICE}"] = (unit, unit_mode) payloads["release-artifact.zip"] = (release_artifact, 0o600) return payloads def build_manifest( *, operation: str, request_id: str, runner_revision: str, release_revision: str, release: Mapping[str, Any], payloads: Mapping[str, tuple[bytes, int]], source_binding: Mapping[str, Any], release_artifact: Mapping[str, Any], ) -> dict[str, Any]: _require(operation in OPERATIONS, "operation was not allowlisted") _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") files = [ { "path": relative, "sha256": _sha256_bytes(content), "size": len(content), "mode": mode, } for relative, (content, mode) in sorted(payloads.items()) ] stable = { "schema": SCHEMA, "operation": operation, "request_id": request_id, "runner_revision": runner_revision, "release_revision": release_revision, "target": { "project": PROJECT, "zone": ZONE, "instance": INSTANCE, "service": SERVICE, "service_account": SERVICE_ACCOUNT, }, "release": { "release_sha256": release["release_sha256"], "image_reference": release["image"]["reference"], "image_digest": release["image"]["digest"], "image_config_digest": release["image"]["config_digest"], }, "installer_source": dict(source_binding), "release_artifact": dict(release_artifact), "files": files, } return {**stable, "manifest_sha256": package.canonical_sha256(stable)} def _tar_info(name: str, *, size: int, mode: int) -> tarfile.TarInfo: _require(name and not name.startswith("/") and ".." not in Path(name).parts, "archive member path was unsafe") info = tarfile.TarInfo(name) info.size = size info.mode = mode info.uid = 0 info.gid = 0 info.uname = "root" info.gname = "root" info.mtime = 0 info.type = tarfile.REGTYPE return info def build_archive( output: Path, *, manifest: Mapping[str, Any], payloads: Mapping[str, tuple[bytes, int]], ) -> str: _require(output.is_absolute(), "archive output path was not absolute") parent = output.parent parent_state = parent.stat(follow_symlinks=False) _require( stat.S_ISDIR(parent_state.st_mode) and parent_state.st_uid == os.geteuid() and parent_state.st_mode & 0o077 == 0, "archive output directory was not private", ) _require(not output.exists() and not output.is_symlink(), "archive output already existed") manifest_bytes = _canonical_json(manifest) members = {"request-manifest.json": (manifest_bytes, 0o600), **payloads} temporary: Path | None = None try: descriptor, temporary_name = tempfile.mkstemp(prefix=".iap-request-", dir=parent) temporary = Path(temporary_name) os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb", closefd=True) as raw: with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive: for relative, (content, mode) in sorted(members.items()): archive.addfile(_tar_info(relative, size=len(content), mode=mode), io.BytesIO(content)) raw.flush() os.fsync(raw.fileno()) _require(temporary.stat().st_size <= MAX_ARCHIVE_BYTES, "request archive exceeded the safety limit") os.link(temporary, output, follow_symlinks=False) temporary.unlink() temporary = None directory = os.open(parent, os.O_RDONLY) try: os.fsync(directory) finally: os.close(directory) return package.sha256_file(output) except OSError as exc: raise IapRequestError("request archive could not be published") from exc finally: if temporary is not None: try: temporary.unlink() except FileNotFoundError: pass def remote_names(request_id: str) -> tuple[str, str]: _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") return ( f"livingip-leoclean-nosend-{request_id}.tar.gz", f"livingip-leoclean-nosend-{request_id}-ssh-key", ) def build_gcloud_argv( *, operation: str, request_id: str, runner_revision: str, release_revision: str, archive: Path, archive_sha256: str, ssh_key: Path, helper_sha256: str, ) -> tuple[list[str], list[str], list[str]]: _require(operation in {"install", "verify", "rollback"}, "plan is local-only") _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid") _require(HEX_40.fullmatch(release_revision) is not None, "release revision was invalid") _require(HEX_64.fullmatch(archive_sha256) is not None, "archive hash was invalid") _require(HEX_64.fullmatch(helper_sha256) is not None, "helper hash was invalid") archive_name, key_name = remote_names(request_id) _require(archive.name == archive_name and ssh_key.name == key_name, "request filenames were not exact") key_flag = f"--ssh-key-file={ssh_key}" scp = [ GCLOUD, "compute", "scp", f"--project={PROJECT}", f"--zone={ZONE}", "--tunnel-through-iap", f"--ssh-key-expire-after={KEY_LIFETIME}", key_flag, "--quiet", str(archive), f"{INSTANCE}:{REMOTE_UPLOAD_ROOT}/", ] remote = [ "sudo", "--", "/usr/bin/python3", "-I", "-c", BOOTSTRAP_CODE, "start", operation, request_id, runner_revision, release_revision, archive_sha256, helper_sha256, ] cleanup_remote = [*remote[:6], "cleanup", *remote[7:]] ssh = [ GCLOUD, "compute", "ssh", INSTANCE, f"--project={PROJECT}", f"--zone={ZONE}", "--tunnel-through-iap", f"--ssh-key-expire-after={KEY_LIFETIME}", key_flag, "--quiet", f"--command={shlex.join(remote)}", ] cleanup = [*ssh[:-1], f"--command={shlex.join(cleanup_remote)}"] return scp, ssh, cleanup def _action_receipts(commands: tuple[list[str], list[str], list[str]]) -> list[dict[str, str]]: labels = ("upload_request", "execute_remote_request", "cleanup_failed_transfer") return [ { "label": label, "argv_sha256": package.canonical_sha256({"argv": argv}), } for label, argv in zip(labels, commands, strict=True) ] def prepare_request( operation: str, request_id: str, runner_revision: str, release_revision: str, bundle: Path, release_artifact_path: Path, output_directory: Path, expected_artifact_sha256: str, expected_release_sha256: str, expected_image_digest: str, expected_image_config_digest: str, *, git_runner: Runner = subprocess.run, ) -> dict[str, Any]: _require(operation in OPERATIONS, "operation was not allowlisted") _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") output_directory = output_directory.absolute() output_state = output_directory.stat(follow_symlinks=False) _require( stat.S_ISDIR(output_state.st_mode) and output_state.st_uid == os.geteuid() and output_state.st_mode & 0o077 == 0, "request output directory was not private", ) expected = _validate_expected_bindings( expected_artifact_sha256=expected_artifact_sha256, expected_release_sha256=expected_release_sha256, expected_image_digest=expected_image_digest, expected_image_config_digest=expected_image_config_digest, ) validate_clean_revision(runner_revision, runner=git_runner) release, unit = validate_release( bundle.absolute(), release_revision, expected_release_sha256=expected_release_sha256, expected_image_digest=expected_image_digest, expected_image_config_digest=expected_image_config_digest, ) release_artifact, release_artifact_bytes = validate_release_artifact( release_artifact_path.absolute(), bundle=bundle.absolute(), release=release, expected_artifact_sha256=expected_artifact_sha256, ) source_binding = installer.reviewed_source_binding() _require( source_binding.get("revision") == runner_revision, "installer source binding differed from runner revision" ) payloads = _member_payloads(bundle.absolute(), unit, release_artifact_bytes) manifest = build_manifest( operation=operation, request_id=request_id, runner_revision=runner_revision, release_revision=release_revision, release=release, payloads=payloads, source_binding=source_binding, release_artifact=release_artifact, ) archive_name, key_name = remote_names(request_id) archive = output_directory / archive_name ssh_key = output_directory / key_name _require( not ssh_key.exists() and not ssh_key.is_symlink() and not Path(f"{ssh_key}.pub").exists() and not Path(f"{ssh_key}.pub").is_symlink(), "request-local SSH key path already existed", ) helper_sha256 = _sha256_bytes(payloads[REMOTE_HELPER_SOURCE][0]) try: archive_sha256 = build_archive(archive, manifest=manifest, payloads=payloads) except Exception: for created in (archive,): try: observed = created.stat(follow_symlinks=False) if stat.S_ISREG(observed.st_mode) and observed.st_uid == os.geteuid() and observed.st_nlink == 1: created.unlink() except FileNotFoundError: pass raise if operation == "plan": actions: list[dict[str, str]] = [] else: commands = build_gcloud_argv( operation=operation, request_id=request_id, runner_revision=runner_revision, release_revision=release_revision, archive=archive, archive_sha256=archive_sha256, ssh_key=ssh_key, helper_sha256=helper_sha256, ) actions = _action_receipts(commands) return { "schema": PLAN_SCHEMA, "status": "pass", "mode": "dry_run", "operation": operation, "request_id": request_id, "runner_revision": runner_revision, "release_revision": release_revision, "target": manifest["target"], "release": manifest["release"], "release_artifact": manifest["release_artifact"], "expected": expected, "request": { "archive": str(archive), "archive_sha256": archive_sha256, "helper_sha256": helper_sha256, "manifest_sha256": manifest["manifest_sha256"], "ssh_key": str(ssh_key), }, "actions": actions, "authority": { "gcp_mutation": operation in {"install", "rollback"}, "service_restart": operation in {"install", "rollback"}, "database_or_secret_provisioning": False, "production_or_transport": False, "vps": False, }, } def _cleanup_local_ssh_key(path: Path) -> None: for candidate in (path, Path(f"{path}.pub")): try: observed = candidate.stat(follow_symlinks=False) except FileNotFoundError: continue _require( stat.S_ISREG(observed.st_mode) and observed.st_uid == os.geteuid() and observed.st_nlink == 1, "request-local SSH key cleanup refused an unsafe path", ) candidate.unlink() def _validate_verification(value: Any, label: str) -> None: _require( isinstance(value, dict) and set(value) == { "active_state", "sub_state", "main_pid", "n_restarts", "invocation_id", "supervising_process", "container_id", "container_pid", "health", "validated_process_environment_fields", "dropins", "environment_files", } and value.get("active_state") == "active" and value.get("sub_state") == "running" and isinstance(value.get("main_pid"), int) and value["main_pid"] > 1 and isinstance(value.get("n_restarts"), int) and value["n_restarts"] >= 0 and re.fullmatch(r"[0-9a-f]{32}", str(value.get("invocation_id"))) is not None and HEX_64.fullmatch(str(value.get("container_id"))) is not None and isinstance(value.get("container_pid"), int) and value["container_pid"] > 1 and value.get("health") == "healthy" and value.get("dropins") == "absent" and value.get("environment_files") == "absent", f"{label} was invalid", ) def _validate_database_summary(value: Any, request_id: str, expected_name: str) -> None: expected_path = f"/var/lib/livingip/leoclean-nosend-iap/requests/{request_id}/{expected_name}" _require( isinstance(value, dict) and set(value) == {"schema", "status", "receipt_sha256", "receipt_path", "proof"} and value.get("schema") == "livingip.leocleanNoSendContainerDatabaseVerification.v1" and value.get("status") == "pass" and HEX_64.fullmatch(str(value.get("receipt_sha256"))) is not None and value.get("receipt_path") == expected_path, "database verification summary was invalid", ) _require( value.get("proof") == { "proposal_staging": "function_only", "proposal_transaction": "rolled_back", "durable_proposal_created": False, "canonical_direct_writes": "denied", "stage_direct_writes": "denied", "role_escalation": "denied", }, "database verification proof summary was invalid", ) def validate_remote_result(plan: Mapping[str, Any], result: Mapping[str, Any]) -> None: _require( set(result) == { "schema", "status", "operation", "request_id", "runner_revision", "release_revision", "release_sha256", "image_digest", "image_config_digest", "release_artifact_sha256", "request_binding", "service", "database", "authority", }, "remote result fields were not exact", ) operation = plan["operation"] request_id = plan["request_id"] request = plan["request"] release = plan["release"] _require( result.get("schema") == REMOTE_RESULT_SCHEMA and result.get("status") == "pass" and result.get("operation") == operation and result.get("request_id") == request_id and result.get("runner_revision") == plan["runner_revision"] and result.get("release_revision") == plan["release_revision"] and result.get("release_sha256") == release["release_sha256"] and result.get("image_digest") == release["image_digest"] and result.get("image_config_digest") == release["image_config_digest"] and result.get("release_artifact_sha256") == plan["release_artifact"]["archive_sha256"] and result.get("request_binding") == { "archive_sha256": request["archive_sha256"], "helper_sha256": request["helper_sha256"], "manifest_sha256": request["manifest_sha256"], } and result.get("authority") == { "sending_enabled": False, "production_traffic_changed": False, "vps_touched": False, "canonical_proposal_approved": False, }, "remote result did not match the request", ) service = result["service"] database = result["database"] if operation == "install": _require( isinstance(service, dict) and set(service) == {"name", "before_restart", "after_restart"}, "install service result was invalid", ) _require(service["name"] == SERVICE, "install service target differed") _validate_verification(service["before_restart"], "pre-restart verification") _validate_verification(service["after_restart"], "post-restart verification") before = service["before_restart"] after = service["after_restart"] _require( before["main_pid"] != after["main_pid"] and before["invocation_id"] != after["invocation_id"] and before["container_id"] != after["container_id"], "restart identities did not change", ) _require( isinstance(database, dict) and set(database) == {"status", "before_restart", "after_restart"} and database.get("status") == "pass", "install database result was invalid", ) _validate_database_summary(database["before_restart"], request_id, "database-before-restart.json") _validate_database_summary(database["after_restart"], request_id, "database-after-restart.json") elif operation == "verify": _require( isinstance(service, dict) and set(service) == {"name", "verification"} and service.get("name") == SERVICE, "verify service result was invalid", ) _validate_verification(service["verification"], "verify running release") _validate_database_summary(database, request_id, "database-verify.json") else: _require( isinstance(service, dict) and set(service) == {"name", "restored_posture", "restored_release_sha256", "verification"} and service.get("name") == SERVICE and service.get("restored_posture") in {"active", "inactive", "absent"} and database == {"status": "not_run_after_explicit_rollback"}, "rollback result was invalid", ) if service["restored_posture"] == "active": _require( HEX_64.fullmatch(str(service["restored_release_sha256"])) is not None, "restored release was invalid" ) _validate_verification(service["verification"], "restored active verification") elif service["restored_posture"] == "inactive": _require( HEX_64.fullmatch(str(service["restored_release_sha256"])) is not None and service["verification"] is None, "restored inactive result was invalid", ) else: _require( service["restored_release_sha256"] is None and service["verification"] is None, "restored absent result was invalid", ) def _write_result_receipt(plan: Mapping[str, Any], result: Mapping[str, Any]) -> tuple[Path, dict[str, Any]]: archive = Path(str(plan["request"]["archive"])) output = archive.parent / f"livingip-leoclean-nosend-result-{plan['request_id']}.json" _require(not output.exists() and not output.is_symlink(), "local result receipt already existed") stable = { "schema": RESULT_RECEIPT_SCHEMA, "status": "pass", "operation": plan["operation"], "request_id": plan["request_id"], "runner_revision": plan["runner_revision"], "release_revision": plan["release_revision"], "request": { "archive_sha256": plan["request"]["archive_sha256"], "helper_sha256": plan["request"]["helper_sha256"], "manifest_sha256": plan["request"]["manifest_sha256"], }, "release": dict(plan["release"]), "release_artifact": dict(plan["release_artifact"]), "expected": dict(plan["expected"]), "remote_result": dict(result), } receipt = {**stable, "receipt_sha256": package.canonical_sha256(stable)} content = _canonical_json(receipt) descriptor, temporary_name = tempfile.mkstemp(prefix=".iap-result-", dir=output.parent) temporary: Path | None = Path(temporary_name) try: os.fchmod(descriptor, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(content) handle.flush() os.fsync(handle.fileno()) os.link(temporary, output, follow_symlinks=False) temporary.unlink() temporary = None directory = os.open(output.parent, os.O_RDONLY) try: os.fsync(directory) finally: os.close(directory) except OSError as exc: raise IapRequestError("local result receipt could not be published") from exc finally: try: if temporary is not None: temporary.unlink() except FileNotFoundError: pass return output, receipt def execute_plan( plan: Mapping[str, Any], *, runner: Runner = subprocess.run, git_runner: Runner = subprocess.run, gcloud_env: Mapping[str, str] | None = None, ) -> dict[str, Any]: _require( set(plan) == { "schema", "status", "mode", "operation", "request_id", "runner_revision", "release_revision", "target", "release", "release_artifact", "expected", "request", "actions", "authority", } and plan.get("schema") == PLAN_SCHEMA and plan.get("status") == "pass" and plan.get("mode") == "dry_run", "request plan fields were not exact", ) operation = plan.get("operation") _require(operation in {"install", "verify", "rollback"}, "operation was not executable") request_id = plan.get("request_id") runner_revision = plan.get("runner_revision") release_revision = plan.get("release_revision") _require(isinstance(request_id, str) and REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") _require( isinstance(runner_revision, str) and HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid", ) _require( isinstance(release_revision, str) and HEX_40.fullmatch(release_revision) is not None, "release revision was invalid", ) _require( plan.get("target") == { "project": PROJECT, "zone": ZONE, "instance": INSTANCE, "service": SERVICE, "service_account": SERVICE_ACCOUNT, }, "request target was not exact", ) expected = plan.get("expected") _require( isinstance(expected, dict) and expected == _validate_expected_bindings( expected_artifact_sha256=str(expected.get("release_artifact_sha256")), expected_release_sha256=str(expected.get("release_sha256")), expected_image_digest=str(expected.get("image_digest")), expected_image_config_digest=str(expected.get("image_config_digest")), ) and plan.get("release") == { "release_sha256": expected["release_sha256"], "image_reference": f"{package.IMAGE_REPOSITORY}@{expected['image_digest']}", "image_digest": expected["image_digest"], "image_config_digest": expected["image_config_digest"], } and isinstance(plan.get("release_artifact"), dict) and plan["release_artifact"].get("archive_sha256") == expected["release_artifact_sha256"], "request expected bindings were not exact", ) request = plan.get("request") _require( isinstance(request, dict) and set(request) == {"archive", "archive_sha256", "helper_sha256", "manifest_sha256", "ssh_key"}, "request files were invalid", ) archive = Path(str(request["archive"])) ssh_key = Path(str(request["ssh_key"])) _require(archive.is_absolute() and ssh_key.is_absolute(), "request file path was not absolute") archive_content, archive_mode = _read_stable_file(archive, root=archive.parent, max_bytes=MAX_ARCHIVE_BYTES) _require( archive_mode == 0o600 and _sha256_bytes(archive_content) == request.get("archive_sha256") and HEX_64.fullmatch(str(request.get("helper_sha256"))) is not None and HEX_64.fullmatch(str(request.get("manifest_sha256"))) is not None, "request files differed from the dry-run plan", ) _require( not ssh_key.exists() and not ssh_key.is_symlink() and not Path(f"{ssh_key}.pub").exists() and not Path(f"{ssh_key}.pub").is_symlink(), "request-local SSH key path was already occupied", ) expected_commands = build_gcloud_argv( operation=str(operation), request_id=request_id, runner_revision=runner_revision, release_revision=release_revision, archive=archive, archive_sha256=str(request["archive_sha256"]), ssh_key=ssh_key, helper_sha256=str(request["helper_sha256"]), ) _require( plan.get("actions") == _action_receipts(expected_commands), "request actions differed from the fixed IAP plan" ) scp, ssh, cleanup = expected_commands validate_remote_main(runner_revision, runner=git_runner) environment = dict(gcloud_env or gcloud_environment()) try: try: _run_bounded(list(scp), runner=runner, label="IAP upload", environment=environment) completed = _run_bounded(list(ssh), runner=runner, label="IAP remote operation", environment=environment) except IapRequestError as exc: try: _run_bounded( list(cleanup), runner=runner, label="IAP failed-transfer cleanup", environment=environment, ) except IapRequestError as cleanup_exc: raise IapRequestError("IAP operation failed and remote upload cleanup was unproven") from cleanup_exc raise exc finally: _cleanup_local_ssh_key(ssh_key) result = _strict_json(completed.stdout, "remote result") validate_remote_result(plan, result) encoded = json.dumps(result, allow_nan=False, sort_keys=True).casefold() _require(not any(marker in encoded for marker in FORBIDDEN_OUTPUT_MARKERS), "remote result was not sanitized") receipt_path, receipt = _write_result_receipt(plan, result) return { "schema": EXECUTION_SCHEMA, "status": "pass", "operation": operation, "request_id": request_id, "runner_revision": runner_revision, "release_revision": release_revision, **expected, "receipt_path": str(receipt_path), "receipt_sha256": receipt["receipt_sha256"], } class _Parser(argparse.ArgumentParser): def error(self, _message: str) -> None: raise IapRequestError("invalid arguments") def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = _Parser(description=__doc__, add_help=False) parser.add_argument("--operation", required=True, choices=OPERATIONS) parser.add_argument("--request-id", required=True) parser.add_argument("--runner-revision", required=True) parser.add_argument("--release-revision", required=True) parser.add_argument("--bundle", required=True, type=Path) parser.add_argument("--release-artifact", required=True, type=Path) parser.add_argument("--output-directory", required=True, type=Path) parser.add_argument("--expected-artifact-sha256", required=True) parser.add_argument("--expected-release-sha256", required=True) parser.add_argument("--expected-image-digest", required=True) parser.add_argument("--expected-image-config-digest", required=True) parser.add_argument("--execute", action="store_true") return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: try: args = parse_args(argv) plan = prepare_request( args.operation, args.request_id, args.runner_revision, args.release_revision, args.bundle, args.release_artifact, args.output_directory, args.expected_artifact_sha256, args.expected_release_sha256, args.expected_image_digest, args.expected_image_config_digest, ) result = execute_plan(plan) if args.execute else plan sys.stdout.write(json.dumps(result, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n") return 0 except (IapRequestError, installer.InstallError, package.PackageError): sys.stderr.write('{"error":"iap_request_failed","status":"fail"}\n') return 65 except Exception: sys.stderr.write('{"error":"iap_request_failed","status":"fail"}\n') return 65 if __name__ == "__main__": raise SystemExit(main())