#!/usr/bin/python3 -I """Execute one root-owned, fixed-target leoclean no-send IAP request.""" from __future__ import annotations import fcntl import hashlib import io import json import os import pwd import re import shutil import stat import subprocess import sys import tarfile import tempfile import zipfile from collections.abc import Callable, Mapping from pathlib import Path from typing import Any SCHEMA = "livingip.leocleanNoSendIapRequest.v1" RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v1" PROJECT = "teleo-501523" ZONE = "europe-west6-a" INSTANCE = "teleo-staging-1" SERVICE = "leoclean-gcp-nosend.service" SERVICE_ACCOUNT = "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com" IMAGE_REPOSITORY = "europe-west6-docker.pkg.dev/teleo-501523/teleo/leoclean-nosend-staging" DOCKER = "/usr/bin/docker" DOCKER_HOST = "unix:///var/run/docker.sock" SERVICE_DOCKER_CONFIG = "/var/lib/livingip/leoclean-nosend/docker-config" METADATA = "http://metadata.google.internal/computeMetadata/v1" RUN_ROOT = Path("/var/lib/livingip/leoclean-nosend-iap/requests") LOCK_PATH = Path("/run/leoclean-gcp-nosend-iap.lock") UPLOAD_ROOT = Path("/tmp") STAGE_ROOT = Path("/run/livingip-leoclean-nosend-iap-bootstrap") REQUEST_ID = re.compile(r"^iap-[a-z0-9]{12,32}$") HEX_40 = re.compile(r"^[0-9a-f]{40}$") HEX_64 = re.compile(r"^[0-9a-f]{64}$") IMAGE_REFERENCE = re.compile(rf"^{re.escape(IMAGE_REPOSITORY)}@sha256:([0-9a-f]{{64}})$") OPERATIONS = ("install", "verify", "rollback") MAX_ARCHIVE_BYTES = 8 * 1024 * 1024 MAX_MEMBER_BYTES = 2 * 1024 * 1024 MAX_MANIFEST_BYTES = 128 * 1024 MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024 MAX_TOKEN_BYTES = 64 * 1024 EXPECTED_MEMBERS = { "request-manifest.json", "release-artifact.zip", "release/release.json", f"release/{SERVICE}", "ops/gcp_leoclean_nosend_package.py", "ops/install_gcp_leoclean_nosend_release.py", "ops/leoclean_nosend_service_control.py", "ops/gcp_leoclean_nosend_iap_remote.py", "ops/run_gcp_leoclean_nosend_iap.py", "scripts/leo_behavior_manifest.py", "scripts/leo_identity_manifest.py", } RELEASE_ARTIFACT_MEMBERS = { "evidence/oci-smoke.json", "evidence/publish-outcome.json", "evidence/runtime-receipt.json", f"final/release-v3/{SERVICE}", "final/release-v3/release.json", "receipts/build-push-receipt.json", } SAFE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" Runner = Callable[..., subprocess.CompletedProcess[bytes]] class RemoteError(RuntimeError): """A remote request failed closed without exposing command output.""" def _require(condition: bool, message: str) -> None: if not condition: raise RemoteError(message) def _sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() def _canonical_sha256(value: Mapping[str, Any]) -> str: try: encoded = json.dumps( value, allow_nan=False, ensure_ascii=True, separators=(",", ":"), sort_keys=True, ).encode("utf-8") except (TypeError, ValueError) as exc: raise RemoteError("request manifest could not be hashed") from exc return _sha256_bytes(encoded) def _strict_json(raw: bytes, label: str, max_bytes: int) -> dict[str, Any]: _require(len(raw) <= max_bytes, f"{label} exceeded the safety limit") def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: value: dict[str, Any] = {} for key, item in pairs: if key in value: raise ValueError("duplicate key") value[key] = item return value try: value = json.loads( raw.decode("utf-8", "strict"), object_pairs_hook=reject_duplicates, parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("constant")), ) except (UnicodeError, ValueError, json.JSONDecodeError) as exc: raise RemoteError(f"{label} was invalid JSON") from exc _require(isinstance(value, dict), f"{label} was not a JSON object") return value def _run( argv: list[str], *, runner: Runner, label: str, input_bytes: bytes | None = None, timeout: int = 120, require_success: bool = True, ) -> subprocess.CompletedProcess[bytes]: try: completed = runner( argv, check=False, stdin=subprocess.DEVNULL if input_bytes is None else None, input=input_bytes, capture_output=True, timeout=timeout, env={"PATH": SAFE_PATH}, ) except (OSError, subprocess.SubprocessError) as exc: raise RemoteError(f"{label} failed") from exc _require( len(completed.stdout) <= MAX_COMMAND_OUTPUT_BYTES and len(completed.stderr) <= MAX_COMMAND_OUTPUT_BYTES, f"{label} output exceeded the safety limit", ) if require_success: _require(completed.returncode == 0, f"{label} failed") return completed def _read_stable_stage(path: Path, expected_sha256: str, max_bytes: int) -> bytes: _require(path.parent.parent == STAGE_ROOT, "staged request path escaped its fixed root") flags = os.O_RDONLY if hasattr(os, "O_CLOEXEC"): flags |= os.O_CLOEXEC if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: descriptor = os.open(path, flags) except OSError as exc: raise RemoteError("request upload was missing or unsafe") from exc try: before = os.fstat(descriptor) _require( stat.S_ISREG(before.st_mode) and before.st_uid == 0 and stat.S_IMODE(before.st_mode) == 0o600 and before.st_nlink == 1 and before.st_size <= max_bytes, "request upload posture was unsafe", ) chunks: list[bytes] = [] remaining = max_bytes + 1 while remaining: chunk = os.read(descriptor, min(65536, remaining)) if not chunk: break chunks.append(chunk) remaining -= len(chunk) content = b"".join(chunks) after = os.fstat(descriptor) named = path.stat(follow_symlinks=False) _require( len(content) == after.st_size and (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns) == (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns) and (after.st_dev, after.st_ino, after.st_mode, after.st_nlink) == (named.st_dev, named.st_ino, named.st_mode, named.st_nlink) and _sha256_bytes(content) == expected_sha256, "request upload changed or differed from its hash", ) return content except OSError as exc: raise RemoteError("request upload could not be read") from exc finally: os.close(descriptor) def _unlink_exact(path: Path, identity: tuple[int, int]) -> None: observed = path.stat(follow_symlinks=False) _require((observed.st_dev, observed.st_ino) == identity and stat.S_ISREG(observed.st_mode), "upload was replaced") path.unlink() def _ensure_root_directory( path: Path, mode: int, *, expected_uid: int = 0, trusted_parent: Path = Path("/var/lib"), ) -> None: _require(path.is_absolute() and path.is_relative_to(trusted_parent), "request directory escaped its fixed root") missing: list[Path] = [] cursor = path while not cursor.exists(): _require(not cursor.is_symlink(), "request directory path was unsafe") missing.append(cursor) cursor = cursor.parent observed = cursor.stat(follow_symlinks=False) _require(stat.S_ISDIR(observed.st_mode), "request directory parent was unsafe") for directory in reversed(missing): directory.mkdir(mode=mode if directory == path else 0o755) os.chown(directory, expected_uid, os.getgid() if expected_uid != 0 else 0) observed = path.stat(follow_symlinks=False) _require( stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0, "request directory was unsafe", ) if path in missing: path.chmod(mode) cursor = path while True: observed = cursor.stat(follow_symlinks=False) _require( stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0, "request directory ancestor was unsafe", ) if cursor == trusted_parent: break cursor = cursor.parent def _acquire_lock(path: Path = LOCK_PATH, *, expected_uid: int = 0) -> int: flags = os.O_RDWR | os.O_CREAT if hasattr(os, "O_CLOEXEC"): flags |= os.O_CLOEXEC if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: descriptor = os.open(path, flags, 0o600) observed = os.fstat(descriptor) named = path.stat(follow_symlinks=False) _require( stat.S_ISREG(observed.st_mode) and observed.st_uid == expected_uid and stat.S_IMODE(observed.st_mode) == 0o600 and observed.st_nlink == 1 and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino), "request lock was unsafe", ) fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) return descriptor except (OSError, RemoteError) as exc: if "descriptor" in locals(): os.close(descriptor) raise RemoteError("another request is active or the request lock was unsafe") from exc def _copy_into_request(path: Path, content: bytes, mode: int = 0o600) -> None: _require(mode in {0o600, 0o644, 0o755}, "request file mode was invalid") flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: descriptor = os.open(path, flags, mode) with os.fdopen(descriptor, "wb") as handle: # receive_request() deliberately runs under umask 077. Apply the # manifest-bound mode explicitly so public release metadata and # executable helpers retain their reviewed 0644/0755 posture. os.fchmod(handle.fileno(), mode) handle.write(content) handle.flush() os.fsync(handle.fileno()) observed = os.fstat(handle.fileno()) named = path.stat(follow_symlinks=False) _require( stat.S_ISREG(observed.st_mode) and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino) and observed.st_uid == os.geteuid() and stat.S_IMODE(observed.st_mode) == mode and stat.S_IMODE(named.st_mode) == mode and observed.st_nlink == 1 and observed.st_size == len(content), "request file posture differed from its manifest", ) except OSError as exc: raise RemoteError("request file could not be secured") from exc def _claim_request_directory( run_root: Path, request_id: str, *, expected_uid: int = 0, trusted_parent: Path = Path("/var/lib"), ) -> Path: _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") _ensure_root_directory(run_root, 0o700, expected_uid=expected_uid, trusted_parent=trusted_parent) request_dir = run_root / request_id _require(not request_dir.exists() and not request_dir.is_symlink(), "request ID was already used") request_dir.mkdir(mode=0o700) os.chown(request_dir, expected_uid, os.getgid() if expected_uid != 0 else 0) return request_dir def _cleanup_upload_if_exact(path: Path, identity: tuple[int, int]) -> None: try: observed = path.stat(follow_symlinks=False) if stat.S_ISREG(observed.st_mode) and (observed.st_dev, observed.st_ino) == identity: path.unlink() except OSError: pass def _validate_member_name(name: str) -> None: path = Path(name) _require( name and not name.startswith("/") and not name.endswith("/") and all(part not in {"", ".", ".."} for part in path.parts), "archive member path was unsafe", ) def extract_archive(archive_path: Path, destination: Path) -> dict[str, Any]: _require(archive_path.stat().st_size <= MAX_ARCHIVE_BYTES, "request archive exceeded the safety limit") try: with tarfile.open(archive_path, mode="r:gz") as archive: return _extract_open_archive(archive, destination) except (OSError, tarfile.TarError) as exc: raise RemoteError("request archive was invalid") from exc def _extract_open_archive(archive: tarfile.TarFile, destination: Path) -> dict[str, Any]: members = archive.getmembers() names = [member.name for member in members] _require(len(names) == len(set(names)) and set(names) == EXPECTED_MEMBERS, "archive member set was not exact") for member in members: _validate_member_name(member.name) _require( member.isfile() and member.size <= MAX_MEMBER_BYTES and member.uid == 0 and member.gid == 0 and member.uname == "root" and member.gname == "root" and member.mtime == 0 and stat.S_IMODE(member.mode) in {0o600, 0o644, 0o755}, "archive member metadata was unsafe", ) manifest_member = archive.getmember("request-manifest.json") _require(manifest_member.size <= MAX_MANIFEST_BYTES, "request manifest exceeded the safety limit") manifest_stream = archive.extractfile(manifest_member) _require(manifest_stream is not None, "request manifest was absent") manifest = _strict_json(manifest_stream.read(), "request manifest", MAX_MANIFEST_BYTES) files = manifest.get("files") _require(isinstance(files, list), "request manifest file list was invalid") file_map: dict[str, dict[str, Any]] = {} for item in files: _require( isinstance(item, dict) and set(item) == {"path", "sha256", "size", "mode"}, "request manifest file record was invalid", ) relative = item.get("path") _require(isinstance(relative, str) and relative not in file_map, "request manifest file path was invalid") file_map[relative] = item _require(set(file_map) == EXPECTED_MEMBERS - {"request-manifest.json"}, "request manifest files were not exact") for relative in sorted(file_map): member = archive.getmember(relative) stream = archive.extractfile(member) _require(stream is not None, "request member was absent") content = stream.read(MAX_MEMBER_BYTES + 1) record = file_map[relative] _require( len(content) == member.size == record.get("size") and stat.S_IMODE(member.mode) == record.get("mode") and _sha256_bytes(content) == record.get("sha256"), "request member differed from its manifest", ) target = destination / relative target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) _require(target.resolve().is_relative_to(destination.resolve()), "request member escaped extraction root") _copy_into_request(target, content, stat.S_IMODE(member.mode)) return manifest def validate_manifest( manifest: Mapping[str, Any], *, operation: str, request_id: str, runner_revision: str, release_revision: str, ) -> None: _require( set(manifest) == { "schema", "operation", "request_id", "runner_revision", "release_revision", "target", "release", "installer_source", "release_artifact", "files", "manifest_sha256", }, "request manifest fields were not exact", ) _require( manifest.get("schema") == SCHEMA and manifest.get("operation") == operation and manifest.get("request_id") == request_id and manifest.get("runner_revision") == runner_revision and manifest.get("release_revision") == release_revision, "request manifest context differed", ) _require( manifest.get("target") == { "project": PROJECT, "zone": ZONE, "instance": INSTANCE, "service": SERVICE, "service_account": SERVICE_ACCOUNT, }, "request target differed", ) stable = {key: value for key, value in manifest.items() if key != "manifest_sha256"} _require(manifest.get("manifest_sha256") == _canonical_sha256(stable), "request manifest hash differed") release = manifest.get("release") _require( isinstance(release, dict) and set(release) == {"release_sha256", "image_reference", "image_digest", "image_config_digest"} and HEX_64.fullmatch(str(release.get("release_sha256"))) is not None and IMAGE_REFERENCE.fullmatch(str(release.get("image_reference"))) is not None and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_digest"))) is not None and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_config_digest"))) is not None, "request release binding was invalid", ) _require( release["image_reference"] == f"{IMAGE_REPOSITORY}@{release['image_digest']}", "request image digest binding differed", ) artifact = manifest.get("release_artifact") _require( isinstance(artifact, dict) and set(artifact) == {"archive_sha256", "archive_size", "members"} and HEX_64.fullmatch(str(artifact.get("archive_sha256"))) is not None and isinstance(artifact.get("archive_size"), int) and 0 < artifact["archive_size"] <= MAX_ARCHIVE_BYTES and isinstance(artifact.get("members"), list), "release artifact binding was invalid", ) def validate_original_release_artifact( artifact_path: Path, *, manifest: Mapping[str, Any], payload_root: Path, ) -> None: raw = artifact_path.read_bytes() binding = manifest["release_artifact"] _require( len(raw) == binding["archive_size"] and _sha256_bytes(raw) == binding["archive_sha256"], "original release artifact archive differed", ) try: archive = zipfile.ZipFile(io.BytesIO(raw), mode="r") except (OSError, zipfile.BadZipFile) as exc: raise RemoteError("original release artifact archive was invalid") from exc with archive: infos = archive.infolist() names = [info.filename for info in infos] _require( len(names) == len(set(names)) and set(names) == RELEASE_ARTIFACT_MEMBERS, "release artifact members differed", ) observed: list[dict[str, Any]] = [] payloads: dict[str, bytes] = {} for info in infos: unix_mode = (info.external_attr >> 16) & 0o177777 _require( not info.is_dir() and info.file_size <= MAX_MEMBER_BYTES and (unix_mode == 0 or stat.S_ISREG(unix_mode)), "release artifact member was unsafe", ) content = archive.read(info) payloads[info.filename] = content observed.append({"path": info.filename, "sha256": _sha256_bytes(content), "size": len(content)}) _require( sorted(observed, key=lambda item: str(item["path"])) == binding["members"], "release artifact member receipt differed", ) _require( payloads["final/release-v3/release.json"] == (payload_root / "release/release.json").read_bytes() and payloads[f"final/release-v3/{SERVICE}"] == (payload_root / f"release/{SERVICE}").read_bytes(), "release artifact differed from extracted release", ) def _metadata(path: str, *, runner: Runner) -> bytes: return _run( [ "/usr/bin/curl", "--fail", "--silent", "--show-error", "--max-time", "5", "--header", "Metadata-Flavor: Google", f"{METADATA}/{path}", ], runner=runner, label="metadata read", timeout=10, ).stdout def verify_service_account(*, runner: Runner) -> None: try: email = _metadata("instance/service-accounts/default/email", runner=runner).decode("ascii", "strict").strip() project = _metadata("project/project-id", runner=runner).decode("ascii", "strict").strip() instance = _metadata("instance/name", runner=runner).decode("ascii", "strict").strip() zone = _metadata("instance/zone", runner=runner).decode("ascii", "strict").strip() except UnicodeError as exc: raise RemoteError("VM target identity was invalid") from exc _require( email == SERVICE_ACCOUNT and project == PROJECT and instance == INSTANCE and zone.endswith(f"/zones/{ZONE}"), "VM target or service account identity differed", ) def _docker_prefix(config: Path) -> list[str]: return [DOCKER, f"--host={DOCKER_HOST}", f"--config={config}"] def pull_immutable_image(reference: str, request_dir: Path, *, runner: Runner) -> None: _require(IMAGE_REFERENCE.fullmatch(reference) is not None, "image reference was not immutable") token_receipt = _strict_json( _metadata("instance/service-accounts/default/token", runner=runner), "metadata token", MAX_TOKEN_BYTES, ) token = token_receipt.get("access_token") _require( isinstance(token, str) and 16 <= len(token) <= 8192 and "\n" not in token and token_receipt.get("token_type") == "Bearer", "metadata token response was invalid", ) config = Path(tempfile.mkdtemp(prefix="docker-auth-", dir=request_dir)) config.chmod(0o700) try: _run( [ *_docker_prefix(config), "login", "--username=oauth2accesstoken", "--password-stdin", "https://europe-west6-docker.pkg.dev", ], runner=runner, label="registry login", input_bytes=(token + "\n").encode("utf-8"), ) _run( [*_docker_prefix(config), "image", "pull", reference], runner=runner, label="immutable image pull", timeout=300, ) finally: try: _run( [*_docker_prefix(config), "logout", "https://europe-west6-docker.pkg.dev"], runner=runner, label="registry logout", ) except RemoteError: pass shutil.rmtree(config, ignore_errors=True) _require(not config.exists(), "ephemeral Docker credentials were not removed") def _load_contract_modules(payload_root: Path) -> tuple[Any, Any]: sys.path.insert(0, str(payload_root)) try: from ops import gcp_leoclean_nosend_package as package from ops import install_gcp_leoclean_nosend_release as installer except Exception as exc: raise RemoteError("reviewed installer sources could not be loaded") from exc _require( package.PROJECT == PROJECT and package.ZONE == ZONE and package.INSTANCE == INSTANCE and package.SERVICE == SERVICE and package.SERVICE_ACCOUNT == SERVICE_ACCOUNT, "reviewed installer target differed", ) return package, installer def _verify_database( *, container_id: str, run_id: str, receipt_name: str, request_dir: Path, package_module: Any, runner: Runner, ) -> dict[str, Any]: completed = _run( [ DOCKER, f"--host={DOCKER_HOST}", f"--config={SERVICE_DOCKER_CONFIG}", "container", "exec", "--user=0", container_id, "/opt/livingip/leoclean-nosend/venv/bin/python", "/opt/livingip/leoclean-nosend/entrypoint.py", "verify-database", run_id, ], runner=runner, label="container database verification", timeout=300, ) receipt = _strict_json(completed.stdout, "database verification receipt", MAX_COMMAND_OUTPUT_BYTES) _require( receipt.get("schema") == "livingip.leocleanNoSendContainerDatabaseVerification.v1" and receipt.get("status") == "pass" and receipt.get("run_id") == run_id, "database verification receipt did not pass", ) stable = {key: value for key, value in receipt.items() if key != "receipt_sha256"} _require(receipt.get("receipt_sha256") == package_module.canonical_sha256(stable), "database receipt hash differed") _require(re.fullmatch(r"[a-z][a-z0-9-]{1,62}\.json", receipt_name) is not None, "receipt name was invalid") receipt_path = request_dir / receipt_name _copy_into_request( receipt_path, (json.dumps(receipt, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600, ) return { "schema": receipt["schema"], "status": "pass", "receipt_sha256": receipt["receipt_sha256"], "receipt_path": str(receipt_path), "proof": { "proposal_staging": "function_only", "proposal_transaction": "rolled_back", "durable_proposal_created": False, "canonical_direct_writes": "denied", "stage_direct_writes": "denied", "role_escalation": "denied", }, } def _execute_explicit_rollback( *, installer_module: Any, release_sha256: str, paths: Any, source_binding: Mapping[str, Any], runner: Runner, ) -> dict[str, Any]: """Call only the receipt-bound explicit rollback API; never model rollback as another install.""" rollback = getattr(installer_module, "rollback_release", None) _require(callable(rollback), "reviewed installer lacked the explicit rollback interface") result = rollback( release_sha256, paths=paths, source_binding=source_binding, runner=lambda argv: _run( argv, runner=runner, label="rollback command", require_success=False, ), execute_rollback=True, ) _require(isinstance(result, dict) and result.get("status") == "pass", "explicit rollback did not pass") return result def _rollback_result( *, manifest: Mapping[str, Any], release: Mapping[str, Any], rollback: Mapping[str, Any], archive_sha256: str, helper_sha256: str, ) -> dict[str, Any]: posture = rollback.get("restored_service_posture") restored_release = rollback.get("restored_release_sha256") verification = rollback.get("verification") _require( rollback.get("schema") == "livingip.leocleanNoSendRollbackResult.v1" and rollback.get("status") == "pass" and rollback.get("mode") == "executed" and rollback.get("rolled_back_release_sha256") == release["release_sha256"] and posture in {"active", "inactive", "absent"}, "explicit rollback result was invalid", ) if posture == "active": _require( isinstance(restored_release, str) and HEX_64.fullmatch(restored_release) is not None and isinstance(verification, dict) and verification.get("active_state") == "active" and verification.get("sub_state") == "running" and isinstance(verification.get("container_id"), str) and HEX_64.fullmatch(verification["container_id"]) is not None, "active rollback result lacked exact restored verification", ) elif posture == "inactive": _require( isinstance(restored_release, str) and HEX_64.fullmatch(restored_release) is not None and verification is None, "inactive rollback result was invalid", ) else: _require(restored_release is None and verification is None, "absent rollback result was invalid") return { "schema": RESULT_SCHEMA, "status": "pass", "operation": "rollback", "request_id": manifest["request_id"], "runner_revision": manifest["runner_revision"], "release_revision": manifest["release_revision"], "release_sha256": release["release_sha256"], "image_digest": release["image"]["digest"], "image_config_digest": release["image"]["config_digest"], "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"], "request_binding": { "archive_sha256": archive_sha256, "helper_sha256": helper_sha256, "manifest_sha256": manifest["manifest_sha256"], }, "service": { "name": SERVICE, "restored_posture": posture, "restored_release_sha256": restored_release, "verification": verification, }, "database": {"status": "not_run_after_explicit_rollback"}, "authority": { "sending_enabled": False, "production_traffic_changed": False, "vps_touched": False, "canonical_proposal_approved": False, }, } def execute_request( manifest: Mapping[str, Any], payload_root: Path, request_dir: Path, archive_sha256: str, helper_sha256: str, *, runner: Runner = subprocess.run, ) -> dict[str, Any]: operation = str(manifest["operation"]) release_binding = manifest["release"] validate_original_release_artifact( payload_root / "release-artifact.zip", manifest=manifest, payload_root=payload_root, ) package_module, installer_module = _load_contract_modules(payload_root) bundle = payload_root / "release" release, unit = installer_module.validate_release_bundle(bundle) _require( release["release_sha256"] == release_binding["release_sha256"] and release["image"]["reference"] == release_binding["image_reference"] and release["image"]["digest"] == release_binding["image_digest"] and release["image"]["config_digest"] == release_binding["image_config_digest"] and unit == package_module.render_systemd_unit(release).encode("utf-8"), "extracted release differed from the request", ) source_binding = manifest["installer_source"] installer_module._validate_source_binding(source_binding) _require( source_binding.get("revision") == manifest["runner_revision"], "installer source revision differed from runner revision", ) _require( release["image_input"]["runtime"]["teleo_git_head"] == manifest["release_revision"], "release source revision differed from release revision", ) paths = installer_module.InstallPaths.under(Path("/")) if operation == "install": pull_immutable_image(release["image"]["reference"], request_dir, runner=runner) install_receipt = installer_module.install_release( bundle, paths=paths, source_binding=source_binding, runner=lambda argv: _run( argv, runner=runner, label="installer command", require_success=False, ), execute_restart=True, ) _require(install_receipt.get("status") == "pass", "installation did not pass") elif operation == "rollback": rollback = _execute_explicit_rollback( installer_module=installer_module, release_sha256=release["release_sha256"], paths=paths, source_binding=source_binding, runner=runner, ) return _rollback_result( manifest=manifest, release=release, rollback=rollback, archive_sha256=archive_sha256, helper_sha256=helper_sha256, ) def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]: return _run( argv, runner=runner, label="release verification command", require_success=False, ) if operation == "install": try: before = installer_module.verify_running_release( release, runner=contract_runner, paths=paths, ) before_database = _verify_database( container_id=before["container_id"], run_id=f"{manifest['request_id']}-pre", receipt_name="database-before-restart.json", request_dir=request_dir, package_module=package_module, runner=runner, ) _run( [installer_module.SYSTEMCTL, "restart", SERVICE], runner=runner, label="fixed service restart", ) after = installer_module.verify_running_release( release, runner=contract_runner, paths=paths, ) _require( before["main_pid"] != after["main_pid"] and before["invocation_id"] != after["invocation_id"] and before["container_id"] != after["container_id"] and after["n_restarts"] >= before["n_restarts"], "post-install restart did not create a new exact service/container identity", ) after_database = _verify_database( container_id=after["container_id"], run_id=f"{manifest['request_id']}-post", receipt_name="database-after-restart.json", request_dir=request_dir, package_module=package_module, runner=runner, ) except Exception as exc: try: recovery = _execute_explicit_rollback( installer_module=installer_module, release_sha256=release["release_sha256"], paths=paths, source_binding=source_binding, runner=runner, ) _rollback_result( manifest=manifest, release=release, rollback=recovery, archive_sha256=archive_sha256, helper_sha256=helper_sha256, ) except Exception as rollback_exc: raise RemoteError("install_failed_rollback_incomplete") from rollback_exc raise RemoteError("install_failed_rolled_back") from exc service = { "name": SERVICE, "before_restart": before, "after_restart": after, } database = { "status": "pass", "before_restart": before_database, "after_restart": after_database, } else: verification = installer_module.verify_running_release( release, runner=contract_runner, paths=paths, ) database = _verify_database( container_id=verification["container_id"], run_id=f"{manifest['request_id']}-verify", receipt_name="database-verify.json", request_dir=request_dir, package_module=package_module, runner=runner, ) service = { "name": SERVICE, "verification": verification, } return { "schema": RESULT_SCHEMA, "status": "pass", "operation": operation, "request_id": manifest["request_id"], "runner_revision": manifest["runner_revision"], "release_revision": manifest["release_revision"], "release_sha256": release["release_sha256"], "image_digest": release["image"]["digest"], "image_config_digest": release["image"]["config_digest"], "release_artifact_sha256": manifest["release_artifact"]["archive_sha256"], "request_binding": { "archive_sha256": archive_sha256, "helper_sha256": helper_sha256, "manifest_sha256": manifest["manifest_sha256"], }, "service": service, "database": database, "authority": { "sending_enabled": False, "production_traffic_changed": False, "vps_touched": False, "canonical_proposal_approved": False, }, } def receive_request( operation: str, request_id: str, runner_revision: str, release_revision: str, archive_sha256: str, helper_sha256: str, *, runner: Runner = subprocess.run, run_root: Path = RUN_ROOT, lock_path: Path = LOCK_PATH, ) -> dict[str, Any]: _require(os.geteuid() == 0, "remote execution required root") _require(operation in OPERATIONS, "operation was not allowlisted") _require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid") _require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid") _require(HEX_40.fullmatch(release_revision) is not None, "release revision was invalid") _require(HEX_64.fullmatch(archive_sha256) is not None, "archive hash was invalid") _require(HEX_64.fullmatch(helper_sha256) is not None, "helper hash was invalid") caller = os.environ.get("SUDO_USER", "") _require(caller and caller != "root", "remote execution lacked a distinct OS Login caller") try: pwd.getpwnam(caller) except KeyError as exc: raise RemoteError("OS Login caller did not resolve") from exc stage_dir = STAGE_ROOT / request_id archive_upload = stage_dir / "request.tar.gz" helper_upload = stage_dir / "remote-helper.py" archive_stat = archive_upload.stat(follow_symlinks=False) helper_stat = helper_upload.stat(follow_symlinks=False) archive_identity = (archive_stat.st_dev, archive_stat.st_ino) helper_identity = (helper_stat.st_dev, helper_stat.st_ino) try: stage_state = stage_dir.stat(follow_symlinks=False) _require( stat.S_ISDIR(stage_state.st_mode) and stage_state.st_uid == 0 and stage_state.st_gid == 0 and stat.S_IMODE(stage_state.st_mode) == 0o700, "root-owned bootstrap stage was unsafe", ) helper_content = _read_stable_stage(helper_upload, helper_sha256, MAX_MEMBER_BYTES) archive_content = _read_stable_stage(archive_upload, archive_sha256, MAX_ARCHIVE_BYTES) _require(_sha256_bytes(Path(__file__).read_bytes()) == helper_sha256, "executing helper differed from request") lock_descriptor = _acquire_lock(lock_path) try: request_dir = _claim_request_directory(run_root, request_id) secured_archive = request_dir / "request.tar.gz" secured_helper = request_dir / "remote-helper.py" _copy_into_request(secured_archive, archive_content) _copy_into_request(secured_helper, helper_content) _unlink_exact(archive_upload, archive_identity) _unlink_exact(helper_upload, helper_identity) stage_dir.rmdir() payload_root = request_dir / "payload" payload_root.mkdir(mode=0o700) manifest = extract_archive(secured_archive, payload_root) validate_manifest( manifest, operation=operation, request_id=request_id, runner_revision=runner_revision, release_revision=release_revision, ) verify_service_account(runner=runner) result = execute_request( manifest, payload_root, request_dir, archive_sha256, helper_sha256, runner=runner, ) result_path = request_dir / "result.json" _copy_into_request( result_path, (json.dumps(result, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"), 0o600, ) return result finally: os.close(lock_descriptor) finally: _cleanup_upload_if_exact(archive_upload, archive_identity) _cleanup_upload_if_exact(helper_upload, helper_identity) def main(argv: list[str] | None = None) -> int: arguments = argv if argv is not None else sys.argv[1:] try: os.umask(0o077) _require(len(arguments) == 6, "remote arguments were not exact") result = receive_request(*arguments) sys.stdout.write(json.dumps(result, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n") return 0 except RemoteError as exc: error = str(exc) if error not in {"install_failed_rolled_back", "install_failed_rollback_incomplete"}: error = "remote_request_failed" sys.stderr.write(json.dumps({"error": error, "status": "fail"}, separators=(",", ":")) + "\n") return 65 except Exception: sys.stderr.write('{"error":"remote_request_failed","status":"fail"}\n') return 65 if __name__ == "__main__": raise SystemExit(main())