1065 lines
40 KiB
Python
1065 lines
40 KiB
Python
#!/usr/bin/python3 -I
|
|
"""Execute one root-owned, fixed-target leoclean no-send IAP request."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import pwd
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import zipfile
|
|
from collections.abc import Callable, Mapping
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
SCHEMA = "livingip.leocleanNoSendIapRequest.v1"
|
|
RESULT_SCHEMA = "livingip.leocleanNoSendIapRemoteResult.v1"
|
|
PROJECT = "teleo-501523"
|
|
ZONE = "europe-west6-a"
|
|
INSTANCE = "teleo-staging-1"
|
|
SERVICE = "leoclean-gcp-nosend.service"
|
|
SERVICE_ACCOUNT = "sa-teleo-staging-vm@teleo-501523.iam.gserviceaccount.com"
|
|
IMAGE_REPOSITORY = "europe-west6-docker.pkg.dev/teleo-501523/teleo/leoclean-nosend-staging"
|
|
DOCKER = "/usr/bin/docker"
|
|
DOCKER_HOST = "unix:///var/run/docker.sock"
|
|
SERVICE_DOCKER_CONFIG = "/var/lib/livingip/leoclean-nosend/docker-config"
|
|
METADATA = "http://metadata.google.internal/computeMetadata/v1"
|
|
RUN_ROOT = Path("/var/lib/livingip/leoclean-nosend-iap/requests")
|
|
LOCK_PATH = Path("/run/leoclean-gcp-nosend-iap.lock")
|
|
UPLOAD_ROOT = Path("/tmp")
|
|
STAGE_ROOT = Path("/run/livingip-leoclean-nosend-iap-bootstrap")
|
|
REQUEST_ID = re.compile(r"^iap-[a-z0-9]{12,32}$")
|
|
HEX_40 = re.compile(r"^[0-9a-f]{40}$")
|
|
HEX_64 = re.compile(r"^[0-9a-f]{64}$")
|
|
IMAGE_REFERENCE = re.compile(rf"^{re.escape(IMAGE_REPOSITORY)}@sha256:([0-9a-f]{{64}})$")
|
|
OPERATIONS = ("install", "verify", "rollback")
|
|
MAX_ARCHIVE_BYTES = 8 * 1024 * 1024
|
|
MAX_MEMBER_BYTES = 2 * 1024 * 1024
|
|
MAX_MANIFEST_BYTES = 128 * 1024
|
|
MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024
|
|
MAX_TOKEN_BYTES = 64 * 1024
|
|
EXPECTED_MEMBERS = {
|
|
"request-manifest.json",
|
|
"release-artifact.zip",
|
|
"release/release.json",
|
|
f"release/{SERVICE}",
|
|
"ops/gcp_leoclean_nosend_package.py",
|
|
"ops/install_gcp_leoclean_nosend_release.py",
|
|
"ops/leoclean_nosend_service_control.py",
|
|
"ops/gcp_leoclean_nosend_iap_remote.py",
|
|
"ops/run_gcp_leoclean_nosend_iap.py",
|
|
"scripts/leo_behavior_manifest.py",
|
|
"scripts/leo_identity_manifest.py",
|
|
}
|
|
RELEASE_ARTIFACT_MEMBERS = {
|
|
"evidence/oci-smoke.json",
|
|
"evidence/publish-outcome.json",
|
|
"evidence/runtime-receipt.json",
|
|
f"final/release-v3/{SERVICE}",
|
|
"final/release-v3/release.json",
|
|
"receipts/build-push-receipt.json",
|
|
}
|
|
SAFE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
|
|
Runner = Callable[..., subprocess.CompletedProcess[bytes]]
|
|
|
|
|
|
class RemoteError(RuntimeError):
|
|
"""A remote request failed closed without exposing command output."""
|
|
|
|
|
|
def _require(condition: bool, message: str) -> None:
|
|
if not condition:
|
|
raise RemoteError(message)
|
|
|
|
|
|
def _sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def _canonical_sha256(value: Mapping[str, Any]) -> str:
|
|
try:
|
|
encoded = json.dumps(
|
|
value,
|
|
allow_nan=False,
|
|
ensure_ascii=True,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
).encode("utf-8")
|
|
except (TypeError, ValueError) as exc:
|
|
raise RemoteError("request manifest could not be hashed") from exc
|
|
return _sha256_bytes(encoded)
|
|
|
|
|
|
def _strict_json(raw: bytes, label: str, max_bytes: int) -> dict[str, Any]:
|
|
_require(len(raw) <= max_bytes, f"{label} exceeded the safety limit")
|
|
|
|
def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
value: dict[str, Any] = {}
|
|
for key, item in pairs:
|
|
if key in value:
|
|
raise ValueError("duplicate key")
|
|
value[key] = item
|
|
return value
|
|
|
|
try:
|
|
value = json.loads(
|
|
raw.decode("utf-8", "strict"),
|
|
object_pairs_hook=reject_duplicates,
|
|
parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("constant")),
|
|
)
|
|
except (UnicodeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise RemoteError(f"{label} was invalid JSON") from exc
|
|
_require(isinstance(value, dict), f"{label} was not a JSON object")
|
|
return value
|
|
|
|
|
|
def _run(
|
|
argv: list[str],
|
|
*,
|
|
runner: Runner,
|
|
label: str,
|
|
input_bytes: bytes | None = None,
|
|
timeout: int = 120,
|
|
require_success: bool = True,
|
|
) -> subprocess.CompletedProcess[bytes]:
|
|
try:
|
|
completed = runner(
|
|
argv,
|
|
check=False,
|
|
stdin=subprocess.DEVNULL if input_bytes is None else None,
|
|
input=input_bytes,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
env={"PATH": SAFE_PATH},
|
|
)
|
|
except (OSError, subprocess.SubprocessError) as exc:
|
|
raise RemoteError(f"{label} failed") from exc
|
|
_require(
|
|
len(completed.stdout) <= MAX_COMMAND_OUTPUT_BYTES and len(completed.stderr) <= MAX_COMMAND_OUTPUT_BYTES,
|
|
f"{label} output exceeded the safety limit",
|
|
)
|
|
if require_success:
|
|
_require(completed.returncode == 0, f"{label} failed")
|
|
return completed
|
|
|
|
|
|
def _read_stable_stage(path: Path, expected_sha256: str, max_bytes: int) -> bytes:
|
|
_require(path.parent.parent == STAGE_ROOT, "staged request path escaped its fixed root")
|
|
flags = os.O_RDONLY
|
|
if hasattr(os, "O_CLOEXEC"):
|
|
flags |= os.O_CLOEXEC
|
|
if hasattr(os, "O_NOFOLLOW"):
|
|
flags |= os.O_NOFOLLOW
|
|
try:
|
|
descriptor = os.open(path, flags)
|
|
except OSError as exc:
|
|
raise RemoteError("request upload was missing or unsafe") from exc
|
|
try:
|
|
before = os.fstat(descriptor)
|
|
_require(
|
|
stat.S_ISREG(before.st_mode)
|
|
and before.st_uid == 0
|
|
and stat.S_IMODE(before.st_mode) == 0o600
|
|
and before.st_nlink == 1
|
|
and before.st_size <= max_bytes,
|
|
"request upload posture was unsafe",
|
|
)
|
|
chunks: list[bytes] = []
|
|
remaining = max_bytes + 1
|
|
while remaining:
|
|
chunk = os.read(descriptor, min(65536, remaining))
|
|
if not chunk:
|
|
break
|
|
chunks.append(chunk)
|
|
remaining -= len(chunk)
|
|
content = b"".join(chunks)
|
|
after = os.fstat(descriptor)
|
|
named = path.stat(follow_symlinks=False)
|
|
_require(
|
|
len(content) == after.st_size
|
|
and (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_size, before.st_mtime_ns)
|
|
== (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_size, after.st_mtime_ns)
|
|
and (after.st_dev, after.st_ino, after.st_mode, after.st_nlink)
|
|
== (named.st_dev, named.st_ino, named.st_mode, named.st_nlink)
|
|
and _sha256_bytes(content) == expected_sha256,
|
|
"request upload changed or differed from its hash",
|
|
)
|
|
return content
|
|
except OSError as exc:
|
|
raise RemoteError("request upload could not be read") from exc
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def _unlink_exact(path: Path, identity: tuple[int, int]) -> None:
|
|
observed = path.stat(follow_symlinks=False)
|
|
_require((observed.st_dev, observed.st_ino) == identity and stat.S_ISREG(observed.st_mode), "upload was replaced")
|
|
path.unlink()
|
|
|
|
|
|
def _ensure_root_directory(
|
|
path: Path,
|
|
mode: int,
|
|
*,
|
|
expected_uid: int = 0,
|
|
trusted_parent: Path = Path("/var/lib"),
|
|
) -> None:
|
|
_require(path.is_absolute() and path.is_relative_to(trusted_parent), "request directory escaped its fixed root")
|
|
missing: list[Path] = []
|
|
cursor = path
|
|
while not cursor.exists():
|
|
_require(not cursor.is_symlink(), "request directory path was unsafe")
|
|
missing.append(cursor)
|
|
cursor = cursor.parent
|
|
observed = cursor.stat(follow_symlinks=False)
|
|
_require(stat.S_ISDIR(observed.st_mode), "request directory parent was unsafe")
|
|
for directory in reversed(missing):
|
|
directory.mkdir(mode=mode if directory == path else 0o755)
|
|
os.chown(directory, expected_uid, os.getgid() if expected_uid != 0 else 0)
|
|
observed = path.stat(follow_symlinks=False)
|
|
_require(
|
|
stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0,
|
|
"request directory was unsafe",
|
|
)
|
|
if path in missing:
|
|
path.chmod(mode)
|
|
cursor = path
|
|
while True:
|
|
observed = cursor.stat(follow_symlinks=False)
|
|
_require(
|
|
stat.S_ISDIR(observed.st_mode) and observed.st_uid == expected_uid and observed.st_mode & 0o022 == 0,
|
|
"request directory ancestor was unsafe",
|
|
)
|
|
if cursor == trusted_parent:
|
|
break
|
|
cursor = cursor.parent
|
|
|
|
|
|
def _acquire_lock(path: Path = LOCK_PATH, *, expected_uid: int = 0) -> int:
|
|
flags = os.O_RDWR | os.O_CREAT
|
|
if hasattr(os, "O_CLOEXEC"):
|
|
flags |= os.O_CLOEXEC
|
|
if hasattr(os, "O_NOFOLLOW"):
|
|
flags |= os.O_NOFOLLOW
|
|
try:
|
|
descriptor = os.open(path, flags, 0o600)
|
|
observed = os.fstat(descriptor)
|
|
named = path.stat(follow_symlinks=False)
|
|
_require(
|
|
stat.S_ISREG(observed.st_mode)
|
|
and observed.st_uid == expected_uid
|
|
and stat.S_IMODE(observed.st_mode) == 0o600
|
|
and observed.st_nlink == 1
|
|
and (observed.st_dev, observed.st_ino) == (named.st_dev, named.st_ino),
|
|
"request lock was unsafe",
|
|
)
|
|
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
return descriptor
|
|
except (OSError, RemoteError) as exc:
|
|
if "descriptor" in locals():
|
|
os.close(descriptor)
|
|
raise RemoteError("another request is active or the request lock was unsafe") from exc
|
|
|
|
|
|
def _copy_into_request(path: Path, content: bytes, mode: int = 0o600) -> None:
|
|
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
if hasattr(os, "O_NOFOLLOW"):
|
|
flags |= os.O_NOFOLLOW
|
|
try:
|
|
descriptor = os.open(path, flags, mode)
|
|
with os.fdopen(descriptor, "wb") as handle:
|
|
handle.write(content)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
except OSError as exc:
|
|
raise RemoteError("request file could not be secured") from exc
|
|
|
|
|
|
def _claim_request_directory(
|
|
run_root: Path,
|
|
request_id: str,
|
|
*,
|
|
expected_uid: int = 0,
|
|
trusted_parent: Path = Path("/var/lib"),
|
|
) -> Path:
|
|
_require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid")
|
|
_ensure_root_directory(run_root, 0o700, expected_uid=expected_uid, trusted_parent=trusted_parent)
|
|
request_dir = run_root / request_id
|
|
_require(not request_dir.exists() and not request_dir.is_symlink(), "request ID was already used")
|
|
request_dir.mkdir(mode=0o700)
|
|
os.chown(request_dir, expected_uid, os.getgid() if expected_uid != 0 else 0)
|
|
return request_dir
|
|
|
|
|
|
def _cleanup_upload_if_exact(path: Path, identity: tuple[int, int]) -> None:
|
|
try:
|
|
observed = path.stat(follow_symlinks=False)
|
|
if stat.S_ISREG(observed.st_mode) and (observed.st_dev, observed.st_ino) == identity:
|
|
path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _validate_member_name(name: str) -> None:
|
|
path = Path(name)
|
|
_require(
|
|
name
|
|
and not name.startswith("/")
|
|
and not name.endswith("/")
|
|
and all(part not in {"", ".", ".."} for part in path.parts),
|
|
"archive member path was unsafe",
|
|
)
|
|
|
|
|
|
def extract_archive(archive_path: Path, destination: Path) -> dict[str, Any]:
|
|
_require(archive_path.stat().st_size <= MAX_ARCHIVE_BYTES, "request archive exceeded the safety limit")
|
|
try:
|
|
with tarfile.open(archive_path, mode="r:gz") as archive:
|
|
return _extract_open_archive(archive, destination)
|
|
except (OSError, tarfile.TarError) as exc:
|
|
raise RemoteError("request archive was invalid") from exc
|
|
|
|
|
|
def _extract_open_archive(archive: tarfile.TarFile, destination: Path) -> dict[str, Any]:
|
|
members = archive.getmembers()
|
|
names = [member.name for member in members]
|
|
_require(len(names) == len(set(names)) and set(names) == EXPECTED_MEMBERS, "archive member set was not exact")
|
|
for member in members:
|
|
_validate_member_name(member.name)
|
|
_require(
|
|
member.isfile()
|
|
and member.size <= MAX_MEMBER_BYTES
|
|
and member.uid == 0
|
|
and member.gid == 0
|
|
and member.uname == "root"
|
|
and member.gname == "root"
|
|
and member.mtime == 0
|
|
and stat.S_IMODE(member.mode) in {0o600, 0o644, 0o755},
|
|
"archive member metadata was unsafe",
|
|
)
|
|
manifest_member = archive.getmember("request-manifest.json")
|
|
_require(manifest_member.size <= MAX_MANIFEST_BYTES, "request manifest exceeded the safety limit")
|
|
manifest_stream = archive.extractfile(manifest_member)
|
|
_require(manifest_stream is not None, "request manifest was absent")
|
|
manifest = _strict_json(manifest_stream.read(), "request manifest", MAX_MANIFEST_BYTES)
|
|
files = manifest.get("files")
|
|
_require(isinstance(files, list), "request manifest file list was invalid")
|
|
file_map: dict[str, dict[str, Any]] = {}
|
|
for item in files:
|
|
_require(
|
|
isinstance(item, dict) and set(item) == {"path", "sha256", "size", "mode"},
|
|
"request manifest file record was invalid",
|
|
)
|
|
relative = item.get("path")
|
|
_require(isinstance(relative, str) and relative not in file_map, "request manifest file path was invalid")
|
|
file_map[relative] = item
|
|
_require(set(file_map) == EXPECTED_MEMBERS - {"request-manifest.json"}, "request manifest files were not exact")
|
|
for relative in sorted(file_map):
|
|
member = archive.getmember(relative)
|
|
stream = archive.extractfile(member)
|
|
_require(stream is not None, "request member was absent")
|
|
content = stream.read(MAX_MEMBER_BYTES + 1)
|
|
record = file_map[relative]
|
|
_require(
|
|
len(content) == member.size == record.get("size")
|
|
and stat.S_IMODE(member.mode) == record.get("mode")
|
|
and _sha256_bytes(content) == record.get("sha256"),
|
|
"request member differed from its manifest",
|
|
)
|
|
target = destination / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
_require(target.resolve().is_relative_to(destination.resolve()), "request member escaped extraction root")
|
|
_copy_into_request(target, content, stat.S_IMODE(member.mode))
|
|
return manifest
|
|
|
|
|
|
def validate_manifest(
|
|
manifest: Mapping[str, Any],
|
|
*,
|
|
operation: str,
|
|
request_id: str,
|
|
runner_revision: str,
|
|
release_revision: str,
|
|
) -> None:
|
|
_require(
|
|
set(manifest)
|
|
== {
|
|
"schema",
|
|
"operation",
|
|
"request_id",
|
|
"runner_revision",
|
|
"release_revision",
|
|
"target",
|
|
"release",
|
|
"installer_source",
|
|
"release_artifact",
|
|
"files",
|
|
"manifest_sha256",
|
|
},
|
|
"request manifest fields were not exact",
|
|
)
|
|
_require(
|
|
manifest.get("schema") == SCHEMA
|
|
and manifest.get("operation") == operation
|
|
and manifest.get("request_id") == request_id
|
|
and manifest.get("runner_revision") == runner_revision
|
|
and manifest.get("release_revision") == release_revision,
|
|
"request manifest context differed",
|
|
)
|
|
_require(
|
|
manifest.get("target")
|
|
== {
|
|
"project": PROJECT,
|
|
"zone": ZONE,
|
|
"instance": INSTANCE,
|
|
"service": SERVICE,
|
|
"service_account": SERVICE_ACCOUNT,
|
|
},
|
|
"request target differed",
|
|
)
|
|
stable = {key: value for key, value in manifest.items() if key != "manifest_sha256"}
|
|
_require(manifest.get("manifest_sha256") == _canonical_sha256(stable), "request manifest hash differed")
|
|
release = manifest.get("release")
|
|
_require(
|
|
isinstance(release, dict)
|
|
and set(release) == {"release_sha256", "image_reference", "image_digest", "image_config_digest"}
|
|
and HEX_64.fullmatch(str(release.get("release_sha256"))) is not None
|
|
and IMAGE_REFERENCE.fullmatch(str(release.get("image_reference"))) is not None
|
|
and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_digest"))) is not None
|
|
and re.fullmatch(r"sha256:[0-9a-f]{64}", str(release.get("image_config_digest"))) is not None,
|
|
"request release binding was invalid",
|
|
)
|
|
_require(
|
|
release["image_reference"] == f"{IMAGE_REPOSITORY}@{release['image_digest']}",
|
|
"request image digest binding differed",
|
|
)
|
|
artifact = manifest.get("release_artifact")
|
|
_require(
|
|
isinstance(artifact, dict)
|
|
and set(artifact) == {"archive_sha256", "archive_size", "members"}
|
|
and HEX_64.fullmatch(str(artifact.get("archive_sha256"))) is not None
|
|
and isinstance(artifact.get("archive_size"), int)
|
|
and 0 < artifact["archive_size"] <= MAX_ARCHIVE_BYTES
|
|
and isinstance(artifact.get("members"), list),
|
|
"release artifact binding was invalid",
|
|
)
|
|
|
|
|
|
def validate_original_release_artifact(
|
|
artifact_path: Path,
|
|
*,
|
|
manifest: Mapping[str, Any],
|
|
payload_root: Path,
|
|
) -> None:
|
|
raw = artifact_path.read_bytes()
|
|
binding = manifest["release_artifact"]
|
|
_require(
|
|
len(raw) == binding["archive_size"] and _sha256_bytes(raw) == binding["archive_sha256"],
|
|
"original release artifact archive differed",
|
|
)
|
|
try:
|
|
archive = zipfile.ZipFile(io.BytesIO(raw), mode="r")
|
|
except (OSError, zipfile.BadZipFile) as exc:
|
|
raise RemoteError("original release artifact archive was invalid") from exc
|
|
with archive:
|
|
infos = archive.infolist()
|
|
names = [info.filename for info in infos]
|
|
_require(
|
|
len(names) == len(set(names)) and set(names) == RELEASE_ARTIFACT_MEMBERS,
|
|
"release artifact members differed",
|
|
)
|
|
observed: list[dict[str, Any]] = []
|
|
payloads: dict[str, bytes] = {}
|
|
for info in infos:
|
|
unix_mode = (info.external_attr >> 16) & 0o177777
|
|
_require(
|
|
not info.is_dir()
|
|
and info.file_size <= MAX_MEMBER_BYTES
|
|
and (unix_mode == 0 or stat.S_ISREG(unix_mode)),
|
|
"release artifact member was unsafe",
|
|
)
|
|
content = archive.read(info)
|
|
payloads[info.filename] = content
|
|
observed.append({"path": info.filename, "sha256": _sha256_bytes(content), "size": len(content)})
|
|
_require(
|
|
sorted(observed, key=lambda item: str(item["path"])) == binding["members"],
|
|
"release artifact member receipt differed",
|
|
)
|
|
_require(
|
|
payloads["final/release-v3/release.json"] == (payload_root / "release/release.json").read_bytes()
|
|
and payloads[f"final/release-v3/{SERVICE}"] == (payload_root / f"release/{SERVICE}").read_bytes(),
|
|
"release artifact differed from extracted release",
|
|
)
|
|
|
|
|
|
def _metadata(path: str, *, runner: Runner) -> bytes:
|
|
return _run(
|
|
[
|
|
"/usr/bin/curl",
|
|
"--fail",
|
|
"--silent",
|
|
"--show-error",
|
|
"--max-time",
|
|
"5",
|
|
"--header",
|
|
"Metadata-Flavor: Google",
|
|
f"{METADATA}/{path}",
|
|
],
|
|
runner=runner,
|
|
label="metadata read",
|
|
timeout=10,
|
|
).stdout
|
|
|
|
|
|
def verify_service_account(*, runner: Runner) -> None:
|
|
try:
|
|
email = _metadata("instance/service-accounts/default/email", runner=runner).decode("ascii", "strict").strip()
|
|
project = _metadata("project/project-id", runner=runner).decode("ascii", "strict").strip()
|
|
instance = _metadata("instance/name", runner=runner).decode("ascii", "strict").strip()
|
|
zone = _metadata("instance/zone", runner=runner).decode("ascii", "strict").strip()
|
|
except UnicodeError as exc:
|
|
raise RemoteError("VM target identity was invalid") from exc
|
|
_require(
|
|
email == SERVICE_ACCOUNT and project == PROJECT and instance == INSTANCE and zone.endswith(f"/zones/{ZONE}"),
|
|
"VM target or service account identity differed",
|
|
)
|
|
|
|
|
|
def _docker_prefix(config: Path) -> list[str]:
|
|
return [DOCKER, f"--host={DOCKER_HOST}", f"--config={config}"]
|
|
|
|
|
|
def pull_immutable_image(reference: str, request_dir: Path, *, runner: Runner) -> None:
|
|
_require(IMAGE_REFERENCE.fullmatch(reference) is not None, "image reference was not immutable")
|
|
token_receipt = _strict_json(
|
|
_metadata("instance/service-accounts/default/token", runner=runner),
|
|
"metadata token",
|
|
MAX_TOKEN_BYTES,
|
|
)
|
|
token = token_receipt.get("access_token")
|
|
_require(
|
|
isinstance(token, str)
|
|
and 16 <= len(token) <= 8192
|
|
and "\n" not in token
|
|
and token_receipt.get("token_type") == "Bearer",
|
|
"metadata token response was invalid",
|
|
)
|
|
config = Path(tempfile.mkdtemp(prefix="docker-auth-", dir=request_dir))
|
|
config.chmod(0o700)
|
|
try:
|
|
_run(
|
|
[
|
|
*_docker_prefix(config),
|
|
"login",
|
|
"--username=oauth2accesstoken",
|
|
"--password-stdin",
|
|
"https://europe-west6-docker.pkg.dev",
|
|
],
|
|
runner=runner,
|
|
label="registry login",
|
|
input_bytes=(token + "\n").encode("utf-8"),
|
|
)
|
|
_run(
|
|
[*_docker_prefix(config), "image", "pull", reference],
|
|
runner=runner,
|
|
label="immutable image pull",
|
|
timeout=300,
|
|
)
|
|
finally:
|
|
try:
|
|
_run(
|
|
[*_docker_prefix(config), "logout", "https://europe-west6-docker.pkg.dev"],
|
|
runner=runner,
|
|
label="registry logout",
|
|
)
|
|
except RemoteError:
|
|
pass
|
|
shutil.rmtree(config, ignore_errors=True)
|
|
_require(not config.exists(), "ephemeral Docker credentials were not removed")
|
|
|
|
|
|
def _load_contract_modules(payload_root: Path) -> tuple[Any, Any]:
|
|
sys.path.insert(0, str(payload_root))
|
|
try:
|
|
from ops import gcp_leoclean_nosend_package as package
|
|
from ops import install_gcp_leoclean_nosend_release as installer
|
|
except Exception as exc:
|
|
raise RemoteError("reviewed installer sources could not be loaded") from exc
|
|
_require(
|
|
package.PROJECT == PROJECT
|
|
and package.ZONE == ZONE
|
|
and package.INSTANCE == INSTANCE
|
|
and package.SERVICE == SERVICE
|
|
and package.SERVICE_ACCOUNT == SERVICE_ACCOUNT,
|
|
"reviewed installer target differed",
|
|
)
|
|
return package, installer
|
|
|
|
|
|
def _verify_database(
|
|
*,
|
|
container_id: str,
|
|
run_id: str,
|
|
receipt_name: str,
|
|
request_dir: Path,
|
|
package_module: Any,
|
|
runner: Runner,
|
|
) -> dict[str, Any]:
|
|
completed = _run(
|
|
[
|
|
DOCKER,
|
|
f"--host={DOCKER_HOST}",
|
|
f"--config={SERVICE_DOCKER_CONFIG}",
|
|
"container",
|
|
"exec",
|
|
"--user=0",
|
|
container_id,
|
|
"/opt/livingip/leoclean-nosend/venv/bin/python",
|
|
"/opt/livingip/leoclean-nosend/entrypoint.py",
|
|
"verify-database",
|
|
run_id,
|
|
],
|
|
runner=runner,
|
|
label="container database verification",
|
|
timeout=300,
|
|
)
|
|
receipt = _strict_json(completed.stdout, "database verification receipt", MAX_COMMAND_OUTPUT_BYTES)
|
|
_require(
|
|
receipt.get("schema") == "livingip.leocleanNoSendContainerDatabaseVerification.v1"
|
|
and receipt.get("status") == "pass"
|
|
and receipt.get("run_id") == run_id,
|
|
"database verification receipt did not pass",
|
|
)
|
|
stable = {key: value for key, value in receipt.items() if key != "receipt_sha256"}
|
|
_require(receipt.get("receipt_sha256") == package_module.canonical_sha256(stable), "database receipt hash differed")
|
|
_require(re.fullmatch(r"[a-z][a-z0-9-]{1,62}\.json", receipt_name) is not None, "receipt name was invalid")
|
|
receipt_path = request_dir / receipt_name
|
|
_copy_into_request(
|
|
receipt_path,
|
|
(json.dumps(receipt, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"),
|
|
0o600,
|
|
)
|
|
return {
|
|
"schema": receipt["schema"],
|
|
"status": "pass",
|
|
"receipt_sha256": receipt["receipt_sha256"],
|
|
"receipt_path": str(receipt_path),
|
|
"proof": {
|
|
"proposal_staging": "function_only",
|
|
"proposal_transaction": "rolled_back",
|
|
"durable_proposal_created": False,
|
|
"canonical_direct_writes": "denied",
|
|
"stage_direct_writes": "denied",
|
|
"role_escalation": "denied",
|
|
},
|
|
}
|
|
|
|
|
|
def _execute_explicit_rollback(
|
|
*,
|
|
installer_module: Any,
|
|
release_sha256: str,
|
|
paths: Any,
|
|
source_binding: Mapping[str, Any],
|
|
runner: Runner,
|
|
) -> dict[str, Any]:
|
|
"""Call only the receipt-bound explicit rollback API; never model rollback as another install."""
|
|
|
|
rollback = getattr(installer_module, "rollback_release", None)
|
|
_require(callable(rollback), "reviewed installer lacked the explicit rollback interface")
|
|
result = rollback(
|
|
release_sha256,
|
|
paths=paths,
|
|
source_binding=source_binding,
|
|
runner=lambda argv: _run(
|
|
argv,
|
|
runner=runner,
|
|
label="rollback command",
|
|
require_success=False,
|
|
),
|
|
execute_rollback=True,
|
|
)
|
|
_require(isinstance(result, dict) and result.get("status") == "pass", "explicit rollback did not pass")
|
|
return result
|
|
|
|
|
|
def _rollback_result(
|
|
*,
|
|
manifest: Mapping[str, Any],
|
|
release: Mapping[str, Any],
|
|
rollback: Mapping[str, Any],
|
|
archive_sha256: str,
|
|
helper_sha256: str,
|
|
) -> dict[str, Any]:
|
|
posture = rollback.get("restored_service_posture")
|
|
restored_release = rollback.get("restored_release_sha256")
|
|
verification = rollback.get("verification")
|
|
_require(
|
|
rollback.get("schema") == "livingip.leocleanNoSendRollbackResult.v1"
|
|
and rollback.get("status") == "pass"
|
|
and rollback.get("mode") == "executed"
|
|
and rollback.get("rolled_back_release_sha256") == release["release_sha256"]
|
|
and posture in {"active", "inactive", "absent"},
|
|
"explicit rollback result was invalid",
|
|
)
|
|
if posture == "active":
|
|
_require(
|
|
isinstance(restored_release, str)
|
|
and HEX_64.fullmatch(restored_release) is not None
|
|
and isinstance(verification, dict)
|
|
and verification.get("active_state") == "active"
|
|
and verification.get("sub_state") == "running"
|
|
and isinstance(verification.get("container_id"), str)
|
|
and HEX_64.fullmatch(verification["container_id"]) is not None,
|
|
"active rollback result lacked exact restored verification",
|
|
)
|
|
elif posture == "inactive":
|
|
_require(
|
|
isinstance(restored_release, str)
|
|
and HEX_64.fullmatch(restored_release) is not None
|
|
and verification is None,
|
|
"inactive rollback result was invalid",
|
|
)
|
|
else:
|
|
_require(restored_release is None and verification is None, "absent rollback result was invalid")
|
|
return {
|
|
"schema": RESULT_SCHEMA,
|
|
"status": "pass",
|
|
"operation": "rollback",
|
|
"request_id": manifest["request_id"],
|
|
"runner_revision": manifest["runner_revision"],
|
|
"release_revision": manifest["release_revision"],
|
|
"release_sha256": release["release_sha256"],
|
|
"image_digest": release["image"]["digest"],
|
|
"image_config_digest": release["image"]["config_digest"],
|
|
"release_artifact_sha256": manifest["release_artifact"]["archive_sha256"],
|
|
"request_binding": {
|
|
"archive_sha256": archive_sha256,
|
|
"helper_sha256": helper_sha256,
|
|
"manifest_sha256": manifest["manifest_sha256"],
|
|
},
|
|
"service": {
|
|
"name": SERVICE,
|
|
"restored_posture": posture,
|
|
"restored_release_sha256": restored_release,
|
|
"verification": verification,
|
|
},
|
|
"database": {"status": "not_run_after_explicit_rollback"},
|
|
"authority": {
|
|
"sending_enabled": False,
|
|
"production_traffic_changed": False,
|
|
"vps_touched": False,
|
|
"canonical_proposal_approved": False,
|
|
},
|
|
}
|
|
|
|
|
|
def execute_request(
|
|
manifest: Mapping[str, Any],
|
|
payload_root: Path,
|
|
request_dir: Path,
|
|
archive_sha256: str,
|
|
helper_sha256: str,
|
|
*,
|
|
runner: Runner = subprocess.run,
|
|
) -> dict[str, Any]:
|
|
operation = str(manifest["operation"])
|
|
release_binding = manifest["release"]
|
|
validate_original_release_artifact(
|
|
payload_root / "release-artifact.zip",
|
|
manifest=manifest,
|
|
payload_root=payload_root,
|
|
)
|
|
package_module, installer_module = _load_contract_modules(payload_root)
|
|
bundle = payload_root / "release"
|
|
release, unit = installer_module.validate_release_bundle(bundle)
|
|
_require(
|
|
release["release_sha256"] == release_binding["release_sha256"]
|
|
and release["image"]["reference"] == release_binding["image_reference"]
|
|
and release["image"]["digest"] == release_binding["image_digest"]
|
|
and release["image"]["config_digest"] == release_binding["image_config_digest"]
|
|
and unit == package_module.render_systemd_unit(release).encode("utf-8"),
|
|
"extracted release differed from the request",
|
|
)
|
|
source_binding = manifest["installer_source"]
|
|
installer_module._validate_source_binding(source_binding)
|
|
_require(
|
|
source_binding.get("revision") == manifest["runner_revision"],
|
|
"installer source revision differed from runner revision",
|
|
)
|
|
_require(
|
|
release["image_input"]["runtime"]["teleo_git_head"] == manifest["release_revision"],
|
|
"release source revision differed from release revision",
|
|
)
|
|
paths = installer_module.InstallPaths.under(Path("/"))
|
|
if operation == "install":
|
|
pull_immutable_image(release["image"]["reference"], request_dir, runner=runner)
|
|
install_receipt = installer_module.install_release(
|
|
bundle,
|
|
paths=paths,
|
|
source_binding=source_binding,
|
|
runner=lambda argv: _run(
|
|
argv,
|
|
runner=runner,
|
|
label="installer command",
|
|
require_success=False,
|
|
),
|
|
execute_restart=True,
|
|
)
|
|
_require(install_receipt.get("status") == "pass", "installation did not pass")
|
|
elif operation == "rollback":
|
|
rollback = _execute_explicit_rollback(
|
|
installer_module=installer_module,
|
|
release_sha256=release["release_sha256"],
|
|
paths=paths,
|
|
source_binding=source_binding,
|
|
runner=runner,
|
|
)
|
|
return _rollback_result(
|
|
manifest=manifest,
|
|
release=release,
|
|
rollback=rollback,
|
|
archive_sha256=archive_sha256,
|
|
helper_sha256=helper_sha256,
|
|
)
|
|
|
|
def contract_runner(argv: list[str]) -> subprocess.CompletedProcess[bytes]:
|
|
return _run(
|
|
argv,
|
|
runner=runner,
|
|
label="release verification command",
|
|
require_success=False,
|
|
)
|
|
|
|
if operation == "install":
|
|
try:
|
|
before = installer_module.verify_running_release(
|
|
release,
|
|
runner=contract_runner,
|
|
paths=paths,
|
|
)
|
|
before_database = _verify_database(
|
|
container_id=before["container_id"],
|
|
run_id=f"{manifest['request_id']}-pre",
|
|
receipt_name="database-before-restart.json",
|
|
request_dir=request_dir,
|
|
package_module=package_module,
|
|
runner=runner,
|
|
)
|
|
_run(
|
|
[installer_module.SYSTEMCTL, "restart", SERVICE],
|
|
runner=runner,
|
|
label="fixed service restart",
|
|
)
|
|
after = installer_module.verify_running_release(
|
|
release,
|
|
runner=contract_runner,
|
|
paths=paths,
|
|
)
|
|
_require(
|
|
before["main_pid"] != after["main_pid"]
|
|
and before["invocation_id"] != after["invocation_id"]
|
|
and before["container_id"] != after["container_id"]
|
|
and after["n_restarts"] >= before["n_restarts"],
|
|
"post-install restart did not create a new exact service/container identity",
|
|
)
|
|
after_database = _verify_database(
|
|
container_id=after["container_id"],
|
|
run_id=f"{manifest['request_id']}-post",
|
|
receipt_name="database-after-restart.json",
|
|
request_dir=request_dir,
|
|
package_module=package_module,
|
|
runner=runner,
|
|
)
|
|
except Exception as exc:
|
|
try:
|
|
recovery = _execute_explicit_rollback(
|
|
installer_module=installer_module,
|
|
release_sha256=release["release_sha256"],
|
|
paths=paths,
|
|
source_binding=source_binding,
|
|
runner=runner,
|
|
)
|
|
_rollback_result(
|
|
manifest=manifest,
|
|
release=release,
|
|
rollback=recovery,
|
|
archive_sha256=archive_sha256,
|
|
helper_sha256=helper_sha256,
|
|
)
|
|
except Exception as rollback_exc:
|
|
raise RemoteError("install_failed_rollback_incomplete") from rollback_exc
|
|
raise RemoteError("install_failed_rolled_back") from exc
|
|
service = {
|
|
"name": SERVICE,
|
|
"before_restart": before,
|
|
"after_restart": after,
|
|
}
|
|
database = {
|
|
"status": "pass",
|
|
"before_restart": before_database,
|
|
"after_restart": after_database,
|
|
}
|
|
else:
|
|
verification = installer_module.verify_running_release(
|
|
release,
|
|
runner=contract_runner,
|
|
paths=paths,
|
|
)
|
|
database = _verify_database(
|
|
container_id=verification["container_id"],
|
|
run_id=f"{manifest['request_id']}-verify",
|
|
receipt_name="database-verify.json",
|
|
request_dir=request_dir,
|
|
package_module=package_module,
|
|
runner=runner,
|
|
)
|
|
service = {
|
|
"name": SERVICE,
|
|
"verification": verification,
|
|
}
|
|
return {
|
|
"schema": RESULT_SCHEMA,
|
|
"status": "pass",
|
|
"operation": operation,
|
|
"request_id": manifest["request_id"],
|
|
"runner_revision": manifest["runner_revision"],
|
|
"release_revision": manifest["release_revision"],
|
|
"release_sha256": release["release_sha256"],
|
|
"image_digest": release["image"]["digest"],
|
|
"image_config_digest": release["image"]["config_digest"],
|
|
"release_artifact_sha256": manifest["release_artifact"]["archive_sha256"],
|
|
"request_binding": {
|
|
"archive_sha256": archive_sha256,
|
|
"helper_sha256": helper_sha256,
|
|
"manifest_sha256": manifest["manifest_sha256"],
|
|
},
|
|
"service": service,
|
|
"database": database,
|
|
"authority": {
|
|
"sending_enabled": False,
|
|
"production_traffic_changed": False,
|
|
"vps_touched": False,
|
|
"canonical_proposal_approved": False,
|
|
},
|
|
}
|
|
|
|
|
|
def receive_request(
|
|
operation: str,
|
|
request_id: str,
|
|
runner_revision: str,
|
|
release_revision: str,
|
|
archive_sha256: str,
|
|
helper_sha256: str,
|
|
*,
|
|
runner: Runner = subprocess.run,
|
|
run_root: Path = RUN_ROOT,
|
|
lock_path: Path = LOCK_PATH,
|
|
) -> dict[str, Any]:
|
|
_require(os.geteuid() == 0, "remote execution required root")
|
|
_require(operation in OPERATIONS, "operation was not allowlisted")
|
|
_require(REQUEST_ID.fullmatch(request_id) is not None, "request ID was invalid")
|
|
_require(HEX_40.fullmatch(runner_revision) is not None, "runner revision was invalid")
|
|
_require(HEX_40.fullmatch(release_revision) is not None, "release revision was invalid")
|
|
_require(HEX_64.fullmatch(archive_sha256) is not None, "archive hash was invalid")
|
|
_require(HEX_64.fullmatch(helper_sha256) is not None, "helper hash was invalid")
|
|
caller = os.environ.get("SUDO_USER", "")
|
|
_require(caller and caller != "root", "remote execution lacked a distinct OS Login caller")
|
|
try:
|
|
pwd.getpwnam(caller)
|
|
except KeyError as exc:
|
|
raise RemoteError("OS Login caller did not resolve") from exc
|
|
stage_dir = STAGE_ROOT / request_id
|
|
archive_upload = stage_dir / "request.tar.gz"
|
|
helper_upload = stage_dir / "remote-helper.py"
|
|
archive_stat = archive_upload.stat(follow_symlinks=False)
|
|
helper_stat = helper_upload.stat(follow_symlinks=False)
|
|
archive_identity = (archive_stat.st_dev, archive_stat.st_ino)
|
|
helper_identity = (helper_stat.st_dev, helper_stat.st_ino)
|
|
try:
|
|
stage_state = stage_dir.stat(follow_symlinks=False)
|
|
_require(
|
|
stat.S_ISDIR(stage_state.st_mode)
|
|
and stage_state.st_uid == 0
|
|
and stage_state.st_gid == 0
|
|
and stat.S_IMODE(stage_state.st_mode) == 0o700,
|
|
"root-owned bootstrap stage was unsafe",
|
|
)
|
|
helper_content = _read_stable_stage(helper_upload, helper_sha256, MAX_MEMBER_BYTES)
|
|
archive_content = _read_stable_stage(archive_upload, archive_sha256, MAX_ARCHIVE_BYTES)
|
|
_require(_sha256_bytes(Path(__file__).read_bytes()) == helper_sha256, "executing helper differed from request")
|
|
lock_descriptor = _acquire_lock(lock_path)
|
|
try:
|
|
request_dir = _claim_request_directory(run_root, request_id)
|
|
secured_archive = request_dir / "request.tar.gz"
|
|
secured_helper = request_dir / "remote-helper.py"
|
|
_copy_into_request(secured_archive, archive_content)
|
|
_copy_into_request(secured_helper, helper_content)
|
|
_unlink_exact(archive_upload, archive_identity)
|
|
_unlink_exact(helper_upload, helper_identity)
|
|
stage_dir.rmdir()
|
|
payload_root = request_dir / "payload"
|
|
payload_root.mkdir(mode=0o700)
|
|
manifest = extract_archive(secured_archive, payload_root)
|
|
validate_manifest(
|
|
manifest,
|
|
operation=operation,
|
|
request_id=request_id,
|
|
runner_revision=runner_revision,
|
|
release_revision=release_revision,
|
|
)
|
|
verify_service_account(runner=runner)
|
|
result = execute_request(
|
|
manifest,
|
|
payload_root,
|
|
request_dir,
|
|
archive_sha256,
|
|
helper_sha256,
|
|
runner=runner,
|
|
)
|
|
result_path = request_dir / "result.json"
|
|
_copy_into_request(
|
|
result_path,
|
|
(json.dumps(result, allow_nan=False, indent=2, sort_keys=True) + "\n").encode("utf-8"),
|
|
0o600,
|
|
)
|
|
return result
|
|
finally:
|
|
os.close(lock_descriptor)
|
|
finally:
|
|
_cleanup_upload_if_exact(archive_upload, archive_identity)
|
|
_cleanup_upload_if_exact(helper_upload, helper_identity)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
arguments = argv if argv is not None else sys.argv[1:]
|
|
try:
|
|
os.umask(0o077)
|
|
_require(len(arguments) == 6, "remote arguments were not exact")
|
|
result = receive_request(*arguments)
|
|
sys.stdout.write(json.dumps(result, allow_nan=False, separators=(",", ":"), sort_keys=True) + "\n")
|
|
return 0
|
|
except RemoteError as exc:
|
|
error = str(exc)
|
|
if error not in {"install_failed_rolled_back", "install_failed_rollback_incomplete"}:
|
|
error = "remote_request_failed"
|
|
sys.stderr.write(json.dumps({"error": error, "status": "fail"}, separators=(",", ":")) + "\n")
|
|
return 65
|
|
except Exception:
|
|
sys.stderr.write('{"error":"remote_request_failed","status":"fail"}\n')
|
|
return 65
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|