import os import resource import signal import subprocess import sys import time from pathlib import Path import pytest REPO_ROOT = Path(__file__).resolve().parents[1] def embedded_service_context_runner_source() -> str: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() marker = "remote_helpers=$(cat <<'REMOTE'\n" start = script.index(marker) + len(marker) end = script.index("\nREMOTE\n)", start) helpers = script[start:end] blocks = [part.split("\nPY", maxsplit=1)[0] for part in helpers.split("<<'PY'\n")[1:]] return next(block for block in blocks if "def run_stopped_child(" in block) def load_service_context_runner() -> dict[str, object]: namespace: dict[str, object] = {"__name__": "embedded_service_context_runner_test"} source = embedded_service_context_runner_source() exec(compile(source, "", "exec"), namespace) return namespace def test_teleo_agent_template_supports_optional_per_agent_env_file(): unit = (REPO_ROOT / "systemd" / "teleo-agent@.service").read_text() assert "Environment=PYTHONUNBUFFERED=1" in unit assert "EnvironmentFile=-/opt/teleo-eval/secrets/teleo-agent-%i.env" in unit def test_teleo_agent_template_allows_transcript_dumps_under_strict_fs(): unit = (REPO_ROOT / "systemd" / "teleo-agent@.service").read_text() assert "ProtectSystem=strict" in unit assert "ReadWritePaths=/opt/teleo-eval/transcripts" in unit def test_deploy_scripts_restart_wallet_test_agent_when_present(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() manual_deploy = (REPO_ROOT / "deploy" / "deploy.sh").read_text() assert "add_restart_if_unit_exists teleo-agent@leo-wallet-test" in auto_deploy assert 'systemctl list-units --all --full "$1.service"' in auto_deploy assert "teleo-agent@leo-wallet-test" in manual_deploy def test_deploy_scripts_do_not_start_inactive_legacy_leo_agent(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() manual_deploy = (REPO_ROOT / "deploy" / "deploy.sh").read_text() assert "add_restart_if_unit_active teleo-agent@leo" in auto_deploy assert "add_restart teleo-agent@leo" not in auto_deploy assert "systemctl is-active --quiet teleo-agent@leo.service" in manual_deploy assert "teleo-pipeline teleo-diagnostics teleo-agent@leo" not in manual_deploy def test_auto_deploy_prefers_github_remote_when_present(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() assert 'DEPLOY_REMOTE="${TELEO_DEPLOY_REMOTE:-}"' in auto_deploy assert "remote get-url github" in auto_deploy assert 'git fetch "$DEPLOY_REMOTE" main' in auto_deploy assert 'git rev-parse "$DEPLOY_REMOTE/main"' in auto_deploy assert 'git merge --ff-only "$DEPLOY_REMOTE/main"' in auto_deploy def test_auto_deploy_executable_safety_net_stays_inside_synced_dirs(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() assert 'for dir in "$PIPELINE_DIR" "$TELEGRAM_DIR" "$DIAGNOSTICS_DIR" "$AGENT_STATE_DIR"' in auto_deploy assert "find /opt/teleo-eval -maxdepth 3 -name '*.sh'" not in auto_deploy assert "backups" in auto_deploy def test_agent_healthcheck_timer_runs_both_live_telegram_agents(): service = (REPO_ROOT / "systemd" / "teleo-agent-healthcheck.service").read_text() timer = (REPO_ROOT / "systemd" / "teleo-agent-healthcheck.timer").read_text() assert "/opt/teleo-eval/telegram/agent_healthcheck.py" in service assert "--agents leo leo-wallet-test" in service assert '--since "20 min ago"' in service assert "OnUnitActiveSec=5min" in timer assert "WantedBy=timers.target" in timer def test_deploy_scripts_install_systemd_units_and_enable_agent_healthcheck(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() manual_deploy = (REPO_ROOT / "deploy" / "deploy.sh").read_text() assert 'SYSTEMD_DIR="/etc/systemd/system"' in auto_deploy assert 'sudo install -m 0644 "$unit" "$SYSTEMD_DIR/$(basename "$unit")"' in auto_deploy assert "sudo systemctl daemon-reload" in auto_deploy assert "sudo systemctl enable --now teleo-agent-healthcheck.timer" in auto_deploy assert 'git diff --name-only "$OLD_SHA" "$NEW_SHA" -- systemd/teleo-agent@.service' in auto_deploy assert 'VPS_SYSTEMD="/etc/systemd/system"' in manual_deploy assert "teleo-agent-healthcheck.timer" in manual_deploy def _leoclean_gateway_restart_required(*changed_paths: str) -> bool: result = subprocess.run( ["bash", str(REPO_ROOT / "deploy" / "leoclean-gateway-restart-required.sh")], input="\n".join(changed_paths), capture_output=True, check=False, text=True, ) assert result.returncode in {0, 1}, result.stderr return result.returncode == 0 def test_deploy_scripts_sync_leoclean_skills_and_restart_gateway_for_runtime_changes(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() manual_deploy = (REPO_ROOT / "deploy" / "deploy.sh").read_text() sudoers = (REPO_ROOT / "deploy" / "sudoers" / "teleo-auto-deploy").read_text() assert 'LEOCLEAN_BIN_DIR="/home/teleo/.hermes/profiles/leoclean/bin"' in auto_deploy assert 'LEOCLEAN_SKILLS_DIR="/home/teleo/.hermes/profiles/leoclean/skills"' in auto_deploy assert 'LEOCLEAN_PLUGINS_DIR="/home/teleo/.hermes/profiles/leoclean/plugins"' in auto_deploy assert "hermes-agent/leoclean-bin/" in auto_deploy assert "hermes-agent/leoclean-skills/vps/" in auto_deploy assert "hermes-agent/leoclean-plugins/vps/" in auto_deploy assert 'HERMES_PATCH_DIR="/home/teleo/.hermes/teleo-runtime-patches"' in auto_deploy assert '"status": "installed_now"' in auto_deploy assert "Hermes response-transform drift repaired" in auto_deploy assert "deploy/leoclean-gateway-restart-required.sh" in auto_deploy assert "TELEO_AUTO_DEPLOY_REEXECED" in auto_deploy assert 'exec bash "$DEPLOY_CHECKOUT/deploy/auto-deploy.sh"' in auto_deploy assert "add_restart_if_unit_active leoclean-gateway" in auto_deploy assert 'sudo systemctl restart "$svc"' in auto_deploy assert "sudo systemctl restart $RESTART" not in auto_deploy assert "NOPASSWD: /usr/bin/systemctl restart leoclean-gateway" in sudoers assert "NOPASSWD: /bin/systemctl restart leoclean-gateway" in sudoers assert 'VPS_LEOCLEAN_BIN="/home/teleo/.hermes/profiles/leoclean/bin"' in manual_deploy assert 'VPS_LEOCLEAN_SKILLS="/home/teleo/.hermes/profiles/leoclean/skills"' in manual_deploy assert 'VPS_LEOCLEAN_PLUGINS="/home/teleo/.hermes/profiles/leoclean/plugins"' in manual_deploy assert "hermes-agent/leoclean-bin/" in manual_deploy assert "hermes-agent/leoclean-skills/vps/" in manual_deploy assert "hermes-agent/leoclean-plugins/vps/" in manual_deploy assert 'VPS_HERMES_PATCHES="/home/teleo/.hermes/teleo-runtime-patches"' in manual_deploy assert "systemctl is-active --quiet leoclean-gateway.service" in manual_deploy def test_deploy_scripts_syntax_check_leoclean_database_runtime_before_restart(): auto_deploy = (REPO_ROOT / "deploy" / "auto-deploy.sh").read_text() manual_deploy = (REPO_ROOT / "deploy" / "deploy.sh").read_text() assert "hermes-agent/leoclean-bin/*.py" in auto_deploy assert "scripts/leo_behavior_manifest.py" in auto_deploy assert '"$REPO_ROOT/hermes-agent/leoclean-bin/"*.py' in manual_deploy assert '"$REPO_ROOT/scripts/leo_behavior_manifest.py"' in manual_deploy def test_auto_deploy_hot_syncs_leoclean_markdown_without_gateway_restart(): assert not _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/teleo-kb-bridge/SKILL.md") assert not _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/live-leo-telegram/SKILL.md") def test_auto_deploy_restarts_gateway_for_leoclean_runtime_changes(): assert _leoclean_gateway_restart_required("hermes-agent/leoclean-bin/kb_tool.py") assert _leoclean_gateway_restart_required("hermes-agent/leoclean-skills/vps/teleo-kb-bridge/runtime_helper.py") assert _leoclean_gateway_restart_required("hermes-agent/leoclean-plugins/vps/leo-db-context/__init__.py") assert _leoclean_gateway_restart_required("hermes-agent/patches/apply_response_transform_hook.py") def test_auto_deploy_sudoers_installer_is_root_only_and_visudo_checked(): installer = (REPO_ROOT / "deploy" / "install-auto-deploy-sudoers.sh").read_text() assert 'TARGET="/etc/sudoers.d/teleo-auto-deploy"' in installer assert "id -u" in installer assert 'install -m 0440 "$SOURCE" "$TARGET"' in installer assert 'visudo -cf "$TARGET"' in installer def test_gcp_leoclean_skill_sync_targets_parallel_runtime_without_secrets(): script_path = REPO_ROOT / "deploy" / "sync-gcp-leoclean-skills.sh" script = script_path.read_text() assert 'PROJECT="${PROJECT:-teleo-501523}"' in script assert 'ZONE="${ZONE:-europe-west6-a}"' in script assert 'INSTANCE="${INSTANCE:-teleo-prod-1}"' in script assert 'SERVICE="${SERVICE:-leoclean-gcp-prod-parallel.service}"' in script assert "hermes-agent/leoclean-skills/gcp" in script assert "--tunnel-through-iap" in script assert "/home/teleo/.hermes/profiles/leoclean/skills" in script assert "/kb/claims/|wrap claim IDs, proposal IDs" in script assert 'COPYFILE_DISABLE=1 tar --format=ustar -C "$SOURCE_DIR" -cf - .' in script assert 'sudo tar -C "\\$remote_skills" -xf -' in script assert "sudo systemctl restart" in script assert "sudo rsync" not in script assert "TELEGRAM_BOT_TOKEN" not in script assert "CLOUDSQL_PASSWORD" not in script def test_gcp_runtime_sync_is_source_bound_and_scoped(): script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() dropin = (REPO_ROOT / "systemd" / "leoclean-gcp-prod-parallel-cloudsql.conf").read_text() assert 'git -C "$REPO_ROOT" rev-parse HEAD' in script assert 'git -C "$REPO_ROOT" diff --quiet HEAD' in script assert "merge-base --is-ancestor" in script assert "Refusing to deploy unmerged revision" in script assert "Refusing to install runtime files without --restart" in script assert 'DROPIN_NAME="10-cloudsql-memory.conf"' in script assert "20-canonical-kb.conf" not in script assert "sha256sum" in script assert "--preflight-only" in script assert "--restart" in script assert '/usr/bin/python3 -I "$tool_path" status --format json --redacted' in script assert "sa-teleo-prod-vm@teleo-501523.iam.gserviceaccount.com" in script assert "169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email" in script assert "ProxyHandler({})" in script assert "env -i" in script assert 'run_scoped_status pre "$payload_dir/$TOOL_REL"' in script assert 'run_permission_verifier pre "$payload_dir/$PERMISSION_VERIFIER_REL"' in script assert 'sudo -n -u teleo mkdir -p -- "$receipt_dir"' in script assert 'if [ "$PREFLIGHT_ONLY" = true ]; then' in script assert "run_service_wrapper_status post" in script assert "run_service_wrapper_status verify" in script assert '"$SERVICE_PROFILE_BIN/teleo-kb" status --format json --redacted' in script wrapper_probe = script.split("run_service_wrapper_status() {", maxsplit=1)[1].split( "\n}\n\nservice_fingerprint", maxsplit=1 )[0] assert 'run_in_service_cgroup "$main_pid"' in wrapper_probe assert 'nsenter -t "$main_pid" -m -n --env' in wrapper_probe assert "/usr/bin/setpriv" in wrapper_probe assert "--reuid=teleo" in wrapper_probe assert "TELEO_CLOUDSQL_" not in wrapper_probe assert "/usr/bin/env -i" not in wrapper_probe assert 'run_permission_verifier post "$runtime_bin/verify_gcp_leoclean_runtime_permissions.py"' in script assert 'SERVICE_ENV_VERIFIER_REL="ops/verify_gcp_leoclean_service_environment.py"' in script assert 'assert_runtime_environment "$runtime_bin/verify_gcp_leoclean_service_environment.py"' in script assert 'assert_runtime_environment "$REMOTE_RUNTIME_BIN/verify_gcp_leoclean_service_environment.py"' in script assert "REMOTE_CLOUDSDK_CONFIG" not in script assert "--property=DropInPaths" in script assert "--property=EnvironmentFiles" in script assert "Service has an effective EnvironmentFile" in script assert "Service has a later drop-in" in script assert "--property=PrivateNetwork" in script assert "--property=PrivateUsers" in script assert "NetworkNamespacePath" in script assert "IPAddressDeny" in script assert "IPAddressAllow" in script assert "RestrictAddressFamilies" in script assert "RestrictNetworkInterfaces" in script assert "SystemCallFilter" in script assert "AppArmorProfile" in script assert "SELinuxContext" in script assert '--property="$network_property"' in script assert "assert_administrator_secret_denied" in script assert '"stdout_discarded": True' in script assert '"credential_context": "systemd_main_pid_environment"' in script administrator_probe = script.split("assert_administrator_secret_denied() {", maxsplit=1)[1].split( "\n}\n\nrun_scoped_status", maxsplit=1 )[0] assert 'run_in_service_cgroup "$main_pid"' in administrator_probe assert 'nsenter -t "$main_pid" -m -n --env' in administrator_probe assert "/usr/bin/setpriv" in administrator_probe assert "env -i" not in administrator_probe permission_probe = script.split("run_permission_verifier() {", maxsplit=1)[1].split( "\n}\n\nassert_trusted_executable", maxsplit=1 )[0] assert 'if [ "$phase" = pre ]; then' in permission_probe assert 'run_in_service_cgroup "$main_pid"' in permission_probe assert 'nsenter -t "$main_pid" -m -n' in permission_probe assert "/usr/bin/env -i" in permission_probe assert 'verification_tmp="$(sudo mktemp -d /run/' in script assert "run_in_service_cgroup() {" in script assert "--property=ControlGroup" in script assert 'Path(f"/proc/{pid}/cgroup")' in script assert 'Path("/sys/fs/cgroup").resolve(strict=True)' in script assert 'cgroup_dir / "cgroup.procs"' in script assert "os.WUNTRACED" in script assert "signal.SIGSTOP" in script assert "signal.SIGCONT" in script assert "os.waitpid(child, 0)" in script assert "PR_SET_PDEATHSIG" in script assert "signal.pthread_sigmask" in script assert "ParentInterrupted" in script assert "kill_and_reap(child)" in script assert "Seccomp_filters" in script assert '"/proc/$main_pid/attr/current"' in script assert "/proc/1/ns/user" in script assert "resource.prlimit" in script assert "copy_process_limits" in script assert "configure_stopped_child" in script assert 'preflight_service_before="$(service_fingerprint)"' in script assert 'assert_service_fingerprint "$preflight_service_before"' in script assert "backup_path" in script assert "backup_directory_state" in script assert "restore_path" in script assert "restore_directory_state" in script assert 'runtime_grandparent="$(dirname "$runtime_parent")"' in script assert 'runtime_anchor="$(dirname "$runtime_grandparent")"' in script assert 'dropin_parent="$(dirname "$dropin_dir")"' in script assert 'backup_directory_state "$runtime_grandparent" runtime_grandparent' in script assert 'restore_directory_state "$runtime_grandparent" runtime_grandparent' in script assert 'sudo test -d "$runtime_anchor"' in script assert 'sudo test -d "$dropin_parent"' in script assert 'sudo rmdir -- "$target"' in script assert "rollback_runtime" in script assert "trap 'on_exit $?' EXIT" in script assert 'backup_path "$dropin_dir/$DROPIN_NAME" dropin' in script assert 'dropin_stage="$dropin_dir/.${DROPIN_NAME}.livingip-new.$$"' in script assert 'sudo mv -f -- "$dropin_stage" "$dropin_dir/$DROPIN_NAME"' in script assert 'active_state" != "active"' in script assert 'sub_state" != "running"' in script assert 'sudo systemctl stop "$SERVICE"' in script assert 'sudo systemctl start "$SERVICE"' in script assert 'REMOTE_RUNTIME_BIN="${REMOTE_RUNTIME_BIN:-/usr/local/libexec/livingip/leoclean-kb}"' in script assert 'sudo install -d -o root -g root -m 0755 "$runtime_parent"' in script assert 'sudo install -d -o root -g root -m 0755 "$runtime_bin"' in script assert 'remote_tmp="$(sudo mktemp -d /run/teleo-gcp-leoclean-runtime.XXXXXX)"' in script assert 'sudo install -d -o root -g root -m 0755 "$payload_dir"' in script assert 'sudo install -d -o root -g root -m 0700 "$backup_dir"' in script assert 'sudo install -d -o teleo -g teleo -m 0700 "$work_dir"' in script assert "sudo tar --no-same-owner --no-same-permissions" in script assert 'sudo chown -R root:root "$payload_dir"' in script assert 'sudo find "$payload_dir" -type d -exec chmod 0555 {} +' in script assert 'sudo find "$payload_dir" -type l -print -quit' in script assert 'sudo -n -u teleo test ! -w "$payload_dir"' in script assert 'sudo -n -u teleo test ! -w "$backup_dir"' in script assert 'sudo touch "$backup_dir/$name.present"' in script assert 'sudo test -f "$backup_dir/$name.present"' in script assert 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL" "$wrapper_stage"' in script assert 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL" "$tool_stage"' in script assert ( 'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL" ' '"$permission_verifier_stage"' in script ) assert ( 'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL" ' '"$service_env_verifier_stage"' in script ) assert "assert_installed_files\nsudo systemctl daemon-reload" in script assert 'tar -C "$remote_tmp" -xf -' not in script assert 'mkdir -p "$backup_dir"' not in script assert 'for protected_file in \\\n "$wrapper" \\' in script assert '"$permission_verifier" \\' in script assert '"$service_env_verifier" \\' in script assert 'sudo -n -u teleo test ! -w "$protected_file"' in script assert "for protected_dir in \\\n /usr \\" in script assert "/usr/local/libexec/livingip" in script assert 'sudo -n -u teleo test ! -w "$protected_dir"' in script assert "for executable_dir in \\\n /usr/local/sbin \\" in script assert "sudo stat -Lc '%U:%G' \"$executable_dir\"" in script assert 'sudo -n -u teleo test ! -w "$executable_dir"' in script assert ( "for protected_file in /bin/bash /usr/bin/curl /usr/bin/env /usr/bin/gcloud " "/usr/bin/psql /usr/bin/python3 /usr/bin/setpriv" in script ) assert "/usr/bin/curl -q --noproxy" in script assert 'test "$(sudo cat "$revision")" = "$SOURCE_REVISION"' in script assert 'assert_existing_root_directory "$existing_directory"' in script assert 'assert_existing_root_file "$existing_file"' in script assert 'validate_restored_path "$runtime_bin/teleo-kb" wrapper' in script assert 'validate_restored_directory "$runtime_bin" runtime_bin' in script rollback = script.split("rollback_runtime() {", maxsplit=1)[1].split("\n}\n\non_exit()", maxsplit=1)[0] assert rollback.index("validate_restored_path") < rollback.index("systemctl daemon-reload") assert rollback.index("systemctl daemon-reload") < rollback.index('systemctl start "$SERVICE"') assert rollback.count('if [ "$rollback_rc" -eq 0 ]; then') >= 3 deploy_tail = script.split("mutation_started=true", maxsplit=1)[1] assert deploy_tail.index("assert_installed_files") < deploy_tail.index("systemctl daemon-reload") assert deploy_tail.index("systemctl daemon-reload") < deploy_tail.index("assert_unit_configuration") assert deploy_tail.index("assert_unit_configuration") < deploy_tail.index('systemctl start "$SERVICE"') for install_line in ( 'sudo install -o root -g root -m 0755 "$payload_dir/$WRAPPER_REL"', 'sudo install -o root -g root -m 0755 "$payload_dir/$TOOL_REL"', 'sudo install -o root -g root -m 0755 "$payload_dir/$PERMISSION_VERIFIER_REL"', 'sudo install -o root -g root -m 0755 "$payload_dir/$SERVICE_ENV_VERIFIER_REL"', 'sudo install -o root -g root -m 0644 "$payload_dir/$DROPIN_REL"', ): assert install_line in deploy_tail assert 'sudo install -o root -g root -m 0755 "$work_dir/' not in script assert 'nsenter -t "$main_pid" -m -- sha256sum "$SERVICE_PROFILE_BIN/teleo-kb"' in script assert 'findmnt -T "$SERVICE_PROFILE_BIN"' in script assert "findmnt -M /home/teleo" in script assert 'findmnt -M "$SERVICE_PROFILE_ROOT/state"' in script assert 'findmnt -M "$SERVICE_PROFILE_ROOT/workspace"' in script assert "for capability_field in CapInh CapPrm CapEff CapBnd CapAmb" in script assert "0000000000000000" in script assert '"current_tier": "T3_live_readonly"' in script assert '"status": "pass"' in script assert "UnsetEnvironment=PGPASSWORD" in dropin assert "UnsetEnvironment=GOOGLE_APPLICATION_CREDENTIALS CLOUDSDK_CORE_PROJECT CLOUDSDK_AUTH_ACCESS_TOKEN" in dropin assert "UnsetEnvironment=BASH_ENV ENV BASHOPTS SHELLOPTS" in dropin assert "UnsetEnvironment=LD_PRELOAD LD_LIBRARY_PATH LD_AUDIT" in dropin assert "UnsetEnvironment=PYTHONHOME PYTHONPATH PYTHONSTARTUP PYTHONINSPECT" in dropin assert "UnsetEnvironment=HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY" in dropin assert "UnsetEnvironment=SSL_CERT_FILE SSL_CERT_DIR REQUESTS_CA_BUNDLE CURL_CA_BUNDLE SSLKEYLOGFILE" in dropin assert ( "UnsetEnvironment=TELEO_CLOUDSQL_CREDENTIAL_MODE TELEO_KB_CLAIM_BASE_URL " "TELEO_KB_READ_ATTEMPTS TELEO_MEMORY_INCLUDE_EXCERPTS TELEO_MEMORY_REDACT" in dropin ) assert ( "BindReadOnlyPaths=/usr/local/libexec/livingip/leoclean-kb:/home/teleo/.hermes/profiles/leoclean/bin" in dropin ) assert "ReadOnlyPaths=/home/teleo" in dropin assert ( "ReadWritePaths=/home/teleo/.hermes/profiles/leoclean/state " "/home/teleo/.hermes/profiles/leoclean/workspace" in dropin ) assert "CapabilityBoundingSet=" in dropin assert "AmbientCapabilities=" in dropin assert "RuntimeDirectory=" not in dropin assert "Environment=CLOUDSDK_CONFIG=/usr/local/libexec/livingip/leoclean-kb/gcloud-config" in dropin assert "Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" in dropin assert "TELEO_KB_MODE=cloudsql" in dropin assert "TELEO_GCP_METADATA_ONLY=1" in dropin assert "TELEO_CLOUDSQL_USER=leoclean_kb_runtime" in dropin assert "gcp-teleo-pgvector-standby-leoclean-kb-runtime-password" in dropin assert "TELEO_CLOUDSQL_ENABLE_AUDIT_FALLBACK=0" in dropin assert not any(line.startswith("Environment=PGPASSWORD") for line in dropin.splitlines()) assert "gcp-teleo-pgvector-standby-postgres-password" not in dropin def test_gcp_runtime_embedded_remote_programs_are_bash_syntax_valid() -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() blocks: dict[str, str] = {} for variable in ("remote_helpers", "verify_body", "sync_body"): marker = f"{variable}=$(cat <<'REMOTE'\n" assert script.count(marker) == 1 start = script.index(marker) + len(marker) end = script.index("\nREMOTE\n)", start) blocks[variable] = script[start:end] for name, body in ( *blocks.items(), ("composed_verify", blocks["remote_helpers"] + "\n" + blocks["verify_body"]), ("composed_sync", blocks["remote_helpers"] + "\n" + blocks["sync_body"]), ): completed = subprocess.run( ["bash", "-n"], input=body, capture_output=True, check=False, text=True, ) assert completed.returncode == 0, f"{name}: {completed.stderr}" embedded_python = [part.split("\nPY", maxsplit=1)[0] for part in blocks["remote_helpers"].split("<<'PY'\n")[1:]] assert len(embedded_python) == 4 for index, body in enumerate(embedded_python): completed = subprocess.run( ["python3", "-c", "import sys; compile(sys.stdin.read(), '', 'exec')"], input=body, capture_output=True, check=False, text=True, ) assert completed.returncode == 0, f"embedded_python_{index}: {completed.stderr}" @pytest.mark.parametrize( ("command", "expected_returncode"), [ (["/bin/sh", "-c", "exit 0"], 0), (["/bin/sh", "-c", "exit 7"], 7), (["/bin/sh", "-c", "kill -TERM $$"], 128 + signal.SIGTERM), ], ) def test_service_context_runner_propagates_result_and_reaps_child( command: list[str], expected_returncode: int, ) -> None: run_stopped_child = load_service_context_runner()["run_stopped_child"] moved_children: list[int] = [] result = run_stopped_child( # type: ignore[operator] command, "/test.service", move_child=moved_children.append, read_child_cgroup=lambda _pid: "/test.service", prepare_child=lambda _parent: None, ) assert result == expected_returncode assert len(moved_children) == 1 with pytest.raises(ProcessLookupError): os.kill(moved_children[0], 0) def test_service_context_runner_reaps_stopped_child_when_move_fails() -> None: run_stopped_child = load_service_context_runner()["run_stopped_child"] stopped_children: list[int] = [] def fail_move(child: int) -> None: stopped_children.append(child) raise RuntimeError("synthetic move failure") with pytest.raises(RuntimeError, match="synthetic move failure"): run_stopped_child( # type: ignore[operator] ["/bin/sh", "-c", "exit 0"], "/test.service", move_child=fail_move, read_child_cgroup=lambda _pid: "/test.service", prepare_child=lambda _parent: None, ) assert len(stopped_children) == 1 with pytest.raises(ProcessLookupError): os.kill(stopped_children[0], 0) @pytest.mark.skipif(not hasattr(resource, "prlimit"), reason="requires Linux prlimit") def test_service_context_runner_copies_mainpid_resource_limits(tmp_path: Path) -> None: namespace = load_service_context_runner() run_stopped_child = namespace["run_stopped_child"] copy_process_limits = namespace["copy_process_limits"] limit_readback = tmp_path / "limits.txt" child_program = ( "import resource,sys; from pathlib import Path; " "limits=resource.getrlimit(resource.RLIMIT_NOFILE); " "Path(sys.argv[1]).write_text(f'{limits[0]}:{limits[1]}', encoding='ascii')" ) def lower_nofile_limit() -> None: resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) source = subprocess.Popen( [sys.executable, "-c", "import time; time.sleep(30)"], preexec_fn=lower_nofile_limit, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) try: result = run_stopped_child( # type: ignore[operator] [sys.executable, "-c", child_program, str(limit_readback)], "/test.service", move_child=lambda _child: None, read_child_cgroup=lambda _pid: "/test.service", prepare_child=lambda _parent: None, configure_stopped_child=lambda child: copy_process_limits(source.pid, child), # type: ignore[operator] ) finally: source.terminate() source.wait(timeout=5) assert result == 0 assert limit_readback.read_text(encoding="ascii") == "64:64" @pytest.mark.parametrize("signum", [signal.SIGTERM, signal.SIGHUP]) def test_service_context_runner_interruption_kills_and_reaps_probe( tmp_path: Path, signum: signal.Signals, ) -> None: run_stopped_child = load_service_context_runner()["run_stopped_child"] child_pid_path = tmp_path / f"probe-{signum.name}.pid" child_program = ( "import os,sys,time; from pathlib import Path; " "Path(sys.argv[1]).write_text(str(os.getpid()), encoding='ascii'); " "time.sleep(30)" ) runner_pid = os.fork() if runner_pid == 0: result = run_stopped_child( # type: ignore[operator] [sys.executable, "-c", child_program, str(child_pid_path)], "/test.service", move_child=lambda _child: None, read_child_cgroup=lambda _pid: "/test.service", prepare_child=lambda _parent: None, ) os._exit(result) runner_reaped = False probe_pid: int | None = None try: deadline = time.monotonic() + 5 probe_pid_text = "" while time.monotonic() < deadline: if child_pid_path.is_file(): probe_pid_text = child_pid_path.read_text(encoding="ascii") if probe_pid_text.isdecimal(): break waited, _status = os.waitpid(runner_pid, os.WNOHANG) if waited == runner_pid: runner_reaped = True pytest.fail("service-context runner exited before probe start") time.sleep(0.01) assert probe_pid_text.isdecimal() probe_pid = int(probe_pid_text) os.kill(runner_pid, signum) waited, status = os.waitpid(runner_pid, 0) runner_reaped = True assert waited == runner_pid assert os.WIFSIGNALED(status) assert os.WTERMSIG(status) == signum with pytest.raises(ProcessLookupError): os.kill(probe_pid, 0) finally: if not runner_reaped: try: os.kill(runner_pid, signal.SIGKILL) except ProcessLookupError: pass try: os.waitpid(runner_pid, 0) except ChildProcessError: pass if probe_pid is not None: try: os.kill(probe_pid, signal.SIGKILL) except ProcessLookupError: pass def test_gcp_runtime_rejects_preexisting_symlink_before_mutation(tmp_path: Path) -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("assert_existing_root_directory() {") function_end = script.index("\n\nbackup_path() {", function_start) trust_functions = script[function_start:function_end] real_directory = tmp_path / "real-runtime" real_directory.mkdir() symlink = tmp_path / "runtime-link" symlink.symlink_to(real_directory, target_is_directory=True) harness = f""" set -euo pipefail sudo() {{ if [ "${{1:-}}" = -n ]; then shift if [ "${{1:-}}" = -u ]; then shift 2; fi fi "$@" }} {trust_functions} assert_existing_root_directory "$1" """ completed = subprocess.run( ["bash", "-s", "--", str(symlink)], input=harness, capture_output=True, check=False, text=True, ) assert completed.returncode != 0 def test_gcp_runtime_executable_configuration_override_gate() -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("assert_unit_configuration() {") function_end = script.index("\n\nassert_runtime_environment() {", function_start) unit_configuration_function = script[function_start:function_end] harness = f""" set -euo pipefail SERVICE=leoclean.service REMOTE_DROPIN_DIR=/etc/systemd/system/leoclean.service.d DROPIN_NAME=10-cloudsql-memory.conf REMOTE_RUNTIME_BIN=/usr/local/libexec/livingip/leoclean-kb SERVICE_PROFILE_ROOT=/home/teleo/.hermes/profiles/leoclean SERVICE_PROFILE_BIN=$SERVICE_PROFILE_ROOT/bin assert_exact_word_set() {{ local actual="$1" expected="$2" label="$3" actual_sorted expected_sorted word actual_sorted="$(for word in $actual; do printf '%s\\n' "${{word#-}}"; done | sort)" expected_sorted="$(for word in $expected; do printf '%s\\n' "${{word#-}}"; done | sort)" [ "$actual_sorted" = "$expected_sorted" ] || {{ echo "Service effective $label differs from the reviewed exact allowlist." >&2 return 1 }} }} systemctl() {{ local argument property= for argument in "$@"; do case "$argument" in --property=*) property=${{argument#--property=}} ;; esac done case "$property" in DropInPaths) printf '%s\\n' "${{FAKE_DROPINS:-}}" ;; EnvironmentFiles) printf '%s\\n' "${{FAKE_ENVIRONMENT_FILES:-}}" ;; LoadCredential|LoadCredentialEncrypted|SetCredential|SetCredentialEncrypted|ImportCredential|ExecCondition|ExecStartPre|ExecStartPost|ExecReload|ExecStop|ExecStopPost) local variable="FAKE_$property" printf '%s\\n' "${{!variable:-}}" ;; CapabilityBoundingSet|AmbientCapabilities|BindPaths|TemporaryFileSystem) printf '\\n' ;; NoNewPrivileges) printf 'yes\\n' ;; PrivateNetwork) printf '%s\\n' "${{FAKE_PrivateNetwork:-no}}" ;; PrivateUsers) printf '%s\\n' "${{FAKE_PrivateUsers:-no}}" ;; NetworkNamespacePath|IPAddressDeny|IPAddressAllow|RestrictAddressFamilies|RestrictNetworkInterfaces|SocketBindAllow|SocketBindDeny|SystemCallFilter|AppArmorProfile|SELinuxContext) local variable="FAKE_$property" printf '%s\\n' "${{!variable:-}}" ;; Group|SupplementaryGroups) printf 'teleo\\n' ;; ReadOnlyPaths) printf '%s\\n' "${{FAKE_ReadOnlyPaths:-/home/teleo}}" ;; ReadWritePaths) printf '%s\\n' "$SERVICE_PROFILE_ROOT/state $SERVICE_PROFILE_ROOT/workspace" ;; BindReadOnlyPaths) printf '%s\\n' "$REMOTE_RUNTIME_BIN:$SERVICE_PROFILE_BIN" ;; InaccessiblePaths) printf '%s\\n' '/home/teleo/.config/gcloud /home/teleo/.pgpass /home/teleo/.pg_service.conf /home/teleo/.postgresql' ;; *) return 90 ;; esac }} {unit_configuration_function} assert_unit_configuration """ reviewed = "/etc/systemd/system/leoclean.service.d/10-cloudsql-memory.conf" cases = ( { "FAKE_DROPINS": f"{reviewed} /etc/systemd/system/leoclean.service.d/99-conflict.conf", "FAKE_ENVIRONMENT_FILES": "", }, {"FAKE_DROPINS": reviewed, "FAKE_ENVIRONMENT_FILES": "/etc/leoclean-runtime.env"}, {"FAKE_DROPINS": reviewed, "FAKE_ExecStartPost": "/usr/local/bin/conflicting-hook"}, {"FAKE_DROPINS": reviewed, "FAKE_ReadOnlyPaths": "/home/teleo /tmp"}, {"FAKE_DROPINS": reviewed, "FAKE_PrivateNetwork": "yes"}, {"FAKE_DROPINS": reviewed, "FAKE_PrivateUsers": "yes"}, {"FAKE_DROPINS": reviewed, "FAKE_NetworkNamespacePath": "/run/netns/restricted"}, {"FAKE_DROPINS": reviewed, "FAKE_IPAddressDeny": "any"}, {"FAKE_DROPINS": reviewed, "FAKE_IPAddressAllow": "10.61.0.4/32"}, {"FAKE_DROPINS": reviewed, "FAKE_RestrictAddressFamilies": "AF_UNIX"}, {"FAKE_DROPINS": reviewed, "FAKE_RestrictNetworkInterfaces": "~ens4"}, {"FAKE_DROPINS": reviewed, "FAKE_SocketBindDeny": "any"}, {"FAKE_DROPINS": reviewed, "FAKE_SystemCallFilter": "@system-service ~@network-io"}, {"FAKE_DROPINS": reviewed, "FAKE_AppArmorProfile": "restricted-profile"}, {"FAKE_DROPINS": reviewed, "FAKE_SELinuxContext": "system_u:system_r:restricted_t:s0"}, ) for environment in cases: completed = subprocess.run( ["bash", "-s"], input=harness, capture_output=True, check=False, env=environment, text=True, ) assert completed.returncode != 0 allowed = subprocess.run( ["bash", "-s"], input=harness, capture_output=True, check=False, env={"FAKE_DROPINS": reviewed}, text=True, ) assert allowed.returncode == 0, allowed.stderr def test_gcp_runtime_effective_execstart_override_is_rejected_before_process_probe() -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("assert_service_running() {") function_end = script.index("\n\nassert_unit_configuration() {", function_start) service_function = script[function_start:function_end] harness = f""" set -euo pipefail SERVICE=leoclean.service EXPECTED_HERMES_EXECUTABLE=/home/teleo/.hermes/hermes-agent/hermes.py EXPECTED_HERMES_PYTHON=/home/teleo/.hermes/hermes-agent/venv/bin/python systemctl() {{ local argument property= for argument in "$@"; do case "$argument" in --property=*) property=${{argument#--property=}} ;; esac done case "$property" in ActiveState) printf 'active\\n' ;; SubState) printf 'running\\n' ;; MainPID) printf '1234\\n' ;; User) printf 'teleo\\n' ;; ExecStart) printf '{{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /tmp/conflicting.py ; }}\\n' ;; *) return 90 ;; esac }} ps() {{ printf 'teleo\\n'; }} {service_function} assert_service_running false """ completed = subprocess.run( ["bash", "-s"], input=harness, capture_output=True, check=False, text=True, ) assert completed.returncode != 0 assert "effective ExecStart is not the reviewed" in completed.stderr def test_gcp_runtime_absence_rollback_removes_every_created_directory(tmp_path: Path) -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("backup_directory_state() {") function_end = script.index("\nrestore_path() {", function_start) directory_functions = script[function_start:function_end] harness = f""" set -euo pipefail sudo() {{ "$@"; }} root="$1" backup_dir="$root/backup" mkdir -p "$backup_dir" "$root/usr/local" "$root/etc/systemd/system" runtime_grandparent="$root/usr/local/libexec" runtime_parent="$runtime_grandparent/livingip" runtime_bin="$runtime_parent/leoclean-kb" dropin_dir="$root/etc/systemd/system/service.d" {directory_functions} backup_directory_state "$runtime_grandparent" runtime_grandparent backup_directory_state "$runtime_parent" runtime_parent backup_directory_state "$runtime_bin" runtime_bin backup_directory_state "$dropin_dir" dropin_dir mkdir -p "$runtime_bin" "$dropin_dir" restore_directory_state "$runtime_bin" runtime_bin restore_directory_state "$runtime_parent" runtime_parent restore_directory_state "$runtime_grandparent" runtime_grandparent restore_directory_state "$dropin_dir" dropin_dir test ! -e "$runtime_grandparent" test ! -e "$dropin_dir" test -d "$root/usr/local" test -d "$root/etc/systemd/system" """ completed = subprocess.run( ["bash", "-s", "--", str(tmp_path)], input=harness, capture_output=True, check=False, text=True, ) assert completed.returncode == 0, completed.stderr def test_gcp_runtime_rollback_never_reloads_or_restarts_after_validation_failure(tmp_path: Path) -> None: script = (REPO_ROOT / "deploy" / "sync-gcp-leoclean-runtime.sh").read_text() function_start = script.index("rollback_runtime() {") function_end = script.index("\n\non_exit() {", function_start) rollback_function = script[function_start:function_end] action_log = tmp_path / "systemctl-actions" harness = f""" set -u SERVICE=leoclean.service runtime_bin=/usr/local/libexec/livingip/leoclean-kb runtime_parent=/usr/local/libexec/livingip runtime_grandparent=/usr/local/libexec dropin_dir=/etc/systemd/system/leoclean.service.d DROPIN_NAME=10-cloudsql-memory.conf action_log="$1" sudo() {{ if [ "${{1:-}}" = systemctl ]; then printf '%s\\n' "$2" >>"$action_log" return 0 fi "$@" }} restore_path() {{ return 0; }} restore_directory_state() {{ return 0; }} validate_restored_path() {{ [ "$2" != wrapper ]; }} validate_restored_directory() {{ return 0; }} assert_unit_configuration() {{ printf '%s\\n' unit-configuration >>"$action_log"; }} assert_service_running() {{ printf '%s\\n' assert-running >>"$action_log"; }} {rollback_function} rollback_runtime """ completed = subprocess.run( ["bash", "-s", "--", str(action_log)], input=harness, capture_output=True, check=False, text=True, ) assert completed.returncode != 0 assert action_log.read_text(encoding="utf-8").splitlines() == ["stop"]