From 32485aaba5809b95c55b8ff2f68a179d63b88a47 Mon Sep 17 00:00:00 2001 From: felipecr Date: Wed, 20 May 2026 09:44:54 +0200 Subject: [PATCH] installer first pass --- .gitignore | 1 + scripts/deploy/installer.py | 81 ++++ scripts/deploy/installer_modules/__init__.py | 1 + scripts/deploy/installer_modules/core.py | 427 ++++++++++++++++++ .../installer_modules/providers/__init__.py | 1 + .../installer_modules/providers/parallax.py | 28 ++ .../providers/podman_static.py | 171 +++++++ .../providers/squashfs_tools.py | 62 +++ .../installer_modules/providers/squashfuse.py | 60 +++ scripts/deploy/manifests/host-tools.yaml | 23 + .../deploy/manifests/sarus-suite-tools.yaml | 9 + scripts/deploy/requirements.txt | 1 + scripts/deploy/test_installer.py | 182 ++++++++ test/vagrant/ubuntu-24.04/.gitignore | 2 + test/vagrant/ubuntu-24.04/Vagrantfile | 91 ++++ .../ubuntu-24.04/prepare-cloud-image.sh | 136 ++++++ .../ubuntu-24.04/provision/test-host-tools.sh | 141 ++++++ 17 files changed, 1417 insertions(+) create mode 100755 scripts/deploy/installer.py create mode 100644 scripts/deploy/installer_modules/__init__.py create mode 100644 scripts/deploy/installer_modules/core.py create mode 100644 scripts/deploy/installer_modules/providers/__init__.py create mode 100644 scripts/deploy/installer_modules/providers/parallax.py create mode 100644 scripts/deploy/installer_modules/providers/podman_static.py create mode 100644 scripts/deploy/installer_modules/providers/squashfs_tools.py create mode 100644 scripts/deploy/installer_modules/providers/squashfuse.py create mode 100644 scripts/deploy/manifests/host-tools.yaml create mode 100644 scripts/deploy/manifests/sarus-suite-tools.yaml create mode 100644 scripts/deploy/requirements.txt create mode 100644 scripts/deploy/test_installer.py create mode 100644 test/vagrant/ubuntu-24.04/.gitignore create mode 100644 test/vagrant/ubuntu-24.04/Vagrantfile create mode 100755 test/vagrant/ubuntu-24.04/prepare-cloud-image.sh create mode 100755 test/vagrant/ubuntu-24.04/provision/test-host-tools.sh diff --git a/.gitignore b/.gitignore index 8b166e1..491e939 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ Cargo.lock !/Cargo.lock .DS_Store +__pycache__/ diff --git a/scripts/deploy/installer.py b/scripts/deploy/installer.py new file mode 100755 index 0000000..1a53fc1 --- /dev/null +++ b/scripts/deploy/installer.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import os +import sys +import tempfile +from pathlib import Path +from typing import List + +try: + from installer_modules.core import ( + DEFAULT_CACHE_DIR, + DEFAULT_INSTALL_PREFIX, + DEFAULT_STATE_ROOT, + DEFAULT_SUPPORT_ROOT, + HostToolsInstaller, + InstallerConfig, + InstallerError, + load_manifest, + ) +except ModuleNotFoundError as exc: + if exc.name == "yaml": + print( + "missing python dependency: PyYAML. Install it with " + "'python3 -m pip install -r scripts/deploy/requirements.txt' " + "or your distro package manager before using this installer.", + file=sys.stderr, + ) + raise SystemExit(2) from exc + raise + + +def parse_args(argv: List[str]) -> InstallerConfig: + def expand_path(path: Path) -> Path: + return Path(os.path.expandvars(str(path))).expanduser() + + parser = argparse.ArgumentParser( + description="Manifest-driven installer for common host tools and Sarus Suite tools." + ) + parser.add_argument("mode", choices=("common", "sarus")) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR) + parser.add_argument("--install-prefix", type=Path, default=DEFAULT_INSTALL_PREFIX) + parser.add_argument("--support-root", type=Path, default=DEFAULT_SUPPORT_ROOT) + parser.add_argument("--state-root", type=Path, default=DEFAULT_STATE_ROOT) + parser.add_argument("--stage-root", type=Path) + parser.add_argument("--write-manifest", type=Path) + parser.add_argument("--dry-run", action="store_true") + ns = parser.parse_args(argv) + + stage_root = ns.stage_root + if stage_root is None: + stage_root = Path(tempfile.mkdtemp(prefix=f"{ns.mode}-installer-")) + + return InstallerConfig( + mode=ns.mode, + manifest_path=expand_path(ns.manifest).resolve(), + cache_dir=expand_path(ns.cache_dir).resolve(), + install_prefix=expand_path(ns.install_prefix), + support_root=expand_path(ns.support_root), + state_root=expand_path(ns.state_root).resolve(), + stage_root=expand_path(stage_root).resolve(), + write_manifest=expand_path(ns.write_manifest).resolve() if ns.write_manifest else None, + dry_run=ns.dry_run, + ) + + +def main(argv: List[str]) -> int: + try: + config = parse_args(argv) + manifest = load_manifest(config.manifest_path) + HostToolsInstaller(config, manifest).run() + return 0 + except InstallerError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/deploy/installer_modules/__init__.py b/scripts/deploy/installer_modules/__init__.py new file mode 100644 index 0000000..4ac4aab --- /dev/null +++ b/scripts/deploy/installer_modules/__init__.py @@ -0,0 +1 @@ +"""Installer modules for general deployment tooling.""" diff --git a/scripts/deploy/installer_modules/core.py b/scripts/deploy/installer_modules/core.py new file mode 100644 index 0000000..5b9748b --- /dev/null +++ b/scripts/deploy/installer_modules/core.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +import importlib +import json +import pkgutil +import platform +import shutil +import subprocess +import tarfile +import urllib.request +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_CACHE_DIR = REPO_ROOT / ".deploy-cache" / "host-tools-installer" +DEFAULT_STATE_ROOT = REPO_ROOT / ".deploy-out" / "install-manifests" +DEFAULT_INSTALL_PREFIX = Path.home() / ".local" / "bin" +DEFAULT_SUPPORT_ROOT = Path.home() / ".local" / "share" / "cluster-tooling" / "host-tools" + + +@dataclass +class InstallerConfig: + mode: str + manifest_path: Path + cache_dir: Path + install_prefix: Path + support_root: Path + state_root: Path + stage_root: Path + write_manifest: Optional[Path] = None + dry_run: bool = False + + +# For tracking installed tools +@dataclass +class InstallRecord: + tool: str + source: str + staged_files: List[str] = field(default_factory=list) + installed_files: List[str] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +# for tracking non-tools (dirs and config files) +@dataclass +class SupportRecord: + name: str + source: str + staged_files: List[str] = field(default_factory=list) + installed_files: List[str] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +class InstallerError(RuntimeError): + pass + + +@dataclass +class ToolProvider: + name: str + domain: str + tools: List[str] + + def install(self, requested_tool: str, ctx: "InstallContext") -> InstallRecord: + raise NotImplementedError + + +# backbone of the installer, and provides some common helper functions: +# - resolve_source, download, unpack_tarball, run_command (for building), stage_binary, stage_file, stage_tree_once +class InstallContext: + def __init__(self, config: InstallerConfig, manifest: dict): + self.config = config + self.manifest = manifest + self.records: List[InstallRecord] = [] + self.support_records: List[SupportRecord] = [] + self.active_tools = self._resolve_active_tools() + self.bundle_state: Dict[str, object] = {} + self.arch = self._detect_architecture() + + def log(self, message: str) -> None: + print(f"[{self.config.mode}-installer] {message}") + + def _detect_architecture(self) -> str: + machine = platform.machine().lower() + arch_aliases = { + "x86_64": "amd64", + "amd64": "amd64", + "aarch64": "arm64", + "arm64": "arm64", + } + if machine not in arch_aliases: + raise InstallerError( + f"unsupported host architecture '{machine}'; expected one of: " + + ", ".join(sorted(arch_aliases)) + ) + return arch_aliases[machine] + + def resolve_source(self, key: str, default: Optional[str] = None) -> str: + sources = self.manifest.get("sources", {}) + if key in sources: + return str(sources[key]) + if default is not None: + return default + raise InstallerError(f"manifest is missing required source setting '{key}'") + + def ensure_dir(self, path: Path) -> None: + if self.config.dry_run: + return + path.mkdir(parents=True, exist_ok=True) + + def download(self, url: str, filename: str) -> Path: + downloads_dir = self.config.cache_dir / "downloads" + dest = downloads_dir / filename + self.log(f"downloading {url}") + if not self.config.dry_run: + downloads_dir.mkdir(parents=True, exist_ok=True) + if not dest.exists(): + urllib.request.urlretrieve(url, dest) + return dest + + def unpack_tarball(self, tarball: Path, destination: Path) -> None: + if self.config.dry_run: + return + shutil.rmtree(destination, ignore_errors=True) + destination.mkdir(parents=True, exist_ok=True) + with tarfile.open(tarball, "r:gz") as archive: + archive.extractall(destination) + + def run_command(self, argv: List[str], cwd: Path) -> None: + self.log(f"running {' '.join(argv)} in {cwd}") + if not self.config.dry_run: + try: + subprocess.run(argv, cwd=cwd, check=True) + except FileNotFoundError as exc: + command = argv[0] if argv else "" + raise InstallerError( + f"required build command '{command}' was not found in PATH; " + "install the system build prerequisites before running the installer" + ) from exc + + def stage_binary( + self, tool: str, src: Path, source: str, note: Optional[str] = None + ) -> InstallRecord: + staged_path = self.config.stage_root / "bin" / tool + installed_path = self.config.install_prefix / tool + + if not src.exists() and not self.config.dry_run: + raise InstallerError(f"expected artifact for {tool} at {src} but it was not found") + + if not self.config.dry_run: + staged_path.parent.mkdir(parents=True, exist_ok=True) + installed_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, staged_path) + staged_path.chmod(0o755) + shutil.copy2(staged_path, installed_path) + installed_path.chmod(0o755) + + notes = [note] if note else [] + return InstallRecord( + tool=tool, + source=source, + staged_files=[str(staged_path)], + installed_files=[str(installed_path)], + notes=notes, + ) + + def stage_script( + self, tool: str, content: str, source: str, note: Optional[str] = None + ) -> InstallRecord: + staged_path = self.config.stage_root / "bin" / tool + installed_path = self.config.install_prefix / tool + + if not self.config.dry_run: + staged_path.parent.mkdir(parents=True, exist_ok=True) + installed_path.parent.mkdir(parents=True, exist_ok=True) + staged_path.write_text(content, encoding="utf-8") + staged_path.chmod(0o755) + shutil.copy2(staged_path, installed_path) + installed_path.chmod(0o755) + + notes = [note] if note else [] + return InstallRecord( + tool=tool, + source=source, + staged_files=[str(staged_path)], + installed_files=[str(installed_path)], + notes=notes, + ) + + def stage_file( + self, + name: str, + content: str, + relative_path: Path, + source: str, + note: Optional[str] = None, + ) -> SupportRecord: + stage_dst = self.config.stage_root / "support" / relative_path + final_dst = self.config.support_root / relative_path + + if not self.config.dry_run: + stage_dst.parent.mkdir(parents=True, exist_ok=True) + final_dst.parent.mkdir(parents=True, exist_ok=True) + stage_dst.write_text(content, encoding="utf-8") + shutil.copy2(stage_dst, final_dst) + + notes = [note] if note else [] + return SupportRecord( + name=name, + source=source, + staged_files=[str(stage_dst)], + installed_files=[str(final_dst)], + notes=notes, + ) + + def stage_tree_once( + self, + state_key: str, + name: str, + src: Path, + relative_root: Path, + source: str, + note: Optional[str] = None, + ) -> SupportRecord: + cached = self.bundle_state.get(state_key) + if isinstance(cached, SupportRecord): + return cached + + if self.config.dry_run: + record = SupportRecord( + name=name, + source=source, + staged_files=[str(self.config.stage_root / "support" / relative_root)], + installed_files=[str(self.config.support_root / relative_root)], + notes=[note] if note else [], + ) + self.bundle_state[state_key] = record + return record + + stage_dst = self.config.stage_root / "support" / relative_root + final_dst = self.config.support_root / relative_root + self.copy_tree(src, stage_dst) + self.copy_tree(stage_dst, final_dst) + record = SupportRecord( + name=name, + source=source, + staged_files=[str(stage_dst)], + installed_files=[str(final_dst)], + notes=[note] if note else [], + ) + self.bundle_state[state_key] = record + return record + + def remember_record(self, record: InstallRecord) -> None: + self.records.append(record) + + def remember_support_record(self, record: SupportRecord) -> None: + if any( + existing.name == record.name and existing.installed_files == record.installed_files + for existing in self.support_records + ): + return + self.support_records.append(record) + + def copy_tree(self, src: Path, dst: Path) -> None: + dst.mkdir(parents=True, exist_ok=True) + for entry in src.iterdir(): + entry_dst = dst / entry.name + if entry.is_dir(): + shutil.copytree(entry, entry_dst, dirs_exist_ok=True) + else: + shutil.copy2(entry, entry_dst) + + def find_single_child(self, root: Path) -> Path: + children = [path for path in root.iterdir()] + if len(children) != 1: + raise InstallerError( + f"expected one unpacked directory under {root}, found {len(children)}" + ) + return children[0] + + def _resolve_active_tools(self) -> Set[str]: + profiles = self.manifest.get("profiles", {}) + if not isinstance(profiles, dict): + raise InstallerError("manifest field 'profiles' must be a mapping") + + active_tools: Set[str] = set() + for profile_name, profile in profiles.items(): + if not isinstance(profile, dict): + raise InstallerError(f"profile '{profile_name}' must be a mapping") + if not profile.get("enabled", False): + continue + tools = profile.get("tools", []) + if not isinstance(tools, list): + raise InstallerError(f"profile '{profile_name}'.tools must be a list") + for tool in tools: + if not isinstance(tool, str): + raise InstallerError( + f"profile '{profile_name}' contains a non-string tool entry" + ) + active_tools.add(tool) + return active_tools + + +# Here we auto discover providers, which dynamically loads providers and associate it to a unique tool +def discover_providers() -> Tuple[List[ToolProvider], Dict[str, ToolProvider]]: + from . import providers as providers_pkg + + providers: List[ToolProvider] = [] + tool_map: Dict[str, ToolProvider] = {} + discovered_modules = sorted(pkgutil.iter_modules(providers_pkg.__path__), key=lambda item: item.name) + + for module_info in discovered_modules: + if module_info.name.startswith("_"): + continue + + module = importlib.import_module(f"{providers_pkg.__name__}.{module_info.name}") + provider = getattr(module, "PROVIDER", None) + if provider is None: + raise InstallerError( + f"provider module '{module.__name__}' does not export a PROVIDER object" + ) + if not isinstance(provider, ToolProvider): + raise InstallerError( + f"provider module '{module.__name__}' exported an invalid PROVIDER object" + ) + if not provider.tools: + raise InstallerError(f"provider '{provider.name}' does not declare any tools") + + providers.append(provider) + for tool in provider.tools: + existing = tool_map.get(tool) + if existing is not None: + raise InstallerError( + f"tool '{tool}' is owned by both provider '{existing.name}' " + f"and provider '{provider.name}'" + ) + tool_map[tool] = provider + + if not providers: + raise InstallerError("no installer providers were discovered") + + return providers, tool_map + + +class HostToolsInstaller: + def __init__(self, config: InstallerConfig, manifest: dict): + self.ctx = InstallContext(config, manifest) + self.providers, self.tool_map = discover_providers() + + def run(self) -> None: + self._validate_manifest() + self._prepare_dirs() + self.ctx.log(f"discovered providers: {', '.join(provider.name for provider in self.providers)}") + self.ctx.log(f"active tools: {', '.join(sorted(self.ctx.active_tools)) or '(none)'}") + + for tool_name in sorted(self.ctx.active_tools): + provider = self.tool_map[tool_name] + record = provider.install(tool_name, self.ctx) + self.ctx.remember_record(record) + + manifest_path = self._write_install_manifest() + self.ctx.log(f"installation manifest written to {manifest_path}") + + def _validate_manifest(self) -> None: + sources = self.ctx.manifest.get("sources", {}) + if not isinstance(sources, dict): + raise InstallerError("manifest field 'sources' must be a mapping") + + unknown = sorted(tool for tool in self.ctx.active_tools if tool not in self.tool_map) + if unknown: + raise InstallerError(f"unknown tools in manifest: {', '.join(unknown)}") + + wrong_domain = sorted( + tool + for tool in self.ctx.active_tools + if self.tool_map[tool].domain != self.ctx.config.mode + ) + if wrong_domain: + if self.ctx.config.mode == "common": + raise InstallerError( + "common installer must not manage Sarus Suite components: " + + ", ".join(wrong_domain) + ) + raise InstallerError( + "sarus installer received non-Sarus tools: " + ", ".join(wrong_domain) + ) + + def _prepare_dirs(self) -> None: + for path in [ + self.ctx.config.cache_dir, + self.ctx.config.stage_root, + self.ctx.config.install_prefix, + self.ctx.config.support_root, + self.ctx.config.state_root, + ]: + self.ctx.ensure_dir(path) + + def _write_install_manifest(self) -> Path: + output_path = self.ctx.config.write_manifest or ( + self.ctx.config.state_root / f"{self.ctx.config.mode}-host-tools-install.json" + ) + payload = { + "mode": self.ctx.config.mode, + "manifest": str(self.ctx.config.manifest_path), + "install_prefix": str(self.ctx.config.install_prefix), + "support_root": str(self.ctx.config.support_root), + "stage_root": str(self.ctx.config.stage_root), + "tools": [asdict(record) for record in self.ctx.records], + "support_artifacts": [asdict(record) for record in self.ctx.support_records], + } + if not self.ctx.config.dry_run: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return output_path + + +def load_manifest(path: Path) -> dict: + with path.open("r", encoding="utf-8") as fh: + data = yaml.safe_load(fh) + if not isinstance(data, dict): + raise InstallerError("manifest root must be a mapping") + return data diff --git a/scripts/deploy/installer_modules/providers/__init__.py b/scripts/deploy/installer_modules/providers/__init__.py new file mode 100644 index 0000000..18af88f --- /dev/null +++ b/scripts/deploy/installer_modules/providers/__init__.py @@ -0,0 +1 @@ +"""Curated tool-family providers for the deployment installer.""" diff --git a/scripts/deploy/installer_modules/providers/parallax.py b/scripts/deploy/installer_modules/providers/parallax.py new file mode 100644 index 0000000..3d2ea23 --- /dev/null +++ b/scripts/deploy/installer_modules/providers/parallax.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from ..core import InstallContext, InstallRecord, ToolProvider + + +class ParallaxProvider(ToolProvider): + DEFAULT_VERSION = "0.10.2" + DEFAULT_PLATFORM = "ubuntu-24.04" + FILENAME_TEMPLATES = { + "parallax": "parallax-v{version}-{platform}-{arch}", + "parallax-mount-program": "parallax-mount-program-v{version}.sh", + } + + def __init__(self) -> None: + super().__init__(name="parallax", domain="sarus", tools=list(self.FILENAME_TEMPLATES)) + + def install(self, requested_tool: str, ctx: InstallContext) -> InstallRecord: + version = ctx.resolve_source("parallax_version", self.DEFAULT_VERSION) + url = ( + "https://github.com/sarus-suite/parallax/releases/download/" + f"v{version}/" + f"{self.FILENAME_TEMPLATES[requested_tool].format(version=version, platform=self.DEFAULT_PLATFORM, arch=ctx.arch)}" + ) + src = ctx.download(url, requested_tool) + return ctx.stage_binary(requested_tool, src, f"parallax:{version}") + + +PROVIDER = ParallaxProvider() diff --git a/scripts/deploy/installer_modules/providers/podman_static.py b/scripts/deploy/installer_modules/providers/podman_static.py new file mode 100644 index 0000000..db7c8cc --- /dev/null +++ b/scripts/deploy/installer_modules/providers/podman_static.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import shlex +from pathlib import Path +from typing import Dict, List + +from ..core import InstallContext, InstallRecord, InstallerError, ToolProvider + + +class PodmanStaticProvider(ToolProvider): + DEFAULT_VERSION = "v5.8.2" + DEFAULT_URL_TEMPLATE = ( + "https://github.com/mgoltzsche/podman-static/releases/download/{version}/" + "podman-linux-{arch}.tar.gz" + ) + ARTIFACT_PATHS: Dict[str, List[str]] = { + "podman": ["usr/local/bin/podman", "usr/bin/podman"], + "conmon": [ + "usr/local/libexec/podman/conmon", + "usr/local/lib/podman/conmon", + "usr/libexec/podman/conmon", + "usr/lib/podman/conmon", + "usr/bin/conmon", + "usr/local/bin/conmon", + ], + "crun": ["usr/local/bin/crun", "usr/bin/crun"], + "fuse-overlayfs": ["usr/local/bin/fuse-overlayfs", "usr/bin/fuse-overlayfs"], + "fusermount3": ["usr/local/bin/fusermount3", "usr/bin/fusermount3"], + "pasta": ["usr/local/bin/pasta", "usr/bin/pasta"], + } + + def __init__(self) -> None: + super().__init__( + name="podman-static", + domain="common", + tools=list(self.ARTIFACT_PATHS), + ) + + def ensure(self, ctx: InstallContext) -> Path: + bundle_root = ctx.bundle_state.get("podman_static_bundle_root") + if isinstance(bundle_root, Path): + return bundle_root + + version = ctx.resolve_source("podman_static_version", self.DEFAULT_VERSION) + url = ctx.resolve_source( + "podman_static_url", + self.DEFAULT_URL_TEMPLATE.format(version=version, arch=ctx.arch), + ) + download_path = ctx.download(url, "podman-static.tar.gz") + unpack_root = ctx.config.cache_dir / "unpacked" / "podman-static" + + if not ctx.config.dry_run: + ctx.unpack_tarball(download_path, unpack_root) + bundle_root = ctx.find_single_child(unpack_root) + else: + bundle_root = unpack_root / "podman-static" + + ctx.bundle_state["podman_static_bundle_root"] = bundle_root + ctx.bundle_state["podman_static_source"] = f"podman-static:{url}" + return bundle_root + + def source_label(self, ctx: InstallContext) -> str: + source = ctx.bundle_state.get("podman_static_source") + if not isinstance(source, str): + self.ensure(ctx) + source = ctx.bundle_state["podman_static_source"] + return str(source) + + def support_root(self, ctx: InstallContext) -> Path: + return ctx.config.support_root / "podman-static" + + def helper_dirs(self, ctx: InstallContext) -> List[Path]: + support_root = self.support_root(ctx) + return [ + support_root / "usr/local/libexec/podman", + support_root / "usr/local/lib/podman", + support_root / "usr/libexec/podman", + support_root / "usr/lib/podman", + support_root / "usr/local/bin", + support_root / "usr/bin", + ] + + def install_support_once(self, ctx: InstallContext) -> None: + bundle_root = self.ensure(ctx) + source = self.source_label(ctx) + for rel in ("usr", "etc"): + src = bundle_root / rel + if src.exists() or ctx.config.dry_run: + record = ctx.stage_tree_once( + state_key=f"podman-static-support:{rel}", + name=f"podman-static-{rel}", + src=src, + relative_root=Path("podman-static") / rel, + source=source, + note="Installed under a private support tree instead of the host root.", + ) + ctx.remember_support_record(record) + + config_record = ctx.stage_file( + name="podman-static-containers-conf", + content=self._render_containers_conf(ctx), + relative_path=Path("podman-static/config/containers.conf"), + source=source, + note="Generated config that points Podman at the private helper-binary tree.", + ) + ctx.remember_support_record(config_record) + + def _render_containers_conf(self, ctx: InstallContext) -> str: + helper_dirs_list = self.helper_dirs(ctx) + helper_dirs = ", ".join(f'"{path}"' for path in helper_dirs_list) + conmon_path = ", ".join(f'"{path / "conmon"}"' for path in helper_dirs_list) + return ( + "[engine]\n" + f"helper_binaries_dir = [{helper_dirs}]\n" + f"conmon_path = [{conmon_path}]\n" + ) + + def render_wrapper(self, ctx: InstallContext, tool_relpath: Path) -> str: + support_root = self.support_root(ctx) + helper_path = ":".join(str(path) for path in self.helper_dirs(ctx)) + containers_conf = support_root / "config" / "containers.conf" + quoted_tool = shlex.quote(str(support_root / tool_relpath)) + quoted_conf = shlex.quote(str(containers_conf)) + quoted_helper_path = shlex.quote(helper_path) + return "\n".join( + [ + "#!/usr/bin/env sh", + "set -eu", + f'export PATH={quoted_helper_path}:"$PATH"', + f'export CONTAINERS_CONF={quoted_conf}', + f'exec {quoted_tool} "$@"', + "", + ] + ) + + def install(self, requested_tool: str, ctx: InstallContext) -> InstallRecord: + if requested_tool not in self.ARTIFACT_PATHS: + raise InstallerError(f"provider '{self.name}' does not manage tool '{requested_tool}'") + + bundle_root = self.ensure(ctx) + self.install_support_once(ctx) + src = self._resolve_artifact_path(ctx, bundle_root, requested_tool) + if requested_tool == "podman": + tool_relpath = src.relative_to(bundle_root) + return ctx.stage_script( + requested_tool, + self.render_wrapper(ctx, tool_relpath), + self.source_label(ctx), + "Wrapper script that runs Podman against the private support tree.", + ) + return ctx.stage_binary(requested_tool, src, self.source_label(ctx)) + + def _resolve_artifact_path( + self, ctx: InstallContext, bundle_root: Path, requested_tool: str + ) -> Path: + artifact_paths = [Path(candidate) for candidate in self.ARTIFACT_PATHS[requested_tool]] + if ctx.config.dry_run: + return bundle_root / artifact_paths[0] + + for artifact_path in artifact_paths: + candidate = bundle_root / artifact_path + if candidate.exists(): + return candidate + + joined = ", ".join(str(path) for path in artifact_paths) + raise InstallerError( + f"could not locate artifact for {requested_tool} in bundle; tried: {joined}" + ) + + +PROVIDER = PodmanStaticProvider() diff --git a/scripts/deploy/installer_modules/providers/squashfs_tools.py b/scripts/deploy/installer_modules/providers/squashfs_tools.py new file mode 100644 index 0000000..3e46dbc --- /dev/null +++ b/scripts/deploy/installer_modules/providers/squashfs_tools.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import os +import shutil +from pathlib import Path + +from ..core import InstallContext, InstallRecord, ToolProvider + + +class SquashfsToolsProvider(ToolProvider): + DEFAULT_VERSION = "4.6.1" + ARTIFACT_PATHS = { + "mksquashfs": "bin/mksquashfs", + "unsquashfs": "bin/unsquashfs", + } + + def __init__(self) -> None: + super().__init__(name="squashfs-tools", domain="common", tools=list(self.ARTIFACT_PATHS)) + + def ensure(self, ctx: InstallContext) -> Path: + install_root = ctx.bundle_state.get("squashfs_tools_install_root") + if isinstance(install_root, Path): + return install_root + + version = ctx.resolve_source("squashfs_tools_version", self.DEFAULT_VERSION) + tarball = ctx.download( + f"https://github.com/plougher/squashfs-tools/archive/refs/tags/{version}.tar.gz", + f"squashfs-tools-{version}.tar.gz", + ) + build_root = ctx.config.cache_dir / "build" / f"squashfs-tools-{version}" + install_root = ctx.config.cache_dir / "prefix" / f"squashfs-tools-{version}" + + if not ctx.config.dry_run and not (install_root / "bin" / "mksquashfs").exists(): + shutil.rmtree(build_root, ignore_errors=True) + shutil.rmtree(install_root, ignore_errors=True) + build_root.mkdir(parents=True, exist_ok=True) + install_root.mkdir(parents=True, exist_ok=True) + ctx.unpack_tarball(tarball, build_root) + srcdir = ctx.find_single_child(build_root) / "squashfs-tools" + ctx.run_command(["make", f"-j{os.cpu_count() or 1}"], cwd=srcdir) + (install_root / "bin").mkdir(parents=True, exist_ok=True) + for tool in ("mksquashfs", "unsquashfs"): + shutil.copy2(srcdir / tool, install_root / "bin" / tool) + + ctx.bundle_state["squashfs_tools_install_root"] = install_root + ctx.bundle_state["squashfs_tools_source"] = f"squashfs-tools:{version}" + return install_root + + def source_label(self, ctx: InstallContext) -> str: + source = ctx.bundle_state.get("squashfs_tools_source") + if not isinstance(source, str): + self.ensure(ctx) + source = ctx.bundle_state["squashfs_tools_source"] + return str(source) + + def install(self, requested_tool: str, ctx: InstallContext) -> InstallRecord: + install_root = self.ensure(ctx) + src = install_root / self.ARTIFACT_PATHS[requested_tool] + return ctx.stage_binary(requested_tool, src, self.source_label(ctx)) + + +PROVIDER = SquashfsToolsProvider() diff --git a/scripts/deploy/installer_modules/providers/squashfuse.py b/scripts/deploy/installer_modules/providers/squashfuse.py new file mode 100644 index 0000000..75227c9 --- /dev/null +++ b/scripts/deploy/installer_modules/providers/squashfuse.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import os +import shutil +from pathlib import Path + +from ..core import InstallContext, InstallRecord, ToolProvider + + +class SquashfuseProvider(ToolProvider): + DEFAULT_VERSION = "0.6.1" + ARTIFACT_PATHS = { + "squashfuse": "bin/squashfuse", + "squashfuse_ll": "bin/squashfuse_ll", + } + + def __init__(self) -> None: + super().__init__(name="squashfuse", domain="common", tools=list(self.ARTIFACT_PATHS)) + + def ensure(self, ctx: InstallContext) -> Path: + install_root = ctx.bundle_state.get("squashfuse_install_root") + if isinstance(install_root, Path): + return install_root + + version = ctx.resolve_source("squashfuse_version", self.DEFAULT_VERSION) + tarball = ctx.download( + f"https://github.com/vasi/squashfuse/releases/download/{version}/squashfuse-{version}.tar.gz", + f"squashfuse-{version}.tar.gz", + ) + build_root = ctx.config.cache_dir / "build" / f"squashfuse-{version}" + install_root = ctx.config.cache_dir / "prefix" / f"squashfuse-{version}" + + if not ctx.config.dry_run and not (install_root / "bin" / "squashfuse").exists(): + shutil.rmtree(build_root, ignore_errors=True) + shutil.rmtree(install_root, ignore_errors=True) + build_root.mkdir(parents=True, exist_ok=True) + ctx.unpack_tarball(tarball, build_root) + srcdir = ctx.find_single_child(build_root) + ctx.run_command(["./configure", f"--prefix={install_root}"], cwd=srcdir) + ctx.run_command(["make", f"-j{os.cpu_count() or 1}"], cwd=srcdir) + ctx.run_command(["make", "install"], cwd=srcdir) + + ctx.bundle_state["squashfuse_install_root"] = install_root + ctx.bundle_state["squashfuse_source"] = f"squashfuse:{version}" + return install_root + + def source_label(self, ctx: InstallContext) -> str: + source = ctx.bundle_state.get("squashfuse_source") + if not isinstance(source, str): + self.ensure(ctx) + source = ctx.bundle_state["squashfuse_source"] + return str(source) + + def install(self, requested_tool: str, ctx: InstallContext) -> InstallRecord: + install_root = self.ensure(ctx) + src = install_root / self.ARTIFACT_PATHS[requested_tool] + return ctx.stage_binary(requested_tool, src, self.source_label(ctx)) + + +PROVIDER = SquashfuseProvider() diff --git a/scripts/deploy/manifests/host-tools.yaml b/scripts/deploy/manifests/host-tools.yaml new file mode 100644 index 0000000..ed5201d --- /dev/null +++ b/scripts/deploy/manifests/host-tools.yaml @@ -0,0 +1,23 @@ +sources: + podman_static_version: v5.8.2 + squashfuse_version: 0.6.1 + squashfs_tools_version: 4.6.1 + +profiles: + runtime: + enabled: true + tools: + - podman + - conmon + - crun + - fuse-overlayfs + - fusermount3 + - pasta + - squashfuse + - squashfuse_ll + + image_build: + enabled: true + tools: + - mksquashfs + - unsquashfs diff --git a/scripts/deploy/manifests/sarus-suite-tools.yaml b/scripts/deploy/manifests/sarus-suite-tools.yaml new file mode 100644 index 0000000..875d181 --- /dev/null +++ b/scripts/deploy/manifests/sarus-suite-tools.yaml @@ -0,0 +1,9 @@ +sources: + parallax_version: 0.10.2 + +profiles: + runtime: + enabled: true + tools: + - parallax + - parallax-mount-program diff --git a/scripts/deploy/requirements.txt b/scripts/deploy/requirements.txt new file mode 100644 index 0000000..5500f00 --- /dev/null +++ b/scripts/deploy/requirements.txt @@ -0,0 +1 @@ +PyYAML diff --git a/scripts/deploy/test_installer.py b/scripts/deploy/test_installer.py new file mode 100644 index 0000000..97a7a37 --- /dev/null +++ b/scripts/deploy/test_installer.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +import sys +import types + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.modules.setdefault("yaml", types.SimpleNamespace(safe_load=lambda _: {})) + +from installer_modules.core import ( + HostToolsInstaller, + InstallerConfig, + InstallerError, + discover_providers, +) +from installer_modules.providers.podman_static import PROVIDER as PODMAN_PROVIDER + + +def make_config(mode: str, dry_run: bool = True) -> InstallerConfig: + temp_root = Path(tempfile.mkdtemp(prefix="deploy-installer-test-")) + return InstallerConfig( + mode=mode, + manifest_path=temp_root / f"{mode}.yaml", + cache_dir=temp_root / "cache", + install_prefix=temp_root / "bin", + support_root=temp_root / "support", + state_root=temp_root / "state", + stage_root=temp_root / "stage", + dry_run=dry_run, + ) + + +class ProviderDiscoveryTests(unittest.TestCase): + def test_discovered_tool_map_covers_current_components(self) -> None: + _, tool_map = discover_providers() + + self.assertEqual( + { + "podman", + "conmon", + "crun", + "fuse-overlayfs", + "fusermount3", + "pasta", + "squashfuse", + "squashfuse_ll", + "mksquashfs", + "unsquashfs", + "parallax", + "parallax-mount-program", + }, + set(tool_map), + ) + + +class PodmanStaticProviderTests(unittest.TestCase): + def test_common_bundle_uses_pinned_release_url(self) -> None: + manifest = { + "sources": {"podman_static_version": "v5.8.2"}, + "profiles": {"runtime": {"enabled": True, "tools": ["podman"]}}, + } + ctx = HostToolsInstaller(make_config("common"), manifest).ctx + + PODMAN_PROVIDER.ensure(ctx) + + source = ctx.bundle_state["podman_static_source"] + self.assertIsInstance(source, str) + self.assertIn("/releases/download/v5.8.2/", source) + self.assertNotIn("/releases/latest/", source) + + def test_common_bundle_allows_manifest_url_override(self) -> None: + manifest = { + "sources": {"podman_static_url": "https://example.invalid/podman.tar.gz"}, + "profiles": {"runtime": {"enabled": True, "tools": ["podman"]}}, + } + ctx = HostToolsInstaller(make_config("common"), manifest).ctx + + PODMAN_PROVIDER.ensure(ctx) + + source = ctx.bundle_state["podman_static_source"] + self.assertEqual(source, "podman-static:https://example.invalid/podman.tar.gz") + + def test_common_bundle_tracks_private_support_tree(self) -> None: + manifest = { + "sources": {"podman_static_version": "v5.8.2"}, + "profiles": {"runtime": {"enabled": True, "tools": ["podman"]}}, + } + ctx = HostToolsInstaller(make_config("common"), manifest).ctx + + PODMAN_PROVIDER.install_support_once(ctx) + + self.assertEqual(len(ctx.support_records), 3) + installed_roots = {record.installed_files[0] for record in ctx.support_records} + self.assertIn(str(ctx.config.support_root / "podman-static" / "usr"), installed_roots) + self.assertIn(str(ctx.config.support_root / "podman-static" / "etc"), installed_roots) + self.assertIn( + str(ctx.config.support_root / "podman-static" / "config" / "containers.conf"), + installed_roots, + ) + + def test_common_bundle_prefers_usr_local_lib_podman_and_usr_bin_fallbacks(self) -> None: + manifest = {"profiles": {"runtime": {"enabled": True, "tools": ["podman", "conmon"]}}} + ctx = HostToolsInstaller(make_config("common", dry_run=False), manifest).ctx + bundle_root = ctx.config.cache_dir / "fake-podman-static" + (bundle_root / "usr/bin").mkdir(parents=True, exist_ok=True) + (bundle_root / "usr/local/lib/podman").mkdir(parents=True, exist_ok=True) + (bundle_root / "usr/bin/podman").write_text("", encoding="utf-8") + (bundle_root / "usr/local/lib/podman/conmon").write_text("", encoding="utf-8") + ctx.bundle_state["podman_static_bundle_root"] = bundle_root + ctx.bundle_state["podman_static_source"] = "podman-static:test" + + podman_record = PODMAN_PROVIDER.install("podman", ctx) + conmon_record = PODMAN_PROVIDER.install("conmon", ctx) + + self.assertEqual(podman_record.tool, "podman") + self.assertEqual(conmon_record.tool, "conmon") + containers_conf = ( + ctx.config.support_root / "podman-static" / "config" / "containers.conf" + ).read_text(encoding="utf-8") + self.assertIn("/usr/local/lib/podman", containers_conf) + self.assertIn("/usr/bin/conmon", containers_conf) + + +class HostToolsInstallerValidationTests(unittest.TestCase): + def test_common_manifest_allows_sources(self) -> None: + manifest = { + "sources": {"podman_static_version": "v5.8.2"}, + "profiles": {"runtime": {"enabled": True, "tools": ["podman"]}}, + } + installer = HostToolsInstaller(make_config("common"), manifest) + + installer._validate_manifest() + + def test_manifest_sources_must_be_a_mapping(self) -> None: + manifest = { + "sources": ["podman_static_version=v5.8.2"], + "profiles": {"runtime": {"enabled": True, "tools": ["podman"]}}, + } + installer = HostToolsInstaller(make_config("common"), manifest) + + with self.assertRaises(InstallerError): + installer._validate_manifest() + + def test_run_command_reports_missing_build_tool_cleanly(self) -> None: + installer = HostToolsInstaller(make_config("common", dry_run=False), {"profiles": {}}) + + with self.assertRaises(InstallerError) as ctx: + installer.ctx.run_command(["definitely-missing-build-tool"], Path("/")) + + self.assertIn("required build command 'definitely-missing-build-tool'", str(ctx.exception)) + + +class HostToolsInstallerManifestTests(unittest.TestCase): + def test_install_manifest_includes_support_artifacts(self) -> None: + config = make_config("common", dry_run=False) + installer = HostToolsInstaller(config, {"profiles": {}}) + installer.ctx.records.append( + installer.ctx.stage_script("podman", "#!/usr/bin/env sh\n", "podman-static:test") + ) + installer.ctx.support_records.append( + installer.ctx.stage_file( + "podman-static-containers-conf", + "[engine]\n", + Path("podman-static/config/containers.conf"), + "podman-static:test", + ) + ) + + installer._prepare_dirs() + manifest_path = installer._write_install_manifest() + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + + self.assertIn("support_artifacts", payload) + self.assertEqual(payload["tools"][0]["tool"], "podman") + self.assertEqual(payload["support_artifacts"][0]["name"], "podman-static-containers-conf") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/vagrant/ubuntu-24.04/.gitignore b/test/vagrant/ubuntu-24.04/.gitignore new file mode 100644 index 0000000..082e360 --- /dev/null +++ b/test/vagrant/ubuntu-24.04/.gitignore @@ -0,0 +1,2 @@ +.cache/ +.vagrant/ diff --git a/test/vagrant/ubuntu-24.04/Vagrantfile b/test/vagrant/ubuntu-24.04/Vagrantfile new file mode 100644 index 0000000..90b3c0e --- /dev/null +++ b/test/vagrant/ubuntu-24.04/Vagrantfile @@ -0,0 +1,91 @@ +require "rbconfig" + +Vagrant.configure("2") do |config| + host_cpu = RbConfig::CONFIG["host_cpu"] + guest_arch = + case host_cpu + when /arm|aarch64/ + "arm64" + else + "amd64" + end + + image_disk_size = ENV["UBUNTU_CLOUD_IMAGE_SIZE"] || "40G" + prepared_image_name = "noble-server-cloudimg-#{guest_arch}-#{image_disk_size.downcase}.qcow2" + image_path = ENV["UBUNTU_CLOUD_IMAGE"] || File.expand_path(".cache/#{prepared_image_name}", __dir__) + seed_path = ENV["UBUNTU_CLOUD_INIT_SEED"] || File.expand_path(".cache/nocloud-seed.iso", __dir__) + qemu_arch = guest_arch == "arm64" ? "aarch64" : "x86_64" + qemu_dir_candidates = [ + ENV["QEMU_DIR"], + "/usr/local/share/qemu", + "/opt/homebrew/share/qemu" + ].compact + qemu_dir = qemu_dir_candidates.find { |path| File.directory?(path) } + qemu_machine = ENV["QEMU_MACHINE"] || (guest_arch == "arm64" ? "virt,accel=hvf,highmem=off" : "q35") + qemu_cpu = ENV["QEMU_CPU"] || (guest_arch == "arm64" ? "cortex-a72" : "qemu64") + qemu_smp = ENV["QEMU_SMP"] || "2" + qemu_memory = ENV["QEMU_MEMORY"] || (guest_arch == "arm64" ? "2G" : "4G") + qemu_ssh_port = ENV["QEMU_SSH_PORT"] || 50022 + + config.vm.hostname = "cluster-tooling-host-tools" + config.vm.boot_timeout = 900 + config.vm.synced_folder ".", "/vagrant", disabled: true + + repo_root = File.expand_path("../../..", __dir__) + config.vm.synced_folder repo_root, "/workspace/cluster-tooling", + type: "rsync", + rsync__exclude: [".git/", "target/", ".vagrant/", ".cache/"] + + config.ssh.username = "vagrant" + config.ssh.private_key_path = File.expand_path("~/.vagrant.d/insecure_private_key") + config.ssh.insert_key = false + + config.vm.provider "qemu" do |qe| + raise <<~MSG unless File.exist?(image_path) + Missing Ubuntu cloud image: #{image_path} + + Run: + ./prepare-cloud-image.sh + + Or set UBUNTU_CLOUD_IMAGE to an existing Ubuntu 24.04 qcow2 image path. + MSG + + raise <<~MSG unless File.exist?(seed_path) + Missing NoCloud seed image: #{seed_path} + + Run: + ./prepare-cloud-image.sh + + Or set UBUNTU_CLOUD_INIT_SEED to an existing seed ISO path. + MSG + + raise <<~MSG if qemu_dir.nil? + Missing QEMU firmware directory. + + Checked: + #{qemu_dir_candidates.join("\n ")} + + Set QEMU_DIR to the directory containing edk2 firmware files such as: + export QEMU_DIR=/usr/local/share/qemu + MSG + + qe.image_path = image_path + qe.arch = qemu_arch + qe.machine = qemu_machine + qe.cpu = qemu_cpu + qe.memory = qemu_memory + qe.smp = qemu_smp + qe.ssh_port = qemu_ssh_port + qe.qemu_dir = qemu_dir + qe.disk_resize = image_disk_size + qe.ssh_auto_correct = true + qe.net_device = guest_arch == "arm64" ? "virtio-net-device" : "virtio-net-pci" + qe.extra_qemu_args = %W(-drive file=#{seed_path},if=virtio,media=cdrom,format=raw) + end + + config.vm.provision( + "shell", + path: "provision/test-host-tools.sh", + args: ["/workspace/cluster-tooling"] + ) +end diff --git a/test/vagrant/ubuntu-24.04/prepare-cloud-image.sh b/test/vagrant/ubuntu-24.04/prepare-cloud-image.sh new file mode 100755 index 0000000..a73894c --- /dev/null +++ b/test/vagrant/ubuntu-24.04/prepare-cloud-image.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CACHE_DIR="${ROOT_DIR}/.cache" +SEED_DIR="${CACHE_DIR}/nocloud" +SEED_IMAGE="${CACHE_DIR}/nocloud-seed.iso" +VAGRANT_INSECURE_PRIVATE_KEY="${HOME}/.vagrant.d/insecure_private_key" +VM_DISK_SIZE="${UBUNTU_CLOUD_IMAGE_SIZE:-40G}" + +log() { + printf '[prepare-cloud-image] %s\n' "$*" +} + +detect_arch() { + case "$(uname -m)" in + arm64|aarch64) + printf 'arm64\n' + ;; + x86_64|amd64) + printf 'amd64\n' + ;; + *) + echo "unsupported host architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +} + +main() { + local arch image_name image_url downloaded_image_path prepared_image_path insecure_pubkey + + arch="$(detect_arch)" + image_name="noble-server-cloudimg-${arch}.img" + image_url="https://cloud-images.ubuntu.com/noble/current/${image_name}" + downloaded_image_path="${CACHE_DIR}/${image_name}" + prepared_image_path="${CACHE_DIR}/noble-server-cloudimg-${arch}-${VM_DISK_SIZE,,}.qcow2" + insecure_pubkey="$(ssh-keygen -y -f "${VAGRANT_INSECURE_PRIVATE_KEY}")" + + mkdir -p "${CACHE_DIR}" + + if [ -f "${downloaded_image_path}" ]; then + log "reusing ${downloaded_image_path}" + else + log "downloading ${image_url}" + curl -fL "${image_url}" -o "${downloaded_image_path}" + log "saved ${downloaded_image_path}" + fi + + if ! command -v qemu-img >/dev/null 2>&1; then + echo "missing qemu-img; install QEMU on the host first" >&2 + exit 1 + fi + + if [ ! -f "${prepared_image_path}" ]; then + log "creating resized guest image ${prepared_image_path}" + cp "${downloaded_image_path}" "${prepared_image_path}" + else + log "reusing ${prepared_image_path}" + fi + + qemu-img resize "${prepared_image_path}" "${VM_DISK_SIZE}" >/dev/null + log "resized ${prepared_image_path} to ${VM_DISK_SIZE}" + + mkdir -p "${SEED_DIR}" + cat > "${SEED_DIR}/user-data" < "${SEED_DIR}/meta-data" </dev/null 2>&1; then + log "building NoCloud seed with cloud-localds" + cloud-localds "${SEED_IMAGE}" "${SEED_DIR}/user-data" "${SEED_DIR}/meta-data" + elif command -v hdiutil >/dev/null 2>&1; then + local tmp_base="${CACHE_DIR}/nocloud-seed" + local generated_path="" + rm -f "${tmp_base}" "${tmp_base}.cdr" "${tmp_base}.iso" "${SEED_IMAGE}" + log "building NoCloud seed with hdiutil" + hdiutil makehybrid \ + -o "${tmp_base}" \ + "${SEED_DIR}" \ + -iso \ + -joliet \ + -default-volume-name cidata \ + >/dev/null + + for candidate in "${tmp_base}" "${tmp_base}.cdr" "${tmp_base}.iso"; do + if [ -f "${candidate}" ]; then + generated_path="${candidate}" + break + fi + done + + if [ -z "${generated_path}" ]; then + echo "hdiutil did not create an output image under ${tmp_base}[.cdr|.iso]" >&2 + exit 1 + fi + + mv "${generated_path}" "${SEED_IMAGE}" + elif command -v genisoimage >/dev/null 2>&1; then + log "building NoCloud seed with genisoimage" + genisoimage -output "${SEED_IMAGE}" -volid cidata -joliet -rock "${SEED_DIR}/user-data" "${SEED_DIR}/meta-data" >/dev/null + elif command -v mkisofs >/dev/null 2>&1; then + log "building NoCloud seed with mkisofs" + mkisofs -output "${SEED_IMAGE}" -volid cidata -joliet -rock "${SEED_DIR}/user-data" "${SEED_DIR}/meta-data" >/dev/null + else + echo "missing tool to build cloud-init seed ISO; install cloud-localds, hdiutil, genisoimage, or mkisofs" >&2 + exit 1 + fi + + log "saved ${SEED_IMAGE}" +} + +main "$@" diff --git a/test/vagrant/ubuntu-24.04/provision/test-host-tools.sh b/test/vagrant/ubuntu-24.04/provision/test-host-tools.sh new file mode 100755 index 0000000..1754d6c --- /dev/null +++ b/test/vagrant/ubuntu-24.04/provision/test-host-tools.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="${1:?missing repo root argument}" +INSTALL_CACHE_ROOT="/var/tmp/cluster-tooling-host-tools-cache" +INSTALL_STATE_ROOT="/var/tmp/cluster-tooling-host-tools-state" +INSTALL_BIN_ROOT="/var/tmp/cluster-tooling-host-tools-bin" +INSTALL_SUPPORT_ROOT="/var/tmp/cluster-tooling-host-tools-support" +COMMON_STAGE_ROOT="/var/tmp/cluster-tooling-common-stage" +SARUS_STAGE_ROOT="/var/tmp/cluster-tooling-sarus-stage" +DEPLOY_INSTALL_CACHE_ROOT="/var/tmp/cluster-tooling-deploy-installer-cache" +DEPLOY_INSTALL_STATE_ROOT="/var/tmp/cluster-tooling-deploy-installer-state" +DEPLOY_INSTALL_BIN_ROOT="/var/tmp/cluster-tooling-deploy-installer-bin" +DEPLOY_INSTALL_SUPPORT_ROOT="/var/tmp/cluster-tooling-deploy-installer-support" +DEPLOY_COMMON_STAGE_ROOT="/var/tmp/cluster-tooling-deploy-common-stage" +DEPLOY_SARUS_STAGE_ROOT="/var/tmp/cluster-tooling-deploy-sarus-stage" +ROOTLESS_PODMAN_USER="vagrant" +ROOTLESS_PODMAN_SUBID_START="100000" +ROOTLESS_PODMAN_SUBID_COUNT="65536" + +log() { + printf '[test-host-tools] %s\n' "$*" +} + +ensure_subid_entry() { + local file="$1" + local user="$2" + local start="$3" + local count="$4" + + if grep -q "^${user}:" "${file}"; then + sed -i "s/^${user}:.*/${user}:${start}:${count}/" "${file}" + return 0 + fi + + printf '%s:%s:%s\n' "${user}" "${start}" "${count}" >> "${file}" +} + +configure_rootless_podman_user() { + log "configuring rootless Podman subordinate ID ranges for ${ROOTLESS_PODMAN_USER}" + ensure_subid_entry /etc/subuid "${ROOTLESS_PODMAN_USER}" "${ROOTLESS_PODMAN_SUBID_START}" "${ROOTLESS_PODMAN_SUBID_COUNT}" + ensure_subid_entry /etc/subgid "${ROOTLESS_PODMAN_USER}" "${ROOTLESS_PODMAN_SUBID_START}" "${ROOTLESS_PODMAN_SUBID_COUNT}" +} + +configure_containers_policy() { + log "installing containers policy for test Podman pulls" + mkdir -p /etc/containers + cat > /etc/containers/policy.json <<'EOF' +{ + "default": [{"type": "insecureAcceptAnything"}] +} +EOF + chmod 0644 /etc/containers/policy.json +} + +publish_test_tool_path() { + cat > /etc/profile.d/cluster-tooling-host-tools.sh <&2 + exit 1 + } +} + +verify_installed_tools() { + local prefix="$1" + + for tool in \ + podman conmon crun fuse-overlayfs fusermount3 pasta \ + squashfuse squashfuse_ll mksquashfs unsquashfs \ + parallax parallax-mount-program + do + require_binary_in_prefix "${prefix}" "${tool}" + done +} + +main() { + log "installing VM prerequisites" + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + libattr1-dev \ + libfuse3-dev \ + liblz4-dev \ + liblzma-dev \ + liblzo2-dev \ + libzstd-dev \ + pkg-config \ + python3-yaml \ + tar \ + uidmap \ + zlib1g-dev + + configure_rootless_podman_user + configure_containers_policy + + mkdir -p \ + "${INSTALL_CACHE_ROOT}" "${INSTALL_STATE_ROOT}" "${INSTALL_BIN_ROOT}" "${INSTALL_SUPPORT_ROOT}" \ + "${DEPLOY_INSTALL_CACHE_ROOT}" "${DEPLOY_INSTALL_STATE_ROOT}" "${DEPLOY_INSTALL_BIN_ROOT}" "${DEPLOY_INSTALL_SUPPORT_ROOT}" + export PATH="${DEPLOY_INSTALL_BIN_ROOT}:${INSTALL_BIN_ROOT}:${PATH}" + + log "running general deploy installer for common host tools" + python3 "${REPO_ROOT}/scripts/deploy/installer.py" common \ + --manifest "${REPO_ROOT}/scripts/deploy/manifests/host-tools.yaml" \ + --cache-dir "${DEPLOY_INSTALL_CACHE_ROOT}/common" \ + --install-prefix "${DEPLOY_INSTALL_BIN_ROOT}" \ + --support-root "${DEPLOY_INSTALL_SUPPORT_ROOT}/common" \ + --stage-root "${DEPLOY_COMMON_STAGE_ROOT}" \ + --state-root "${DEPLOY_INSTALL_STATE_ROOT}" + + log "running general deploy installer for Sarus Suite tools" + python3 "${REPO_ROOT}/scripts/deploy/installer.py" sarus \ + --manifest "${REPO_ROOT}/scripts/deploy/manifests/sarus-suite-tools.yaml" \ + --cache-dir "${DEPLOY_INSTALL_CACHE_ROOT}/sarus" \ + --install-prefix "${DEPLOY_INSTALL_BIN_ROOT}" \ + --support-root "${DEPLOY_INSTALL_SUPPORT_ROOT}/sarus" \ + --stage-root "${DEPLOY_SARUS_STAGE_ROOT}" \ + --state-root "${DEPLOY_INSTALL_STATE_ROOT}" + + log "verifying installed binaries from deploy installer" + verify_installed_tools "${DEPLOY_INSTALL_BIN_ROOT}" + + log "installed manifests from deploy installer" + ls -l "${DEPLOY_INSTALL_STATE_ROOT}" + + log "publishing test tool PATH for login shells" + publish_test_tool_path +} + +main "$@"