diff --git a/fboss-image/distro_cli/builder/__init__.py b/fboss-image/distro_cli/builder/__init__.py new file mode 100644 index 0000000000000..162bd57c07b74 --- /dev/null +++ b/fboss-image/distro_cli/builder/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) 2004-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +"""FBOSS Distribution CLI builder package.""" diff --git a/fboss-image/distro_cli/builder/component.py b/fboss-image/distro_cli/builder/component.py new file mode 100644 index 0000000000000..43ebce5feeb3c --- /dev/null +++ b/fboss-image/distro_cli/builder/component.py @@ -0,0 +1,342 @@ +# Copyright (c) 2004-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +"""Generic component builder for FBOSS image components.""" + +import hashlib +import logging +from os.path import commonpath +from pathlib import Path + +from distro_cli.lib.artifact import find_artifact_in_dir +from distro_cli.lib.constants import FBOSS_BUILDER_IMAGE +from distro_cli.lib.download import download_artifact +from distro_cli.lib.exceptions import ComponentError +from distro_cli.lib.execute import execute_build_in_container +from distro_cli.lib.paths import get_abs_path + +logger = logging.getLogger(__name__) + + +def _get_component_directory(component_name: str, script_path: str) -> str: + """Determine the component directory for build artifacts. + + For scripts_path that has the component_name, we return the path in script_path + leading to the component_name. Otherwise, the script's parent directory is returned. + + Examples: + kernel component: + component_name="kernel" + script_path="fboss-image/kernel/scripts/build_kernel.sh" + returns: "fboss-image/kernel" (component_name found in path) + + sai component: + component_name="sai" + script_path="broadcom-sai-sdk/build_fboss_sai.sh" + returns: "broadcom-sai-sdk" (fallback to script's parent) + + Args: + component_name: Base component name (without array index) + script_path: Path to the build script from the execute directive + + Returns: + Component directory path (relative to workspace root) + + """ + script_path_obj = Path(script_path) + + # Check if component_name appears in the script path + if component_name in script_path_obj.parts: + # Find the last occurrence of component_name in the path + # Handle cases where component name appears multiple times + # e.g., /src/kernel/fboss/12345/kernel/build.sh -> use the last "kernel" + parts = script_path_obj.parts + # Find last occurrence by reversing and using index + component_index = len(parts) - 1 - parts[::-1].index(component_name) + # Return the path up to and including the component_name + return str(Path(*parts[: component_index + 1])) + + # Fall back to script's parent directory + return str(script_path_obj.parent) + + +class ComponentBuilder: + """Generic builder for FBOSS image components. + + Supports two modes: + - download: Download pre-built artifact from URL + - execute: Build component using a build script in Docker container + + Component-specific logic (argument parsing, paths, etc.) should be + embedded in the component build script, not in this builder. + """ + + def __init__( + self, + component_name: str, + component_data: dict, + manifest_dir: Path, + store, + artifact_pattern: str | None = None, + dependency_artifacts: dict[str, Path] | None = None, + artifact_key_salt: str | None = None, + ): + """Initialize the component builder. + + Args: + component_name: Name of the component + component_data: Component data dict from manifest + manifest_dir: Path to the manifest directory + store: ArtifactStore instance + artifact_pattern: Glob pattern for finding build artifacts (e.g., "kernel-*.rpms.tar.zst") + If None, component cannot use execute mode + dependency_artifacts: Optional dict mapping dependency names to their artifact paths + artifact_key_salt: Salt added to artifact store key to differentiate variants + """ + self.component_name = component_name + self.component_data = component_data + self.manifest_dir = manifest_dir + self.store = store + self.artifact_pattern = artifact_pattern + self.dependency_artifacts = dependency_artifacts or {} + self.artifact_key_salt = artifact_key_salt + + def build(self) -> Path: + """Build or download the component. + + Returns: + Path to the component artifact (or None for empty components) + + Raises: + ComponentError: If component has invalid structure + """ + if self.component_data is None: + raise ComponentError(f"Component '{self.component_name}' has no data") + + # ComponentBuilder handles single component instances only + # Array components should be handled at a higher level (e.g., ImageBuilder) + # by creating one ComponentBuilder instance per array element + if isinstance(self.component_data, list): + raise ComponentError( + f"Component '{self.component_name}' data is an array. " + "ComponentBuilder only handles single component instances. " + "Create one ComponentBuilder per array element instead." + ) + + # Check for both download and execute (invalid) + has_download = "download" in self.component_data + has_execute = "execute" in self.component_data + + if has_download and has_execute: + raise ComponentError( + f"Component '{self.component_name}' has both 'download' and 'execute' fields. " + "Only one is allowed." + ) + + # Allow empty components + if not has_download and not has_execute: + logger.info(f"Component '{self.component_name}' is empty, skipping") + return None + + if has_download: + return self._download_component(self.component_data["download"]) + + # this must be an "execute" + # Use artifact pattern from manifest if specified, otherwise use default + artifact_pattern = self.component_data.get("artifact", self.artifact_pattern) + return self._execute_component(self.component_data["execute"], artifact_pattern) + + def _download_component(self, url: str) -> Path: + """Download component artifact from URL. + + Args: + url: URL to download from + + Returns: + Path to downloaded artifact (cached) + """ + store_key = ( + f"{self.component_name}-download-{hashlib.sha256(url.encode()).hexdigest()}" + ) + data_files, metadata_files = self.store.get( + store_key, + lambda data, meta: download_artifact(url, self.manifest_dir, data, meta), + ) + + if not data_files: + raise ComponentError(f"No artifact files found for {self.component_name}") + + artifact_path = data_files[0] + logger.info(f"{self.component_name} artifact ready: {artifact_path}") + if metadata_files: + logger.debug(f" with {len(metadata_files)} metadata file(s)") + return artifact_path + + def _execute_component( + self, cmd_line: str | list[str], artifact_pattern: str | None = None + ) -> Path: + """Execute component build in Docker container. + + Args: + cmd_line: Command line to execute (string or list of strings) + artifact_pattern: Optional artifact pattern override from manifest + + Returns: + Path to build artifact in cache + """ + # For store key, convert to string (works for both str and list) + # Include artifact_key_salt to differentiate cache variants (e.g., compressed vs uncompressed) + store_key_str = f"{cmd_line}-salt={self.artifact_key_salt}" + + # _execute_build as a fetch_fn always starts a build expecting the underlying + # build system to provide build specific optimizations. The objects are returned + # back to the store with a store-miss indication. + store_key = f"{self.component_name}-build-{hashlib.sha256(store_key_str.encode()).hexdigest()[:8]}" + data_files, _ = self.store.get( + store_key, + lambda _data, _meta: ( + False, + [self._execute_build(cmd_line, artifact_pattern)], + [], + ), + ) + + if not data_files: + raise ComponentError(f"No artifact files found for {self.component_name}") + + artifact_path = data_files[0] + logger.info(f"{self.component_name} build complete: {artifact_path}") + return artifact_path + + def _execute_build( + self, cmd_line: str | list[str], artifact_pattern: str | None = None + ) -> Path: + """Execute build in Docker container. + + Args: + cmd_line: Command line to execute (string or list of strings) + artifact_pattern: Optional artifact pattern override from manifest + + Returns: + Path to build artifact + + Raises: + ComponentError: If build fails or artifact not found + """ + # Get script path from command line + script_path_str = ( + cmd_line[0] if isinstance(cmd_line, list) else cmd_line.split()[0] + ) + + # Resolve script path: if absolute, use as-is; if relative, resolve from manifest_dir + script_path = Path(script_path_str) + resolved_script_path = ( + script_path + if script_path.is_absolute() + else (self.manifest_dir / script_path).resolve() + ) + + # Verify the script exists + if not resolved_script_path.exists(): + raise ComponentError( + f"Build script not found: {resolved_script_path} " + f"(from manifest path: {script_path_str})" + ) + + # We mount the common parent of the script path and manifest dir + src_dir = Path(commonpath([resolved_script_path, self.manifest_dir])) + script_relative_to_src = resolved_script_path.relative_to(src_dir) + container_script_path = Path("/src") / script_relative_to_src + + # For array elements, extract the base name + base_name = ( + self.component_name.split("[")[0] + if "[" in self.component_name + else self.component_name + ) + + # Determine component directory (component root if known, else script's parent) + # Use the resolved script path relative to src_dir + script_relative_to_src = resolved_script_path.relative_to(src_dir) + component_dir = _get_component_directory(base_name, str(script_relative_to_src)) + + # Create build and dist directories under the component directory + artifact_base_dir = src_dir / component_dir + + # Use artifact_pattern from parameter, or fall back to instance pattern, or use generic pattern + # Generic pattern uses .tar (will match both .tar and .tar.zst via compression variant finder) + if artifact_pattern is None: + artifact_pattern = self.artifact_pattern or "*.tar" + if not artifact_pattern: + logger.warning( + f"Component '{self.component_name}' has no artifact_pattern specified. " + f"Using generic pattern: {artifact_pattern}" + ) + + build_dir = artifact_base_dir / ".build" + build_dir.mkdir(parents=True, exist_ok=True) + + dist_dir = artifact_base_dir / "dist" + dist_dir.mkdir(parents=True, exist_ok=True) + + # Mount src_dir as /src in the container + logger.info(f"Mounting {src_dir} as /src") + + # Mount distro_cli/tools as /tools for build utilities + tools_dir = get_abs_path("fboss-image/distro_cli/tools") + + # Mount fboss/oss/scripts for common build utilities (sccache config, etc.) + common_scripts_dir = get_abs_path("fboss/oss/scripts") + + volumes = { + src_dir: Path("/src"), + build_dir: Path("/build"), + dist_dir: Path("/output"), + tools_dir: Path("/tools"), + common_scripts_dir: Path("/fboss/oss/scripts"), + } + + # Mount dependency artifacts into the container + dependency_install_paths = {} + for dep_name, dep_artifact in self.dependency_artifacts.items(): + dep_mount_point = Path(f"/deps/{dep_name}") + volumes[dep_artifact] = dep_mount_point + dependency_install_paths[dep_name] = dep_mount_point + logger.info( + f"Mounting dependency '{dep_name}' at {dep_mount_point}: {dep_artifact}" + ) + + # Working directory is the parent of the script + working_dir = str(container_script_path.parent) + + # Build the container command using the container script path + if isinstance(cmd_line, list): + # Replace first element with container path, keep the rest + container_cmd = [str(container_script_path), *cmd_line[1:]] + else: + # For string commands, replace the script path with the in-container version + container_cmd = [str(container_script_path)] + + logger.info(f"Container command: {container_cmd}") + + # Execute build command + execute_build_in_container( + image_name=FBOSS_BUILDER_IMAGE, + command=container_cmd, + volumes=volumes, + component_name=self.component_name, + working_dir=working_dir, + dependency_install_paths=( + dependency_install_paths if dependency_install_paths else None + ), + ) + + return find_artifact_in_dir( + output_dir=dist_dir, + pattern=artifact_pattern, + component_name=self.component_name.capitalize(), + ) diff --git a/fboss-image/distro_cli/lib/execute.py b/fboss-image/distro_cli/lib/execute.py index 868806a239967..65fe99bc90547 100644 --- a/fboss-image/distro_cli/lib/execute.py +++ b/fboss-image/distro_cli/lib/execute.py @@ -23,7 +23,6 @@ def execute_build_in_container( command: list[str], volumes: dict[Path, Path], component_name: str, - privileged: bool = False, working_dir: str | None = None, dependency_install_paths: dict[str, Path] | None = None, ) -> None: @@ -34,7 +33,6 @@ def execute_build_in_container( command: Command to execute as list volumes: Host to container path mappings component_name: Component name - privileged: Run in privileged mode working_dir: Working directory in container dependency_install_paths: Dict mapping dependency names to their container mount paths (e.g., {'kernel': Path('/deps/kernel')}) @@ -66,7 +64,7 @@ def execute_build_in_container( image=image_name, command=cmd_list, volumes=volumes, - privileged=privileged, + privileged=True, working_dir=working_dir, ) diff --git a/fboss-image/distro_cli/lib/manifest.py b/fboss-image/distro_cli/lib/manifest.py index ce0d6b61f4fb6..e3e07b121f90a 100644 --- a/fboss-image/distro_cli/lib/manifest.py +++ b/fboss-image/distro_cli/lib/manifest.py @@ -56,6 +56,12 @@ def has_component(self, component: str) -> bool: """Check if component is present in manifest.""" return component in self.data + def get_component(self, component: str) -> dict | None: + """Return component data in manifest.""" + if component not in self.data: + return None + return self.data[component] + def resolve_path(self, path: str) -> Path: """Resolve a path relative to the manifest file.""" if path.startswith("http://") or path.startswith("https://"): diff --git a/fboss-image/distro_cli/tests/data/kernel-test.tar.zst b/fboss-image/distro_cli/tests/data/kernel-test.tar.zst new file mode 100644 index 0000000000000..a50fdf2254085 Binary files /dev/null and b/fboss-image/distro_cli/tests/data/kernel-test.tar.zst differ diff --git a/fboss-image/distro_cli/tests/kernel_download_test.py b/fboss-image/distro_cli/tests/kernel_download_test.py new file mode 100644 index 0000000000000..df551d4cc090a --- /dev/null +++ b/fboss-image/distro_cli/tests/kernel_download_test.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +# Copyright (c) 2004-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +"""Test kernel component download functionality with caching.""" + +import hashlib +import http.server +import json +import shutil +import socketserver +import tempfile +import threading +import unittest +from pathlib import Path + +from distro_cli.builder.component import ComponentBuilder +from distro_cli.lib.artifact import ArtifactStore +from distro_cli.lib.manifest import ImageManifest + + +class SimpleHTTPServer: + """Simple HTTP server for testing downloads.""" + + def __init__(self, directory, port=0): + """Initialize HTTP server. + + Args: + directory: Directory to serve files from + port: Port to listen on (default: 0 = let OS assign random port) + """ + self.directory = directory + self.port = port + self.server = None + self.server_thread = None + self._ready = threading.Event() + + def start(self): + """Start the HTTP server in a background thread.""" + # Enable SO_REUSEADDR to avoid "Address already in use" errors + socketserver.TCPServer.allow_reuse_address = True + + self.server = socketserver.TCPServer( + ("127.0.0.1", self.port), + lambda *args, **kwargs: http.server.SimpleHTTPRequestHandler( + *args, directory=str(self.directory), **kwargs + ), + ) + + # Get the actual port assigned by the OS (important when port=0) + self.port = self.server.server_address[1] + + def serve_with_ready_signal(): + """Serve forever and signal when ready.""" + self._ready.set() + self.server.serve_forever() + + self.server_thread = threading.Thread( + target=serve_with_ready_signal, daemon=True + ) + self.server_thread.start() + + # Wait for server to be ready (with timeout) + if not self._ready.wait(timeout=180): # 3 minutes max + raise RuntimeError("HTTP server failed to start within 3 minutes") + + def stop(self): + """Stop the HTTP server.""" + if self.server: + self.server.shutdown() + self.server.server_close() + + def get_url(self, filename): + """Get URL for a file served by this server. + + Args: + filename: Name of file to get URL for + + Returns: + Full HTTP URL to the file + """ + return f"http://127.0.0.1:{self.port}/{filename}" + + +class KernelDownloadTestHelper: + """Helper class with common test logic for kernel download tests. + + This is NOT a TestCase - it's a helper that test classes use via delegation. + """ + + def __init__( + self, + test_case, + manifest_path, + store_dir, + source_file=None, + temp_source_dir=None, + ): + """Initialize helper. + + Args: + test_case: The unittest.TestCase instance (for assertions) + manifest_path: Path to manifest file + store_dir: Path to store directory + source_file: Original source file to copy from (for file:// tests) + temp_source_dir: Temp directory where source file is staged (for file:// tests) + """ + self.test_case = test_case + self.manifest_path = manifest_path + self.store_dir = store_dir + self.source_file = source_file + self.temp_source_dir = temp_source_dir + + def build_kernel(self): + """Build kernel with caching. + + For file:// tests, ensures source file exists by copying from original if needed. + + Returns: + Path to built/cached artifact + """ + # For file:// tests, ensure source file exists + # (it may have been moved to artifact store on previous build) + if self.source_file and self.temp_source_dir: + temp_source_file = self.temp_source_dir / self.source_file.name + if not temp_source_file.exists(): + shutil.copy2(self.source_file, temp_source_file) + + manifest = ImageManifest(self.manifest_path) + # Override class attribute for testing + ArtifactStore.ARTIFACT_STORE_DIR = self.store_dir + store = ArtifactStore() + + # Get kernel component data + kernel_data = manifest.get_component("kernel") + + kernel_builder = ComponentBuilder( + component_name="kernel", + component_data=kernel_data, + manifest_dir=manifest.manifest_dir, + store=store, + artifact_pattern="kernel-*.rpms.tar", + ) + return kernel_builder.build() + + def get_store_key(self): + """Get store key for kernel component in manifest. + + Returns: + Store key string + """ + manifest = ImageManifest(self.manifest_path) + component_data = manifest.get_component("kernel") + url = component_data["download"] + return f"kernel-download-{hashlib.sha256(url.encode()).hexdigest()}" + + def test_download_and_cache(self): + """Test that kernel download works and caching is effective.""" + # First build - should download and cache + artifact_path_1 = self.build_kernel() + + # Verify artifact exists and is in cache + self.test_case.assertTrue( + artifact_path_1.exists(), f"Artifact not found: {artifact_path_1}" + ) + self.test_case.assertTrue( + str(artifact_path_1).startswith(str(self.store_dir)), + f"Artifact not in store dir: {artifact_path_1}", + ) + + # Second build - should hit cache (no download) + artifact_path_2 = self.build_kernel() + + # Verify we got the same cached artifact + self.test_case.assertEqual( + artifact_path_1, + artifact_path_2, + "Second build should return same cached artifact", + ) + self.test_case.assertTrue( + artifact_path_2.exists(), f"Cached artifact not found: {artifact_path_2}" + ) + + def test_cache_invalidation(self): + """Test that cache invalidation works.""" + # Build and cache artifact + artifact_path = self.build_kernel() + self.test_case.assertTrue(artifact_path.exists()) + + # Get store and invalidate + ArtifactStore.ARTIFACT_STORE_DIR = self.store_dir + store = ArtifactStore() + store_key = self.get_store_key() + store.invalidate(store_key) + + # Verify artifact is gone + self.test_case.assertFalse( + artifact_path.exists(), "Artifact should be removed after invalidation" + ) + + def test_cache_clear(self): + """Test that clearing cache removes all artifacts.""" + # Build and cache artifact + artifact_path = self.build_kernel() + self.test_case.assertTrue(artifact_path.exists()) + + # Clear the store + ArtifactStore.ARTIFACT_STORE_DIR = self.store_dir + store = ArtifactStore() + store.clear() + + # Verify store directory is empty + store_contents = list(self.store_dir.glob("*")) + self.test_case.assertEqual( + len(store_contents), + 0, + f"Store should be empty after clear, but contains: {store_contents}", + ) + + +class TestKernelDownloadFile(unittest.TestCase): + """Test kernel component download from file:// URL.""" + + def setUp(self): + """Set up test fixtures.""" + self.test_dir = Path(__file__).parent + self.store_dir = Path(tempfile.mkdtemp(prefix="test-store-")) + + # Create temp directory for source files (on same filesystem as artifact store) + # This ensures the file can be moved efficiently during download + self.temp_source_dir = Path(tempfile.mkdtemp(prefix="test-source-")) + + # Copy test data file to temp source directory + # This way the original test data file is preserved when download moves it + # Downloads should use compressed files to save bandwidth + self.source_file = self.test_dir / "data" / "kernel-test.tar.zst" + temp_source_file = self.temp_source_dir / "kernel-test.tar.zst" + shutil.copy2(self.source_file, temp_source_file) + + # Create test manifest that points to temp source file + self.manifest_path = self.temp_source_dir / "test-manifest.json" + manifest_data = { + "kernel": {"download": f"file:{temp_source_file}"}, + "distribution_formats": {"usb": "output/test-download.iso"}, + } + with open(self.manifest_path, "w") as f: + json.dump(manifest_data, f, indent=2) + + # Create helper with delegation, passing source file info + self.helper = KernelDownloadTestHelper( + self, + self.manifest_path, + self.store_dir, + source_file=self.source_file, + temp_source_dir=self.temp_source_dir, + ) + + def tearDown(self): + """Clean up test fixtures.""" + if self.store_dir.exists(): + shutil.rmtree(self.store_dir) + if self.temp_source_dir.exists(): + shutil.rmtree(self.temp_source_dir) + + def test_download_and_cache(self): + """Test that kernel download works and caching is effective.""" + self.helper.test_download_and_cache() + + def test_cache_invalidation(self): + """Test that cache invalidation works.""" + self.helper.test_cache_invalidation() + + def test_cache_clear(self): + """Test that clearing cache removes all artifacts.""" + self.helper.test_cache_clear() + + +class TestKernelDownloadHTTP(unittest.TestCase): + """Test kernel component download from HTTP server.""" + + @classmethod + def setUpClass(cls): + """Start HTTP server for all tests.""" + cls.test_dir = Path(__file__).parent + cls.data_dir = cls.test_dir / "data" + + # Start HTTP server using helper (port=0 lets OS assign random available port) + cls.http_server = SimpleHTTPServer(cls.data_dir) + cls.http_server.start() + + @classmethod + def tearDownClass(cls): + """Stop HTTP server.""" + cls.http_server.stop() + + def setUp(self): + """Set up test fixtures.""" + self.temp_dir = Path(tempfile.mkdtemp(prefix="test-http-")) + self.store_dir = Path(tempfile.mkdtemp(prefix="test-store-")) + + # Create test manifest with HTTP URL + self.manifest_path = self.temp_dir / "test-http-manifest.json" + manifest_data = { + "name": "test-http-download", + "version": "1.0.0", + "kernel": {"download": self.http_server.get_url("kernel-test.tar.zst")}, + "distribution_formats": {"usb": "output/test.iso"}, + } + + with open(self.manifest_path, "w") as f: + json.dump(manifest_data, f, indent=2) + + # Create helper with delegation (HTTP tests don't need source file restoration) + self.helper = KernelDownloadTestHelper(self, self.manifest_path, self.store_dir) + + def tearDown(self): + """Clean up test fixtures.""" + if self.temp_dir.exists(): + shutil.rmtree(self.temp_dir) + if self.store_dir.exists(): + shutil.rmtree(self.store_dir) + + def test_download_and_cache(self): + """Test that kernel download works and caching is effective.""" + self.helper.test_download_and_cache() + + def test_cache_invalidation(self): + """Test that cache invalidation works.""" + self.helper.test_cache_invalidation() + + def test_cache_clear(self): + """Test that clearing cache removes all artifacts.""" + self.helper.test_cache_clear() + + +if __name__ == "__main__": + unittest.main() diff --git a/fboss/oss/docker/Dockerfile b/fboss/oss/docker/Dockerfile index 245b565ca0830..5f491cc860920 100644 --- a/fboss/oss/docker/Dockerfile +++ b/fboss/oss/docker/Dockerfile @@ -99,4 +99,7 @@ RUN if [ "$USERNAME" != "root" ] && [ "$USER_UID" != "0" ]; then \ chown -R $USERNAME /var/FBOSS; \ fi +# Support sudo in a privileged container +RUN chmod 600 /etc/shadow + USER ${USERNAME}