From ba17fed0e033e0f238c46d8c312deb1606f53b7a Mon Sep 17 00:00:00 2001 From: Sanjay Bhatnagar Date: Sun, 2 Jul 2023 21:29:43 -0600 Subject: [PATCH 01/15] Changes to implement suggestions in Issue #75. Seems to fix the problem reported in #73. --- src/exodus_bundler/bundling.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index dd8f469..2f754dc 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -513,7 +513,7 @@ def create_entry_point(self, working_directory, bundle_root): if not os.path.exists(bin_directory): os.makedirs(bin_directory) entry_point_path = os.path.join(bin_directory, self.entry_point) - relative_destination_path = os.path.relpath(source_path, bin_directory) + relative_destination_path = os.path.relpath(source_path, bin_directory)+".sh" os.symlink(relative_destination_path, entry_point_path) def create_launcher(self, working_directory, bundle_root, linker_basename, symlink_basename, @@ -599,9 +599,10 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli launcher_content = construct_bash_launcher( linker=linker, library_path=library_path, executable=executable, full_linker=full_linker) - with open(source_path, 'w') as f: + tt=source_path+".sh" + with open(tt, 'w') as f: f.write(launcher_content) - shutil.copymode(self.path, source_path) + shutil.copymode(self.path, tt) return os.path.normpath(os.path.abspath(source_path)) @@ -781,6 +782,7 @@ def create_bundle(self, shell_launchers=False): if file.no_symlink: # We'll need to copy the actual file into the bundle subdirectory in this # case so that it can locate resources using paths relative to the executable. + parent_directory = os.path.dirname(file_path) if not os.path.exists(parent_directory): os.makedirs(parent_directory) @@ -819,14 +821,15 @@ def create_bundle(self, shell_launchers=False): # We'll again attempt to find a unique available name, this time for the symlink # to the executable. file_basename = file.entry_point or os.path.basename(file.path) - desired_symlink_path = os.path.join(directory, '%s-x' % file_basename) + #desired_symlink_path = os.path.join(directory, '%s-x' % file_basename) + desired_symlink_path = os.path.join(directory, '%s' % file_basename) symlink_path = desired_symlink_path - iteration = 2 - while symlink_path in file_paths: - symlink_path = '%s-%d' % (desired_symlink_path, iteration) - iteration += 1 + # iteration = 2 + # while symlink_path in file_paths: + # symlink_path = '%s-%d' % (desired_symlink_path, iteration) + # iteration += 1 file_paths.add(symlink_path) - symlink_basename = os.path.basename(symlink_path) + symlink_basename = os.path.basename(symlink_path); file.create_launcher(self.working_directory, self.bundle_root, linker_basename, symlink_basename, shell_launcher=shell_launchers) From 790f496d15ae52b353881079f107542e03dc5a59 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 21:32:42 -0600 Subject: [PATCH 02/15] Removing python 2 to 3.9 dependencies. Added github ci for building. --- .github/workflows/release.yml | 70 ++++++++++++ .github/workflows/test.yml | 124 +++++++++++++++++++++ development-requirements.txt | 29 +++-- pyproject.toml | 91 +++++++++++++++ setup.py | 20 +--- src/exodus_bundler/__init__.py | 2 +- src/exodus_bundler/bundling.py | 41 ++++--- src/exodus_bundler/dependency_detection.py | 2 +- src/exodus_bundler/launchers.py | 11 +- src/exodus_bundler/templating.py | 2 +- tests/test_bundling.py | 6 +- tox.ini | 36 +++--- 12 files changed, 361 insertions(+), 73 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..abb9894 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,70 @@ +name: Release + +on: + push: + tags: + - 'v*' + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y dietlibc-dev gcc musl musl-tools + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r development-requirements.txt + pip install -e . + + - name: Run tests + run: pytest --cov -v + + build-and-publish: + needs: test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Check package + run: twine check dist/* + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: dist/* + generate_release_notes: true + + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: twine upload dist/* \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..305acde --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,124 @@ +name: Test + +on: + push: + branches: [ master, main ] + pull_request: + branches: [ master, main ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y dietlibc-dev gcc musl musl-tools + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + .tox + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/development-requirements.txt', '**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.python-version }}- + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r development-requirements.txt + pip install -e . + + - name: Run linting + run: | + flake8 src tests setup.py + isort --verbose --check-only --diff src tests setup.py + + - name: Run tests with pytest + run: | + pytest --cov --cov-report=term-missing --cov-report=xml -v + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + fail_ci_if_error: false + + - name: Test package build + run: | + python -m build + + - name: Test basic functionality + run: | + exodus --help + echo "Testing basic bundling (will use shell launchers without musl/diet)" + # Test bundling a simple command - this will fail gracefully if binary not found + timeout 30s exodus --shell-launchers /bin/echo --output test-bundle.sh || echo "Expected: binary bundling test completed" + + test-tox: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y dietlibc-dev gcc musl musl-tools + + - name: Install tox + run: | + python -m pip install --upgrade pip + pip install tox + + - name: Run tox + run: tox + + package: + runs-on: ubuntu-latest + needs: [test, test-tox] + if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Check package + run: twine check dist/* + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: packages + path: dist/ \ No newline at end of file diff --git a/development-requirements.txt b/development-requirements.txt index 9eeef10..51a780c 100644 --- a/development-requirements.txt +++ b/development-requirements.txt @@ -1,12 +1,17 @@ -bumpversion==0.5.3 -coverage>=5.4 -m2r>=0.1.12 -pluggy==0.5.2 -py==1.4.34 -pytest==3.2.3 -pytest-sugar==0.9.0 -pytest-watch==4.1.0 -six==1.11.0 -tox==2.9.1 -twine==1.9.1 -virtualenv==15.1.0 +bumpversion>=0.6.0 +build>=1.0 +coverage>=7.0 +pytest>=8.0 +pytest-sugar>=1.0 +pytest-watch>=4.2 +pytest-cov>=5.0 +tox>=4.0 +twine>=5.0 +flake8>=7.0 +flake8-commas>=4.0 +flake8-quotes>=3.4 +isort>=5.13 +check-manifest>=0.49 +docutils>=0.20 +readme-renderer>=43.0 +pygments>=2.17 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c07b8c8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,91 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "exodus-bundler" +version = "3.0.0" +description = "The exodus application bundler." +readme = "README.md" +license = {text = "BSD"} +authors = [ + {name = "Intoli", email = "contact@intoli.com"} +] +maintainers = [ + {name = "Intoli", email = "contact@intoli.com"} +] +keywords = ["linux", "executable", "elf", "binaries"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: System :: Archiving :: Packaging", + "Topic :: Utilities", +] +requires-python = ">=3.10" +dependencies = [] + +[project.urls] +Homepage = "https://github.com/intoli/exodus" +Repository = "https://github.com/intoli/exodus" +Issues = "https://github.com/intoli/exodus/issues" + +[project.scripts] +exodus = "exodus_bundler.cli:main" + +[tool.setuptools] +package-dir = {"" = "src"} +include-package-data = true +zip-safe = false + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.flake8] +ignore = ["E128", "W504"] +max-line-length = 100 + +[tool.isort] +force_single_line = true +line_length = 100 +lines_after_imports = 2 +known_first_party = ["exodus_bundler"] +default_section = "THIRDPARTY" +skip_gitignore = false + +[tool.pytest.ini_options] +norecursedirs = [".git", ".tox", ".env", "dist", "build"] +python_files = ["test_*.py", "*_test.py", "tests.py"] +addopts = [ + "-rxEfsw", + "--strict-markers", + "--doctest-modules", + "--doctest-glob=*.rst", + "--tb=short" +] +testpaths = ["tests"] + +[tool.coverage.run] +source = ["src"] +omit = ["*/tests/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError" +] \ No newline at end of file diff --git a/setup.py b/setup.py index 2409e74..082ac76 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function from glob import glob from os.path import basename @@ -13,7 +11,7 @@ setup( name='exodus-bundler', - version='2.0.4', + version='3.0.0', license='BSD', platforms=['Linux'], description='The exodus application bundler.', @@ -37,18 +35,11 @@ 'License :: OSI Approved :: BSD License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.0', - 'Programming Language :: Python :: 3.1', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: System :: Archiving :: Packaging', @@ -57,6 +48,7 @@ keywords=[ 'linux', 'executable', 'elf', 'binaries', ], + python_requires='>=3.10', install_requires=[ ], entry_points={ diff --git a/src/exodus_bundler/__init__.py b/src/exodus_bundler/__init__.py index 529155b..882a45a 100644 --- a/src/exodus_bundler/__init__.py +++ b/src/exodus_bundler/__init__.py @@ -1,7 +1,7 @@ import logging -__version__ = '2.0.4' +__version__ = '3.0.0' root_logger = logging.getLogger(__name__) root_logger.handlers = [logging.NullHandler()] diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 2f754dc..da15254 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -67,7 +67,7 @@ def create_bundle(executables, output, tarball=False, rename=[], chroot=None, ad # Configure the appropriate output mechanism. if output_filename == '-': - output_file = getattr(sys.stdout, 'buffer', sys.stdout) + output_file = sys.stdout.buffer else: output_file = open(output_filename, 'wb') @@ -86,9 +86,9 @@ def create_bundle(executables, output, tarball=False, rename=[], chroot=None, ad output_file.write(tar_stream.getvalue()) # Write out the success message. - logger.info('Successfully created "%s".' % output_filename) + logger.info(f'Successfully created "{output_filename}".') return True - except: # noqa: E722 + except Exception: raise finally: if root_directory: @@ -146,7 +146,7 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_sym bundle.create_bundle(shell_launchers=shell_launchers) return bundle.working_directory - except: # noqa: E722 + except Exception: bundle.delete_working_directory() raise @@ -154,7 +154,7 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_sym def detect_elf_binary(filename): """Returns `True` if a file has an ELF header.""" if not os.path.exists(filename): - raise MissingFileError('The "%s" file was not found.' % filename) + raise MissingFileError(f'The "{filename}" file was not found.') with open(filename, 'rb') as f: first_four_bytes = f.read(4) @@ -191,7 +191,7 @@ def resolve_binary(binary): if os.path.exists(absolute_binary_path): break else: - raise MissingFileError('The "%s" binary could not be found in $PATH.' % binary) + raise MissingFileError(f'The "{binary}" binary could not be found in $PATH.') return absolute_binary_path @@ -208,23 +208,23 @@ def resolve_file_path(path, search_environment_path=False): if search_environment_path: path = resolve_binary(path) if not os.path.exists(path): - raise MissingFileError('The "%s" file was not found.' % path) + raise MissingFileError(f'The "{path}" file was not found.') if os.path.isdir(path): - raise UnexpectedDirectoryError('"%s" is a directory, not a file.' % path) + raise UnexpectedDirectoryError(f'"{path}" is a directory, not a file.') return os.path.normpath(os.path.abspath(path)) def run_ldd(ldd, binary): """Runs `ldd` and gets the combined stdout/stderr output as a list of lines.""" if not detect_elf_binary(resolve_binary(binary)): - raise InvalidElfBinaryError('The "%s" file is not a binary ELF file.' % binary) + raise InvalidElfBinaryError(f'The "{binary}" file is not a binary ELF file.') process = Popen([ldd, binary], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() return stdout.decode('utf-8').split('\n') + stderr.decode('utf-8').split('\n') -class stored_property(object): +class stored_property: """Simple decorator for a class property that will be cached indefinitely.""" def __init__(self, function): self.__doc__ = getattr(function, '__doc__') @@ -235,7 +235,7 @@ def __get__(self, instance, type): return result -class Elf(object): +class Elf: """Parses basic attributes from the ELF header of a file. Attributes: @@ -256,7 +256,7 @@ def __init__(self, path, chroot=None, file_factory=None): file_factory (function, optional): A function to use when creating new `File` instances. """ if not os.path.exists(path): - raise MissingFileError('The "%s" file was not found.' % path) + raise MissingFileError(f'The "{path}" file was not found.') self.path = path self.chroot = chroot self.file_factory = file_factory or File @@ -265,7 +265,7 @@ def __init__(self, path, chroot=None, file_factory=None): # Make sure that this is actually an ELF binary. first_four_bytes = f.read(4) if first_four_bytes != b'\x7fELF': - raise InvalidElfBinaryError('The "%s" file is not a binary ELF file.' % path) + raise InvalidElfBinaryError(f'The "{path}" file is not a binary ELF file.') # Determine whether this is a 32-bit or 64-bit file. format_byte = f.read(1) @@ -357,7 +357,7 @@ def __hash__(self): return hash(self.path) def __repr__(self): - return '' % self.path + return f'' def find_direct_dependencies(self, linker_file=None): """Runs the specified linker and returns a set of the dependencies as `File` instances.""" @@ -414,7 +414,7 @@ def direct_dependencies(self): return self.find_direct_dependencies() -class File(object): +class File: """Represents a file on disk and provides access to relevant properties and actions. Note: @@ -474,7 +474,7 @@ def __hash__(self): return hash((self.path, self.entry_point)) def __repr__(self): - return '' % self.path + return f'=8.0 + pytest-sugar>=1.0 + pytest-cov>=5.0 commands = {posargs:py.test --cov --cov-report=term-missing -vv} [testenv:check] deps = - docutils - check-manifest - flake8 - flake8-commas - flake8-quotes - readme-renderer - pygments - isort + docutils>=0.20 + check-manifest>=0.49 + flake8>=7.0 + flake8-commas>=4.0 + flake8-quotes>=3.4 + readme-renderer>=43.0 + pygments>=2.17 + isort>=5.13 skip_install = true commands = flake8 src tests setup.py @@ -41,7 +43,7 @@ commands = check-manifest {toxinidir} [testenv:report] -deps = coverage +deps = coverage>=7.0 skip_install = true commands = coverage report @@ -50,7 +52,7 @@ commands = [testenv:clean] commands = coverage erase skip_install = true -deps = coverage +deps = coverage>=7.0 [flake8] ignore = E128,W504 From 1f7e709c7762d6e106cb12796cf48f8c7ab38a57 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 21:48:03 -0600 Subject: [PATCH 03/15] Fixing variable scope for tt --- src/exodus_bundler/bundling.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index da15254..e018a93 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -590,6 +590,7 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli full_linker=full_linker) with open(source_path, 'wb') as f: f.write(launcher_content) + tt = source_path except CompilerNotFoundError: if not shell_launcher: logger.warning(( From cd7bf57359b9aa710a28ab6b8d4b9f8de65f69de Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 22:24:38 -0600 Subject: [PATCH 04/15] Updated formatting and linting --- conftest.py | 2 +- development-requirements.txt | 7 +- pyproject.toml | 17 +- setup.cfg | 14 +- setup.py | 74 ++-- src/exodus_bundler/__init__.py | 2 +- src/exodus_bundler/bundling.py | 365 +++++++++++------- src/exodus_bundler/cli.py | 147 +++++--- src/exodus_bundler/dependency_detection.py | 41 +- src/exodus_bundler/errors.py | 6 + src/exodus_bundler/input_parsing.py | 44 +-- src/exodus_bundler/launchers.py | 69 ++-- src/exodus_bundler/templating.py | 6 +- tests/test_bundling.py | 417 ++++++++++++--------- tests/test_cli.py | 101 ++--- tests/test_dependency_detection.py | 11 +- tests/test_input_parsing.py | 75 ++-- tests/test_launchers.py | 47 +-- tests/test_pytest.py | 2 +- tests/test_templating.py | 14 +- 20 files changed, 818 insertions(+), 643 deletions(-) diff --git a/conftest.py b/conftest.py index 98c75b2..044963b 100644 --- a/conftest.py +++ b/conftest.py @@ -1 +1 @@ -collect_ignore = ['setup.py'] +collect_ignore = ["setup.py"] diff --git a/development-requirements.txt b/development-requirements.txt index 51a780c..1b7ac21 100644 --- a/development-requirements.txt +++ b/development-requirements.txt @@ -7,11 +7,8 @@ pytest-watch>=4.2 pytest-cov>=5.0 tox>=4.0 twine>=5.0 -flake8>=7.0 -flake8-commas>=4.0 -flake8-quotes>=3.4 -isort>=5.13 +ruff>=0.4.0 check-manifest>=0.49 docutils>=0.20 readme-renderer>=43.0 -pygments>=2.17 \ No newline at end of file +pygments>=2.17 diff --git a/pyproject.toml b/pyproject.toml index c07b8c8..2354adf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,18 +54,13 @@ zip-safe = false [tool.setuptools.packages.find] where = ["src"] -[tool.flake8] -ignore = ["E128", "W504"] -max-line-length = 100 - -[tool.isort] -force_single_line = true -line_length = 100 -lines_after_imports = 2 -known_first_party = ["exodus_bundler"] -default_section = "THIRDPARTY" -skip_gitignore = false +[tool.ruff] +line-length = 100 +target-version = "py310" +[tool.ruff.lint] +select = ["E", "F", "W"] +ignore = ["E501"] [tool.pytest.ini_options] norecursedirs = [".git", ".tox", ".env", "dist", "build"] python_files = ["test_*.py", "*_test.py", "tests.py"] diff --git a/setup.cfg b/setup.cfg index a530891..7447b35 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,10 +4,6 @@ universal = 1 [metadata] license_file = LICENSE.md -[flake8] -ignore = E128 -max-line-length = 100 - [tool:pytest] norecursedirs = .git @@ -24,12 +20,4 @@ addopts = --strict --doctest-modules --doctest-glob=\*.rst - --tb=short - -[isort] -force_single_line = True -line_length = 100 -lines_after_imports = 2 -known_first_party = exodus_bundler -default_section = THIRDPARTY -not_skip = __init__.py + --tb=short \ No newline at end of file diff --git a/setup.py b/setup.py index 082ac76..2544daf 100644 --- a/setup.py +++ b/setup.py @@ -10,50 +10,52 @@ setup( - name='exodus-bundler', - version='3.0.0', - license='BSD', - platforms=['Linux'], - description='The exodus application bundler.', - long_description='See the documentation for details.', - author='Intoli', - author_email='contact@intoli.com', - url='https://github.com/intoli/exodus', - packages=find_packages('src'), - package_dir={'': 'src'}, - py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + name="exodus-bundler", + version="3.0.0", + license="BSD", + platforms=["Linux"], + description="The exodus application bundler.", + long_description="See the documentation for details.", + author="Intoli", + author_email="contact@intoli.com", + url="https://github.com/intoli/exodus", + packages=find_packages("src"), + package_dir={"": "src"}, + py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, zip_safe=False, classifiers=[ # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: BSD License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: System :: Archiving :: Packaging', - 'Topic :: Utilities', + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: System :: Archiving :: Packaging", + "Topic :: Utilities", ], keywords=[ - 'linux', 'executable', 'elf', 'binaries', - ], - python_requires='>=3.10', - install_requires=[ + "linux", + "executable", + "elf", + "binaries", ], + python_requires=">=3.10", + install_requires=[], entry_points={ - 'console_scripts': [ - 'exodus = exodus_bundler.cli:main', + "console_scripts": [ + "exodus = exodus_bundler.cli:main", ], }, ) diff --git a/src/exodus_bundler/__init__.py b/src/exodus_bundler/__init__.py index 882a45a..47125e9 100644 --- a/src/exodus_bundler/__init__.py +++ b/src/exodus_bundler/__init__.py @@ -1,7 +1,7 @@ import logging -__version__ = '3.0.0' +__version__ = "3.0.0" root_logger = logging.getLogger(__name__) root_logger.handlers = [logging.NullHandler()] diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index e018a93..a4f2a9a 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -32,54 +32,70 @@ logger = logging.getLogger(__name__) -def bytes_to_int(bytes, byteorder='big'): +def bytes_to_int(bytes, byteorder="big"): """Simple helper function to convert byte strings into integers.""" - endian = {'big': '>', 'little': '<'}[byteorder] - chars = struct.unpack(endian + ('B' * len(bytes)), bytes) - if byteorder == 'big': + endian = {"big": ">", "little": "<"}[byteorder] + chars = struct.unpack(endian + ("B" * len(bytes)), bytes) + if byteorder == "big": chars = chars[::-1] - return sum(int(char) * 256 ** i for (i, char) in enumerate(chars)) - - -def create_bundle(executables, output, tarball=False, rename=[], chroot=None, add=[], - no_symlink=[], shell_launchers=False, detect=False): + return sum(int(char) * 256**i for (i, char) in enumerate(chars)) + + +def create_bundle( + executables, + output, + tarball=False, + rename=[], + chroot=None, + add=[], + no_symlink=[], + shell_launchers=False, + detect=False, +): """Handles the creation of the full bundle.""" # Initialize these ahead of time so they're always available for error handling. output_filename, output_file, root_directory = None, None, None try: - # Create a temporary unpackaged bundle for the executables. root_directory = create_unpackaged_bundle( - executables, rename=rename, chroot=chroot, add=add, no_symlink=no_symlink, - shell_launchers=shell_launchers, detect=detect, + executables, + rename=rename, + chroot=chroot, + add=add, + no_symlink=no_symlink, + shell_launchers=shell_launchers, + detect=detect, ) # Populate the filename template. - output_filename = render_template(output, - executables=('-'.join(os.path.basename(executable) for executable in executables)), - extension=('tgz' if tarball else 'sh'), + output_filename = render_template( + output, + executables=("-".join(os.path.basename(executable) for executable in executables)), + extension=("tgz" if tarball else "sh"), ) # Store a gzipped tarball of the bundle in memory. tar_stream = io.BytesIO() - with tarfile.open(fileobj=tar_stream, mode='w:gz') as tar: - tar.add(root_directory, arcname='exodus') + with tarfile.open(fileobj=tar_stream, mode="w:gz") as tar: + tar.add(root_directory, arcname="exodus") # Configure the appropriate output mechanism. - if output_filename == '-': + if output_filename == "-": output_file = sys.stdout.buffer else: - output_file = open(output_filename, 'wb') + output_file = open(output_filename, "wb") # Construct the installation script and write it out. if not tarball: - if output_filename == '-': - base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode('utf-8') - script_content = render_template_file('install-bundle-noninteractive.sh', - base64_encoded_tarball=base64_encoded_tarball) - output_file.write(script_content.encode('utf-8')) + if output_filename == "-": + base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode("utf-8") + script_content = render_template_file( + "install-bundle-noninteractive.sh", + base64_encoded_tarball=base64_encoded_tarball, + ) + output_file.write(script_content.encode("utf-8")) else: - output_file.write(render_template_file('install-bundle.sh').encode('utf-8')) + output_file.write(render_template_file("install-bundle.sh").encode("utf-8")) output_file.write(tar_stream.getvalue()) else: # Or just write out the tarball. @@ -95,25 +111,33 @@ def create_bundle(executables, output, tarball=False, rename=[], chroot=None, ad shutil.rmtree(root_directory) if output_file and output_filename: output_file.close() - if not tarball and output_filename not in ['-', '/dev/null']: + if not tarball and output_filename not in ["-", "/dev/null"]: st = os.stat(output_filename) os.chmod(output_filename, st.st_mode | stat.S_IEXEC) -def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_symlink=[], - shell_launchers=False, detect=False): +def create_unpackaged_bundle( + executables, + rename=[], + chroot=None, + add=[], + no_symlink=[], + shell_launchers=False, + detect=False, +): """Creates a temporary directory containing the unpackaged contents of the bundle.""" bundle = Bundle(chroot=chroot, working_directory=True) try: # Sanitize the inputs. - assert len(executables), 'No executables were specified.' - assert len(executables) >= len(rename), \ - 'More renamed options were included than executables.' + assert len(executables), "No executables were specified." + assert len(executables) >= len( + rename + ), "More renamed options were included than executables." # Pad the rename's with `True` so that `entry_point` can be specified. entry_points = rename + [True for i in range(len(executables) - len(rename))] # Populate the bundle with main executable files and their dependencies. - for (executable, entry_point) in zip(executables, entry_points): + for executable, entry_point in zip(executables, entry_points): file = bundle.add_file(executable, entry_point=entry_point) # We'll only auto-detect dependencies for these entry points as well. @@ -122,11 +146,13 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_sym dependency_paths = detect_dependencies(file.path) if not dependency_paths: raise DependencyDetectionError( - ('Automatic dependency detection failed. Either "%s" ' % file.path) + - 'is not tracked by your package manager, or your operating system ' - 'is not currently compatible with the `--detect` option. If not, please ' - "create an issue at https://github.com/intoli/exodus and we'll try our " - ' to add support for it in the future.', + ( + 'Automatic dependency detection failed. Either "%s" ' % file.path + + "is not tracked by your package manager, or your operating system " + + "is not currently compatible with the `--detect` option. If not, please " + + "create an issue at https://github.com/intoli/exodus and we'll try our " + + " to add support for it in the future." + ), ) for path in dependency_paths: @@ -156,26 +182,26 @@ def detect_elf_binary(filename): if not os.path.exists(filename): raise MissingFileError(f'The "{filename}" file was not found.') - with open(filename, 'rb') as f: + with open(filename, "rb") as f: first_four_bytes = f.read(4) - return first_four_bytes == b'\x7fELF' + return first_four_bytes == b"\x7fELF" def parse_dependencies_from_ldd_output(content): """Takes the output of `ldd` as a string or list of lines and parses the dependencies.""" - if type(content) == str: - content = content.split('\n') + if isinstance(content, str): + content = content.split("\n") dependencies = [] for line in content: # This first one is a special case of invoking the linker as `ldd`. - if re.search(r'^\s*(/.*?)\s*=>\s*ldd\s*\(', line): + if re.search(r"^\s*(/.*?)\s*=>\s*ldd\s*\(", line): # We'll exclude this because it's the hardcoded INTERP path, and it would be # impossible to get the full path from this command output. continue - match = re.search(r'=>\s*(/.*?)\s*\(', line) - match = match or re.search(r'\s*(/.*?)\s*\(', line) + match = re.search(r"=>\s*(/.*?)\s*\(", line) + match = match or re.search(r"\s*(/.*?)\s*\(", line) if match: dependencies.append(match.group(1)) @@ -186,7 +212,7 @@ def resolve_binary(binary): """Attempts to find the absolute path to the binary.""" absolute_binary_path = os.path.normpath(os.path.abspath(binary)) if not os.path.exists(absolute_binary_path): - for path in os.getenv('PATH', '/bin/:/usr/bin/').split(os.pathsep): + for path in os.getenv("PATH", "/bin/:/usr/bin/").split(os.pathsep): absolute_binary_path = os.path.normpath(os.path.abspath(os.path.join(path, binary))) if os.path.exists(absolute_binary_path): break @@ -221,13 +247,14 @@ def run_ldd(ldd, binary): process = Popen([ldd, binary], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - return stdout.decode('utf-8').split('\n') + stderr.decode('utf-8').split('\n') + return stdout.decode("utf-8").split("\n") + stderr.decode("utf-8").split("\n") class stored_property: """Simple decorator for a class property that will be cached indefinitely.""" + def __init__(self, function): - self.__doc__ = getattr(function, '__doc__') + self.__doc__ = getattr(function, "__doc__") self.function = function def __get__(self, instance, type): @@ -246,6 +273,7 @@ class Elf: path (str): The path to the file. type (str): The binary type, one of 'relocatable', 'executable', 'shared', or 'core'. """ + def __init__(self, path, chroot=None, file_factory=None): """Constructs the `Elf` instance. @@ -261,56 +289,60 @@ def __init__(self, path, chroot=None, file_factory=None): self.chroot = chroot self.file_factory = file_factory or File - with open(path, 'rb') as f: + with open(path, "rb") as f: # Make sure that this is actually an ELF binary. first_four_bytes = f.read(4) - if first_four_bytes != b'\x7fELF': + if first_four_bytes != b"\x7fELF": raise InvalidElfBinaryError(f'The "{path}" file is not a binary ELF file.') # Determine whether this is a 32-bit or 64-bit file. format_byte = f.read(1) - self.bits = {b'\x01': 32, b'\x02': 64}.get(format_byte) + self.bits = {b"\x01": 32, b"\x02": 64}.get(format_byte) if not self.bits: raise UnsupportedArchitectureError( - ('The "%s" file does not appear to be either 32 or 64 bits. ' % path) + - 'Other architectures are not currently supported, but you can open an ' - 'issue at https://github.com/intoli/exodus stating your use-case and ' - 'support might get extended in the future.', + ( + 'The "%s" file does not appear to be either 32 or 64 bits. ' % path + + "Other architectures are not currently supported, but you can open an " + + "issue at https://github.com/intoli/exodus stating your use-case and " + + "support might get extended in the future." + ), ) # Determine whether it's big or little endian and construct an integer parsing function. endian_byte = f.read(1) - byteorder = {b'\x01': 'little', b'\x02': 'big'}[endian_byte] - assert byteorder == 'little', 'Big endian is not supported right now.' + byteorder = {b"\x01": "little", b"\x02": "big"}[endian_byte] + assert byteorder == "little", "Big endian is not supported right now." if not byteorder: raise UnsupportedArchitectureError( - ('The "%s" file does not appear to be little endian, ' % path) + - 'and big endian binaries are not currently supported. You can open an ' - 'issue at https://github.com/intoli/exodus stating your use-case and ' - 'support might get extended in the future.', + ( + 'The "%s" file does not appear to be little endian, ' % path + + "and big endian binaries are not currently supported. You can open an " + + "issue at https://github.com/intoli/exodus stating your use-case and " + + "support might get extended in the future." + ), ) def hex(bytes): return bytes_to_int(bytes, byteorder=byteorder) # Determine the type of the binary. - f.seek(hex(b'\x10')) + f.seek(hex(b"\x10")) e_type = hex(f.read(2)) - self.type = {1: 'relocatable', 2: 'executable', 3: 'shared', 4: 'core'}[e_type] + self.type = {1: "relocatable", 2: "executable", 3: "shared", 4: "core"}[e_type] # Find the program header offset. - e_phoff_start = {32: hex(b'\x1c'), 64: hex(b'\x20')}[self.bits] + e_phoff_start = {32: hex(b"\x1c"), 64: hex(b"\x20")}[self.bits] e_phoff_length = {32: 4, 64: 8}[self.bits] f.seek(e_phoff_start) e_phoff = hex(f.read(e_phoff_length)) # Determine the size of a program header entry. - e_phentsize_start = {32: hex(b'\x2a'), 64: hex(b'\x36')}[self.bits] + e_phentsize_start = {32: hex(b"\x2a"), 64: hex(b"\x36")}[self.bits] f.seek(e_phentsize_start) e_phentsize = hex(f.read(2)) # Determine the number of program header entries. - e_phnum_start = {32: hex(b'\x2c'), 64: hex(b'\x38')}[self.bits] + e_phnum_start = {32: hex(b"\x2c"), 64: hex(b"\x38")}[self.bits] f.seek(e_phnum_start) e_phnum = hex(f.read(2)) @@ -323,17 +355,17 @@ def hex(bytes): # A p_type of \x03 corresponds to a PT_INTERP header (e.g. the linker). if len(p_type) == 0: break - if not p_type == b'\x03\x00\x00\x00': + if not p_type == b"\x03\x00\x00\x00": continue # Determine the offset for the segment. - p_offset_start = header_start + {32: hex(b'\04'), 64: hex(b'\x08')}[self.bits] + p_offset_start = header_start + {32: hex(b"\04"), 64: hex(b"\x08")}[self.bits] p_offset_length = {32: 4, 64: 8}[self.bits] f.seek(p_offset_start) p_offset = hex(f.read(p_offset_length)) # Determine the size of the segment. - p_filesz_start = header_start + {32: hex(b'\x10'), 64: hex(b'\x20')}[self.bits] + p_filesz_start = header_start + {32: hex(b"\x10"), 64: hex(b"\x20")}[self.bits] p_filesz_length = {32: 4, 64: 8}[self.bits] f.seek(p_filesz_start) p_filesz = hex(f.read(p_filesz_length)) @@ -342,11 +374,11 @@ def hex(bytes): f.seek(p_offset) segment = f.read(p_filesz) # It should be null-terminated (b'\x00' in Python 2, 0 in Python 3). - assert segment[-1] in [b'\x00', 0], 'The string should be null terminated.' - assert self.linker_file is None, 'More than one linker found.' - linker_path = segment[:-1].decode('ascii') + assert segment[-1] in [b"\x00", 0], "The string should be null terminated." + assert self.linker_file is None, "More than one linker found." + linker_path = segment[:-1].decode("ascii") if chroot: - linker_path = os.path.join(chroot, os.path.relpath(linker_path, '/')) + linker_path = os.path.join(chroot, os.path.relpath(linker_path, "/")) self.linker_file = self.file_factory(linker_path, chroot=self.chroot) def __eq__(self, other): @@ -367,31 +399,37 @@ def find_direct_dependencies(self, linker_file=None): linker_path = linker_file.path environment = {} environment.update(os.environ) - environment['LD_TRACE_LOADED_OBJECTS'] = '1' + environment["LD_TRACE_LOADED_OBJECTS"] = "1" extra_ldd_arguments = [] if self.chroot: - ld_library_path = '/lib64:/usr/lib64:/lib/:/usr/lib:/lib32/:/usr/lib32/:' - ld_library_path += environment.get('LD_LIBRARY_PATH', '') + ld_library_path = "/lib64:/usr/lib64:/lib/:/usr/lib:/lib32/:/usr/lib32/:" + ld_library_path += environment.get("LD_LIBRARY_PATH", "") directories = [] - for directory in ld_library_path.split(':'): + for directory in ld_library_path.split(":"): if os.path.isabs(directory): - directory = os.path.join(self.chroot, os.path.relpath(directory, '/')) + directory = os.path.join(self.chroot, os.path.relpath(directory, "/")) directories.append(directory) - ld_library_path = ':'.join(directories) - environment['LD_LIBRARY_PATH'] = ld_library_path + ld_library_path = ":".join(directories) + environment["LD_LIBRARY_PATH"] = ld_library_path # We only need to avoid including system dependencies if there's a chroot set. - extra_ldd_arguments += ['--inhibit-cache', '--inhibit-rpath', ''] - - process = Popen(['ldd'] + extra_ldd_arguments + [self.path], - executable=linker_path, stdout=PIPE, stderr=PIPE, env=environment) + extra_ldd_arguments += ["--inhibit-cache", "--inhibit-rpath", ""] + + process = Popen( + ["ldd"] + extra_ldd_arguments + [self.path], + executable=linker_path, + stdout=PIPE, + stderr=PIPE, + env=environment, + ) stdout, stderr = process.communicate() - combined_output = stdout.decode('utf-8').split('\n') + stderr.decode('utf-8').split('\n') + combined_output = stdout.decode("utf-8").split("\n") + stderr.decode("utf-8").split("\n") # Note that we're explicitly adding the linker because when we invoke it as `ldd` we can't # extract the real path from the trace output. Even if it were here twice, it would be # deduplicated though the use of a set. filenames = parse_dependencies_from_ldd_output(combined_output) + [linker_path] - return set(self.file_factory(filename, chroot=self.chroot, library=True) - for filename in filenames) + return set( + self.file_factory(filename, chroot=self.chroot, library=True) for filename in filenames + ) @stored_property def dependencies(self): @@ -404,7 +442,8 @@ def dependencies(self): for dependency in unprocessed_dependencies: if dependency.elf: new_dependencies |= set( - dependency.elf.find_direct_dependencies(self.linker_file)) + dependency.elf.find_direct_dependencies(self.linker_file) + ) unprocessed_dependencies = new_dependencies - all_dependencies return all_dependencies @@ -450,7 +489,7 @@ def __init__(self, path, entry_point=None, chroot=None, library=False, file_fact # Set the entry point for the file. if entry_point is True: - self.entry_point = os.path.basename(self.path).replace(os.sep, '') + self.entry_point = os.path.basename(self.path).replace(os.sep, "") else: self.entry_point = entry_point or None @@ -466,8 +505,11 @@ def __init__(self, path, entry_point=None, chroot=None, library=False, file_fact self.no_symlink = self.entry_point and not self.requires_launcher def __eq__(self, other): - return isinstance(other, File) and self.path == self.path and \ - self.entry_point == self.entry_point + return ( + isinstance(other, File) + and self.path == other.path + and self.entry_point == other.entry_point + ) def __hash__(self): """Computes a hash for the instance unique up to the file path and entry point.""" @@ -509,15 +551,21 @@ def create_entry_point(self, working_directory, bundle_root): bundle_root (str): The root that `source` will be joined with. """ source_path = os.path.join(bundle_root, self.source) - bin_directory = os.path.join(working_directory, 'bin') + bin_directory = os.path.join(working_directory, "bin") if not os.path.exists(bin_directory): os.makedirs(bin_directory) entry_point_path = os.path.join(bin_directory, self.entry_point) - relative_destination_path = os.path.relpath(source_path, bin_directory)+".sh" + relative_destination_path = os.path.relpath(source_path, bin_directory) + ".sh" os.symlink(relative_destination_path, entry_point_path) - def create_launcher(self, working_directory, bundle_root, linker_basename, symlink_basename, - shell_launcher=False): + def create_launcher( + self, + working_directory, + bundle_root, + linker_basename, + symlink_basename, + shell_launcher=False, + ): """Creates a launcher at `source` for `destination`. Note: @@ -542,21 +590,29 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli relative_destination_path = os.path.relpath(destination_path, source_parent) symlink_path = os.path.join(source_parent, symlink_basename) os.symlink(relative_destination_path, symlink_path) - executable = os.path.join('.', symlink_basename) + executable = os.path.join(".", symlink_basename) # Copy over the linker. linker_path = os.path.join(source_parent, linker_basename) if not os.path.exists(linker_path): shutil.copy(self.elf.linker_file.path, linker_path) else: - assert filecmp.cmp(self.elf.linker_file.path, linker_path), \ + assert filecmp.cmp(self.elf.linker_file.path, linker_path), ( 'The "%s" linker file already exists and has differing contents.' % linker_path - linker = os.path.join('.', linker_basename) + ) + linker = os.path.join(".", linker_basename) # Construct the library path original_file_parent = os.path.dirname(self.path) - library_paths = os.environ.get('LD_LIBRARY_PATH', '').split(':') - library_paths += ['/lib64', '/usr/lib64', '/lib', '/usr/lib', '/lib32', '/usr/lib32'] + library_paths = os.environ.get("LD_LIBRARY_PATH", "").split(":") + library_paths += [ + "/lib64", + "/usr/lib64", + "/lib", + "/usr/lib", + "/lib32", + "/usr/lib32", + ] for dependency in self.elf.dependencies: library_paths.append(os.path.dirname(dependency.path)) relative_library_paths = [] @@ -567,18 +623,18 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli # Get the actual absolute path for the library directory. directory = os.path.normpath(os.path.abspath(directory)) if self.chroot: - directory = os.path.join(self.chroot, os.path.relpath(directory, '/')) + directory = os.path.join(self.chroot, os.path.relpath(directory, "/")) # Convert it into a path relative to the launcher/source. relative_library_path = os.path.relpath(directory, original_file_parent) if relative_library_path not in relative_library_paths: relative_library_paths.append(relative_library_path) - library_path = ':'.join(relative_library_paths) + library_path = ":".join(relative_library_paths) # Determine whether this is a "full" linker (*e.g.* GNU linker). - with open(self.elf.linker_file.path, 'rb') as f: + with open(self.elf.linker_file.path, "rb") as f: linker_content = f.read() - full_linker = (linker_content.find(b'inhibit-rpath') > -1) + full_linker = linker_content.find(b"inhibit-rpath") > -1 # Try a c launcher first and fallback. try: @@ -586,22 +642,30 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli raise CompilerNotFoundError() launcher_content = construct_binary_launcher( - linker=linker, library_path=library_path, executable=executable, - full_linker=full_linker) - with open(source_path, 'wb') as f: + linker=linker, + library_path=library_path, + executable=executable, + full_linker=full_linker, + ) + with open(source_path, "wb") as f: f.write(launcher_content) tt = source_path except CompilerNotFoundError: if not shell_launcher: - logger.warning(( - 'Installing either the musl or diet C libraries will result in more efficient ' - 'launchers (currently using bash fallbacks instead).' - )) + logger.warning( + ( + "Installing either the musl or diet C libraries will result in more efficient " + "launchers (currently using bash fallbacks instead)." + ) + ) launcher_content = construct_bash_launcher( - linker=linker, library_path=library_path, executable=executable, - full_linker=full_linker) - tt=source_path+".sh" - with open(tt, 'w') as f: + linker=linker, + library_path=library_path, + executable=executable, + full_linker=full_linker, + ) + tt = source_path + ".sh" + with open(tt, "w") as f: f.write(launcher_content) shutil.copymode(self.path, tt) @@ -634,7 +698,7 @@ def symlink(self, working_directory, bundle_root): @stored_property def destination(self): """str: The relative path for the destination of the actual file contents.""" - return os.path.join('.', 'data', self.hash) + return os.path.join(".", "data", self.hash) @stored_property def executable(self): @@ -648,7 +712,7 @@ def elf(self): @stored_property def hash(self): """str: Computes a hash based on the file content, useful for file deduplication.""" - with open(self.path, 'rb') as f: + with open(self.path, "rb") as f: return hashlib.sha256(f.read()).hexdigest() @stored_property @@ -660,14 +724,14 @@ def requires_launcher(self): # The easy ones. if self.library or not self.elf or not self.elf.linker_file or not self.executable: return False - if self.elf.type == 'executable': + if self.elf.type == "executable": return True if self.entry_point: return True # These will hopefully do more good than harm. - bin_directories = ['/bin/', '/bin32/', '/bin64/'] - lib_directories = ['/lib/', '/lib32/', '/lib64/'] + bin_directories = ["/bin/", "/bin32/", "/bin64/"] + lib_directories = ["/lib/", "/lib32/", "/lib64/"] in_bin_directory = any(directory in self.path for directory in bin_directories) in_lib_directory = any(directory in self.path for directory in lib_directories) if in_bin_directory and not in_lib_directory: @@ -676,12 +740,14 @@ def requires_launcher(self): return False # Most libraries will include `.so` in the filename. - return re.search(r'\.so(?:\.|$)', self.path) + if re.search(r"\.so(?:\.|$)", self.path): + return True + return False @stored_property def source(self): """str: The relative path for the source of the actual file contents.""" - return os.path.relpath(self.path, '/') + return os.path.relpath(self.path, "/") class Bundle: @@ -693,6 +759,7 @@ class Bundle: linker_files (:obj:`set` of :obj:`File`): A list of observed linker files. working_directory (str): The root directory where the bundles will be written and packaged. """ + def __init__(self, working_directory=None, chroot=None): """Constructor for the `Bundle` class. @@ -705,7 +772,7 @@ def __init__(self, working_directory=None, chroot=None): """ self.working_directory = working_directory if working_directory is True: - self.working_directory = tempfile.mkdtemp(prefix='exodus-bundle-') + self.working_directory = tempfile.mkdtemp(prefix="exodus-bundle-") # The permissions on the `mkdtemp()` directory will be extremely restricted by default, # so we'll modify them to to reflect the current umask. umask = os.umask(0) @@ -755,10 +822,10 @@ def add_file(self, path, entry_point=None): # We definitely don't want a launcher for this file, so clear the linker. file.elf.linker_file = None else: - logger.warning(( - 'An ELF binary without a suitable linker candidate was encountered. ' - 'Either no linker was found or there are multiple conflicting linkers.' - )) + logger.warning( + "An ELF binary without a suitable linker candidate was encountered. " + "Either no linker was found or there are multiple conflicting linkers." + ) return file @@ -795,21 +862,27 @@ def create_bundle(self, shell_launchers=False): if file.requires_launcher: # These are kind of complicated, we'll just store the requirements for now. - directory_and_linker = (os.path.dirname(file_path), file.elf.linker_file) + directory_and_linker = ( + os.path.dirname(file_path), + file.elf.linker_file, + ) files_needing_launchers[directory_and_linker].add(file) else: - file.symlink(working_directory=self.working_directory, bundle_root=self.bundle_root) + file.symlink( + working_directory=self.working_directory, + bundle_root=self.bundle_root, + ) # Now we need to write out one unique copy of each linker in each directory where it's # required. This is necessary so that `readlink("/proc/self/exe")` will return the correct # directory when programs use that to construct relative paths to resources. - for ((directory, linker), executable_files) in files_needing_launchers.items(): + for (directory, linker), executable_files in files_needing_launchers.items(): # First, we'll find a unique name for the linker in this directory and write it out. - desired_linker_path = os.path.join(directory, f'linker-{linker.hash}') + desired_linker_path = os.path.join(directory, f"linker-{linker.hash}") linker_path = desired_linker_path iteration = 2 while linker_path in file_paths: - linker_path = f'{desired_linker_path}-{iteration}' + linker_path = f"{desired_linker_path}-{iteration}" iteration += 1 file_paths.add(linker_path) linker_dirname, linker_basename = os.path.split(linker_path) @@ -822,17 +895,21 @@ def create_bundle(self, shell_launchers=False): # We'll again attempt to find a unique available name, this time for the symlink # to the executable. file_basename = file.entry_point or os.path.basename(file.path) - desired_symlink_path = os.path.join(directory, f'{file_basename}-x') + desired_symlink_path = os.path.join(directory, f"{file_basename}-x") symlink_path = desired_symlink_path # iteration = 2 # while symlink_path in file_paths: # symlink_path = '%s-%d' % (desired_symlink_path, iteration) # iteration += 1 file_paths.add(symlink_path) - symlink_basename = os.path.basename(symlink_path); - file.create_launcher(self.working_directory, self.bundle_root, - linker_basename, symlink_basename, - shell_launcher=shell_launchers) + symlink_basename = os.path.basename(symlink_path) + file.create_launcher( + self.working_directory, + self.bundle_root, + linker_basename, + symlink_basename, + shell_launcher=shell_launchers, + ) def delete_working_directory(self): """Recursively deletes the working directory.""" @@ -856,13 +933,15 @@ def file_factory(self, path, entry_point=None, chroot=None, library=False, file_ path = resolve_file_path(path, search_environment_path=entry_point is not None) file = next((file for file in self.files if file.path == path), None) if file is not None: - assert entry_point == file.entry_point or not entry_point or not file.entry_point, \ - "The entry point property should always persist, but can't conflict." + assert ( + entry_point == file.entry_point or not entry_point or not file.entry_point + ), "The entry point property should always persist, but can't conflict." file.entry_point = file.entry_point or entry_point - assert chroot == file.chroot, 'The chroot must match.' + assert chroot == file.chroot, "The chroot must match." file.library = file.library or library - assert not file.entry_point or not file.library, \ - "A file can't be both an entry point and a library." + assert ( + not file.entry_point or not file.library + ), "A file can't be both an entry point and a library." return file return File(path, entry_point, chroot, library, file_factory) @@ -870,12 +949,12 @@ def file_factory(self, path, entry_point=None, chroot=None, library=False, file_ @property def bundle_root(self): """str: The root directory of the bundle where the original file structure is mirrored.""" - path = os.path.join(self.working_directory, 'bundles', self.hash) + path = os.path.join(self.working_directory, "bundles", self.hash) return os.path.normpath(os.path.abspath(path)) @property def hash(self): """str: Computes a hash based on the current contents of the bundle.""" file_hashes = sorted(file.hash for file in self.files) - combined_hashes = '\n'.join(file_hashes).encode('utf-8') + combined_hashes = "\n".join(file_hashes).encode("utf-8") return hashlib.sha256(combined_hashes).hexdigest() diff --git a/src/exodus_bundler/cli.py b/src/exodus_bundler/cli.py index c83dbea..fe51cb5 100644 --- a/src/exodus_bundler/cli.py +++ b/src/exodus_bundler/cli.py @@ -17,81 +17,122 @@ def parse_args(args=None, namespace=None): to parse the arguments from `sys.argv`. A dictionary is returned rather than the typical namespace produced by `argparse`.""" formatter = argparse.ArgumentDefaultsHelpFormatter - parser = argparse.ArgumentParser(formatter_class=formatter, description=( - 'Bundle ELF binary executables with all of their runtime dependencies ' - 'so that they can be relocated to other systems with incompatible system ' - 'libraries.' - )) + parser = argparse.ArgumentParser( + formatter_class=formatter, + description=( + "Bundle ELF binary executables with all of their runtime dependencies " + "so that they can be relocated to other systems with incompatible system " + "libraries." + ), + ) - parser.add_argument('executables', metavar='EXECUTABLE', nargs='+', help=( - 'One or more ELF executables to include in the exodus bundle.' - )) + parser.add_argument( + "executables", + metavar="EXECUTABLE", + nargs="+", + help=("One or more ELF executables to include in the exodus bundle."), + ) - parser.add_argument('-c', '--chroot', metavar='CHROOT_PATH', + parser.add_argument( + "-c", + "--chroot", + metavar="CHROOT_PATH", default=None, help=( - 'A directory that will be treated as the root during linking. Useful for testing and ' - 'bundling extracted packages that won\t run without a chroot.' + "A directory that will be treated as the root during linking. Useful for testing and " + "bundling extracted packages that won\t run without a chroot." ), ) - parser.add_argument('-a', '--add', '--additional-file', metavar='DEPENDENCY', action='append', + parser.add_argument( + "-a", + "--add", + "--additional-file", + metavar="DEPENDENCY", + action="append", default=[], help=( - 'Specifies an additional file to include in the bundle, useful for adding ' - 'programatically loaded libraries and other non-library dependencies. ' - 'The argument can be used more than once to include multiple files, and ' - 'directories will be included recursively.' + "Specifies an additional file to include in the bundle, useful for adding " + "programatically loaded libraries and other non-library dependencies. " + "The argument can be used more than once to include multiple files, and " + "directories will be included recursively." ), ) - parser.add_argument('-d', '--detect', action='store_true', help=( - 'Attempt to autodetect direct dependencies using the system package manager. ' - 'Operating system support is limited.' - )) + parser.add_argument( + "-d", + "--detect", + action="store_true", + help=( + "Attempt to autodetect direct dependencies using the system package manager. " + "Operating system support is limited." + ), + ) - parser.add_argument('--no-symlink', metavar='FILE', action='append', + parser.add_argument( + "--no-symlink", + metavar="FILE", + action="append", default=[], help=( - 'Signifies that a file must not be symlinked to the deduplicated data directory. This ' - 'is useful if a file looks for other resources based on paths relative its own ' - 'location. This is enabled by default for executables.' + "Signifies that a file must not be symlinked to the deduplicated data directory. This " + "is useful if a file looks for other resources based on paths relative its own " + "location. This is enabled by default for executables." ), ) - parser.add_argument('-o', '--output', metavar='OUTPUT_FILE', + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT_FILE", default=None, help=( - 'The file where the bundle will be written out to. The extension depends on the ' + "The file where the bundle will be written out to. The extension depends on the " 'output type. The "{{executables}}" and "{{extension}}" template strings can be ' - ' used in the provided filename. If omitted, the output will go to stdout when ' + " used in the provided filename. If omitted, the output will go to stdout when " 'it is being piped, or to "./exodus-{{executables}}-bundle.{{extension}}" otherwise.' ), ) - parser.add_argument('-q', '--quiet', action='store_true', help=( - 'Suppress warning messages.' - )) + parser.add_argument("-q", "--quiet", action="store_true", help=("Suppress warning messages.")) - parser.add_argument('-r', '--rename', metavar='NEW_NAME', nargs='?', action='append', - default=[], help=( - 'Renames the binary executable(s) before packaging. The order of rename tags must ' - 'match the order of positional executable arguments.' + parser.add_argument( + "-r", + "--rename", + metavar="NEW_NAME", + nargs="?", + action="append", + default=[], + help=( + "Renames the binary executable(s) before packaging. The order of rename tags must " + "match the order of positional executable arguments." ), ) - parser.add_argument('--shell-launchers', action='store_true', help=( - 'Force the use of shell launchers instead of attempting to compile statically linked ones.' - )) + parser.add_argument( + "--shell-launchers", + action="store_true", + help=( + "Force the use of shell launchers instead of attempting to compile statically linked ones." + ), + ) - parser.add_argument('-t', '--tarball', action='store_true', help=( - 'Creates a tarball for manual extraction instead of an installation script. ' - 'Note that this will change the output extension from ".sh" to ".tgz".' - )) + parser.add_argument( + "-t", + "--tarball", + action="store_true", + help=( + "Creates a tarball for manual extraction instead of an installation script. " + 'Note that this will change the output extension from ".sh" to ".tgz".' + ), + ) - parser.add_argument('-v', '--verbose', action='store_true', help=( - 'Output additional informational messages.' - )) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help=("Output additional informational messages."), + ) return vars(parser.parse_args(args, namespace)) @@ -110,7 +151,7 @@ def filter(self, record): return record.levelno in (logging.WARN, logging.ERROR) stderr_handler = logging.StreamHandler(sys.stderr) - stderr_formatter = logging.Formatter('%(levelname)s: %(message)s') + stderr_formatter = logging.Formatter("%(levelname)s: %(message)s") stderr_handler.setFormatter(stderr_formatter) stderr_handler.addFilter(StderrFilter()) root_logger.addHandler(stderr_handler) @@ -123,7 +164,7 @@ class StdoutFilter(logging.Filter): def filter(self, record): return record.levelno in (logging.DEBUG, logging.INFO) - stdout_formatter = logging.Formatter('%(message)s') + stdout_formatter = logging.Formatter("%(message)s") stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(stdout_formatter) stdout_handler.addFilter(StdoutFilter()) @@ -134,25 +175,25 @@ def main(args=None, namespace=None): args = parse_args(args, namespace) # Dynamically set the default output to stdout if it is being piped. - if args['output'] is None: + if args["output"] is None: if sys.stdout.isatty(): - args['output'] = './exodus-{{executables}}-bundle.{{extension}}' + args["output"] = "./exodus-{{executables}}-bundle.{{extension}}" else: - args['output'] = '-' + args["output"] = "-" # Handle the CLI specific options here, removing them from `args` in the process. - quiet, verbose = args.pop('quiet'), args.pop('verbose') - suppress_stdout = args['output'] == '-' + quiet, verbose = args.pop("quiet"), args.pop("verbose") + suppress_stdout = args["output"] == "-" configure_logging(quiet=quiet, verbose=verbose, suppress_stdout=suppress_stdout) # Allow piping in additional files. if not sys.stdin.isatty(): - args['add'] += extract_paths(sys.stdin.read()) + args["add"] += extract_paths(sys.stdin.read()) # Create the bundle with all of the arguments. try: create_bundle(**args) except FatalError as fatal_error: - logger.error('Fatal error encountered, exiting.') + logger.error("Fatal error encountered, exiting.") logger.error(fatal_error, exc_info=verbose) sys.exit(1) diff --git a/src/exodus_bundler/dependency_detection.py b/src/exodus_bundler/dependency_detection.py index f5459ca..8c1a735 100644 --- a/src/exodus_bundler/dependency_detection.py +++ b/src/exodus_bundler/dependency_detection.py @@ -21,11 +21,12 @@ class PackageManager: package that owns a specific file. owner_regex (str): A regex to extract the package name from the output of the owner command. """ + cache_directory = None list_command = None - list_regex = '(.*)' + list_regex = "(.*)" owner_command = None - owner_regex = '(.*)' + owner_regex = "(.*)" def find_dependencies(self, path): """Finds a list of all of the files contained with the package containing a file.""" @@ -37,7 +38,7 @@ def find_dependencies(self, path): process = subprocess.Popen(args, stdout=subprocess.PIPE) stdout, stderr = process.communicate() dependencies = [] - for line in stdout.decode('utf-8').split('\n'): + for line in stdout.decode("utf-8").split("\n"): match = re.search(self.list_regex, line.strip()) if match: dependency_path = match.groups()[0] @@ -52,10 +53,10 @@ def find_owner(self, path): return None args = self.owner_command + [path] env = os.environ.copy() - env['LC_ALL'] = 'C' + env["LC_ALL"] = "C" process = subprocess.Popen(args, stdout=subprocess.PIPE, env=env) stdout, stderr = process.communicate() - output = stdout.decode('utf-8').strip() + output = stdout.decode("utf-8").strip() match = re.search(self.owner_regex, output) if match: return match.groups()[0].strip() @@ -73,27 +74,27 @@ def commands_exist(self): class Apt(PackageManager): - cache_directory = '/var/cache/apt' - list_command = ['dpkg-query', '-L'] - list_regex = '(.+)' - owner_command = ['dpkg', '-S'] - owner_regex = '(.+): ' + cache_directory = "/var/cache/apt" + list_command = ["dpkg-query", "-L"] + list_regex = "(.+)" + owner_command = ["dpkg", "-S"] + owner_regex = "(.+): " class Pacman(PackageManager): - cache_directory = '/var/cache/pacman' - list_command = ['pacman', '-Ql'] - list_regex = r'.*\s+(\/.+)' - owner_command = ['pacman', '-Qo'] - owner_regex = r' is owned by (.*)\s+.*' + cache_directory = "/var/cache/pacman" + list_command = ["pacman", "-Ql"] + list_regex = r".*\s+(\/.+)" + owner_command = ["pacman", "-Qo"] + owner_regex = r" is owned by (.*)\s+.*" class Yum(PackageManager): - cache_directory = '/var/cache/yum' - list_command = ['rpm', '-ql'] - list_regex = r'(.+)' - owner_command = ['rpm', '-qf'] - owner_regex = r'(.+)' + cache_directory = "/var/cache/yum" + list_command = ["rpm", "-ql"] + list_regex = r"(.+)" + owner_command = ["rpm", "-qf"] + owner_regex = r"(.+)" package_managers = [ diff --git a/src/exodus_bundler/errors.py b/src/exodus_bundler/errors.py index 4ae1301..1d4ae3f 100644 --- a/src/exodus_bundler/errors.py +++ b/src/exodus_bundler/errors.py @@ -1,29 +1,35 @@ # -*- coding: utf-8 -*- class FatalError(Exception): """Base class for exceptions that should terminate program execution.""" + pass class DependencyDetectionError(FatalError): """Signifies that the dependency detection process failed.""" + pass class InvalidElfBinaryError(FatalError): """Signifies that a file was expected to be an ELF binary, but wasn't.""" + pass class MissingFileError(FatalError): """Signifies that a file was not found.""" + pass class UnexpectedDirectoryError(FatalError): """Signifies that a path was unexpectedly a directory.""" + pass class UnsupportedArchitectureError(FatalError): """Signifies that a binary has an unexpected architecture.""" + pass diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index a9ee725..91b3354 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -5,23 +5,23 @@ # We don't actually want to include anything in these directories in bundles. blacklisted_directories = [ - '/dev/', - '/proc/', - '/run/', - '/sys/', + "/dev/", + "/proc/", + "/run/", + "/sys/", # This isn't a directory exactly, but it will filter out active bundling. - '/tmp/exodus-bundle-', + "/tmp/exodus-bundle-", ] exec_methods = [ - 'execve', - 'exec', - 'execl', - 'execlp', - 'execle', - 'execv', - 'execvp', - 'execvpe', + "execve", + "exec", + "execl", + "execlp", + "execle", + "execv", + "execvp", + "execvpe", ] @@ -31,7 +31,7 @@ def extract_exec_path(line): for method in exec_methods: prefix = method + '("' if line.startswith(prefix): - line = line[len(prefix):] + line = line[len(prefix) :] parts = line.split('", ') if len(parts) > 1: return parts[0] @@ -43,14 +43,14 @@ def extract_open_path(line): line = strip_pid_prefix(line) for prefix in ['openat(AT_FDCWD, "', 'open("']: if line.startswith(prefix): - parts = line[len(prefix):].split('", ') + parts = line[len(prefix) :].split('", ') if len(parts) != 2: continue - if 'ENOENT' in parts[1]: + if "ENOENT" in parts[1]: continue - if 'O_RDONLY' not in parts[1]: + if "O_RDONLY" not in parts[1]: continue - if 'O_DIRECTORY' in parts[1]: + if "O_DIRECTORY" in parts[1]: continue return parts[0] return None @@ -61,8 +61,8 @@ def extract_stat_path(line): line = strip_pid_prefix(line) prefix = 'stat("' if line.startswith(prefix): - parts = line[len(prefix):].split('", ') - if len(parts) == 2 and 'ENOENT' not in parts[1]: + parts = line[len(prefix) :].split('", ') + if len(parts) == 2 and "ENOENT" not in parts[1]: return parts[0] return None @@ -104,7 +104,7 @@ def extract_paths(content, existing_only=True): def strip_pid_prefix(line): """Strips out the `[pid XXX] ` prefix if present.""" - match = re.match(r'\[pid\s+\d+\]\s*', line) + match = re.match(r"\[pid\s+\d+\]\s*", line) if match: - return line[len(match.group()):] + return line[len(match.group()) :] return line diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index bde3a35..31b412d 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Methods to produce launchers that will invoke the relocated executables with the proper linker and library paths.""" + import os import re import shutil @@ -21,8 +22,8 @@ class CompilerNotFoundError(Exception): # This is kind of a hack to find things in PATH inside of bundles. def find_executable(binary_name, skip_original_for_testing=False): # This won't be set on Alpine Linux, but it's required for the `find_executable()` calls. - if 'PATH' not in os.environ: - os.environ['PATH'] = '/bin/:/usr/bin/' + if "PATH" not in os.environ: + os.environ["PATH"] = "/bin/:/usr/bin/" executable = shutil.which(binary_name) if executable and not skip_original_for_testing: return executable @@ -33,16 +34,15 @@ def find_executable(binary_name, skip_original_for_testing=False): if not len(basename): break # The bundle directory. - if re.match('[A-Fa-f0-9]{64}', basename): - for bin_directory in os.environ['PATH'].split(':'): + if re.match("[A-Fa-f0-9]{64}", basename): + for bin_directory in os.environ["PATH"].split(":"): if os.path.isabs(bin_directory): - bin_directory = os.path.relpath(bin_directory, '/') - candidate_executable = os.path.join(directory, basename, - bin_directory, binary_name) + bin_directory = os.path.relpath(bin_directory, "/") + candidate_executable = os.path.join(directory, basename, bin_directory, binary_name) if os.path.exists(candidate_executable): return candidate_executable # Also check for shell launcher version (.sh extension) - candidate_executable_sh = candidate_executable + '.sh' + candidate_executable_sh = candidate_executable + ".sh" if os.path.exists(candidate_executable_sh): return candidate_executable_sh return None @@ -55,33 +55,32 @@ def compile(code): try: return compile_diet(code) except CompilerNotFoundError: - raise CompilerNotFoundError('No suiteable C compiler was found.') + raise CompilerNotFoundError("No suiteable C compiler was found.") def compile_diet(code): - diet = find_executable('diet') - gcc = find_executable('gcc') + diet = find_executable("diet") + gcc = find_executable("gcc") if diet is None or gcc is None: - raise CompilerNotFoundError('The diet compiler was not found.') - return compile_helper(code, [diet, 'gcc']) + raise CompilerNotFoundError("The diet compiler was not found.") + return compile_helper(code, [diet, "gcc"]) def compile_helper(code, initial_args): - f, input_filename = tempfile.mkstemp(prefix='exodus-bundle-', suffix='.c') + f, input_filename = tempfile.mkstemp(prefix="exodus-bundle-", suffix=".c") os.close(f) - f, output_filename = tempfile.mkstemp(prefix='exodus-bundle-') + f, output_filename = tempfile.mkstemp(prefix="exodus-bundle-") os.close(f) try: - with open(input_filename, 'w') as input_file: + with open(input_filename, "w") as input_file: input_file.write(code) - args = initial_args + ['-static', '-O3', input_filename, '-o', output_filename] + args = initial_args + ["-static", "-O3", input_filename, "-o", output_filename] process = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert process.returncode == 0, \ - f'There was an error compiling: {stderr.decode("utf-8")}' + assert process.returncode == 0, f"There was an error compiling: {stderr.decode('utf-8')}" - with open(output_filename, 'rb') as output_file: + with open(output_filename, "rb") as output_file: return output_file.read() finally: os.remove(input_filename) @@ -89,24 +88,34 @@ def compile_helper(code, initial_args): def compile_musl(code): - musl = find_executable('musl-gcc') + musl = find_executable("musl-gcc") if musl is None: - raise CompilerNotFoundError('The musl compiler was not found.') + raise CompilerNotFoundError("The musl compiler was not found.") return compile_helper(code, [musl]) def construct_bash_launcher(linker, library_path, executable, full_linker=True): linker_dirname, linker_basename = os.path.split(linker) - full_linker = 'true' if full_linker else 'false' - return render_template_file('launcher.sh', linker_basename=linker_basename, - linker_dirname=linker_dirname, library_path=library_path, - executable=executable, full_linker=full_linker) + full_linker = "true" if full_linker else "false" + return render_template_file( + "launcher.sh", + linker_basename=linker_basename, + linker_dirname=linker_dirname, + library_path=library_path, + executable=executable, + full_linker=full_linker, + ) def construct_binary_launcher(linker, library_path, executable, full_linker=True): linker_dirname, linker_basename = os.path.split(linker) - full_linker = '1' if full_linker else '0' - code = render_template_file('launcher.c', linker_basename=linker_basename, - linker_dirname=linker_dirname, library_path=library_path, - executable=executable, full_linker=full_linker) + full_linker = "1" if full_linker else "0" + code = render_template_file( + "launcher.c", + linker_basename=linker_basename, + linker_dirname=linker_dirname, + library_path=library_path, + executable=executable, + full_linker=full_linker, + ) return compile(code) diff --git a/src/exodus_bundler/templating.py b/src/exodus_bundler/templating.py index ca470e5..59d330d 100644 --- a/src/exodus_bundler/templating.py +++ b/src/exodus_bundler/templating.py @@ -7,17 +7,17 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -template_directory = os.path.join(parent_directory, 'templates') +template_directory = os.path.join(parent_directory, "templates") def render_template(string, **context): for key, value in context.items(): - string = string.replace(f'{{{{{key}}}}}', value) + string = string.replace(f"{{{{{key}}}}}", value) return string def render_template_file(filename, **context): if not os.path.isabs(filename): filename = os.path.join(template_directory, filename) - with open(filename, 'r') as f: + with open(filename, "r") as f: return render_template(f.read(), **context) diff --git a/tests/test_bundling.py b/tests/test_bundling.py index 2df3528..b8139fa 100644 --- a/tests/test_bundling.py +++ b/tests/test_bundling.py @@ -20,55 +20,60 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -ldd_output_directory = os.path.join(parent_directory, 'data', 'ldd-output') -chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') -ldd = os.path.join(chroot, 'bin', 'ldd') -echo_args_glibc_32 = os.path.join(chroot, 'bin', 'echo-args-glibc-32') -echo_proc_self_exe_glibc_32 = os.path.join(chroot, 'bin', 'echo-proc-self-exe-glibc-32') -fizz_buzz_glibc_32 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32') -fizz_buzz_glibc_32_exe = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32-exe') -fizz_buzz_glibc_64 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-64') -fizz_buzz_musl_64 = os.path.join(chroot, 'bin', 'fizz-buzz-musl-64') - - -@pytest.mark.parametrize('path,expected_file_count', [ - (fizz_buzz_glibc_32, 3), - (fizz_buzz_glibc_64, 3), - (fizz_buzz_musl_64, 2), - (ldd, 1), -]) +ldd_output_directory = os.path.join(parent_directory, "data", "ldd-output") +chroot = os.path.join(parent_directory, "data", "binaries", "chroot") +ldd = os.path.join(chroot, "bin", "ldd") +echo_args_glibc_32 = os.path.join(chroot, "bin", "echo-args-glibc-32") +echo_proc_self_exe_glibc_32 = os.path.join(chroot, "bin", "echo-proc-self-exe-glibc-32") +fizz_buzz_glibc_32 = os.path.join(chroot, "bin", "fizz-buzz-glibc-32") +fizz_buzz_glibc_32_exe = os.path.join(chroot, "bin", "fizz-buzz-glibc-32-exe") +fizz_buzz_glibc_64 = os.path.join(chroot, "bin", "fizz-buzz-glibc-64") +fizz_buzz_musl_64 = os.path.join(chroot, "bin", "fizz-buzz-musl-64") + + +@pytest.mark.parametrize( + "path,expected_file_count", + [ + (fizz_buzz_glibc_32, 3), + (fizz_buzz_glibc_64, 3), + (fizz_buzz_musl_64, 2), + (ldd, 1), + ], +) def test_bundle_add_file(path, expected_file_count): bundle = Bundle(chroot=chroot) - assert len(bundle.files) == 0, 'The initial bundle should contain no files.' + assert len(bundle.files) == 0, "The initial bundle should contain no files." bundle.add_file(path) - assert len(bundle.files) == expected_file_count, \ - 'The bundle should include %d files.' % expected_file_count + assert len(bundle.files) == expected_file_count, ( + "The bundle should include %d files." % expected_file_count + ) def test_bundle_add_file_recursively(): bundle = Bundle(chroot=chroot) - assert len(bundle.files) == 0, 'The initial bundle should contain no files.' + assert len(bundle.files) == 0, "The initial bundle should contain no files." bundle.add_file(chroot) second_bundle = Bundle(chroot=chroot) for path in [ldd, fizz_buzz_glibc_32, fizz_buzz_glibc_32_exe, fizz_buzz_musl_64]: second_bundle.add_file(path) - assert second_bundle.files.issubset(bundle.files), \ - 'All of the executables and their dependencies should be in the first bundle.' + assert second_bundle.files.issubset(bundle.files), ( + "All of the executables and their dependencies should be in the first bundle." + ) def test_bundle_delete_working_directory(): bundle = Bundle() - assert bundle.working_directory is None, \ - 'A directory should only be created if passed `working_directory=True`.' + assert bundle.working_directory is None, ( + "A directory should only be created if passed `working_directory=True`." + ) bundle = Bundle(working_directory=True) working_directory = bundle.working_directory - assert os.path.exists(working_directory), \ - 'A working directory should have been created.' + assert os.path.exists(working_directory), "A working directory should have been created." bundle.delete_working_directory() - assert not os.path.exists(working_directory), \ - 'The working directory should have been deleted.' - assert bundle.working_directory is None, \ - 'The working directory should have been cleared after deletion.' + assert not os.path.exists(working_directory), "The working directory should have been deleted." + assert bundle.working_directory is None, ( + "The working directory should have been cleared after deletion." + ) def test_bundle_file_factory(): @@ -77,8 +82,7 @@ def test_bundle_file_factory(): # Note that `ldd` is a shell script, and should bring in no dependencies. [file] = bundle.files new_file = bundle.file_factory(ldd) - assert new_file is file, \ - 'The same file should be returned instead of making a new one.' + assert new_file is file, "The same file should be returned instead of making a new one." def test_bundle_hash(): @@ -87,40 +91,47 @@ def test_bundle_hash(): for filename in [fizz_buzz_glibc_32, fizz_buzz_glibc_64, fizz_buzz_musl_64]: bundle.add_file(filename) hashes.append(bundle.hash) - assert len(hashes) == len(set(hashes)), 'All of the hashes should be unique.' - assert all(len(hash) == 64 for hash in hashes), 'All of the hashes should have length 64.' + assert len(hashes) == len(set(hashes)), "All of the hashes should be unique." + assert all(len(hash) == 64 for hash in hashes), "All of the hashes should have length 64." def test_bundle_root(): try: bundle = Bundle(working_directory=True) - assert bundle.hash in bundle.bundle_root, 'Bundle path should include the hash.' - assert bundle.bundle_root.startswith(bundle.working_directory), \ - 'The bundle root should be a subdirectory of the working directory.' + assert bundle.hash in bundle.bundle_root, "Bundle path should include the hash." + assert bundle.bundle_root.startswith(bundle.working_directory), ( + "The bundle root should be a subdirectory of the working directory." + ) except Exception: raise finally: bundle.delete_working_directory() -@pytest.mark.parametrize('int,bytes,byteorder', [ - (1234567890, b'\xd2\x02\x96I\x00\x00\x00\x00', 'little'), - (1234567890, b'\x00\x00\x00\x00I\x96\x02\xd2', 'big'), - (9876543210, b'\xea\x16\xb0L\x02\x00\x00\x00', 'little'), - (9876543210, b'\x00\x00\x00\x02L\xb0\x16\xea', 'big'), -]) +@pytest.mark.parametrize( + "int,bytes,byteorder", + [ + (1234567890, b"\xd2\x02\x96I\x00\x00\x00\x00", "little"), + (1234567890, b"\x00\x00\x00\x00I\x96\x02\xd2", "big"), + (9876543210, b"\xea\x16\xb0L\x02\x00\x00\x00", "little"), + (9876543210, b"\x00\x00\x00\x02L\xb0\x16\xea", "big"), + ], +) def test_bytes_to_int(int, bytes, byteorder): - assert bytes_to_int(bytes, byteorder=byteorder) == int, 'Byte conversion should work.' - - -@pytest.mark.parametrize('fizz_buzz,shell_launchers', [ - (fizz_buzz_glibc_32, True), - (fizz_buzz_glibc_32, False), - (fizz_buzz_glibc_64, True), - (fizz_buzz_glibc_64, False), - (fizz_buzz_musl_64, True), - (fizz_buzz_musl_64, False), -]) + assert bytes_to_int(bytes, byteorder=byteorder) == int, "Byte conversion should work." + + +@pytest.mark.parametrize( + "fizz_buzz,shell_launchers", + [ + (fizz_buzz_glibc_32, True), + (fizz_buzz_glibc_32, False), + (fizz_buzz_glibc_64, True), + (fizz_buzz_glibc_64, False), + (fizz_buzz_musl_64, True), + (fizz_buzz_musl_64, False), + ], +) def test_create_unpackaged_bundle(fizz_buzz, shell_launchers): """This tests that the packaged executable runs as expected. At the very least, this tests that the symbolic links and launcher are functioning correctly. Unfortunately, @@ -128,197 +139,232 @@ def test_create_unpackaged_bundle(fizz_buzz, shell_launchers): present on the current system. FWIW, the CircleCI docker image being used is incompatible, so the continuous integration tests are more meaningful.""" root_directory = create_unpackaged_bundle( - rename=[], executables=[fizz_buzz], chroot=chroot, shell_launchers=shell_launchers) + rename=[], + executables=[fizz_buzz], + chroot=chroot, + shell_launchers=shell_launchers, + ) try: - binary_path = os.path.join(root_directory, 'bin', os.path.basename(fizz_buzz)) + binary_path = os.path.join(root_directory, "bin", os.path.basename(fizz_buzz)) process = Popen([binary_path], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert 'FIZZBUZZ' in stdout.decode('utf-8') - assert len(stderr.decode('utf-8')) == 0 + assert "FIZZBUZZ" in stdout.decode("utf-8") + assert len(stderr.decode("utf-8")) == 0 finally: - assert root_directory.startswith('/tmp/') + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) -@pytest.mark.parametrize('detect', [False, True]) +@pytest.mark.parametrize("detect", [False, True]) def test_create_unpackaged_bundle_detects_dependencies(detect): - binary_name = 'ls' - root_directory = create_unpackaged_bundle( - rename=[], executables=[binary_name], detect=detect) + binary_name = "ls" + root_directory = create_unpackaged_bundle(rename=[], executables=[binary_name], detect=detect) try: # Determine the bundle root. - binary_symlink = os.path.join(root_directory, 'bin', binary_name) + binary_symlink = os.path.join(root_directory, "bin", binary_name) binary_path = os.path.realpath(binary_symlink) dirname, basename = os.path.split(binary_path) while len(basename) != 64: dirname, basename = os.path.split(dirname) bundle_root = os.path.join(dirname, basename) - man_directory = os.path.join(bundle_root, 'usr', 'share', 'man') - assert os.path.exists(man_directory) == detect, \ - 'The man directory should only exist when `detect=True`.' + man_directory = os.path.join(bundle_root, "usr", "share", "man") + assert os.path.exists(man_directory) == detect, ( + "The man directory should only exist when `detect=True`." + ) finally: - assert root_directory.startswith('/tmp/') + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) def test_create_unpackaged_bundle_has_correct_args(): root_directory = create_unpackaged_bundle( - rename=[], executables=[echo_args_glibc_32], chroot=chroot) + rename=[], executables=[echo_args_glibc_32], chroot=chroot + ) try: - binary_path = os.path.join(root_directory, 'bin', os.path.basename(echo_args_glibc_32)) + binary_path = os.path.join(root_directory, "bin", os.path.basename(echo_args_glibc_32)) - process = Popen([binary_path, 'arg1', 'arg2'], stdout=PIPE, stderr=PIPE) + process = Popen([binary_path, "arg1", "arg2"], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert len(stderr.decode('utf-8')) == 0 - args = stdout.decode('utf-8').split('\n') - assert os.path.basename(args[0]) == '%s-x' % os.path.basename(echo_args_glibc_32), \ - 'The value of argv[0] should correspond to the local symlink.' - assert args[1] == 'arg1' and args[2] == 'arg2', \ - 'The other arguments should be passed through to the child process.' + assert len(stderr.decode("utf-8")) == 0 + args = stdout.decode("utf-8").split("\n") + assert os.path.basename(args[0]) == "%s-x" % os.path.basename(echo_args_glibc_32), ( + "The value of argv[0] should correspond to the local symlink." + ) + assert args[1] == "arg1" and args[2] == "arg2", ( + "The other arguments should be passed through to the child process." + ) finally: - assert root_directory.startswith('/tmp/') + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) def test_create_unpackaged_bundle_has_correct_proc_self_exe(): root_directory = create_unpackaged_bundle( - rename=[], executables=[echo_proc_self_exe_glibc_32], chroot=chroot) + rename=[], executables=[echo_proc_self_exe_glibc_32], chroot=chroot + ) try: - binary_path = os.path.join(root_directory, 'bin', - os.path.basename(echo_proc_self_exe_glibc_32)) + binary_path = os.path.join( + root_directory, "bin", os.path.basename(echo_proc_self_exe_glibc_32) + ) process = Popen([binary_path], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert len(stderr.decode('utf-8')) == 0 - proc_self_exe = stdout.decode('utf-8').strip() - assert os.path.basename(proc_self_exe).startswith('linker-'), \ - 'The linker should be the executing process.' + assert len(stderr.decode("utf-8")) == 0 + proc_self_exe = stdout.decode("utf-8").strip() + assert os.path.basename(proc_self_exe).startswith("linker-"), ( + "The linker should be the executing process." + ) relative_path = os.path.relpath(proc_self_exe, root_directory) - assert relative_path.startswith('bundles/'), \ - 'The process should be in the bundles directory.' + assert relative_path.startswith("bundles/"), ( + "The process should be in the bundles directory." + ) finally: - assert root_directory.startswith('/tmp/') + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) def test_detect_elf_binary(): - assert detect_elf_binary(fizz_buzz_glibc_32), 'The `fizz-buzz` file should be an ELF binary.' - assert not detect_elf_binary(ldd), 'The `ldd` file should be a shell script.' - - -@pytest.mark.parametrize('fizz_buzz,bits', [ - (fizz_buzz_glibc_32, 32), - (fizz_buzz_glibc_64, 64), - (fizz_buzz_musl_64, 64), -]) + assert detect_elf_binary(fizz_buzz_glibc_32), "The `fizz-buzz` file should be an ELF binary." + assert not detect_elf_binary(ldd), "The `ldd` file should be a shell script." + + +@pytest.mark.parametrize( + "fizz_buzz,bits", + [ + (fizz_buzz_glibc_32, 32), + (fizz_buzz_glibc_64, 64), + (fizz_buzz_musl_64, 64), + ], +) def test_elf_bits(fizz_buzz, bits): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) # Can be checked by running `file fizz-buzz`. - assert fizz_buzz_elf.bits == bits, \ - 'The fizz buzz executable should be %d-bit.' % bits + assert fizz_buzz_elf.bits == bits, "The fizz buzz executable should be %d-bit." % bits -@pytest.mark.parametrize('fizz_buzz', [ - (fizz_buzz_glibc_32), - (fizz_buzz_glibc_64), -]) +@pytest.mark.parametrize( + "fizz_buzz", + [ + (fizz_buzz_glibc_32), + (fizz_buzz_glibc_64), + ], +) def test_elf_dependencies(fizz_buzz): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) direct_dependencies = fizz_buzz_elf.direct_dependencies all_dependencies = fizz_buzz_elf.dependencies - assert set(direct_dependencies).issubset(all_dependencies), \ - 'The direct dependencies should be a subset of all dependencies.' - - -@pytest.mark.parametrize('fizz_buzz', [ - (fizz_buzz_glibc_32), - (fizz_buzz_glibc_64), - (fizz_buzz_musl_64), -]) + assert set(direct_dependencies).issubset(all_dependencies), ( + "The direct dependencies should be a subset of all dependencies." + ) + + +@pytest.mark.parametrize( + "fizz_buzz", + [ + (fizz_buzz_glibc_32), + (fizz_buzz_glibc_64), + (fizz_buzz_musl_64), + ], +) def test_elf_direct_dependencies(fizz_buzz): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) dependencies = fizz_buzz_elf.direct_dependencies - assert all(file.path.startswith(chroot) for file in dependencies), \ - 'All dependencies should be located within the chroot.' - assert len(dependencies), 'There should be at least one dependency.' + assert all(file.path.startswith(chroot) for file in dependencies), ( + "All dependencies should be located within the chroot." + ) + assert len(dependencies), "There should be at least one dependency." # These don't apply to the musl binary. - if 'glib' in fizz_buzz: - assert len(dependencies) == 2, 'The linker and libc should be the only dependencies.' - assert any('libc.so' in file.path for file in dependencies), \ + if "glib" in fizz_buzz: + assert len(dependencies) == 2, "The linker and libc should be the only dependencies." + assert any("libc.so" in file.path for file in dependencies), ( '"libc" was not found as a direct dependency of the executable.' + ) -@pytest.mark.parametrize('fizz_buzz,expected_linker_path', [ - (fizz_buzz_glibc_32, '/lib/ld-linux.so.2'), - (fizz_buzz_glibc_64, '/lib64/ld-linux-x86-64.so.2'), - (fizz_buzz_musl_64, '/lib/ld-musl-x86_64.so.1'), -]) +@pytest.mark.parametrize( + "fizz_buzz,expected_linker_path", + [ + (fizz_buzz_glibc_32, "/lib/ld-linux.so.2"), + (fizz_buzz_glibc_64, "/lib64/ld-linux-x86-64.so.2"), + (fizz_buzz_musl_64, "/lib/ld-musl-x86_64.so.1"), + ], +) def test_elf_linker(fizz_buzz, expected_linker_path): # Found by running `readelf -l fizz-buzz`. fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) - expected_linker_path = os.path.join(chroot, os.path.relpath(expected_linker_path, '/')) - assert fizz_buzz_elf.linker_file.path == expected_linker_path, \ - 'The correct linker should be extracted from the ELF program header.' - - -@pytest.mark.parametrize('fizz_buzz, expected_type', [ - (fizz_buzz_glibc_32, 'shared'), - (fizz_buzz_glibc_32_exe, 'executable'), - (fizz_buzz_glibc_64, 'shared'), -]) + expected_linker_path = os.path.join(chroot, os.path.relpath(expected_linker_path, "/")) + assert fizz_buzz_elf.linker_file.path == expected_linker_path, ( + "The correct linker should be extracted from the ELF program header." + ) + + +@pytest.mark.parametrize( + "fizz_buzz, expected_type", + [ + (fizz_buzz_glibc_32, "shared"), + (fizz_buzz_glibc_32_exe, "executable"), + (fizz_buzz_glibc_64, "shared"), + ], +) def test_elf_type(fizz_buzz, expected_type): elf = Elf(fizz_buzz, chroot=chroot) - assert elf.type == expected_type, 'Fizz buzz should match the expected ELF binary type.' + assert elf.type == expected_type, "Fizz buzz should match the expected ELF binary type." def test_file_destination(): - arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) + arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) arch_directory = os.path.dirname(arch_file.destination) fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) fizz_buzz_directory = os.path.dirname(fizz_buzz_file.destination) - assert arch_directory == fizz_buzz_directory, \ - 'Executable and non-executable files should be written to the same directory.' + assert arch_directory == fizz_buzz_directory, ( + "Executable and non-executable files should be written to the same directory." + ) def test_file_executable(): fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) - arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) - assert fizz_buzz_file.executable, 'The fizz buzz executable should be executable.' - assert not arch_file.executable, 'The arch text file should not be executable.' + arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) + assert fizz_buzz_file.executable, "The fizz buzz executable should be executable." + assert not arch_file.executable, "The arch text file should not be executable." def test_file_elf(): fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) - arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) - assert fizz_buzz_file.elf, 'The fizz buzz executable should be an ELF binary.' - assert not arch_file.elf, 'The arch text file should not be an ELF binary.' + arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) + assert fizz_buzz_file.elf, "The fizz buzz executable should be an ELF binary." + assert not arch_file.elf, "The arch text file should not be an ELF binary." def test_file_hash(): - amazon_file = File(os.path.join(ldd_output_directory, 'htop-amazon-linux.txt')) - arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) - assert amazon_file.hash != arch_file.hash, 'The hashes should differ.' - assert len(amazon_file.hash) == len(arch_file.hash) == 64, \ - 'The hashes should have a consistent length of 64 characters.' + amazon_file = File(os.path.join(ldd_output_directory, "htop-amazon-linux.txt")) + arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) + assert amazon_file.hash != arch_file.hash, "The hashes should differ." + assert len(amazon_file.hash) == len(arch_file.hash) == 64, ( + "The hashes should have a consistent length of 64 characters." + ) # Found by executing `sha256sum fizz-buzz`. - expected_hash = 'd54ab4714215d7822bf490df5cdf49bc3f32b4c85a439b109fc7581355f9d9c5' - assert File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash, 'Hashes should match.' - - -@pytest.mark.parametrize('fizz_buzz', [ - (fizz_buzz_glibc_32), - (fizz_buzz_glibc_64), - (fizz_buzz_musl_64), -]) + expected_hash = "d54ab4714215d7822bf490df5cdf49bc3f32b4c85a439b109fc7581355f9d9c5" + assert File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash, "Hashes should match." + + +@pytest.mark.parametrize( + "fizz_buzz", + [ + (fizz_buzz_glibc_32), + (fizz_buzz_glibc_64), + (fizz_buzz_musl_64), + ], +) def test_file_requires_launcher(fizz_buzz): file = File(fizz_buzz, chroot=chroot) - assert file.requires_launcher, 'Fizz buzz should require a launcher.' - assert all(not dependency.requires_launcher for dependency in file.elf.dependencies), \ - 'All of the dependencies should not require launchers.' + assert file.requires_launcher, "Fizz buzz should require a launcher." + assert all(not dependency.requires_launcher for dependency in file.elf.dependencies), ( + "All of the dependencies should not require launchers." + ) def test_file_symlink(): @@ -328,58 +374,65 @@ def test_file_symlink(): file = next(iter(bundle.files)) file.copy(bundle.working_directory) symlink = file.symlink(bundle.working_directory, bundle.bundle_root) - assert os.path.islink(symlink), 'A symlink should have been created.' - assert os.path.exists(symlink), 'The symlink should point to the actual file.' + assert os.path.islink(symlink), "A symlink should have been created." + assert os.path.exists(symlink), "The symlink should point to the actual file." except Exception: raise finally: bundle.delete_working_directory() -@pytest.mark.parametrize('filename_prefix', [ - 'htop-amazon-linux', - 'htop-arch', - 'htop-ubuntu-14.04', -]) +@pytest.mark.parametrize( + "filename_prefix", + [ + "htop-amazon-linux", + "htop-arch", + "htop-ubuntu-14.04", + ], +) def test_parse_dependencies_from_ldd_output(filename_prefix): - ldd_output_filename = filename_prefix + '.txt' + ldd_output_filename = filename_prefix + ".txt" with open(os.path.join(ldd_output_directory, ldd_output_filename)) as f: ldd_output = f.read() dependencies = parse_dependencies_from_ldd_output(ldd_output) - ldd_results_filename = filename_prefix + '-dependencies.txt' + ldd_results_filename = filename_prefix + "-dependencies.txt" with open(os.path.join(ldd_output_directory, ldd_results_filename)) as f: - expected_dependencies = [line for line in f.read().split('\n') if len(line)] + expected_dependencies = [line for line in f.read().split("\n") if len(line)] - assert set(dependencies) == set(expected_dependencies), \ + assert set(dependencies) == set(expected_dependencies), ( 'The dependencies were not parsed correctly from ldd output for "%s"' % filename_prefix + ) def test_resolve_binary(): binary_directory = os.path.dirname(fizz_buzz_glibc_32) binary = os.path.basename(fizz_buzz_glibc_32) - old_path = os.getenv('PATH', '') + old_path = os.getenv("PATH", "") try: - os.environ['PATH'] = '%s%s%s' % (binary_directory, os.pathsep, old_path) + os.environ["PATH"] = "%s%s%s" % (binary_directory, os.pathsep, old_path) resolved_binary = resolve_binary(binary) - assert resolved_binary == os.path.normpath(fizz_buzz_glibc_32), \ - 'The full binary path was not resolved correctly.' + assert resolved_binary == os.path.normpath(fizz_buzz_glibc_32), ( + "The full binary path was not resolved correctly." + ) finally: - os.environ['PATH'] = old_path + os.environ["PATH"] = old_path def test_resolve_file_path(): with pytest.raises(Exception): resolve_file_path(chroot) with pytest.raises(Exception): - resolve_file_path(os.path.join(chroot, 'non-existent-file')) - assert os.path.isabs(resolve_file_path(fizz_buzz_glibc_32)), \ - 'The resolved path should be absolute.' + resolve_file_path(os.path.join(chroot, "non-existent-file")) + assert os.path.isabs(resolve_file_path(fizz_buzz_glibc_32)), ( + "The resolved path should be absolute." + ) def test_run_ldd(): - assert any('libc.so' in line for line in run_ldd(ldd, fizz_buzz_glibc_32)), \ + assert any("libc.so" in line for line in run_ldd(ldd, fizz_buzz_glibc_32)), ( '"libc" was not found in the output of "ldd" for the executable.' + ) def test_stored_property(): @@ -394,4 +447,4 @@ def next(self): incrementer = Incrementer() for i in range(10): - assert incrementer.next == 1, '`Incrementer.next` should not change.' + assert incrementer.next == 1, "`Incrementer.next` should not change." diff --git a/tests/test_cli.py b/tests/test_cli.py index 9fc5ac6..ad7bd82 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,136 +13,137 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') -fizz_buzz_glibc_32 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32') -fizz_buzz_glibc_32_exe = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32-exe') -fizz_buzz_glibc_64 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-64') -fizz_buzz_musl_64 = os.path.join(chroot, 'bin', 'fizz-buzz-musl-64') +chroot = os.path.join(parent_directory, "data", "binaries", "chroot") +fizz_buzz_glibc_32 = os.path.join(chroot, "bin", "fizz-buzz-glibc-32") +fizz_buzz_glibc_32_exe = os.path.join(chroot, "bin", "fizz-buzz-glibc-32-exe") +fizz_buzz_glibc_64 = os.path.join(chroot, "bin", "fizz-buzz-glibc-64") +fizz_buzz_musl_64 = os.path.join(chroot, "bin", "fizz-buzz-musl-64") def run_exodus(args, **options): - options['universal_newlines'] = options.get('universal_newlines', True) + options["universal_newlines"] = options.get("universal_newlines", True) # Allow specifying content to pipe into stdin, with options['stdin'] - if 'stdin' in options: - input = options['stdin'].encode('utf-8') - options['stdin'] = subprocess.PIPE + if "stdin" in options: + input = options["stdin"].encode("utf-8") + options["stdin"] = subprocess.PIPE else: input = None process = subprocess.Popen( - ['exodus'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **options) + ["exodus"] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **options + ) stdout, stderr = process.communicate(input=input) return process.returncode, stdout, stderr def test_adding_additional_files(capsys): - args = ['--chroot', chroot, '--output', '-', '--tarball', fizz_buzz_glibc_32] - stdin = '\n'.join((fizz_buzz_glibc_32_exe, fizz_buzz_glibc_64)) + args = ["--chroot", chroot, "--output", "-", "--tarball", fizz_buzz_glibc_32] + stdin = "\n".join((fizz_buzz_glibc_32_exe, fizz_buzz_glibc_64)) returncode, stdout, stderr = run_exodus(args, universal_newlines=False, stdin=stdin) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." stream = io.BytesIO(stdout) - with tarfile.open(fileobj=stream, mode='r:gz') as f: + with tarfile.open(fileobj=stream, mode="r:gz") as f: names = f.getnames() - assert 'exodus/bin/fizz-buzz-glibc-32' in names, stderr + assert "exodus/bin/fizz-buzz-glibc-32" in names, stderr # These shouldn't be entrypoints, but should be included - assert 'exodus/bin/fizz-buzz-glibc-32-exe' not in names, stderr - assert 'exodus/bin/fizz-buzz-glibc-64' not in names, stderr + assert "exodus/bin/fizz-buzz-glibc-32-exe" not in names, stderr + assert "exodus/bin/fizz-buzz-glibc-64" not in names, stderr assert any(fizz_buzz_glibc_32_exe in name for name in names), stderr assert any(fizz_buzz_glibc_64 in name for name in names), stderr def test_logging_outputs(capsys): # There should be no output before configuring the logger. - logger.error('error') + logger.error("error") out, err = capsys.readouterr() print(out, err) assert len(out) == len(err) == 0 # The different levels should be routed separately to stdout/stderr. configure_logging(verbose=True, quiet=False) - logger.debug('debug') - logger.warning('warn') - logger.info('info') - logger.error('error') + logger.debug("debug") + logger.warning("warn") + logger.info("info") + logger.error("error") out, err = capsys.readouterr() - assert all(output in out for output in ('info')) - assert all(output not in out for output in ('debug', 'warn', 'error')) - assert all(output in err for output in ('warn', 'error')) - assert all(output not in err for output in ('info', 'debug')) + assert all(output in out for output in ("info")) + assert all(output not in out for output in ("debug", "warn", "error")) + assert all(output in err for output in ("warn", "error")) + assert all(output not in err for output in ("info", "debug")) def test_missing_binary(capsys): # Without the --verbose flag. - command = 'this-is-almost-definitely-not-going-to-be-a-command-anywhere' + command = "this-is-almost-definitely-not-going-to-be-a-command-anywhere" returncode, stdout, stderr = run_exodus([command]) - assert returncode != 0, 'Running exodus should have failed.' - assert 'Traceback' not in stderr, 'Traceback should not be included without the --verbose flag.' + assert returncode != 0, "Running exodus should have failed." + assert "Traceback" not in stderr, "Traceback should not be included without the --verbose flag." # With the --verbose flag. - returncode, stdout, stderr = run_exodus(['--verbose', command]) - assert returncode != 0, 'Running exodus should have failed.' - assert 'Traceback' in stderr, 'Traceback should be included with the --verbose flag.' + returncode, stdout, stderr = run_exodus(["--verbose", command]) + assert returncode != 0, "Running exodus should have failed." + assert "Traceback" in stderr, "Traceback should be included with the --verbose flag." def test_required_argument(): with pytest.raises(SystemExit): parse_args([]) - parse_args(['/bin/bash']) + parse_args(["/bin/bash"]) def test_return_type_is_dict(): - assert type(parse_args(['/bin/bash'])) == dict + assert isinstance(parse_args(["/bin/bash"]), dict) def test_quiet_and_verbose_flags(): - result = parse_args(['--quiet', '/bin/bash']) - assert result['quiet'] and not result['verbose'] - result = parse_args(['--verbose', '/bin/bash']) - assert result['verbose'] and not result['quiet'] + result = parse_args(["--quiet", "/bin/bash"]) + assert result["quiet"] and not result["verbose"] + result = parse_args(["--verbose", "/bin/bash"]) + assert result["verbose"] and not result["quiet"] def test_writing_bundle_to_disk(): - f, filename = tempfile.mkstemp(suffix='.sh') + f, filename = tempfile.mkstemp(suffix=".sh") os.close(f) - args = ['--chroot', chroot, '--output', filename, fizz_buzz_glibc_32] + args = ["--chroot", chroot, "--output", filename, fizz_buzz_glibc_32] try: returncode, stdout, stderr = run_exodus(args) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." - with open(filename, 'rb') as f_in: + with open(filename, "rb") as f_in: first_line = f_in.readline().strip() - assert first_line == b'#! /bin/bash', stderr + assert first_line == b"#! /bin/bash", stderr finally: if os.path.exists(filename): os.unlink(filename) def test_writing_bundle_to_stdout(): - args = ['--chroot', chroot, '--output', '-', fizz_buzz_glibc_32] + args = ["--chroot", chroot, "--output", "-", fizz_buzz_glibc_32] returncode, stdout, stderr = run_exodus(args) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." - assert stdout.startswith('#! /bin/sh'), stderr + assert stdout.startswith("#! /bin/sh"), stderr def test_writing_tarball_to_disk(): - f, filename = tempfile.mkstemp(suffix='.tgz') + f, filename = tempfile.mkstemp(suffix=".tgz") os.close(f) - args = ['--chroot', chroot, '--output', filename, '--tarball', fizz_buzz_glibc_32] + args = ["--chroot", chroot, "--output", filename, "--tarball", fizz_buzz_glibc_32] try: returncode, stdout, stderr = run_exodus(args) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." assert tarfile.is_tarfile(filename), stderr - with tarfile.open(filename, mode='r:gz') as f_in: - assert 'exodus/bin/fizz-buzz-glibc-32' in f_in.getnames() + with tarfile.open(filename, mode="r:gz") as f_in: + assert "exodus/bin/fizz-buzz-glibc-32" in f_in.getnames() finally: if os.path.exists(filename): os.unlink(filename) def test_writing_tarball_to_stdout(): - args = ['--chroot', chroot, '--output', '-', '--tarball', fizz_buzz_glibc_32] + args = ["--chroot", chroot, "--output", "-", "--tarball", fizz_buzz_glibc_32] returncode, stdout, stderr = run_exodus(args, universal_newlines=False) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." stream = io.BytesIO(stdout) - with tarfile.open(fileobj=stream, mode='r:gz') as f: - assert 'exodus/bin/fizz-buzz-glibc-32' in f.getnames(), stderr + with tarfile.open(fileobj=stream, mode="r:gz") as f: + assert "exodus/bin/fizz-buzz-glibc-32" in f.getnames(), stderr diff --git a/tests/test_dependency_detection.py b/tests/test_dependency_detection.py index ee11e9f..995e4bd 100644 --- a/tests/test_dependency_detection.py +++ b/tests/test_dependency_detection.py @@ -6,11 +6,12 @@ def test_detect_dependencies(): # This is a little janky, but the test suite won't run anywhere where it's not true. - ls = '/usr/bin/ls' + ls = "/usr/bin/ls" if not os.path.exists(ls): - ls = '/bin/ls' - assert os.path.exists(ls), 'This test assumes that `ls` is installed on the system.' + ls = "/bin/ls" + assert os.path.exists(ls), "This test assumes that `ls` is installed on the system." dependencies = detect_dependencies(ls) - assert any(ls in dependency for dependency in dependencies), \ - '`%s` should have been detected as a dependency for `ls`.' % ls + assert any(ls in dependency for dependency in dependencies), ( + "`%s` should have been detected as a dependency for `ls`." % ls + ) diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index 8eb06bb..e450ac9 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -9,91 +9,90 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -strace_output_directory = os.path.join(parent_directory, 'data', 'strace-output') -exodus_strace = os.path.join(strace_output_directory, 'exodus-output.txt') +strace_output_directory = os.path.join(parent_directory, "data", "strace-output") +exodus_strace = os.path.join(strace_output_directory, "exodus-output.txt") def test_extract_exec_path(): line = 'execve("/usr/bin/ls", ["ls"], 0x7ffea775ad70 /* 113 vars */) = 0' - assert extract_exec_path(line) == '/usr/bin/ls', \ - 'It should have extracted the path to the ls executable.' - assert extract_exec_path('blah') is None, \ - 'It should return `None` when there is no match.' + assert extract_exec_path(line) == "/usr/bin/ls", ( + "It should have extracted the path to the ls executable." + ) + assert extract_exec_path("blah") is None, "It should return `None` when there is no match." def test_extract_no_paths(): - input_paths = extract_paths('') - assert input_paths == [], 'It should return an empty list.' + input_paths = extract_paths("") + assert input_paths == [], "It should return an empty list." def test_extract_open_path(): line = ( 'openat(AT_FDCWD, "/usr/lib/root/tls/x86_64/libcap.so.2", O_RDONLY|O_CLOEXEC) ' - '= -1 ENOENT (No such file or directory)' + "= -1 ENOENT (No such file or directory)" ) - assert extract_open_path(line) is None, 'Missing files should not return paths.' + assert extract_open_path(line) is None, "Missing files should not return paths." line = 'open(".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4' - assert extract_open_path(line) is None, 'Opened directories should not return paths.' + assert extract_open_path(line) is None, "Opened directories should not return paths." line = 'open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 4' - assert extract_open_path(line) == '/usr/lib/locale/locale-archive', \ - 'An open() call should return a path.' + assert extract_open_path(line) == "/usr/lib/locale/locale-archive", ( + "An open() call should return a path." + ) line = 'openat(AT_FDCWD, "/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 4' - assert extract_open_path(line) == '/usr/lib/libc.so.6', \ - 'An openat() call relative to the current directory should return a path.' + assert extract_open_path(line) == "/usr/lib/libc.so.6", ( + "An openat() call relative to the current directory should return a path." + ) def test_extract_raw_paths(): input_paths = [ - '/absolute/path/to/file', - './relative/path', - '/another/absolute/path', + "/absolute/path/to/file", + "./relative/path", + "/another/absolute/path", ] - input_paths_with_whitespace = \ - [' ', ''] + [input_paths[0]] + [' '] + input_paths[1:] - input_content = '\n'.join(input_paths_with_whitespace) + input_paths_with_whitespace = [" ", ""] + [input_paths[0]] + [" "] + input_paths[1:] + input_content = "\n".join(input_paths_with_whitespace) extracted_paths = extract_paths(input_content) - assert set(input_paths) == set(extracted_paths), \ - 'The paths should have been extracted without the whitespace.' + assert set(input_paths) == set(extracted_paths), ( + "The paths should have been extracted without the whitespace." + ) def test_extract_stat_path(): line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.py", ' - '{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0' + "{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0" ) - expected_path = '/usr/local/lib/python3.6/encodings/__init__.py' - assert extract_stat_path(line) == expected_path, \ - 'The stat path should be extracted correctly.' + expected_path = "/usr/local/lib/python3.6/encodings/__init__.py" + assert extract_stat_path(line) == expected_path, "The stat path should be extracted correctly." line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.abi3.so", 0x7ffc9d6a0160) = -1 ' - 'ENOENT (No such file or directory)' + "ENOENT (No such file or directory)" ) - assert extract_stat_path(line) is None, \ - 'Non-existent files should not be extracted.' + assert extract_stat_path(line) is None, "Non-existent files should not be extracted." def test_extract_strace_paths(): - with open(exodus_strace, 'r') as f: + with open(exodus_strace, "r") as f: content = f.read() extracted_paths = extract_paths(content, existing_only=False) expected_paths = [ # `execve()` call - '/home/sangaline/projects/exodus/.env/bin/exodus', + "/home/sangaline/projects/exodus/.env/bin/exodus", # `openat()` call - '/usr/lib/libpthread.so.0', + "/usr/lib/libpthread.so.0", # `open()` call - '/usr/lib/gconv/gconv-modules', + "/usr/lib/gconv/gconv-modules", ] for path in expected_paths: - assert path in extracted_paths, \ - '"%s" should be present in the extracted paths.' % path + assert path in extracted_paths, '"%s" should be present in the extracted paths.' % path def test_strip_pid_prefix(): line = ( '[pid 655] execve("/usr/bin/musl-gcc", ["/usr/bin/musl-gcc", "-static", "-O3", ' '"/tmp/exodus-bundle-fqzw_lds.c", "-o", "/tmp/exodus-bundle-3p_c0osh"], [/* 45 vars */] ' - '' + "" ) - assert strip_pid_prefix(line).startswith('execve('), 'The PID prefix should be stripped.' + assert strip_pid_prefix(line).startswith("execve("), "The PID prefix should be stripped." diff --git a/tests/test_launchers.py b/tests/test_launchers.py index 9d1d0c2..9a62e21 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -18,26 +18,27 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') -echo_args_glibc_32 = os.path.join(chroot, 'bin', 'echo-args-glibc-32') -fizz_buzz_source_file = os.path.join(parent_directory, 'data', 'binaries', 'fizz-buzz.c') +chroot = os.path.join(parent_directory, "data", "binaries", "chroot") +echo_args_glibc_32 = os.path.join(chroot, "bin", "echo-args-glibc-32") +fizz_buzz_source_file = os.path.join(parent_directory, "data", "binaries", "fizz-buzz.c") def test_construct_bash_launcher(): - linker, library_path, executable = '../lib/ld-linux.so.2', '../lib/', 'grep' - script_content = construct_bash_launcher(linker=linker, library_path=library_path, - executable=executable) - assert script_content.startswith('#! /bin/bash\n') + linker, library_path, executable = "../lib/ld-linux.so.2", "../lib/", "grep" + script_content = construct_bash_launcher( + linker=linker, library_path=library_path, executable=executable + ) + assert script_content.startswith("#! /bin/bash\n") assert linker in script_content assert executable in script_content -@pytest.mark.parametrize('compiler', ['diet', 'musl']) +@pytest.mark.parametrize("compiler", ["diet", "musl"]) def test_compile(compiler): - with open(fizz_buzz_source_file, 'r') as f: + with open(fizz_buzz_source_file, "r") as f: code = f.read() - compile = compile_diet if compiler == 'diet' else None - compile = compile or (compile_musl if compiler == 'musl' else None) + compile = compile_diet if compiler == "diet" else None + compile = compile or (compile_musl if compiler == "musl" else None) try: content = compile(code) except CompilerNotFoundError: @@ -47,34 +48,36 @@ def test_compile(compiler): f, filename = tempfile.mkstemp() os.close(f) - with open(filename, 'wb') as f: + with open(filename, "wb") as f: f.write(content) st = os.stat(filename) os.chmod(f.name, st.st_mode | stat.S_IXUSR) process = Popen(f.name, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert 'FIZZBUZZ' in stdout.decode('utf-8') - assert len(stderr.decode('utf-8')) == 0 + assert "FIZZBUZZ" in stdout.decode("utf-8") + assert len(stderr.decode("utf-8")) == 0 def test_find_executable(): - original_environment = os.environ.get('PATH') + original_environment = os.environ.get("PATH") original_parent_directory = launchers.parent_directory root_directory = create_unpackaged_bundle( - rename=[], executables=[echo_args_glibc_32], chroot=chroot) + rename=[], executables=[echo_args_glibc_32], chroot=chroot + ) try: binary_name = os.path.basename(echo_args_glibc_32) - binary_symlink = os.path.join(root_directory, 'bin', binary_name) + binary_symlink = os.path.join(root_directory, "bin", binary_name) binary_path = os.path.realpath(binary_symlink) # This is a pretend directory, but it doesn't check. - launchers.parent_directory = os.path.join(os.path.dirname(binary_path), 'somewhere', 'else') - os.environ['PATH'] = os.path.dirname(echo_args_glibc_32) - assert find_executable(binary_name, skip_original_for_testing=True) == binary_path, \ + launchers.parent_directory = os.path.join(os.path.dirname(binary_path), "somewhere", "else") + os.environ["PATH"] = os.path.dirname(echo_args_glibc_32) + assert find_executable(binary_name, skip_original_for_testing=True) == binary_path, ( 'It should have found the binary path "%s".' % binary_path + ) finally: launchers.parent_directory = original_parent_directory - os.environ['PATH'] = original_environment - assert root_directory.startswith('/tmp/') + os.environ["PATH"] = original_environment + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) diff --git a/tests/test_pytest.py b/tests/test_pytest.py index 55c5967..547f985 100644 --- a/tests/test_pytest.py +++ b/tests/test_pytest.py @@ -5,4 +5,4 @@ def test_catching_a_value_error(): """Temporary test to make sure that we can run tests.""" with pytest.raises(KeyError): - {}['no-matching-key'] + {}["no-matching-key"] diff --git a/tests/test_templating.py b/tests/test_templating.py index 4a7d565..d4b0228 100644 --- a/tests/test_templating.py +++ b/tests/test_templating.py @@ -6,19 +6,19 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -data_directory = os.path.join(parent_directory, 'data') +data_directory = os.path.join(parent_directory, "data") def test_render_template(): - template = '{{greeting}}, my name is {{name}}.' - expected = 'Hello, my name is Evan.' - result = render_template(template, greeting='Hello', name='Evan') + template = "{{greeting}}, my name is {{name}}." + expected = "Hello, my name is Evan." + result = render_template(template, greeting="Hello", name="Evan") assert expected == result def test_render_template_file(): - template_file = os.path.join(data_directory, 'template.txt') - result = render_template_file(template_file, noun='word', location='here') - with open(os.path.join(data_directory, 'template-result.txt'), 'r') as f: + template_file = os.path.join(data_directory, "template.txt") + result = render_template_file(template_file, noun="word", location="here") + with open(os.path.join(data_directory, "template-result.txt"), "r") as f: expected_result = f.read() assert result == expected_result From b9e0c5459be7ed31418c083593024f69880a4d35 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 22:26:32 -0600 Subject: [PATCH 05/15] removing linting with flake8 --- .github/workflows/test.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 305acde..68fb501 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,11 +43,6 @@ jobs: pip install -r development-requirements.txt pip install -e . - - name: Run linting - run: | - flake8 src tests setup.py - isort --verbose --check-only --diff src tests setup.py - - name: Run tests with pytest run: | pytest --cov --cov-report=term-missing --cov-report=xml -v From 081b62e3040cac71e2ba9477b237bb6bed5fc979 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 22:32:17 -0600 Subject: [PATCH 06/15] Fixing file path when file has .sh --- src/exodus_bundler/bundling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index a4f2a9a..d38212a 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -669,7 +669,7 @@ def create_launcher( f.write(launcher_content) shutil.copymode(self.path, tt) - return os.path.normpath(os.path.abspath(source_path)) + return os.path.normpath(os.path.abspath(tt)) def symlink(self, working_directory, bundle_root): """Creates a relative symlink from the `source` to the `destination`. From f5f44bd36c8b8f296cb00c40dac0847b1c5dee64 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 22:36:46 -0600 Subject: [PATCH 07/15] Updating bundling and entry point paths --- src/exodus_bundler/bundling.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index d38212a..53ac50e 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -541,7 +541,7 @@ def copy(self, working_directory): return full_destination - def create_entry_point(self, working_directory, bundle_root): + def create_entry_point(self, working_directory, bundle_root, launcher_path=None): """Creates a symlink in `bin/` to the executable or its launcher. Note: @@ -549,13 +549,17 @@ def create_entry_point(self, working_directory, bundle_root): Args: working_directory (str): The root that the `destination` will be joined with. bundle_root (str): The root that `source` will be joined with. + launcher_path (str, optional): The actual path to the launcher if one was created. """ - source_path = os.path.join(bundle_root, self.source) + if launcher_path: + source_path = launcher_path + else: + source_path = os.path.join(bundle_root, self.source) bin_directory = os.path.join(working_directory, "bin") if not os.path.exists(bin_directory): os.makedirs(bin_directory) entry_point_path = os.path.join(bin_directory, self.entry_point) - relative_destination_path = os.path.relpath(source_path, bin_directory) + ".sh" + relative_destination_path = os.path.relpath(source_path, bin_directory) os.symlink(relative_destination_path, entry_point_path) def create_launcher( @@ -838,14 +842,13 @@ def create_bundle(self, shell_launchers=False): """ file_paths = set() files_needing_launchers = defaultdict(set) + entry_points_needing_launchers = {} # Map file to launcher path for file in self.files: # Store the file path to avoid collisions later. file_path = os.path.join(self.bundle_root, file.source) file_paths.add(file_path) - # Create a symlink in `./bin/` if an entry point is specified. - if file.entry_point: - file.create_entry_point(self.working_directory, self.bundle_root) + # Defer entry point creation until after launchers are created if file.no_symlink: # We'll need to copy the actual file into the bundle subdirectory in this @@ -872,6 +875,9 @@ def create_bundle(self, shell_launchers=False): working_directory=self.working_directory, bundle_root=self.bundle_root, ) + # Create entry point for non-launcher files + if file.entry_point: + file.create_entry_point(self.working_directory, self.bundle_root) # Now we need to write out one unique copy of each linker in each directory where it's # required. This is necessary so that `readlink("/proc/self/exe")` will return the correct @@ -903,13 +909,20 @@ def create_bundle(self, shell_launchers=False): # iteration += 1 file_paths.add(symlink_path) symlink_basename = os.path.basename(symlink_path) - file.create_launcher( + launcher_path = file.create_launcher( self.working_directory, self.bundle_root, linker_basename, symlink_basename, shell_launcher=shell_launchers, ) + # Store launcher path for entry point creation + if file.entry_point: + entry_points_needing_launchers[file] = launcher_path + + # Create entry points for files that needed launchers + for file, launcher_path in entry_points_needing_launchers.items(): + file.create_entry_point(self.working_directory, self.bundle_root, launcher_path) def delete_working_directory(self): """Recursively deletes the working directory.""" From c1696b4a8598581906f6f5532676b677e28b4e63 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 22:42:03 -0600 Subject: [PATCH 08/15] remove flake8 linting --- pyproject.toml | 5 ++++- tox.ini | 9 +++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2354adf..12938bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,10 @@ target-version = "py310" [tool.ruff.lint] select = ["E", "F", "W"] -ignore = ["E501"] +ignore = ["E501"] # Ignore line length for now + +[tool.ruff.format] +line-ending = "lf" [tool.pytest.ini_options] norecursedirs = [".git", ".tox", ".env", "dist", "build"] python_files = ["test_*.py", "*_test.py", "tests.py"] diff --git a/tox.ini b/tox.ini index 6f306c9..7f95340 100644 --- a/tox.ini +++ b/tox.ini @@ -29,16 +29,13 @@ commands = deps = docutils>=0.20 check-manifest>=0.49 - flake8>=7.0 - flake8-commas>=4.0 - flake8-quotes>=3.4 + ruff>=0.4.0 readme-renderer>=43.0 pygments>=2.17 - isort>=5.13 skip_install = true commands = - flake8 src tests setup.py - isort --verbose --check-only --diff --recursive src tests setup.py + ruff check src tests setup.py + ruff format --check src tests setup.py python setup.py check --strict --metadata --restructuredtext check-manifest {toxinidir} From 6c72ae10d766b7be18bf89538a549f9349cba29e Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 22:45:16 -0600 Subject: [PATCH 09/15] Fixed formatting --- src/exodus_bundler/bundling.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 53ac50e..dabf464 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -130,9 +130,9 @@ def create_unpackaged_bundle( try: # Sanitize the inputs. assert len(executables), "No executables were specified." - assert len(executables) >= len( - rename - ), "More renamed options were included than executables." + assert len(executables) >= len(rename), ( + "More renamed options were included than executables." + ) # Pad the rename's with `True` so that `entry_point` can be specified. entry_points = rename + [True for i in range(len(executables) - len(rename))] @@ -946,15 +946,15 @@ def file_factory(self, path, entry_point=None, chroot=None, library=False, file_ path = resolve_file_path(path, search_environment_path=entry_point is not None) file = next((file for file in self.files if file.path == path), None) if file is not None: - assert ( - entry_point == file.entry_point or not entry_point or not file.entry_point - ), "The entry point property should always persist, but can't conflict." + assert entry_point == file.entry_point or not entry_point or not file.entry_point, ( + "The entry point property should always persist, but can't conflict." + ) file.entry_point = file.entry_point or entry_point assert chroot == file.chroot, "The chroot must match." file.library = file.library or library - assert ( - not file.entry_point or not file.library - ), "A file can't be both an entry point and a library." + assert not file.entry_point or not file.library, ( + "A file can't be both an entry point and a library." + ) return file return File(path, entry_point, chroot, library, file_factory) From afc8354ae479ec9a82a5ebc556ade7e357b2ad60 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 22:50:12 -0600 Subject: [PATCH 10/15] removing RST check for readme --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7f95340..630ee6c 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ skip_install = true commands = ruff check src tests setup.py ruff format --check src tests setup.py - python setup.py check --strict --metadata --restructuredtext + python setup.py check --strict --metadata check-manifest {toxinidir} [testenv:report] From 57f795bf07fc6f5f0bf6eb4260a44dab09722fc0 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Mon, 11 Aug 2025 22:54:01 -0600 Subject: [PATCH 11/15] Specify readme in MD --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 12938bc..94c23a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "exodus-bundler" version = "3.0.0" description = "The exodus application bundler." -readme = "README.md" +readme = {file = "README.md", content-type = "text/markdown"} license = {text = "BSD"} authors = [ {name = "Intoli", email = "contact@intoli.com"} From b7ce273cd8bfb572144610b7dbdfbe42d5211c2d Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Wed, 26 Nov 2025 13:29:25 -0700 Subject: [PATCH 12/15] Removing the -x from the bundle path --- src/exodus_bundler/bundling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index dabf464..d87135c 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -901,7 +901,7 @@ def create_bundle(self, shell_launchers=False): # We'll again attempt to find a unique available name, this time for the symlink # to the executable. file_basename = file.entry_point or os.path.basename(file.path) - desired_symlink_path = os.path.join(directory, f"{file_basename}-x") + desired_symlink_path = os.path.join(directory, file_basename) symlink_path = desired_symlink_path # iteration = 2 # while symlink_path in file_paths: From 19942fa9d552806dda597c7ce4db0375d908c3ba Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Wed, 26 Nov 2025 13:33:20 -0700 Subject: [PATCH 13/15] Updated formatting --- src/exodus_bundler/bundling.py | 108 +++++++++++++++------ src/exodus_bundler/cli.py | 4 +- src/exodus_bundler/dependency_detection.py | 8 +- src/exodus_bundler/input_parsing.py | 16 ++- src/exodus_bundler/launchers.py | 8 +- 5 files changed, 105 insertions(+), 39 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index d87135c..a0ab39a 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -70,7 +70,9 @@ def create_bundle( # Populate the filename template. output_filename = render_template( output, - executables=("-".join(os.path.basename(executable) for executable in executables)), + executables=( + "-".join(os.path.basename(executable) for executable in executables) + ), extension=("tgz" if tarball else "sh"), ) @@ -88,14 +90,18 @@ def create_bundle( # Construct the installation script and write it out. if not tarball: if output_filename == "-": - base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode("utf-8") + base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode( + "utf-8" + ) script_content = render_template_file( "install-bundle-noninteractive.sh", base64_encoded_tarball=base64_encoded_tarball, ) output_file.write(script_content.encode("utf-8")) else: - output_file.write(render_template_file("install-bundle.sh").encode("utf-8")) + output_file.write( + render_template_file("install-bundle.sh").encode("utf-8") + ) output_file.write(tar_stream.getvalue()) else: # Or just write out the tarball. @@ -130,9 +136,9 @@ def create_unpackaged_bundle( try: # Sanitize the inputs. assert len(executables), "No executables were specified." - assert len(executables) >= len(rename), ( - "More renamed options were included than executables." - ) + assert len(executables) >= len( + rename + ), "More renamed options were included than executables." # Pad the rename's with `True` so that `entry_point` can be specified. entry_points = rename + [True for i in range(len(executables) - len(rename))] @@ -147,7 +153,8 @@ def create_unpackaged_bundle( if not dependency_paths: raise DependencyDetectionError( ( - 'Automatic dependency detection failed. Either "%s" ' % file.path + 'Automatic dependency detection failed. Either "%s" ' + % file.path + "is not tracked by your package manager, or your operating system " + "is not currently compatible with the `--detect` option. If not, please " + "create an issue at https://github.com/intoli/exodus and we'll try our " @@ -213,11 +220,15 @@ def resolve_binary(binary): absolute_binary_path = os.path.normpath(os.path.abspath(binary)) if not os.path.exists(absolute_binary_path): for path in os.getenv("PATH", "/bin/:/usr/bin/").split(os.pathsep): - absolute_binary_path = os.path.normpath(os.path.abspath(os.path.join(path, binary))) + absolute_binary_path = os.path.normpath( + os.path.abspath(os.path.join(path, binary)) + ) if os.path.exists(absolute_binary_path): break else: - raise MissingFileError(f'The "{binary}" binary could not be found in $PATH.') + raise MissingFileError( + f'The "{binary}" binary could not be found in $PATH.' + ) return absolute_binary_path @@ -293,7 +304,9 @@ def __init__(self, path, chroot=None, file_factory=None): # Make sure that this is actually an ELF binary. first_four_bytes = f.read(4) if first_four_bytes != b"\x7fELF": - raise InvalidElfBinaryError(f'The "{path}" file is not a binary ELF file.') + raise InvalidElfBinaryError( + f'The "{path}" file is not a binary ELF file.' + ) # Determine whether this is a 32-bit or 64-bit file. format_byte = f.read(1) @@ -301,7 +314,8 @@ def __init__(self, path, chroot=None, file_factory=None): if not self.bits: raise UnsupportedArchitectureError( ( - 'The "%s" file does not appear to be either 32 or 64 bits. ' % path + 'The "%s" file does not appear to be either 32 or 64 bits. ' + % path + "Other architectures are not currently supported, but you can open an " + "issue at https://github.com/intoli/exodus stating your use-case and " + "support might get extended in the future." @@ -328,7 +342,9 @@ def hex(bytes): # Determine the type of the binary. f.seek(hex(b"\x10")) e_type = hex(f.read(2)) - self.type = {1: "relocatable", 2: "executable", 3: "shared", 4: "core"}[e_type] + self.type = {1: "relocatable", 2: "executable", 3: "shared", 4: "core"}[ + e_type + ] # Find the program header offset. e_phoff_start = {32: hex(b"\x1c"), 64: hex(b"\x20")}[self.bits] @@ -359,13 +375,17 @@ def hex(bytes): continue # Determine the offset for the segment. - p_offset_start = header_start + {32: hex(b"\04"), 64: hex(b"\x08")}[self.bits] + p_offset_start = ( + header_start + {32: hex(b"\04"), 64: hex(b"\x08")}[self.bits] + ) p_offset_length = {32: 4, 64: 8}[self.bits] f.seek(p_offset_start) p_offset = hex(f.read(p_offset_length)) # Determine the size of the segment. - p_filesz_start = header_start + {32: hex(b"\x10"), 64: hex(b"\x20")}[self.bits] + p_filesz_start = ( + header_start + {32: hex(b"\x10"), 64: hex(b"\x20")}[self.bits] + ) p_filesz_length = {32: 4, 64: 8}[self.bits] f.seek(p_filesz_start) p_filesz = hex(f.read(p_filesz_length)) @@ -374,11 +394,16 @@ def hex(bytes): f.seek(p_offset) segment = f.read(p_filesz) # It should be null-terminated (b'\x00' in Python 2, 0 in Python 3). - assert segment[-1] in [b"\x00", 0], "The string should be null terminated." + assert segment[-1] in [ + b"\x00", + 0, + ], "The string should be null terminated." assert self.linker_file is None, "More than one linker found." linker_path = segment[:-1].decode("ascii") if chroot: - linker_path = os.path.join(chroot, os.path.relpath(linker_path, "/")) + linker_path = os.path.join( + chroot, os.path.relpath(linker_path, "/") + ) self.linker_file = self.file_factory(linker_path, chroot=self.chroot) def __eq__(self, other): @@ -407,7 +432,9 @@ def find_direct_dependencies(self, linker_file=None): directories = [] for directory in ld_library_path.split(":"): if os.path.isabs(directory): - directory = os.path.join(self.chroot, os.path.relpath(directory, "/")) + directory = os.path.join( + self.chroot, os.path.relpath(directory, "/") + ) directories.append(directory) ld_library_path = ":".join(directories) environment["LD_LIBRARY_PATH"] = ld_library_path @@ -422,13 +449,16 @@ def find_direct_dependencies(self, linker_file=None): env=environment, ) stdout, stderr = process.communicate() - combined_output = stdout.decode("utf-8").split("\n") + stderr.decode("utf-8").split("\n") + combined_output = stdout.decode("utf-8").split("\n") + stderr.decode( + "utf-8" + ).split("\n") # Note that we're explicitly adding the linker because when we invoke it as `ldd` we can't # extract the real path from the trace output. Even if it were here twice, it would be # deduplicated though the use of a set. filenames = parse_dependencies_from_ldd_output(combined_output) + [linker_path] return set( - self.file_factory(filename, chroot=self.chroot, library=True) for filename in filenames + self.file_factory(filename, chroot=self.chroot, library=True) + for filename in filenames ) @stored_property @@ -470,7 +500,9 @@ class File: path (str): The absolute normalized path to the file on disk. """ - def __init__(self, path, entry_point=None, chroot=None, library=False, file_factory=None): + def __init__( + self, path, entry_point=None, chroot=None, library=False, file_factory=None + ): """Constructor for the `File` class. Note: @@ -485,7 +517,9 @@ def __init__(self, path, entry_point=None, chroot=None, library=False, file_fact file_factory (function, optional): A function to use when creating new `File` instances. """ # Find the full path to the file. - self.path = resolve_file_path(path, search_environment_path=(entry_point is not None)) + self.path = resolve_file_path( + path, search_environment_path=(entry_point is not None) + ) # Set the entry point for the file. if entry_point is True: @@ -602,7 +636,8 @@ def create_launcher( shutil.copy(self.elf.linker_file.path, linker_path) else: assert filecmp.cmp(self.elf.linker_file.path, linker_path), ( - 'The "%s" linker file already exists and has differing contents.' % linker_path + 'The "%s" linker file already exists and has differing contents.' + % linker_path ) linker = os.path.join(".", linker_basename) @@ -726,7 +761,12 @@ def requires_launcher(self): # as shared libraries, and many mostly-libraries are executable (*e.g.* glibc). # The easy ones. - if self.library or not self.elf or not self.elf.linker_file or not self.executable: + if ( + self.library + or not self.elf + or not self.elf.linker_file + or not self.executable + ): return False if self.elf.type == "executable": return True @@ -922,14 +962,18 @@ def create_bundle(self, shell_launchers=False): # Create entry points for files that needed launchers for file, launcher_path in entry_points_needing_launchers.items(): - file.create_entry_point(self.working_directory, self.bundle_root, launcher_path) + file.create_entry_point( + self.working_directory, self.bundle_root, launcher_path + ) def delete_working_directory(self): """Recursively deletes the working directory.""" shutil.rmtree(self.working_directory) self.working_directory = None - def file_factory(self, path, entry_point=None, chroot=None, library=False, file_factory=None): + def file_factory( + self, path, entry_point=None, chroot=None, library=False, file_factory=None + ): """Either creates a new `File`, or updates and returns one from `files`. This method can be used in place of `File.__init__()` when it is known that the `File` @@ -946,15 +990,17 @@ def file_factory(self, path, entry_point=None, chroot=None, library=False, file_ path = resolve_file_path(path, search_environment_path=entry_point is not None) file = next((file for file in self.files if file.path == path), None) if file is not None: - assert entry_point == file.entry_point or not entry_point or not file.entry_point, ( - "The entry point property should always persist, but can't conflict." - ) + assert ( + entry_point == file.entry_point + or not entry_point + or not file.entry_point + ), "The entry point property should always persist, but can't conflict." file.entry_point = file.entry_point or entry_point assert chroot == file.chroot, "The chroot must match." file.library = file.library or library - assert not file.entry_point or not file.library, ( - "A file can't be both an entry point and a library." - ) + assert ( + not file.entry_point or not file.library + ), "A file can't be both an entry point and a library." return file return File(path, entry_point, chroot, library, file_factory) diff --git a/src/exodus_bundler/cli.py b/src/exodus_bundler/cli.py index fe51cb5..e5fc462 100644 --- a/src/exodus_bundler/cli.py +++ b/src/exodus_bundler/cli.py @@ -94,7 +94,9 @@ def parse_args(args=None, namespace=None): ), ) - parser.add_argument("-q", "--quiet", action="store_true", help=("Suppress warning messages.")) + parser.add_argument( + "-q", "--quiet", action="store_true", help=("Suppress warning messages.") + ) parser.add_argument( "-r", diff --git a/src/exodus_bundler/dependency_detection.py b/src/exodus_bundler/dependency_detection.py index 8c1a735..9ba89fd 100644 --- a/src/exodus_bundler/dependency_detection.py +++ b/src/exodus_bundler/dependency_detection.py @@ -42,7 +42,9 @@ def find_dependencies(self, path): match = re.search(self.list_regex, line.strip()) if match: dependency_path = match.groups()[0] - if os.path.exists(dependency_path) and not os.path.isdir(dependency_path): + if os.path.exists(dependency_path) and not os.path.isdir( + dependency_path + ): dependencies.append(dependency_path) return dependencies @@ -64,7 +66,9 @@ def find_owner(self, path): @property def cache_exists(self): """Whether or not the expected package cache directory exists.""" - return os.path.exists(self.cache_directory) and os.path.isdir(self.cache_directory) + return os.path.exists(self.cache_directory) and os.path.isdir( + self.cache_directory + ) @property def commands_exist(self): diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 91b3354..70331d9 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -89,14 +89,24 @@ def extract_paths(content, existing_only=True): # Extract files from `open()`, `openat()`, and `exec()` calls. paths = set() for line in lines: - path = extract_exec_path(line) or extract_open_path(line) or extract_stat_path(line) + path = ( + extract_exec_path(line) + or extract_open_path(line) + or extract_stat_path(line) + ) if path: - blacklisted = any(path.startswith(directory) for directory in blacklisted_directories) + blacklisted = any( + path.startswith(directory) for directory in blacklisted_directories + ) if not blacklisted: if not existing_only: paths.add(path) continue - if os.path.exists(path) and os.access(path, os.R_OK) and not os.path.isdir(path): + if ( + os.path.exists(path) + and os.access(path, os.R_OK) + and not os.path.isdir(path) + ): paths.add(path) return list(paths) diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 31b412d..b9d8972 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -38,7 +38,9 @@ def find_executable(binary_name, skip_original_for_testing=False): for bin_directory in os.environ["PATH"].split(":"): if os.path.isabs(bin_directory): bin_directory = os.path.relpath(bin_directory, "/") - candidate_executable = os.path.join(directory, basename, bin_directory, binary_name) + candidate_executable = os.path.join( + directory, basename, bin_directory, binary_name + ) if os.path.exists(candidate_executable): return candidate_executable # Also check for shell launcher version (.sh extension) @@ -78,7 +80,9 @@ def compile_helper(code, initial_args): args = initial_args + ["-static", "-O3", input_filename, "-o", output_filename] process = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert process.returncode == 0, f"There was an error compiling: {stderr.decode('utf-8')}" + assert ( + process.returncode == 0 + ), f"There was an error compiling: {stderr.decode('utf-8')}" with open(output_filename, "rb") as output_file: return output_file.read() From ddd8c8fe1cc6f39c90554ee3e1b713e0d36226aa Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Wed, 26 Nov 2025 14:12:43 -0700 Subject: [PATCH 14/15] Fixing tests. The internal binary inside bundles/bin/appname will point to bundles/hash/.../appname.bin for the binary --- src/exodus_bundler/bundling.py | 4 +- src/exodus_bundler/launchers.py | 9 +- tests/test_bundling.py | 167 +++++++++++++++++++------------- tests/test_cli.py | 28 ++++-- tests/test_input_parsing.py | 52 ++++++---- tests/test_launchers.py | 14 ++- 6 files changed, 169 insertions(+), 105 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index a0ab39a..8226c86 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -686,9 +686,9 @@ def create_launcher( executable=executable, full_linker=full_linker, ) - with open(source_path, "wb") as f: + tt = source_path + ".bin" + with open(tt, "wb") as f: f.write(launcher_content) - tt = source_path except CompilerNotFoundError: if not shell_launcher: logger.warning( diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index b9d8972..83c15bb 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -41,12 +41,17 @@ def find_executable(binary_name, skip_original_for_testing=False): candidate_executable = os.path.join( directory, basename, bin_directory, binary_name ) - if os.path.exists(candidate_executable): - return candidate_executable + # Check for binary launcher version (.bin extension) + candidate_executable_bin = candidate_executable + ".bin" + if os.path.exists(candidate_executable_bin): + return candidate_executable_bin # Also check for shell launcher version (.sh extension) candidate_executable_sh = candidate_executable + ".sh" if os.path.exists(candidate_executable_sh): return candidate_executable_sh + # Finally check for exact match (could be a symlink) + if os.path.exists(candidate_executable): + return candidate_executable return None diff --git a/tests/test_bundling.py b/tests/test_bundling.py index b8139fa..9fc22fa 100644 --- a/tests/test_bundling.py +++ b/tests/test_bundling.py @@ -56,24 +56,28 @@ def test_bundle_add_file_recursively(): second_bundle = Bundle(chroot=chroot) for path in [ldd, fizz_buzz_glibc_32, fizz_buzz_glibc_32_exe, fizz_buzz_musl_64]: second_bundle.add_file(path) - assert second_bundle.files.issubset(bundle.files), ( - "All of the executables and their dependencies should be in the first bundle." - ) + assert second_bundle.files.issubset( + bundle.files + ), "All of the executables and their dependencies should be in the first bundle." def test_bundle_delete_working_directory(): bundle = Bundle() - assert bundle.working_directory is None, ( - "A directory should only be created if passed `working_directory=True`." - ) + assert ( + bundle.working_directory is None + ), "A directory should only be created if passed `working_directory=True`." bundle = Bundle(working_directory=True) working_directory = bundle.working_directory - assert os.path.exists(working_directory), "A working directory should have been created." + assert os.path.exists( + working_directory + ), "A working directory should have been created." bundle.delete_working_directory() - assert not os.path.exists(working_directory), "The working directory should have been deleted." - assert bundle.working_directory is None, ( - "The working directory should have been cleared after deletion." - ) + assert not os.path.exists( + working_directory + ), "The working directory should have been deleted." + assert ( + bundle.working_directory is None + ), "The working directory should have been cleared after deletion." def test_bundle_file_factory(): @@ -82,7 +86,9 @@ def test_bundle_file_factory(): # Note that `ldd` is a shell script, and should bring in no dependencies. [file] = bundle.files new_file = bundle.file_factory(ldd) - assert new_file is file, "The same file should be returned instead of making a new one." + assert ( + new_file is file + ), "The same file should be returned instead of making a new one." def test_bundle_hash(): @@ -92,16 +98,18 @@ def test_bundle_hash(): bundle.add_file(filename) hashes.append(bundle.hash) assert len(hashes) == len(set(hashes)), "All of the hashes should be unique." - assert all(len(hash) == 64 for hash in hashes), "All of the hashes should have length 64." + assert all( + len(hash) == 64 for hash in hashes + ), "All of the hashes should have length 64." def test_bundle_root(): try: bundle = Bundle(working_directory=True) assert bundle.hash in bundle.bundle_root, "Bundle path should include the hash." - assert bundle.bundle_root.startswith(bundle.working_directory), ( - "The bundle root should be a subdirectory of the working directory." - ) + assert bundle.bundle_root.startswith( + bundle.working_directory + ), "The bundle root should be a subdirectory of the working directory." except Exception: raise finally: @@ -118,7 +126,9 @@ def test_bundle_root(): ], ) def test_bytes_to_int(int, bytes, byteorder): - assert bytes_to_int(bytes, byteorder=byteorder) == int, "Byte conversion should work." + assert ( + bytes_to_int(bytes, byteorder=byteorder) == int + ), "Byte conversion should work." @pytest.mark.parametrize( @@ -159,7 +169,9 @@ def test_create_unpackaged_bundle(fizz_buzz, shell_launchers): @pytest.mark.parametrize("detect", [False, True]) def test_create_unpackaged_bundle_detects_dependencies(detect): binary_name = "ls" - root_directory = create_unpackaged_bundle(rename=[], executables=[binary_name], detect=detect) + root_directory = create_unpackaged_bundle( + rename=[], executables=[binary_name], detect=detect + ) try: # Determine the bundle root. binary_symlink = os.path.join(root_directory, "bin", binary_name) @@ -170,9 +182,9 @@ def test_create_unpackaged_bundle_detects_dependencies(detect): bundle_root = os.path.join(dirname, basename) man_directory = os.path.join(bundle_root, "usr", "share", "man") - assert os.path.exists(man_directory) == detect, ( - "The man directory should only exist when `detect=True`." - ) + assert ( + os.path.exists(man_directory) == detect + ), "The man directory should only exist when `detect=True`." finally: assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) @@ -183,18 +195,20 @@ def test_create_unpackaged_bundle_has_correct_args(): rename=[], executables=[echo_args_glibc_32], chroot=chroot ) try: - binary_path = os.path.join(root_directory, "bin", os.path.basename(echo_args_glibc_32)) + binary_path = os.path.join( + root_directory, "bin", os.path.basename(echo_args_glibc_32) + ) process = Popen([binary_path, "arg1", "arg2"], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() assert len(stderr.decode("utf-8")) == 0 args = stdout.decode("utf-8").split("\n") - assert os.path.basename(args[0]) == "%s-x" % os.path.basename(echo_args_glibc_32), ( - "The value of argv[0] should correspond to the local symlink." - ) - assert args[1] == "arg1" and args[2] == "arg2", ( - "The other arguments should be passed through to the child process." - ) + assert os.path.basename(args[0]) == os.path.basename( + echo_args_glibc_32 + ), "The value of argv[0] should correspond to the local symlink." + assert ( + args[1] == "arg1" and args[2] == "arg2" + ), "The other arguments should be passed through to the child process." finally: assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) @@ -213,20 +227,22 @@ def test_create_unpackaged_bundle_has_correct_proc_self_exe(): stdout, stderr = process.communicate() assert len(stderr.decode("utf-8")) == 0 proc_self_exe = stdout.decode("utf-8").strip() - assert os.path.basename(proc_self_exe).startswith("linker-"), ( - "The linker should be the executing process." - ) + assert os.path.basename(proc_self_exe).startswith( + "linker-" + ), "The linker should be the executing process." relative_path = os.path.relpath(proc_self_exe, root_directory) - assert relative_path.startswith("bundles/"), ( - "The process should be in the bundles directory." - ) + assert relative_path.startswith( + "bundles/" + ), "The process should be in the bundles directory." finally: assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) def test_detect_elf_binary(): - assert detect_elf_binary(fizz_buzz_glibc_32), "The `fizz-buzz` file should be an ELF binary." + assert detect_elf_binary( + fizz_buzz_glibc_32 + ), "The `fizz-buzz` file should be an ELF binary." assert not detect_elf_binary(ldd), "The `ldd` file should be a shell script." @@ -241,7 +257,9 @@ def test_detect_elf_binary(): def test_elf_bits(fizz_buzz, bits): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) # Can be checked by running `file fizz-buzz`. - assert fizz_buzz_elf.bits == bits, "The fizz buzz executable should be %d-bit." % bits + assert fizz_buzz_elf.bits == bits, ( + "The fizz buzz executable should be %d-bit." % bits + ) @pytest.mark.parametrize( @@ -255,9 +273,9 @@ def test_elf_dependencies(fizz_buzz): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) direct_dependencies = fizz_buzz_elf.direct_dependencies all_dependencies = fizz_buzz_elf.dependencies - assert set(direct_dependencies).issubset(all_dependencies), ( - "The direct dependencies should be a subset of all dependencies." - ) + assert set(direct_dependencies).issubset( + all_dependencies + ), "The direct dependencies should be a subset of all dependencies." @pytest.mark.parametrize( @@ -271,17 +289,19 @@ def test_elf_dependencies(fizz_buzz): def test_elf_direct_dependencies(fizz_buzz): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) dependencies = fizz_buzz_elf.direct_dependencies - assert all(file.path.startswith(chroot) for file in dependencies), ( - "All dependencies should be located within the chroot." - ) + assert all( + file.path.startswith(chroot) for file in dependencies + ), "All dependencies should be located within the chroot." assert len(dependencies), "There should be at least one dependency." # These don't apply to the musl binary. if "glib" in fizz_buzz: - assert len(dependencies) == 2, "The linker and libc should be the only dependencies." - assert any("libc.so" in file.path for file in dependencies), ( - '"libc" was not found as a direct dependency of the executable.' - ) + assert ( + len(dependencies) == 2 + ), "The linker and libc should be the only dependencies." + assert any( + "libc.so" in file.path for file in dependencies + ), '"libc" was not found as a direct dependency of the executable.' @pytest.mark.parametrize( @@ -295,10 +315,12 @@ def test_elf_direct_dependencies(fizz_buzz): def test_elf_linker(fizz_buzz, expected_linker_path): # Found by running `readelf -l fizz-buzz`. fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) - expected_linker_path = os.path.join(chroot, os.path.relpath(expected_linker_path, "/")) - assert fizz_buzz_elf.linker_file.path == expected_linker_path, ( - "The correct linker should be extracted from the ELF program header." + expected_linker_path = os.path.join( + chroot, os.path.relpath(expected_linker_path, "/") ) + assert ( + fizz_buzz_elf.linker_file.path == expected_linker_path + ), "The correct linker should be extracted from the ELF program header." @pytest.mark.parametrize( @@ -311,7 +333,9 @@ def test_elf_linker(fizz_buzz, expected_linker_path): ) def test_elf_type(fizz_buzz, expected_type): elf = Elf(fizz_buzz, chroot=chroot) - assert elf.type == expected_type, "Fizz buzz should match the expected ELF binary type." + assert ( + elf.type == expected_type + ), "Fizz buzz should match the expected ELF binary type." def test_file_destination(): @@ -319,9 +343,9 @@ def test_file_destination(): arch_directory = os.path.dirname(arch_file.destination) fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) fizz_buzz_directory = os.path.dirname(fizz_buzz_file.destination) - assert arch_directory == fizz_buzz_directory, ( - "Executable and non-executable files should be written to the same directory." - ) + assert ( + arch_directory == fizz_buzz_directory + ), "Executable and non-executable files should be written to the same directory." def test_file_executable(): @@ -342,13 +366,15 @@ def test_file_hash(): amazon_file = File(os.path.join(ldd_output_directory, "htop-amazon-linux.txt")) arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) assert amazon_file.hash != arch_file.hash, "The hashes should differ." - assert len(amazon_file.hash) == len(arch_file.hash) == 64, ( - "The hashes should have a consistent length of 64 characters." - ) + assert ( + len(amazon_file.hash) == len(arch_file.hash) == 64 + ), "The hashes should have a consistent length of 64 characters." # Found by executing `sha256sum fizz-buzz`. expected_hash = "d54ab4714215d7822bf490df5cdf49bc3f32b4c85a439b109fc7581355f9d9c5" - assert File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash, "Hashes should match." + assert ( + File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash + ), "Hashes should match." @pytest.mark.parametrize( @@ -362,9 +388,9 @@ def test_file_hash(): def test_file_requires_launcher(fizz_buzz): file = File(fizz_buzz, chroot=chroot) assert file.requires_launcher, "Fizz buzz should require a launcher." - assert all(not dependency.requires_launcher for dependency in file.elf.dependencies), ( - "All of the dependencies should not require launchers." - ) + assert all( + not dependency.requires_launcher for dependency in file.elf.dependencies + ), "All of the dependencies should not require launchers." def test_file_symlink(): @@ -401,7 +427,8 @@ def test_parse_dependencies_from_ldd_output(filename_prefix): expected_dependencies = [line for line in f.read().split("\n") if len(line)] assert set(dependencies) == set(expected_dependencies), ( - 'The dependencies were not parsed correctly from ldd output for "%s"' % filename_prefix + 'The dependencies were not parsed correctly from ldd output for "%s"' + % filename_prefix ) @@ -412,9 +439,9 @@ def test_resolve_binary(): try: os.environ["PATH"] = "%s%s%s" % (binary_directory, os.pathsep, old_path) resolved_binary = resolve_binary(binary) - assert resolved_binary == os.path.normpath(fizz_buzz_glibc_32), ( - "The full binary path was not resolved correctly." - ) + assert resolved_binary == os.path.normpath( + fizz_buzz_glibc_32 + ), "The full binary path was not resolved correctly." finally: os.environ["PATH"] = old_path @@ -424,15 +451,15 @@ def test_resolve_file_path(): resolve_file_path(chroot) with pytest.raises(Exception): resolve_file_path(os.path.join(chroot, "non-existent-file")) - assert os.path.isabs(resolve_file_path(fizz_buzz_glibc_32)), ( - "The resolved path should be absolute." - ) + assert os.path.isabs( + resolve_file_path(fizz_buzz_glibc_32) + ), "The resolved path should be absolute." def test_run_ldd(): - assert any("libc.so" in line for line in run_ldd(ldd, fizz_buzz_glibc_32)), ( - '"libc" was not found in the output of "ldd" for the executable.' - ) + assert any( + "libc.so" in line for line in run_ldd(ldd, fizz_buzz_glibc_32) + ), '"libc" was not found in the output of "ldd" for the executable.' def test_stored_property(): diff --git a/tests/test_cli.py b/tests/test_cli.py index ad7bd82..c20a28c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,7 +41,9 @@ def test_adding_additional_files(capsys): args = ["--chroot", chroot, "--output", "-", "--tarball", fizz_buzz_glibc_32] stdin = "\n".join((fizz_buzz_glibc_32_exe, fizz_buzz_glibc_64)) returncode, stdout, stderr = run_exodus(args, universal_newlines=False, stdin=stdin) - assert returncode == 0, "Exodus should have exited with a success status code, but didn't." + assert ( + returncode == 0 + ), "Exodus should have exited with a success status code, but didn't." stream = io.BytesIO(stdout) with tarfile.open(fileobj=stream, mode="r:gz") as f: names = f.getnames() @@ -78,12 +80,16 @@ def test_missing_binary(capsys): command = "this-is-almost-definitely-not-going-to-be-a-command-anywhere" returncode, stdout, stderr = run_exodus([command]) assert returncode != 0, "Running exodus should have failed." - assert "Traceback" not in stderr, "Traceback should not be included without the --verbose flag." + assert ( + "Traceback" not in stderr + ), "Traceback should not be included without the --verbose flag." # With the --verbose flag. returncode, stdout, stderr = run_exodus(["--verbose", command]) assert returncode != 0, "Running exodus should have failed." - assert "Traceback" in stderr, "Traceback should be included with the --verbose flag." + assert ( + "Traceback" in stderr + ), "Traceback should be included with the --verbose flag." def test_required_argument(): @@ -109,7 +115,9 @@ def test_writing_bundle_to_disk(): args = ["--chroot", chroot, "--output", filename, fizz_buzz_glibc_32] try: returncode, stdout, stderr = run_exodus(args) - assert returncode == 0, "Exodus should have exited with a success status code, but didn't." + assert ( + returncode == 0 + ), "Exodus should have exited with a success status code, but didn't." with open(filename, "rb") as f_in: first_line = f_in.readline().strip() assert first_line == b"#! /bin/bash", stderr @@ -121,7 +129,9 @@ def test_writing_bundle_to_disk(): def test_writing_bundle_to_stdout(): args = ["--chroot", chroot, "--output", "-", fizz_buzz_glibc_32] returncode, stdout, stderr = run_exodus(args) - assert returncode == 0, "Exodus should have exited with a success status code, but didn't." + assert ( + returncode == 0 + ), "Exodus should have exited with a success status code, but didn't." assert stdout.startswith("#! /bin/sh"), stderr @@ -131,7 +141,9 @@ def test_writing_tarball_to_disk(): args = ["--chroot", chroot, "--output", filename, "--tarball", fizz_buzz_glibc_32] try: returncode, stdout, stderr = run_exodus(args) - assert returncode == 0, "Exodus should have exited with a success status code, but didn't." + assert ( + returncode == 0 + ), "Exodus should have exited with a success status code, but didn't." assert tarfile.is_tarfile(filename), stderr with tarfile.open(filename, mode="r:gz") as f_in: assert "exodus/bin/fizz-buzz-glibc-32" in f_in.getnames() @@ -143,7 +155,9 @@ def test_writing_tarball_to_disk(): def test_writing_tarball_to_stdout(): args = ["--chroot", chroot, "--output", "-", "--tarball", fizz_buzz_glibc_32] returncode, stdout, stderr = run_exodus(args, universal_newlines=False) - assert returncode == 0, "Exodus should have exited with a success status code, but didn't." + assert ( + returncode == 0 + ), "Exodus should have exited with a success status code, but didn't." stream = io.BytesIO(stdout) with tarfile.open(fileobj=stream, mode="r:gz") as f: assert "exodus/bin/fizz-buzz-glibc-32" in f.getnames(), stderr diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index e450ac9..2ca3453 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -15,10 +15,12 @@ def test_extract_exec_path(): line = 'execve("/usr/bin/ls", ["ls"], 0x7ffea775ad70 /* 113 vars */) = 0' - assert extract_exec_path(line) == "/usr/bin/ls", ( - "It should have extracted the path to the ls executable." - ) - assert extract_exec_path("blah") is None, "It should return `None` when there is no match." + assert ( + extract_exec_path(line) == "/usr/bin/ls" + ), "It should have extracted the path to the ls executable." + assert ( + extract_exec_path("blah") is None + ), "It should return `None` when there is no match." def test_extract_no_paths(): @@ -33,15 +35,17 @@ def test_extract_open_path(): ) assert extract_open_path(line) is None, "Missing files should not return paths." line = 'open(".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4' - assert extract_open_path(line) is None, "Opened directories should not return paths." + assert ( + extract_open_path(line) is None + ), "Opened directories should not return paths." line = 'open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 4' - assert extract_open_path(line) == "/usr/lib/locale/locale-archive", ( - "An open() call should return a path." - ) + assert ( + extract_open_path(line) == "/usr/lib/locale/locale-archive" + ), "An open() call should return a path." line = 'openat(AT_FDCWD, "/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 4' - assert extract_open_path(line) == "/usr/lib/libc.so.6", ( - "An openat() call relative to the current directory should return a path." - ) + assert ( + extract_open_path(line) == "/usr/lib/libc.so.6" + ), "An openat() call relative to the current directory should return a path." def test_extract_raw_paths(): @@ -50,12 +54,14 @@ def test_extract_raw_paths(): "./relative/path", "/another/absolute/path", ] - input_paths_with_whitespace = [" ", ""] + [input_paths[0]] + [" "] + input_paths[1:] + input_paths_with_whitespace = ( + [" ", ""] + [input_paths[0]] + [" "] + input_paths[1:] + ) input_content = "\n".join(input_paths_with_whitespace) extracted_paths = extract_paths(input_content) - assert set(input_paths) == set(extracted_paths), ( - "The paths should have been extracted without the whitespace." - ) + assert set(input_paths) == set( + extracted_paths + ), "The paths should have been extracted without the whitespace." def test_extract_stat_path(): @@ -64,12 +70,16 @@ def test_extract_stat_path(): "{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0" ) expected_path = "/usr/local/lib/python3.6/encodings/__init__.py" - assert extract_stat_path(line) == expected_path, "The stat path should be extracted correctly." + assert ( + extract_stat_path(line) == expected_path + ), "The stat path should be extracted correctly." line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.abi3.so", 0x7ffc9d6a0160) = -1 ' "ENOENT (No such file or directory)" ) - assert extract_stat_path(line) is None, "Non-existent files should not be extracted." + assert ( + extract_stat_path(line) is None + ), "Non-existent files should not be extracted." def test_extract_strace_paths(): @@ -86,7 +96,9 @@ def test_extract_strace_paths(): ] for path in expected_paths: - assert path in extracted_paths, '"%s" should be present in the extracted paths.' % path + assert path in extracted_paths, ( + '"%s" should be present in the extracted paths.' % path + ) def test_strip_pid_prefix(): @@ -95,4 +107,6 @@ def test_strip_pid_prefix(): '"/tmp/exodus-bundle-fqzw_lds.c", "-o", "/tmp/exodus-bundle-3p_c0osh"], [/* 45 vars */] ' "" ) - assert strip_pid_prefix(line).startswith("execve("), "The PID prefix should be stripped." + assert strip_pid_prefix(line).startswith( + "execve(" + ), "The PID prefix should be stripped." diff --git a/tests/test_launchers.py b/tests/test_launchers.py index 9a62e21..d0b73b0 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -20,7 +20,9 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) chroot = os.path.join(parent_directory, "data", "binaries", "chroot") echo_args_glibc_32 = os.path.join(chroot, "bin", "echo-args-glibc-32") -fizz_buzz_source_file = os.path.join(parent_directory, "data", "binaries", "fizz-buzz.c") +fizz_buzz_source_file = os.path.join( + parent_directory, "data", "binaries", "fizz-buzz.c" +) def test_construct_bash_launcher(): @@ -71,11 +73,13 @@ def test_find_executable(): binary_symlink = os.path.join(root_directory, "bin", binary_name) binary_path = os.path.realpath(binary_symlink) # This is a pretend directory, but it doesn't check. - launchers.parent_directory = os.path.join(os.path.dirname(binary_path), "somewhere", "else") - os.environ["PATH"] = os.path.dirname(echo_args_glibc_32) - assert find_executable(binary_name, skip_original_for_testing=True) == binary_path, ( - 'It should have found the binary path "%s".' % binary_path + launchers.parent_directory = os.path.join( + os.path.dirname(binary_path), "somewhere", "else" ) + os.environ["PATH"] = os.path.dirname(echo_args_glibc_32) + assert ( + find_executable(binary_name, skip_original_for_testing=True) == binary_path + ), ('It should have found the binary path "%s".' % binary_path) finally: launchers.parent_directory = original_parent_directory os.environ["PATH"] = original_environment From dcd55c34085f86fa631fa1d0de698933fab95528 Mon Sep 17 00:00:00 2001 From: Preshanth Jagannathan Date: Wed, 26 Nov 2025 14:19:18 -0700 Subject: [PATCH 15/15] Updating to pass tox checks for ruff formatting --- src/exodus_bundler/bundling.py | 103 ++++--------- src/exodus_bundler/cli.py | 4 +- src/exodus_bundler/dependency_detection.py | 8 +- src/exodus_bundler/input_parsing.py | 16 +- src/exodus_bundler/launchers.py | 8 +- tests/test_bundling.py | 167 +++++++++------------ tests/test_cli.py | 28 +--- tests/test_input_parsing.py | 52 +++---- tests/test_launchers.py | 14 +- 9 files changed, 139 insertions(+), 261 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 8226c86..4da55eb 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -70,9 +70,7 @@ def create_bundle( # Populate the filename template. output_filename = render_template( output, - executables=( - "-".join(os.path.basename(executable) for executable in executables) - ), + executables=("-".join(os.path.basename(executable) for executable in executables)), extension=("tgz" if tarball else "sh"), ) @@ -90,18 +88,14 @@ def create_bundle( # Construct the installation script and write it out. if not tarball: if output_filename == "-": - base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode( - "utf-8" - ) + base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode("utf-8") script_content = render_template_file( "install-bundle-noninteractive.sh", base64_encoded_tarball=base64_encoded_tarball, ) output_file.write(script_content.encode("utf-8")) else: - output_file.write( - render_template_file("install-bundle.sh").encode("utf-8") - ) + output_file.write(render_template_file("install-bundle.sh").encode("utf-8")) output_file.write(tar_stream.getvalue()) else: # Or just write out the tarball. @@ -136,9 +130,9 @@ def create_unpackaged_bundle( try: # Sanitize the inputs. assert len(executables), "No executables were specified." - assert len(executables) >= len( - rename - ), "More renamed options were included than executables." + assert len(executables) >= len(rename), ( + "More renamed options were included than executables." + ) # Pad the rename's with `True` so that `entry_point` can be specified. entry_points = rename + [True for i in range(len(executables) - len(rename))] @@ -153,8 +147,7 @@ def create_unpackaged_bundle( if not dependency_paths: raise DependencyDetectionError( ( - 'Automatic dependency detection failed. Either "%s" ' - % file.path + 'Automatic dependency detection failed. Either "%s" ' % file.path + "is not tracked by your package manager, or your operating system " + "is not currently compatible with the `--detect` option. If not, please " + "create an issue at https://github.com/intoli/exodus and we'll try our " @@ -220,15 +213,11 @@ def resolve_binary(binary): absolute_binary_path = os.path.normpath(os.path.abspath(binary)) if not os.path.exists(absolute_binary_path): for path in os.getenv("PATH", "/bin/:/usr/bin/").split(os.pathsep): - absolute_binary_path = os.path.normpath( - os.path.abspath(os.path.join(path, binary)) - ) + absolute_binary_path = os.path.normpath(os.path.abspath(os.path.join(path, binary))) if os.path.exists(absolute_binary_path): break else: - raise MissingFileError( - f'The "{binary}" binary could not be found in $PATH.' - ) + raise MissingFileError(f'The "{binary}" binary could not be found in $PATH.') return absolute_binary_path @@ -304,9 +293,7 @@ def __init__(self, path, chroot=None, file_factory=None): # Make sure that this is actually an ELF binary. first_four_bytes = f.read(4) if first_four_bytes != b"\x7fELF": - raise InvalidElfBinaryError( - f'The "{path}" file is not a binary ELF file.' - ) + raise InvalidElfBinaryError(f'The "{path}" file is not a binary ELF file.') # Determine whether this is a 32-bit or 64-bit file. format_byte = f.read(1) @@ -314,8 +301,7 @@ def __init__(self, path, chroot=None, file_factory=None): if not self.bits: raise UnsupportedArchitectureError( ( - 'The "%s" file does not appear to be either 32 or 64 bits. ' - % path + 'The "%s" file does not appear to be either 32 or 64 bits. ' % path + "Other architectures are not currently supported, but you can open an " + "issue at https://github.com/intoli/exodus stating your use-case and " + "support might get extended in the future." @@ -342,9 +328,7 @@ def hex(bytes): # Determine the type of the binary. f.seek(hex(b"\x10")) e_type = hex(f.read(2)) - self.type = {1: "relocatable", 2: "executable", 3: "shared", 4: "core"}[ - e_type - ] + self.type = {1: "relocatable", 2: "executable", 3: "shared", 4: "core"}[e_type] # Find the program header offset. e_phoff_start = {32: hex(b"\x1c"), 64: hex(b"\x20")}[self.bits] @@ -375,17 +359,13 @@ def hex(bytes): continue # Determine the offset for the segment. - p_offset_start = ( - header_start + {32: hex(b"\04"), 64: hex(b"\x08")}[self.bits] - ) + p_offset_start = header_start + {32: hex(b"\04"), 64: hex(b"\x08")}[self.bits] p_offset_length = {32: 4, 64: 8}[self.bits] f.seek(p_offset_start) p_offset = hex(f.read(p_offset_length)) # Determine the size of the segment. - p_filesz_start = ( - header_start + {32: hex(b"\x10"), 64: hex(b"\x20")}[self.bits] - ) + p_filesz_start = header_start + {32: hex(b"\x10"), 64: hex(b"\x20")}[self.bits] p_filesz_length = {32: 4, 64: 8}[self.bits] f.seek(p_filesz_start) p_filesz = hex(f.read(p_filesz_length)) @@ -401,9 +381,7 @@ def hex(bytes): assert self.linker_file is None, "More than one linker found." linker_path = segment[:-1].decode("ascii") if chroot: - linker_path = os.path.join( - chroot, os.path.relpath(linker_path, "/") - ) + linker_path = os.path.join(chroot, os.path.relpath(linker_path, "/")) self.linker_file = self.file_factory(linker_path, chroot=self.chroot) def __eq__(self, other): @@ -432,9 +410,7 @@ def find_direct_dependencies(self, linker_file=None): directories = [] for directory in ld_library_path.split(":"): if os.path.isabs(directory): - directory = os.path.join( - self.chroot, os.path.relpath(directory, "/") - ) + directory = os.path.join(self.chroot, os.path.relpath(directory, "/")) directories.append(directory) ld_library_path = ":".join(directories) environment["LD_LIBRARY_PATH"] = ld_library_path @@ -449,16 +425,13 @@ def find_direct_dependencies(self, linker_file=None): env=environment, ) stdout, stderr = process.communicate() - combined_output = stdout.decode("utf-8").split("\n") + stderr.decode( - "utf-8" - ).split("\n") + combined_output = stdout.decode("utf-8").split("\n") + stderr.decode("utf-8").split("\n") # Note that we're explicitly adding the linker because when we invoke it as `ldd` we can't # extract the real path from the trace output. Even if it were here twice, it would be # deduplicated though the use of a set. filenames = parse_dependencies_from_ldd_output(combined_output) + [linker_path] return set( - self.file_factory(filename, chroot=self.chroot, library=True) - for filename in filenames + self.file_factory(filename, chroot=self.chroot, library=True) for filename in filenames ) @stored_property @@ -500,9 +473,7 @@ class File: path (str): The absolute normalized path to the file on disk. """ - def __init__( - self, path, entry_point=None, chroot=None, library=False, file_factory=None - ): + def __init__(self, path, entry_point=None, chroot=None, library=False, file_factory=None): """Constructor for the `File` class. Note: @@ -517,9 +488,7 @@ def __init__( file_factory (function, optional): A function to use when creating new `File` instances. """ # Find the full path to the file. - self.path = resolve_file_path( - path, search_environment_path=(entry_point is not None) - ) + self.path = resolve_file_path(path, search_environment_path=(entry_point is not None)) # Set the entry point for the file. if entry_point is True: @@ -636,8 +605,7 @@ def create_launcher( shutil.copy(self.elf.linker_file.path, linker_path) else: assert filecmp.cmp(self.elf.linker_file.path, linker_path), ( - 'The "%s" linker file already exists and has differing contents.' - % linker_path + 'The "%s" linker file already exists and has differing contents.' % linker_path ) linker = os.path.join(".", linker_basename) @@ -761,12 +729,7 @@ def requires_launcher(self): # as shared libraries, and many mostly-libraries are executable (*e.g.* glibc). # The easy ones. - if ( - self.library - or not self.elf - or not self.elf.linker_file - or not self.executable - ): + if self.library or not self.elf or not self.elf.linker_file or not self.executable: return False if self.elf.type == "executable": return True @@ -962,18 +925,14 @@ def create_bundle(self, shell_launchers=False): # Create entry points for files that needed launchers for file, launcher_path in entry_points_needing_launchers.items(): - file.create_entry_point( - self.working_directory, self.bundle_root, launcher_path - ) + file.create_entry_point(self.working_directory, self.bundle_root, launcher_path) def delete_working_directory(self): """Recursively deletes the working directory.""" shutil.rmtree(self.working_directory) self.working_directory = None - def file_factory( - self, path, entry_point=None, chroot=None, library=False, file_factory=None - ): + def file_factory(self, path, entry_point=None, chroot=None, library=False, file_factory=None): """Either creates a new `File`, or updates and returns one from `files`. This method can be used in place of `File.__init__()` when it is known that the `File` @@ -990,17 +949,15 @@ def file_factory( path = resolve_file_path(path, search_environment_path=entry_point is not None) file = next((file for file in self.files if file.path == path), None) if file is not None: - assert ( - entry_point == file.entry_point - or not entry_point - or not file.entry_point - ), "The entry point property should always persist, but can't conflict." + assert entry_point == file.entry_point or not entry_point or not file.entry_point, ( + "The entry point property should always persist, but can't conflict." + ) file.entry_point = file.entry_point or entry_point assert chroot == file.chroot, "The chroot must match." file.library = file.library or library - assert ( - not file.entry_point or not file.library - ), "A file can't be both an entry point and a library." + assert not file.entry_point or not file.library, ( + "A file can't be both an entry point and a library." + ) return file return File(path, entry_point, chroot, library, file_factory) diff --git a/src/exodus_bundler/cli.py b/src/exodus_bundler/cli.py index e5fc462..fe51cb5 100644 --- a/src/exodus_bundler/cli.py +++ b/src/exodus_bundler/cli.py @@ -94,9 +94,7 @@ def parse_args(args=None, namespace=None): ), ) - parser.add_argument( - "-q", "--quiet", action="store_true", help=("Suppress warning messages.") - ) + parser.add_argument("-q", "--quiet", action="store_true", help=("Suppress warning messages.")) parser.add_argument( "-r", diff --git a/src/exodus_bundler/dependency_detection.py b/src/exodus_bundler/dependency_detection.py index 9ba89fd..8c1a735 100644 --- a/src/exodus_bundler/dependency_detection.py +++ b/src/exodus_bundler/dependency_detection.py @@ -42,9 +42,7 @@ def find_dependencies(self, path): match = re.search(self.list_regex, line.strip()) if match: dependency_path = match.groups()[0] - if os.path.exists(dependency_path) and not os.path.isdir( - dependency_path - ): + if os.path.exists(dependency_path) and not os.path.isdir(dependency_path): dependencies.append(dependency_path) return dependencies @@ -66,9 +64,7 @@ def find_owner(self, path): @property def cache_exists(self): """Whether or not the expected package cache directory exists.""" - return os.path.exists(self.cache_directory) and os.path.isdir( - self.cache_directory - ) + return os.path.exists(self.cache_directory) and os.path.isdir(self.cache_directory) @property def commands_exist(self): diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 70331d9..91b3354 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -89,24 +89,14 @@ def extract_paths(content, existing_only=True): # Extract files from `open()`, `openat()`, and `exec()` calls. paths = set() for line in lines: - path = ( - extract_exec_path(line) - or extract_open_path(line) - or extract_stat_path(line) - ) + path = extract_exec_path(line) or extract_open_path(line) or extract_stat_path(line) if path: - blacklisted = any( - path.startswith(directory) for directory in blacklisted_directories - ) + blacklisted = any(path.startswith(directory) for directory in blacklisted_directories) if not blacklisted: if not existing_only: paths.add(path) continue - if ( - os.path.exists(path) - and os.access(path, os.R_OK) - and not os.path.isdir(path) - ): + if os.path.exists(path) and os.access(path, os.R_OK) and not os.path.isdir(path): paths.add(path) return list(paths) diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 83c15bb..0ace939 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -38,9 +38,7 @@ def find_executable(binary_name, skip_original_for_testing=False): for bin_directory in os.environ["PATH"].split(":"): if os.path.isabs(bin_directory): bin_directory = os.path.relpath(bin_directory, "/") - candidate_executable = os.path.join( - directory, basename, bin_directory, binary_name - ) + candidate_executable = os.path.join(directory, basename, bin_directory, binary_name) # Check for binary launcher version (.bin extension) candidate_executable_bin = candidate_executable + ".bin" if os.path.exists(candidate_executable_bin): @@ -85,9 +83,7 @@ def compile_helper(code, initial_args): args = initial_args + ["-static", "-O3", input_filename, "-o", output_filename] process = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert ( - process.returncode == 0 - ), f"There was an error compiling: {stderr.decode('utf-8')}" + assert process.returncode == 0, f"There was an error compiling: {stderr.decode('utf-8')}" with open(output_filename, "rb") as output_file: return output_file.read() diff --git a/tests/test_bundling.py b/tests/test_bundling.py index 9fc22fa..e86b5c5 100644 --- a/tests/test_bundling.py +++ b/tests/test_bundling.py @@ -56,28 +56,24 @@ def test_bundle_add_file_recursively(): second_bundle = Bundle(chroot=chroot) for path in [ldd, fizz_buzz_glibc_32, fizz_buzz_glibc_32_exe, fizz_buzz_musl_64]: second_bundle.add_file(path) - assert second_bundle.files.issubset( - bundle.files - ), "All of the executables and their dependencies should be in the first bundle." + assert second_bundle.files.issubset(bundle.files), ( + "All of the executables and their dependencies should be in the first bundle." + ) def test_bundle_delete_working_directory(): bundle = Bundle() - assert ( - bundle.working_directory is None - ), "A directory should only be created if passed `working_directory=True`." + assert bundle.working_directory is None, ( + "A directory should only be created if passed `working_directory=True`." + ) bundle = Bundle(working_directory=True) working_directory = bundle.working_directory - assert os.path.exists( - working_directory - ), "A working directory should have been created." + assert os.path.exists(working_directory), "A working directory should have been created." bundle.delete_working_directory() - assert not os.path.exists( - working_directory - ), "The working directory should have been deleted." - assert ( - bundle.working_directory is None - ), "The working directory should have been cleared after deletion." + assert not os.path.exists(working_directory), "The working directory should have been deleted." + assert bundle.working_directory is None, ( + "The working directory should have been cleared after deletion." + ) def test_bundle_file_factory(): @@ -86,9 +82,7 @@ def test_bundle_file_factory(): # Note that `ldd` is a shell script, and should bring in no dependencies. [file] = bundle.files new_file = bundle.file_factory(ldd) - assert ( - new_file is file - ), "The same file should be returned instead of making a new one." + assert new_file is file, "The same file should be returned instead of making a new one." def test_bundle_hash(): @@ -98,18 +92,16 @@ def test_bundle_hash(): bundle.add_file(filename) hashes.append(bundle.hash) assert len(hashes) == len(set(hashes)), "All of the hashes should be unique." - assert all( - len(hash) == 64 for hash in hashes - ), "All of the hashes should have length 64." + assert all(len(hash) == 64 for hash in hashes), "All of the hashes should have length 64." def test_bundle_root(): try: bundle = Bundle(working_directory=True) assert bundle.hash in bundle.bundle_root, "Bundle path should include the hash." - assert bundle.bundle_root.startswith( - bundle.working_directory - ), "The bundle root should be a subdirectory of the working directory." + assert bundle.bundle_root.startswith(bundle.working_directory), ( + "The bundle root should be a subdirectory of the working directory." + ) except Exception: raise finally: @@ -126,9 +118,7 @@ def test_bundle_root(): ], ) def test_bytes_to_int(int, bytes, byteorder): - assert ( - bytes_to_int(bytes, byteorder=byteorder) == int - ), "Byte conversion should work." + assert bytes_to_int(bytes, byteorder=byteorder) == int, "Byte conversion should work." @pytest.mark.parametrize( @@ -169,9 +159,7 @@ def test_create_unpackaged_bundle(fizz_buzz, shell_launchers): @pytest.mark.parametrize("detect", [False, True]) def test_create_unpackaged_bundle_detects_dependencies(detect): binary_name = "ls" - root_directory = create_unpackaged_bundle( - rename=[], executables=[binary_name], detect=detect - ) + root_directory = create_unpackaged_bundle(rename=[], executables=[binary_name], detect=detect) try: # Determine the bundle root. binary_symlink = os.path.join(root_directory, "bin", binary_name) @@ -182,9 +170,9 @@ def test_create_unpackaged_bundle_detects_dependencies(detect): bundle_root = os.path.join(dirname, basename) man_directory = os.path.join(bundle_root, "usr", "share", "man") - assert ( - os.path.exists(man_directory) == detect - ), "The man directory should only exist when `detect=True`." + assert os.path.exists(man_directory) == detect, ( + "The man directory should only exist when `detect=True`." + ) finally: assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) @@ -195,20 +183,18 @@ def test_create_unpackaged_bundle_has_correct_args(): rename=[], executables=[echo_args_glibc_32], chroot=chroot ) try: - binary_path = os.path.join( - root_directory, "bin", os.path.basename(echo_args_glibc_32) - ) + binary_path = os.path.join(root_directory, "bin", os.path.basename(echo_args_glibc_32)) process = Popen([binary_path, "arg1", "arg2"], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() assert len(stderr.decode("utf-8")) == 0 args = stdout.decode("utf-8").split("\n") - assert os.path.basename(args[0]) == os.path.basename( - echo_args_glibc_32 - ), "The value of argv[0] should correspond to the local symlink." - assert ( - args[1] == "arg1" and args[2] == "arg2" - ), "The other arguments should be passed through to the child process." + assert os.path.basename(args[0]) == os.path.basename(echo_args_glibc_32), ( + "The value of argv[0] should correspond to the local symlink." + ) + assert args[1] == "arg1" and args[2] == "arg2", ( + "The other arguments should be passed through to the child process." + ) finally: assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) @@ -227,22 +213,20 @@ def test_create_unpackaged_bundle_has_correct_proc_self_exe(): stdout, stderr = process.communicate() assert len(stderr.decode("utf-8")) == 0 proc_self_exe = stdout.decode("utf-8").strip() - assert os.path.basename(proc_self_exe).startswith( - "linker-" - ), "The linker should be the executing process." + assert os.path.basename(proc_self_exe).startswith("linker-"), ( + "The linker should be the executing process." + ) relative_path = os.path.relpath(proc_self_exe, root_directory) - assert relative_path.startswith( - "bundles/" - ), "The process should be in the bundles directory." + assert relative_path.startswith("bundles/"), ( + "The process should be in the bundles directory." + ) finally: assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) def test_detect_elf_binary(): - assert detect_elf_binary( - fizz_buzz_glibc_32 - ), "The `fizz-buzz` file should be an ELF binary." + assert detect_elf_binary(fizz_buzz_glibc_32), "The `fizz-buzz` file should be an ELF binary." assert not detect_elf_binary(ldd), "The `ldd` file should be a shell script." @@ -257,9 +241,7 @@ def test_detect_elf_binary(): def test_elf_bits(fizz_buzz, bits): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) # Can be checked by running `file fizz-buzz`. - assert fizz_buzz_elf.bits == bits, ( - "The fizz buzz executable should be %d-bit." % bits - ) + assert fizz_buzz_elf.bits == bits, "The fizz buzz executable should be %d-bit." % bits @pytest.mark.parametrize( @@ -273,9 +255,9 @@ def test_elf_dependencies(fizz_buzz): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) direct_dependencies = fizz_buzz_elf.direct_dependencies all_dependencies = fizz_buzz_elf.dependencies - assert set(direct_dependencies).issubset( - all_dependencies - ), "The direct dependencies should be a subset of all dependencies." + assert set(direct_dependencies).issubset(all_dependencies), ( + "The direct dependencies should be a subset of all dependencies." + ) @pytest.mark.parametrize( @@ -289,19 +271,17 @@ def test_elf_dependencies(fizz_buzz): def test_elf_direct_dependencies(fizz_buzz): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) dependencies = fizz_buzz_elf.direct_dependencies - assert all( - file.path.startswith(chroot) for file in dependencies - ), "All dependencies should be located within the chroot." + assert all(file.path.startswith(chroot) for file in dependencies), ( + "All dependencies should be located within the chroot." + ) assert len(dependencies), "There should be at least one dependency." # These don't apply to the musl binary. if "glib" in fizz_buzz: - assert ( - len(dependencies) == 2 - ), "The linker and libc should be the only dependencies." - assert any( - "libc.so" in file.path for file in dependencies - ), '"libc" was not found as a direct dependency of the executable.' + assert len(dependencies) == 2, "The linker and libc should be the only dependencies." + assert any("libc.so" in file.path for file in dependencies), ( + '"libc" was not found as a direct dependency of the executable.' + ) @pytest.mark.parametrize( @@ -315,12 +295,10 @@ def test_elf_direct_dependencies(fizz_buzz): def test_elf_linker(fizz_buzz, expected_linker_path): # Found by running `readelf -l fizz-buzz`. fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) - expected_linker_path = os.path.join( - chroot, os.path.relpath(expected_linker_path, "/") + expected_linker_path = os.path.join(chroot, os.path.relpath(expected_linker_path, "/")) + assert fizz_buzz_elf.linker_file.path == expected_linker_path, ( + "The correct linker should be extracted from the ELF program header." ) - assert ( - fizz_buzz_elf.linker_file.path == expected_linker_path - ), "The correct linker should be extracted from the ELF program header." @pytest.mark.parametrize( @@ -333,9 +311,7 @@ def test_elf_linker(fizz_buzz, expected_linker_path): ) def test_elf_type(fizz_buzz, expected_type): elf = Elf(fizz_buzz, chroot=chroot) - assert ( - elf.type == expected_type - ), "Fizz buzz should match the expected ELF binary type." + assert elf.type == expected_type, "Fizz buzz should match the expected ELF binary type." def test_file_destination(): @@ -343,9 +319,9 @@ def test_file_destination(): arch_directory = os.path.dirname(arch_file.destination) fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) fizz_buzz_directory = os.path.dirname(fizz_buzz_file.destination) - assert ( - arch_directory == fizz_buzz_directory - ), "Executable and non-executable files should be written to the same directory." + assert arch_directory == fizz_buzz_directory, ( + "Executable and non-executable files should be written to the same directory." + ) def test_file_executable(): @@ -366,15 +342,13 @@ def test_file_hash(): amazon_file = File(os.path.join(ldd_output_directory, "htop-amazon-linux.txt")) arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) assert amazon_file.hash != arch_file.hash, "The hashes should differ." - assert ( - len(amazon_file.hash) == len(arch_file.hash) == 64 - ), "The hashes should have a consistent length of 64 characters." + assert len(amazon_file.hash) == len(arch_file.hash) == 64, ( + "The hashes should have a consistent length of 64 characters." + ) # Found by executing `sha256sum fizz-buzz`. expected_hash = "d54ab4714215d7822bf490df5cdf49bc3f32b4c85a439b109fc7581355f9d9c5" - assert ( - File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash - ), "Hashes should match." + assert File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash, "Hashes should match." @pytest.mark.parametrize( @@ -388,9 +362,9 @@ def test_file_hash(): def test_file_requires_launcher(fizz_buzz): file = File(fizz_buzz, chroot=chroot) assert file.requires_launcher, "Fizz buzz should require a launcher." - assert all( - not dependency.requires_launcher for dependency in file.elf.dependencies - ), "All of the dependencies should not require launchers." + assert all(not dependency.requires_launcher for dependency in file.elf.dependencies), ( + "All of the dependencies should not require launchers." + ) def test_file_symlink(): @@ -427,8 +401,7 @@ def test_parse_dependencies_from_ldd_output(filename_prefix): expected_dependencies = [line for line in f.read().split("\n") if len(line)] assert set(dependencies) == set(expected_dependencies), ( - 'The dependencies were not parsed correctly from ldd output for "%s"' - % filename_prefix + 'The dependencies were not parsed correctly from ldd output for "%s"' % filename_prefix ) @@ -439,9 +412,9 @@ def test_resolve_binary(): try: os.environ["PATH"] = "%s%s%s" % (binary_directory, os.pathsep, old_path) resolved_binary = resolve_binary(binary) - assert resolved_binary == os.path.normpath( - fizz_buzz_glibc_32 - ), "The full binary path was not resolved correctly." + assert resolved_binary == os.path.normpath(fizz_buzz_glibc_32), ( + "The full binary path was not resolved correctly." + ) finally: os.environ["PATH"] = old_path @@ -451,15 +424,15 @@ def test_resolve_file_path(): resolve_file_path(chroot) with pytest.raises(Exception): resolve_file_path(os.path.join(chroot, "non-existent-file")) - assert os.path.isabs( - resolve_file_path(fizz_buzz_glibc_32) - ), "The resolved path should be absolute." + assert os.path.isabs(resolve_file_path(fizz_buzz_glibc_32)), ( + "The resolved path should be absolute." + ) def test_run_ldd(): - assert any( - "libc.so" in line for line in run_ldd(ldd, fizz_buzz_glibc_32) - ), '"libc" was not found in the output of "ldd" for the executable.' + assert any("libc.so" in line for line in run_ldd(ldd, fizz_buzz_glibc_32)), ( + '"libc" was not found in the output of "ldd" for the executable.' + ) def test_stored_property(): diff --git a/tests/test_cli.py b/tests/test_cli.py index c20a28c..ad7bd82 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,9 +41,7 @@ def test_adding_additional_files(capsys): args = ["--chroot", chroot, "--output", "-", "--tarball", fizz_buzz_glibc_32] stdin = "\n".join((fizz_buzz_glibc_32_exe, fizz_buzz_glibc_64)) returncode, stdout, stderr = run_exodus(args, universal_newlines=False, stdin=stdin) - assert ( - returncode == 0 - ), "Exodus should have exited with a success status code, but didn't." + assert returncode == 0, "Exodus should have exited with a success status code, but didn't." stream = io.BytesIO(stdout) with tarfile.open(fileobj=stream, mode="r:gz") as f: names = f.getnames() @@ -80,16 +78,12 @@ def test_missing_binary(capsys): command = "this-is-almost-definitely-not-going-to-be-a-command-anywhere" returncode, stdout, stderr = run_exodus([command]) assert returncode != 0, "Running exodus should have failed." - assert ( - "Traceback" not in stderr - ), "Traceback should not be included without the --verbose flag." + assert "Traceback" not in stderr, "Traceback should not be included without the --verbose flag." # With the --verbose flag. returncode, stdout, stderr = run_exodus(["--verbose", command]) assert returncode != 0, "Running exodus should have failed." - assert ( - "Traceback" in stderr - ), "Traceback should be included with the --verbose flag." + assert "Traceback" in stderr, "Traceback should be included with the --verbose flag." def test_required_argument(): @@ -115,9 +109,7 @@ def test_writing_bundle_to_disk(): args = ["--chroot", chroot, "--output", filename, fizz_buzz_glibc_32] try: returncode, stdout, stderr = run_exodus(args) - assert ( - returncode == 0 - ), "Exodus should have exited with a success status code, but didn't." + assert returncode == 0, "Exodus should have exited with a success status code, but didn't." with open(filename, "rb") as f_in: first_line = f_in.readline().strip() assert first_line == b"#! /bin/bash", stderr @@ -129,9 +121,7 @@ def test_writing_bundle_to_disk(): def test_writing_bundle_to_stdout(): args = ["--chroot", chroot, "--output", "-", fizz_buzz_glibc_32] returncode, stdout, stderr = run_exodus(args) - assert ( - returncode == 0 - ), "Exodus should have exited with a success status code, but didn't." + assert returncode == 0, "Exodus should have exited with a success status code, but didn't." assert stdout.startswith("#! /bin/sh"), stderr @@ -141,9 +131,7 @@ def test_writing_tarball_to_disk(): args = ["--chroot", chroot, "--output", filename, "--tarball", fizz_buzz_glibc_32] try: returncode, stdout, stderr = run_exodus(args) - assert ( - returncode == 0 - ), "Exodus should have exited with a success status code, but didn't." + assert returncode == 0, "Exodus should have exited with a success status code, but didn't." assert tarfile.is_tarfile(filename), stderr with tarfile.open(filename, mode="r:gz") as f_in: assert "exodus/bin/fizz-buzz-glibc-32" in f_in.getnames() @@ -155,9 +143,7 @@ def test_writing_tarball_to_disk(): def test_writing_tarball_to_stdout(): args = ["--chroot", chroot, "--output", "-", "--tarball", fizz_buzz_glibc_32] returncode, stdout, stderr = run_exodus(args, universal_newlines=False) - assert ( - returncode == 0 - ), "Exodus should have exited with a success status code, but didn't." + assert returncode == 0, "Exodus should have exited with a success status code, but didn't." stream = io.BytesIO(stdout) with tarfile.open(fileobj=stream, mode="r:gz") as f: assert "exodus/bin/fizz-buzz-glibc-32" in f.getnames(), stderr diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index 2ca3453..e450ac9 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -15,12 +15,10 @@ def test_extract_exec_path(): line = 'execve("/usr/bin/ls", ["ls"], 0x7ffea775ad70 /* 113 vars */) = 0' - assert ( - extract_exec_path(line) == "/usr/bin/ls" - ), "It should have extracted the path to the ls executable." - assert ( - extract_exec_path("blah") is None - ), "It should return `None` when there is no match." + assert extract_exec_path(line) == "/usr/bin/ls", ( + "It should have extracted the path to the ls executable." + ) + assert extract_exec_path("blah") is None, "It should return `None` when there is no match." def test_extract_no_paths(): @@ -35,17 +33,15 @@ def test_extract_open_path(): ) assert extract_open_path(line) is None, "Missing files should not return paths." line = 'open(".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4' - assert ( - extract_open_path(line) is None - ), "Opened directories should not return paths." + assert extract_open_path(line) is None, "Opened directories should not return paths." line = 'open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 4' - assert ( - extract_open_path(line) == "/usr/lib/locale/locale-archive" - ), "An open() call should return a path." + assert extract_open_path(line) == "/usr/lib/locale/locale-archive", ( + "An open() call should return a path." + ) line = 'openat(AT_FDCWD, "/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 4' - assert ( - extract_open_path(line) == "/usr/lib/libc.so.6" - ), "An openat() call relative to the current directory should return a path." + assert extract_open_path(line) == "/usr/lib/libc.so.6", ( + "An openat() call relative to the current directory should return a path." + ) def test_extract_raw_paths(): @@ -54,14 +50,12 @@ def test_extract_raw_paths(): "./relative/path", "/another/absolute/path", ] - input_paths_with_whitespace = ( - [" ", ""] + [input_paths[0]] + [" "] + input_paths[1:] - ) + input_paths_with_whitespace = [" ", ""] + [input_paths[0]] + [" "] + input_paths[1:] input_content = "\n".join(input_paths_with_whitespace) extracted_paths = extract_paths(input_content) - assert set(input_paths) == set( - extracted_paths - ), "The paths should have been extracted without the whitespace." + assert set(input_paths) == set(extracted_paths), ( + "The paths should have been extracted without the whitespace." + ) def test_extract_stat_path(): @@ -70,16 +64,12 @@ def test_extract_stat_path(): "{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0" ) expected_path = "/usr/local/lib/python3.6/encodings/__init__.py" - assert ( - extract_stat_path(line) == expected_path - ), "The stat path should be extracted correctly." + assert extract_stat_path(line) == expected_path, "The stat path should be extracted correctly." line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.abi3.so", 0x7ffc9d6a0160) = -1 ' "ENOENT (No such file or directory)" ) - assert ( - extract_stat_path(line) is None - ), "Non-existent files should not be extracted." + assert extract_stat_path(line) is None, "Non-existent files should not be extracted." def test_extract_strace_paths(): @@ -96,9 +86,7 @@ def test_extract_strace_paths(): ] for path in expected_paths: - assert path in extracted_paths, ( - '"%s" should be present in the extracted paths.' % path - ) + assert path in extracted_paths, '"%s" should be present in the extracted paths.' % path def test_strip_pid_prefix(): @@ -107,6 +95,4 @@ def test_strip_pid_prefix(): '"/tmp/exodus-bundle-fqzw_lds.c", "-o", "/tmp/exodus-bundle-3p_c0osh"], [/* 45 vars */] ' "" ) - assert strip_pid_prefix(line).startswith( - "execve(" - ), "The PID prefix should be stripped." + assert strip_pid_prefix(line).startswith("execve("), "The PID prefix should be stripped." diff --git a/tests/test_launchers.py b/tests/test_launchers.py index d0b73b0..9a62e21 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -20,9 +20,7 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) chroot = os.path.join(parent_directory, "data", "binaries", "chroot") echo_args_glibc_32 = os.path.join(chroot, "bin", "echo-args-glibc-32") -fizz_buzz_source_file = os.path.join( - parent_directory, "data", "binaries", "fizz-buzz.c" -) +fizz_buzz_source_file = os.path.join(parent_directory, "data", "binaries", "fizz-buzz.c") def test_construct_bash_launcher(): @@ -73,13 +71,11 @@ def test_find_executable(): binary_symlink = os.path.join(root_directory, "bin", binary_name) binary_path = os.path.realpath(binary_symlink) # This is a pretend directory, but it doesn't check. - launchers.parent_directory = os.path.join( - os.path.dirname(binary_path), "somewhere", "else" - ) + launchers.parent_directory = os.path.join(os.path.dirname(binary_path), "somewhere", "else") os.environ["PATH"] = os.path.dirname(echo_args_glibc_32) - assert ( - find_executable(binary_name, skip_original_for_testing=True) == binary_path - ), ('It should have found the binary path "%s".' % binary_path) + assert find_executable(binary_name, skip_original_for_testing=True) == binary_path, ( + 'It should have found the binary path "%s".' % binary_path + ) finally: launchers.parent_directory = original_parent_directory os.environ["PATH"] = original_environment