diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..abb9894 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,70 @@ +name: Release + +on: + push: + tags: + - 'v*' + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y dietlibc-dev gcc musl musl-tools + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r development-requirements.txt + pip install -e . + + - name: Run tests + run: pytest --cov -v + + build-and-publish: + needs: test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Check package + run: twine check dist/* + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: dist/* + generate_release_notes: true + + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: twine upload dist/* \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..68fb501 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,119 @@ +name: Test + +on: + push: + branches: [ master, main ] + pull_request: + branches: [ master, main ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y dietlibc-dev gcc musl musl-tools + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + .tox + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/development-requirements.txt', '**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.python-version }}- + ${{ runner.os }}-pip- + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r development-requirements.txt + pip install -e . + + - name: Run tests with pytest + run: | + pytest --cov --cov-report=term-missing --cov-report=xml -v + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + fail_ci_if_error: false + + - name: Test package build + run: | + python -m build + + - name: Test basic functionality + run: | + exodus --help + echo "Testing basic bundling (will use shell launchers without musl/diet)" + # Test bundling a simple command - this will fail gracefully if binary not found + timeout 30s exodus --shell-launchers /bin/echo --output test-bundle.sh || echo "Expected: binary bundling test completed" + + test-tox: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y dietlibc-dev gcc musl musl-tools + + - name: Install tox + run: | + python -m pip install --upgrade pip + pip install tox + + - name: Run tox + run: tox + + package: + runs-on: ubuntu-latest + needs: [test, test-tox] + if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Check package + run: twine check dist/* + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: packages + path: dist/ \ No newline at end of file diff --git a/conftest.py b/conftest.py index 98c75b2..044963b 100644 --- a/conftest.py +++ b/conftest.py @@ -1 +1 @@ -collect_ignore = ['setup.py'] +collect_ignore = ["setup.py"] diff --git a/development-requirements.txt b/development-requirements.txt index 9eeef10..1b7ac21 100644 --- a/development-requirements.txt +++ b/development-requirements.txt @@ -1,12 +1,14 @@ -bumpversion==0.5.3 -coverage>=5.4 -m2r>=0.1.12 -pluggy==0.5.2 -py==1.4.34 -pytest==3.2.3 -pytest-sugar==0.9.0 -pytest-watch==4.1.0 -six==1.11.0 -tox==2.9.1 -twine==1.9.1 -virtualenv==15.1.0 +bumpversion>=0.6.0 +build>=1.0 +coverage>=7.0 +pytest>=8.0 +pytest-sugar>=1.0 +pytest-watch>=4.2 +pytest-cov>=5.0 +tox>=4.0 +twine>=5.0 +ruff>=0.4.0 +check-manifest>=0.49 +docutils>=0.20 +readme-renderer>=43.0 +pygments>=2.17 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..94c23a5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,89 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "exodus-bundler" +version = "3.0.0" +description = "The exodus application bundler." +readme = {file = "README.md", content-type = "text/markdown"} +license = {text = "BSD"} +authors = [ + {name = "Intoli", email = "contact@intoli.com"} +] +maintainers = [ + {name = "Intoli", email = "contact@intoli.com"} +] +keywords = ["linux", "executable", "elf", "binaries"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: System :: Archiving :: Packaging", + "Topic :: Utilities", +] +requires-python = ">=3.10" +dependencies = [] + +[project.urls] +Homepage = "https://github.com/intoli/exodus" +Repository = "https://github.com/intoli/exodus" +Issues = "https://github.com/intoli/exodus/issues" + +[project.scripts] +exodus = "exodus_bundler.cli:main" + +[tool.setuptools] +package-dir = {"" = "src"} +include-package-data = true +zip-safe = false + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "W"] +ignore = ["E501"] # Ignore line length for now + +[tool.ruff.format] +line-ending = "lf" +[tool.pytest.ini_options] +norecursedirs = [".git", ".tox", ".env", "dist", "build"] +python_files = ["test_*.py", "*_test.py", "tests.py"] +addopts = [ + "-rxEfsw", + "--strict-markers", + "--doctest-modules", + "--doctest-glob=*.rst", + "--tb=short" +] +testpaths = ["tests"] + +[tool.coverage.run] +source = ["src"] +omit = ["*/tests/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError" +] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index a530891..7447b35 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,10 +4,6 @@ universal = 1 [metadata] license_file = LICENSE.md -[flake8] -ignore = E128 -max-line-length = 100 - [tool:pytest] norecursedirs = .git @@ -24,12 +20,4 @@ addopts = --strict --doctest-modules --doctest-glob=\*.rst - --tb=short - -[isort] -force_single_line = True -line_length = 100 -lines_after_imports = 2 -known_first_party = exodus_bundler -default_section = THIRDPARTY -not_skip = __init__.py + --tb=short \ No newline at end of file diff --git a/setup.py b/setup.py index 2409e74..2544daf 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function from glob import glob from os.path import basename @@ -12,56 +10,52 @@ setup( - name='exodus-bundler', - version='2.0.4', - license='BSD', - platforms=['Linux'], - description='The exodus application bundler.', - long_description='See the documentation for details.', - author='Intoli', - author_email='contact@intoli.com', - url='https://github.com/intoli/exodus', - packages=find_packages('src'), - package_dir={'': 'src'}, - py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + name="exodus-bundler", + version="3.0.0", + license="BSD", + platforms=["Linux"], + description="The exodus application bundler.", + long_description="See the documentation for details.", + author="Intoli", + author_email="contact@intoli.com", + url="https://github.com/intoli/exodus", + packages=find_packages("src"), + package_dir={"": "src"}, + py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], include_package_data=True, zip_safe=False, classifiers=[ # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: BSD License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.0', - 'Programming Language :: Python :: 3.1', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: System :: Archiving :: Packaging', - 'Topic :: Utilities', + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: System :: Archiving :: Packaging", + "Topic :: Utilities", ], keywords=[ - 'linux', 'executable', 'elf', 'binaries', - ], - install_requires=[ + "linux", + "executable", + "elf", + "binaries", ], + python_requires=">=3.10", + install_requires=[], entry_points={ - 'console_scripts': [ - 'exodus = exodus_bundler.cli:main', + "console_scripts": [ + "exodus = exodus_bundler.cli:main", ], }, ) diff --git a/src/exodus_bundler/__init__.py b/src/exodus_bundler/__init__.py index 529155b..47125e9 100644 --- a/src/exodus_bundler/__init__.py +++ b/src/exodus_bundler/__init__.py @@ -1,7 +1,7 @@ import logging -__version__ = '2.0.4' +__version__ = "3.0.0" root_logger = logging.getLogger(__name__) root_logger.handlers = [logging.NullHandler()] diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index dd8f469..4da55eb 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -32,88 +32,112 @@ logger = logging.getLogger(__name__) -def bytes_to_int(bytes, byteorder='big'): +def bytes_to_int(bytes, byteorder="big"): """Simple helper function to convert byte strings into integers.""" - endian = {'big': '>', 'little': '<'}[byteorder] - chars = struct.unpack(endian + ('B' * len(bytes)), bytes) - if byteorder == 'big': + endian = {"big": ">", "little": "<"}[byteorder] + chars = struct.unpack(endian + ("B" * len(bytes)), bytes) + if byteorder == "big": chars = chars[::-1] - return sum(int(char) * 256 ** i for (i, char) in enumerate(chars)) - - -def create_bundle(executables, output, tarball=False, rename=[], chroot=None, add=[], - no_symlink=[], shell_launchers=False, detect=False): + return sum(int(char) * 256**i for (i, char) in enumerate(chars)) + + +def create_bundle( + executables, + output, + tarball=False, + rename=[], + chroot=None, + add=[], + no_symlink=[], + shell_launchers=False, + detect=False, +): """Handles the creation of the full bundle.""" # Initialize these ahead of time so they're always available for error handling. output_filename, output_file, root_directory = None, None, None try: - # Create a temporary unpackaged bundle for the executables. root_directory = create_unpackaged_bundle( - executables, rename=rename, chroot=chroot, add=add, no_symlink=no_symlink, - shell_launchers=shell_launchers, detect=detect, + executables, + rename=rename, + chroot=chroot, + add=add, + no_symlink=no_symlink, + shell_launchers=shell_launchers, + detect=detect, ) # Populate the filename template. - output_filename = render_template(output, - executables=('-'.join(os.path.basename(executable) for executable in executables)), - extension=('tgz' if tarball else 'sh'), + output_filename = render_template( + output, + executables=("-".join(os.path.basename(executable) for executable in executables)), + extension=("tgz" if tarball else "sh"), ) # Store a gzipped tarball of the bundle in memory. tar_stream = io.BytesIO() - with tarfile.open(fileobj=tar_stream, mode='w:gz') as tar: - tar.add(root_directory, arcname='exodus') + with tarfile.open(fileobj=tar_stream, mode="w:gz") as tar: + tar.add(root_directory, arcname="exodus") # Configure the appropriate output mechanism. - if output_filename == '-': - output_file = getattr(sys.stdout, 'buffer', sys.stdout) + if output_filename == "-": + output_file = sys.stdout.buffer else: - output_file = open(output_filename, 'wb') + output_file = open(output_filename, "wb") # Construct the installation script and write it out. if not tarball: - if output_filename == '-': - base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode('utf-8') - script_content = render_template_file('install-bundle-noninteractive.sh', - base64_encoded_tarball=base64_encoded_tarball) - output_file.write(script_content.encode('utf-8')) + if output_filename == "-": + base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode("utf-8") + script_content = render_template_file( + "install-bundle-noninteractive.sh", + base64_encoded_tarball=base64_encoded_tarball, + ) + output_file.write(script_content.encode("utf-8")) else: - output_file.write(render_template_file('install-bundle.sh').encode('utf-8')) + output_file.write(render_template_file("install-bundle.sh").encode("utf-8")) output_file.write(tar_stream.getvalue()) else: # Or just write out the tarball. output_file.write(tar_stream.getvalue()) # Write out the success message. - logger.info('Successfully created "%s".' % output_filename) + logger.info(f'Successfully created "{output_filename}".') return True - except: # noqa: E722 + except Exception: raise finally: if root_directory: shutil.rmtree(root_directory) if output_file and output_filename: output_file.close() - if not tarball and output_filename not in ['-', '/dev/null']: + if not tarball and output_filename not in ["-", "/dev/null"]: st = os.stat(output_filename) os.chmod(output_filename, st.st_mode | stat.S_IEXEC) -def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_symlink=[], - shell_launchers=False, detect=False): +def create_unpackaged_bundle( + executables, + rename=[], + chroot=None, + add=[], + no_symlink=[], + shell_launchers=False, + detect=False, +): """Creates a temporary directory containing the unpackaged contents of the bundle.""" bundle = Bundle(chroot=chroot, working_directory=True) try: # Sanitize the inputs. - assert len(executables), 'No executables were specified.' - assert len(executables) >= len(rename), \ - 'More renamed options were included than executables.' + assert len(executables), "No executables were specified." + assert len(executables) >= len(rename), ( + "More renamed options were included than executables." + ) # Pad the rename's with `True` so that `entry_point` can be specified. entry_points = rename + [True for i in range(len(executables) - len(rename))] # Populate the bundle with main executable files and their dependencies. - for (executable, entry_point) in zip(executables, entry_points): + for executable, entry_point in zip(executables, entry_points): file = bundle.add_file(executable, entry_point=entry_point) # We'll only auto-detect dependencies for these entry points as well. @@ -122,11 +146,13 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_sym dependency_paths = detect_dependencies(file.path) if not dependency_paths: raise DependencyDetectionError( - ('Automatic dependency detection failed. Either "%s" ' % file.path) + - 'is not tracked by your package manager, or your operating system ' - 'is not currently compatible with the `--detect` option. If not, please ' - "create an issue at https://github.com/intoli/exodus and we'll try our " - ' to add support for it in the future.', + ( + 'Automatic dependency detection failed. Either "%s" ' % file.path + + "is not tracked by your package manager, or your operating system " + + "is not currently compatible with the `--detect` option. If not, please " + + "create an issue at https://github.com/intoli/exodus and we'll try our " + + " to add support for it in the future." + ), ) for path in dependency_paths: @@ -146,7 +172,7 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_sym bundle.create_bundle(shell_launchers=shell_launchers) return bundle.working_directory - except: # noqa: E722 + except Exception: bundle.delete_working_directory() raise @@ -154,28 +180,28 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_sym def detect_elf_binary(filename): """Returns `True` if a file has an ELF header.""" if not os.path.exists(filename): - raise MissingFileError('The "%s" file was not found.' % filename) + raise MissingFileError(f'The "{filename}" file was not found.') - with open(filename, 'rb') as f: + with open(filename, "rb") as f: first_four_bytes = f.read(4) - return first_four_bytes == b'\x7fELF' + return first_four_bytes == b"\x7fELF" def parse_dependencies_from_ldd_output(content): """Takes the output of `ldd` as a string or list of lines and parses the dependencies.""" - if type(content) == str: - content = content.split('\n') + if isinstance(content, str): + content = content.split("\n") dependencies = [] for line in content: # This first one is a special case of invoking the linker as `ldd`. - if re.search(r'^\s*(/.*?)\s*=>\s*ldd\s*\(', line): + if re.search(r"^\s*(/.*?)\s*=>\s*ldd\s*\(", line): # We'll exclude this because it's the hardcoded INTERP path, and it would be # impossible to get the full path from this command output. continue - match = re.search(r'=>\s*(/.*?)\s*\(', line) - match = match or re.search(r'\s*(/.*?)\s*\(', line) + match = re.search(r"=>\s*(/.*?)\s*\(", line) + match = match or re.search(r"\s*(/.*?)\s*\(", line) if match: dependencies.append(match.group(1)) @@ -186,12 +212,12 @@ def resolve_binary(binary): """Attempts to find the absolute path to the binary.""" absolute_binary_path = os.path.normpath(os.path.abspath(binary)) if not os.path.exists(absolute_binary_path): - for path in os.getenv('PATH', '/bin/:/usr/bin/').split(os.pathsep): + for path in os.getenv("PATH", "/bin/:/usr/bin/").split(os.pathsep): absolute_binary_path = os.path.normpath(os.path.abspath(os.path.join(path, binary))) if os.path.exists(absolute_binary_path): break else: - raise MissingFileError('The "%s" binary could not be found in $PATH.' % binary) + raise MissingFileError(f'The "{binary}" binary could not be found in $PATH.') return absolute_binary_path @@ -208,26 +234,27 @@ def resolve_file_path(path, search_environment_path=False): if search_environment_path: path = resolve_binary(path) if not os.path.exists(path): - raise MissingFileError('The "%s" file was not found.' % path) + raise MissingFileError(f'The "{path}" file was not found.') if os.path.isdir(path): - raise UnexpectedDirectoryError('"%s" is a directory, not a file.' % path) + raise UnexpectedDirectoryError(f'"{path}" is a directory, not a file.') return os.path.normpath(os.path.abspath(path)) def run_ldd(ldd, binary): """Runs `ldd` and gets the combined stdout/stderr output as a list of lines.""" if not detect_elf_binary(resolve_binary(binary)): - raise InvalidElfBinaryError('The "%s" file is not a binary ELF file.' % binary) + raise InvalidElfBinaryError(f'The "{binary}" file is not a binary ELF file.') process = Popen([ldd, binary], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - return stdout.decode('utf-8').split('\n') + stderr.decode('utf-8').split('\n') + return stdout.decode("utf-8").split("\n") + stderr.decode("utf-8").split("\n") -class stored_property(object): +class stored_property: """Simple decorator for a class property that will be cached indefinitely.""" + def __init__(self, function): - self.__doc__ = getattr(function, '__doc__') + self.__doc__ = getattr(function, "__doc__") self.function = function def __get__(self, instance, type): @@ -235,7 +262,7 @@ def __get__(self, instance, type): return result -class Elf(object): +class Elf: """Parses basic attributes from the ELF header of a file. Attributes: @@ -246,6 +273,7 @@ class Elf(object): path (str): The path to the file. type (str): The binary type, one of 'relocatable', 'executable', 'shared', or 'core'. """ + def __init__(self, path, chroot=None, file_factory=None): """Constructs the `Elf` instance. @@ -256,61 +284,65 @@ def __init__(self, path, chroot=None, file_factory=None): file_factory (function, optional): A function to use when creating new `File` instances. """ if not os.path.exists(path): - raise MissingFileError('The "%s" file was not found.' % path) + raise MissingFileError(f'The "{path}" file was not found.') self.path = path self.chroot = chroot self.file_factory = file_factory or File - with open(path, 'rb') as f: + with open(path, "rb") as f: # Make sure that this is actually an ELF binary. first_four_bytes = f.read(4) - if first_four_bytes != b'\x7fELF': - raise InvalidElfBinaryError('The "%s" file is not a binary ELF file.' % path) + if first_four_bytes != b"\x7fELF": + raise InvalidElfBinaryError(f'The "{path}" file is not a binary ELF file.') # Determine whether this is a 32-bit or 64-bit file. format_byte = f.read(1) - self.bits = {b'\x01': 32, b'\x02': 64}.get(format_byte) + self.bits = {b"\x01": 32, b"\x02": 64}.get(format_byte) if not self.bits: raise UnsupportedArchitectureError( - ('The "%s" file does not appear to be either 32 or 64 bits. ' % path) + - 'Other architectures are not currently supported, but you can open an ' - 'issue at https://github.com/intoli/exodus stating your use-case and ' - 'support might get extended in the future.', + ( + 'The "%s" file does not appear to be either 32 or 64 bits. ' % path + + "Other architectures are not currently supported, but you can open an " + + "issue at https://github.com/intoli/exodus stating your use-case and " + + "support might get extended in the future." + ), ) # Determine whether it's big or little endian and construct an integer parsing function. endian_byte = f.read(1) - byteorder = {b'\x01': 'little', b'\x02': 'big'}[endian_byte] - assert byteorder == 'little', 'Big endian is not supported right now.' + byteorder = {b"\x01": "little", b"\x02": "big"}[endian_byte] + assert byteorder == "little", "Big endian is not supported right now." if not byteorder: raise UnsupportedArchitectureError( - ('The "%s" file does not appear to be little endian, ' % path) + - 'and big endian binaries are not currently supported. You can open an ' - 'issue at https://github.com/intoli/exodus stating your use-case and ' - 'support might get extended in the future.', + ( + 'The "%s" file does not appear to be little endian, ' % path + + "and big endian binaries are not currently supported. You can open an " + + "issue at https://github.com/intoli/exodus stating your use-case and " + + "support might get extended in the future." + ), ) def hex(bytes): return bytes_to_int(bytes, byteorder=byteorder) # Determine the type of the binary. - f.seek(hex(b'\x10')) + f.seek(hex(b"\x10")) e_type = hex(f.read(2)) - self.type = {1: 'relocatable', 2: 'executable', 3: 'shared', 4: 'core'}[e_type] + self.type = {1: "relocatable", 2: "executable", 3: "shared", 4: "core"}[e_type] # Find the program header offset. - e_phoff_start = {32: hex(b'\x1c'), 64: hex(b'\x20')}[self.bits] + e_phoff_start = {32: hex(b"\x1c"), 64: hex(b"\x20")}[self.bits] e_phoff_length = {32: 4, 64: 8}[self.bits] f.seek(e_phoff_start) e_phoff = hex(f.read(e_phoff_length)) # Determine the size of a program header entry. - e_phentsize_start = {32: hex(b'\x2a'), 64: hex(b'\x36')}[self.bits] + e_phentsize_start = {32: hex(b"\x2a"), 64: hex(b"\x36")}[self.bits] f.seek(e_phentsize_start) e_phentsize = hex(f.read(2)) # Determine the number of program header entries. - e_phnum_start = {32: hex(b'\x2c'), 64: hex(b'\x38')}[self.bits] + e_phnum_start = {32: hex(b"\x2c"), 64: hex(b"\x38")}[self.bits] f.seek(e_phnum_start) e_phnum = hex(f.read(2)) @@ -323,17 +355,17 @@ def hex(bytes): # A p_type of \x03 corresponds to a PT_INTERP header (e.g. the linker). if len(p_type) == 0: break - if not p_type == b'\x03\x00\x00\x00': + if not p_type == b"\x03\x00\x00\x00": continue # Determine the offset for the segment. - p_offset_start = header_start + {32: hex(b'\04'), 64: hex(b'\x08')}[self.bits] + p_offset_start = header_start + {32: hex(b"\04"), 64: hex(b"\x08")}[self.bits] p_offset_length = {32: 4, 64: 8}[self.bits] f.seek(p_offset_start) p_offset = hex(f.read(p_offset_length)) # Determine the size of the segment. - p_filesz_start = header_start + {32: hex(b'\x10'), 64: hex(b'\x20')}[self.bits] + p_filesz_start = header_start + {32: hex(b"\x10"), 64: hex(b"\x20")}[self.bits] p_filesz_length = {32: 4, 64: 8}[self.bits] f.seek(p_filesz_start) p_filesz = hex(f.read(p_filesz_length)) @@ -342,11 +374,14 @@ def hex(bytes): f.seek(p_offset) segment = f.read(p_filesz) # It should be null-terminated (b'\x00' in Python 2, 0 in Python 3). - assert segment[-1] in [b'\x00', 0], 'The string should be null terminated.' - assert self.linker_file is None, 'More than one linker found.' - linker_path = segment[:-1].decode('ascii') + assert segment[-1] in [ + b"\x00", + 0, + ], "The string should be null terminated." + assert self.linker_file is None, "More than one linker found." + linker_path = segment[:-1].decode("ascii") if chroot: - linker_path = os.path.join(chroot, os.path.relpath(linker_path, '/')) + linker_path = os.path.join(chroot, os.path.relpath(linker_path, "/")) self.linker_file = self.file_factory(linker_path, chroot=self.chroot) def __eq__(self, other): @@ -357,7 +392,7 @@ def __hash__(self): return hash(self.path) def __repr__(self): - return '' % self.path + return f'' def find_direct_dependencies(self, linker_file=None): """Runs the specified linker and returns a set of the dependencies as `File` instances.""" @@ -367,31 +402,37 @@ def find_direct_dependencies(self, linker_file=None): linker_path = linker_file.path environment = {} environment.update(os.environ) - environment['LD_TRACE_LOADED_OBJECTS'] = '1' + environment["LD_TRACE_LOADED_OBJECTS"] = "1" extra_ldd_arguments = [] if self.chroot: - ld_library_path = '/lib64:/usr/lib64:/lib/:/usr/lib:/lib32/:/usr/lib32/:' - ld_library_path += environment.get('LD_LIBRARY_PATH', '') + ld_library_path = "/lib64:/usr/lib64:/lib/:/usr/lib:/lib32/:/usr/lib32/:" + ld_library_path += environment.get("LD_LIBRARY_PATH", "") directories = [] - for directory in ld_library_path.split(':'): + for directory in ld_library_path.split(":"): if os.path.isabs(directory): - directory = os.path.join(self.chroot, os.path.relpath(directory, '/')) + directory = os.path.join(self.chroot, os.path.relpath(directory, "/")) directories.append(directory) - ld_library_path = ':'.join(directories) - environment['LD_LIBRARY_PATH'] = ld_library_path + ld_library_path = ":".join(directories) + environment["LD_LIBRARY_PATH"] = ld_library_path # We only need to avoid including system dependencies if there's a chroot set. - extra_ldd_arguments += ['--inhibit-cache', '--inhibit-rpath', ''] - - process = Popen(['ldd'] + extra_ldd_arguments + [self.path], - executable=linker_path, stdout=PIPE, stderr=PIPE, env=environment) + extra_ldd_arguments += ["--inhibit-cache", "--inhibit-rpath", ""] + + process = Popen( + ["ldd"] + extra_ldd_arguments + [self.path], + executable=linker_path, + stdout=PIPE, + stderr=PIPE, + env=environment, + ) stdout, stderr = process.communicate() - combined_output = stdout.decode('utf-8').split('\n') + stderr.decode('utf-8').split('\n') + combined_output = stdout.decode("utf-8").split("\n") + stderr.decode("utf-8").split("\n") # Note that we're explicitly adding the linker because when we invoke it as `ldd` we can't # extract the real path from the trace output. Even if it were here twice, it would be # deduplicated though the use of a set. filenames = parse_dependencies_from_ldd_output(combined_output) + [linker_path] - return set(self.file_factory(filename, chroot=self.chroot, library=True) - for filename in filenames) + return set( + self.file_factory(filename, chroot=self.chroot, library=True) for filename in filenames + ) @stored_property def dependencies(self): @@ -404,7 +445,8 @@ def dependencies(self): for dependency in unprocessed_dependencies: if dependency.elf: new_dependencies |= set( - dependency.elf.find_direct_dependencies(self.linker_file)) + dependency.elf.find_direct_dependencies(self.linker_file) + ) unprocessed_dependencies = new_dependencies - all_dependencies return all_dependencies @@ -414,7 +456,7 @@ def direct_dependencies(self): return self.find_direct_dependencies() -class File(object): +class File: """Represents a file on disk and provides access to relevant properties and actions. Note: @@ -450,7 +492,7 @@ def __init__(self, path, entry_point=None, chroot=None, library=False, file_fact # Set the entry point for the file. if entry_point is True: - self.entry_point = os.path.basename(self.path).replace(os.sep, '') + self.entry_point = os.path.basename(self.path).replace(os.sep, "") else: self.entry_point = entry_point or None @@ -466,15 +508,18 @@ def __init__(self, path, entry_point=None, chroot=None, library=False, file_fact self.no_symlink = self.entry_point and not self.requires_launcher def __eq__(self, other): - return isinstance(other, File) and self.path == self.path and \ - self.entry_point == self.entry_point + return ( + isinstance(other, File) + and self.path == other.path + and self.entry_point == other.entry_point + ) def __hash__(self): """Computes a hash for the instance unique up to the file path and entry point.""" return hash((self.path, self.entry_point)) def __repr__(self): - return '' % self.path + return f' -1) + full_linker = linker_content.find(b"inhibit-rpath") > -1 # Try a c launcher first and fallback. try: @@ -586,24 +649,34 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli raise CompilerNotFoundError() launcher_content = construct_binary_launcher( - linker=linker, library_path=library_path, executable=executable, - full_linker=full_linker) - with open(source_path, 'wb') as f: + linker=linker, + library_path=library_path, + executable=executable, + full_linker=full_linker, + ) + tt = source_path + ".bin" + with open(tt, "wb") as f: f.write(launcher_content) except CompilerNotFoundError: if not shell_launcher: - logger.warning(( - 'Installing either the musl or diet C libraries will result in more efficient ' - 'launchers (currently using bash fallbacks instead).' - )) + logger.warning( + ( + "Installing either the musl or diet C libraries will result in more efficient " + "launchers (currently using bash fallbacks instead)." + ) + ) launcher_content = construct_bash_launcher( - linker=linker, library_path=library_path, executable=executable, - full_linker=full_linker) - with open(source_path, 'w') as f: + linker=linker, + library_path=library_path, + executable=executable, + full_linker=full_linker, + ) + tt = source_path + ".sh" + with open(tt, "w") as f: f.write(launcher_content) - shutil.copymode(self.path, source_path) + shutil.copymode(self.path, tt) - return os.path.normpath(os.path.abspath(source_path)) + return os.path.normpath(os.path.abspath(tt)) def symlink(self, working_directory, bundle_root): """Creates a relative symlink from the `source` to the `destination`. @@ -632,7 +705,7 @@ def symlink(self, working_directory, bundle_root): @stored_property def destination(self): """str: The relative path for the destination of the actual file contents.""" - return os.path.join('.', 'data', self.hash) + return os.path.join(".", "data", self.hash) @stored_property def executable(self): @@ -646,7 +719,7 @@ def elf(self): @stored_property def hash(self): """str: Computes a hash based on the file content, useful for file deduplication.""" - with open(self.path, 'rb') as f: + with open(self.path, "rb") as f: return hashlib.sha256(f.read()).hexdigest() @stored_property @@ -658,14 +731,14 @@ def requires_launcher(self): # The easy ones. if self.library or not self.elf or not self.elf.linker_file or not self.executable: return False - if self.elf.type == 'executable': + if self.elf.type == "executable": return True if self.entry_point: return True # These will hopefully do more good than harm. - bin_directories = ['/bin/', '/bin32/', '/bin64/'] - lib_directories = ['/lib/', '/lib32/', '/lib64/'] + bin_directories = ["/bin/", "/bin32/", "/bin64/"] + lib_directories = ["/lib/", "/lib32/", "/lib64/"] in_bin_directory = any(directory in self.path for directory in bin_directories) in_lib_directory = any(directory in self.path for directory in lib_directories) if in_bin_directory and not in_lib_directory: @@ -674,15 +747,17 @@ def requires_launcher(self): return False # Most libraries will include `.so` in the filename. - return re.search(r'\.so(?:\.|$)', self.path) + if re.search(r"\.so(?:\.|$)", self.path): + return True + return False @stored_property def source(self): """str: The relative path for the source of the actual file contents.""" - return os.path.relpath(self.path, '/') + return os.path.relpath(self.path, "/") -class Bundle(object): +class Bundle: """A collection of files to be included in a bundle and utilities for creating bundles. Attributes: @@ -691,6 +766,7 @@ class Bundle(object): linker_files (:obj:`set` of :obj:`File`): A list of observed linker files. working_directory (str): The root directory where the bundles will be written and packaged. """ + def __init__(self, working_directory=None, chroot=None): """Constructor for the `Bundle` class. @@ -703,7 +779,7 @@ def __init__(self, working_directory=None, chroot=None): """ self.working_directory = working_directory if working_directory is True: - self.working_directory = tempfile.mkdtemp(prefix='exodus-bundle-') + self.working_directory = tempfile.mkdtemp(prefix="exodus-bundle-") # The permissions on the `mkdtemp()` directory will be extremely restricted by default, # so we'll modify them to to reflect the current umask. umask = os.umask(0) @@ -753,10 +829,10 @@ def add_file(self, path, entry_point=None): # We definitely don't want a launcher for this file, so clear the linker. file.elf.linker_file = None else: - logger.warning(( - 'An ELF binary without a suitable linker candidate was encountered. ' - 'Either no linker was found or there are multiple conflicting linkers.' - )) + logger.warning( + "An ELF binary without a suitable linker candidate was encountered. " + "Either no linker was found or there are multiple conflicting linkers." + ) return file @@ -769,18 +845,18 @@ def create_bundle(self, shell_launchers=False): """ file_paths = set() files_needing_launchers = defaultdict(set) + entry_points_needing_launchers = {} # Map file to launcher path for file in self.files: # Store the file path to avoid collisions later. file_path = os.path.join(self.bundle_root, file.source) file_paths.add(file_path) - # Create a symlink in `./bin/` if an entry point is specified. - if file.entry_point: - file.create_entry_point(self.working_directory, self.bundle_root) + # Defer entry point creation until after launchers are created if file.no_symlink: # We'll need to copy the actual file into the bundle subdirectory in this # case so that it can locate resources using paths relative to the executable. + parent_directory = os.path.dirname(file_path) if not os.path.exists(parent_directory): os.makedirs(parent_directory) @@ -792,21 +868,30 @@ def create_bundle(self, shell_launchers=False): if file.requires_launcher: # These are kind of complicated, we'll just store the requirements for now. - directory_and_linker = (os.path.dirname(file_path), file.elf.linker_file) + directory_and_linker = ( + os.path.dirname(file_path), + file.elf.linker_file, + ) files_needing_launchers[directory_and_linker].add(file) else: - file.symlink(working_directory=self.working_directory, bundle_root=self.bundle_root) + file.symlink( + working_directory=self.working_directory, + bundle_root=self.bundle_root, + ) + # Create entry point for non-launcher files + if file.entry_point: + file.create_entry_point(self.working_directory, self.bundle_root) # Now we need to write out one unique copy of each linker in each directory where it's # required. This is necessary so that `readlink("/proc/self/exe")` will return the correct # directory when programs use that to construct relative paths to resources. - for ((directory, linker), executable_files) in files_needing_launchers.items(): + for (directory, linker), executable_files in files_needing_launchers.items(): # First, we'll find a unique name for the linker in this directory and write it out. - desired_linker_path = os.path.join(directory, 'linker-%s' % linker.hash) + desired_linker_path = os.path.join(directory, f"linker-{linker.hash}") linker_path = desired_linker_path iteration = 2 while linker_path in file_paths: - linker_path = '%s-%d' % (desired_linker_path, iteration) + linker_path = f"{desired_linker_path}-{iteration}" iteration += 1 file_paths.add(linker_path) linker_dirname, linker_basename = os.path.split(linker_path) @@ -819,17 +904,28 @@ def create_bundle(self, shell_launchers=False): # We'll again attempt to find a unique available name, this time for the symlink # to the executable. file_basename = file.entry_point or os.path.basename(file.path) - desired_symlink_path = os.path.join(directory, '%s-x' % file_basename) + desired_symlink_path = os.path.join(directory, file_basename) symlink_path = desired_symlink_path - iteration = 2 - while symlink_path in file_paths: - symlink_path = '%s-%d' % (desired_symlink_path, iteration) - iteration += 1 + # iteration = 2 + # while symlink_path in file_paths: + # symlink_path = '%s-%d' % (desired_symlink_path, iteration) + # iteration += 1 file_paths.add(symlink_path) symlink_basename = os.path.basename(symlink_path) - file.create_launcher(self.working_directory, self.bundle_root, - linker_basename, symlink_basename, - shell_launcher=shell_launchers) + launcher_path = file.create_launcher( + self.working_directory, + self.bundle_root, + linker_basename, + symlink_basename, + shell_launcher=shell_launchers, + ) + # Store launcher path for entry point creation + if file.entry_point: + entry_points_needing_launchers[file] = launcher_path + + # Create entry points for files that needed launchers + for file, launcher_path in entry_points_needing_launchers.items(): + file.create_entry_point(self.working_directory, self.bundle_root, launcher_path) def delete_working_directory(self): """Recursively deletes the working directory.""" @@ -853,13 +949,15 @@ def file_factory(self, path, entry_point=None, chroot=None, library=False, file_ path = resolve_file_path(path, search_environment_path=entry_point is not None) file = next((file for file in self.files if file.path == path), None) if file is not None: - assert entry_point == file.entry_point or not entry_point or not file.entry_point, \ + assert entry_point == file.entry_point or not entry_point or not file.entry_point, ( "The entry point property should always persist, but can't conflict." + ) file.entry_point = file.entry_point or entry_point - assert chroot == file.chroot, 'The chroot must match.' + assert chroot == file.chroot, "The chroot must match." file.library = file.library or library - assert not file.entry_point or not file.library, \ + assert not file.entry_point or not file.library, ( "A file can't be both an entry point and a library." + ) return file return File(path, entry_point, chroot, library, file_factory) @@ -867,12 +965,12 @@ def file_factory(self, path, entry_point=None, chroot=None, library=False, file_ @property def bundle_root(self): """str: The root directory of the bundle where the original file structure is mirrored.""" - path = os.path.join(self.working_directory, 'bundles', self.hash) + path = os.path.join(self.working_directory, "bundles", self.hash) return os.path.normpath(os.path.abspath(path)) @property def hash(self): """str: Computes a hash based on the current contents of the bundle.""" file_hashes = sorted(file.hash for file in self.files) - combined_hashes = '\n'.join(file_hashes).encode('utf-8') + combined_hashes = "\n".join(file_hashes).encode("utf-8") return hashlib.sha256(combined_hashes).hexdigest() diff --git a/src/exodus_bundler/cli.py b/src/exodus_bundler/cli.py index c83dbea..fe51cb5 100644 --- a/src/exodus_bundler/cli.py +++ b/src/exodus_bundler/cli.py @@ -17,81 +17,122 @@ def parse_args(args=None, namespace=None): to parse the arguments from `sys.argv`. A dictionary is returned rather than the typical namespace produced by `argparse`.""" formatter = argparse.ArgumentDefaultsHelpFormatter - parser = argparse.ArgumentParser(formatter_class=formatter, description=( - 'Bundle ELF binary executables with all of their runtime dependencies ' - 'so that they can be relocated to other systems with incompatible system ' - 'libraries.' - )) + parser = argparse.ArgumentParser( + formatter_class=formatter, + description=( + "Bundle ELF binary executables with all of their runtime dependencies " + "so that they can be relocated to other systems with incompatible system " + "libraries." + ), + ) - parser.add_argument('executables', metavar='EXECUTABLE', nargs='+', help=( - 'One or more ELF executables to include in the exodus bundle.' - )) + parser.add_argument( + "executables", + metavar="EXECUTABLE", + nargs="+", + help=("One or more ELF executables to include in the exodus bundle."), + ) - parser.add_argument('-c', '--chroot', metavar='CHROOT_PATH', + parser.add_argument( + "-c", + "--chroot", + metavar="CHROOT_PATH", default=None, help=( - 'A directory that will be treated as the root during linking. Useful for testing and ' - 'bundling extracted packages that won\t run without a chroot.' + "A directory that will be treated as the root during linking. Useful for testing and " + "bundling extracted packages that won\t run without a chroot." ), ) - parser.add_argument('-a', '--add', '--additional-file', metavar='DEPENDENCY', action='append', + parser.add_argument( + "-a", + "--add", + "--additional-file", + metavar="DEPENDENCY", + action="append", default=[], help=( - 'Specifies an additional file to include in the bundle, useful for adding ' - 'programatically loaded libraries and other non-library dependencies. ' - 'The argument can be used more than once to include multiple files, and ' - 'directories will be included recursively.' + "Specifies an additional file to include in the bundle, useful for adding " + "programatically loaded libraries and other non-library dependencies. " + "The argument can be used more than once to include multiple files, and " + "directories will be included recursively." ), ) - parser.add_argument('-d', '--detect', action='store_true', help=( - 'Attempt to autodetect direct dependencies using the system package manager. ' - 'Operating system support is limited.' - )) + parser.add_argument( + "-d", + "--detect", + action="store_true", + help=( + "Attempt to autodetect direct dependencies using the system package manager. " + "Operating system support is limited." + ), + ) - parser.add_argument('--no-symlink', metavar='FILE', action='append', + parser.add_argument( + "--no-symlink", + metavar="FILE", + action="append", default=[], help=( - 'Signifies that a file must not be symlinked to the deduplicated data directory. This ' - 'is useful if a file looks for other resources based on paths relative its own ' - 'location. This is enabled by default for executables.' + "Signifies that a file must not be symlinked to the deduplicated data directory. This " + "is useful if a file looks for other resources based on paths relative its own " + "location. This is enabled by default for executables." ), ) - parser.add_argument('-o', '--output', metavar='OUTPUT_FILE', + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT_FILE", default=None, help=( - 'The file where the bundle will be written out to. The extension depends on the ' + "The file where the bundle will be written out to. The extension depends on the " 'output type. The "{{executables}}" and "{{extension}}" template strings can be ' - ' used in the provided filename. If omitted, the output will go to stdout when ' + " used in the provided filename. If omitted, the output will go to stdout when " 'it is being piped, or to "./exodus-{{executables}}-bundle.{{extension}}" otherwise.' ), ) - parser.add_argument('-q', '--quiet', action='store_true', help=( - 'Suppress warning messages.' - )) + parser.add_argument("-q", "--quiet", action="store_true", help=("Suppress warning messages.")) - parser.add_argument('-r', '--rename', metavar='NEW_NAME', nargs='?', action='append', - default=[], help=( - 'Renames the binary executable(s) before packaging. The order of rename tags must ' - 'match the order of positional executable arguments.' + parser.add_argument( + "-r", + "--rename", + metavar="NEW_NAME", + nargs="?", + action="append", + default=[], + help=( + "Renames the binary executable(s) before packaging. The order of rename tags must " + "match the order of positional executable arguments." ), ) - parser.add_argument('--shell-launchers', action='store_true', help=( - 'Force the use of shell launchers instead of attempting to compile statically linked ones.' - )) + parser.add_argument( + "--shell-launchers", + action="store_true", + help=( + "Force the use of shell launchers instead of attempting to compile statically linked ones." + ), + ) - parser.add_argument('-t', '--tarball', action='store_true', help=( - 'Creates a tarball for manual extraction instead of an installation script. ' - 'Note that this will change the output extension from ".sh" to ".tgz".' - )) + parser.add_argument( + "-t", + "--tarball", + action="store_true", + help=( + "Creates a tarball for manual extraction instead of an installation script. " + 'Note that this will change the output extension from ".sh" to ".tgz".' + ), + ) - parser.add_argument('-v', '--verbose', action='store_true', help=( - 'Output additional informational messages.' - )) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help=("Output additional informational messages."), + ) return vars(parser.parse_args(args, namespace)) @@ -110,7 +151,7 @@ def filter(self, record): return record.levelno in (logging.WARN, logging.ERROR) stderr_handler = logging.StreamHandler(sys.stderr) - stderr_formatter = logging.Formatter('%(levelname)s: %(message)s') + stderr_formatter = logging.Formatter("%(levelname)s: %(message)s") stderr_handler.setFormatter(stderr_formatter) stderr_handler.addFilter(StderrFilter()) root_logger.addHandler(stderr_handler) @@ -123,7 +164,7 @@ class StdoutFilter(logging.Filter): def filter(self, record): return record.levelno in (logging.DEBUG, logging.INFO) - stdout_formatter = logging.Formatter('%(message)s') + stdout_formatter = logging.Formatter("%(message)s") stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(stdout_formatter) stdout_handler.addFilter(StdoutFilter()) @@ -134,25 +175,25 @@ def main(args=None, namespace=None): args = parse_args(args, namespace) # Dynamically set the default output to stdout if it is being piped. - if args['output'] is None: + if args["output"] is None: if sys.stdout.isatty(): - args['output'] = './exodus-{{executables}}-bundle.{{extension}}' + args["output"] = "./exodus-{{executables}}-bundle.{{extension}}" else: - args['output'] = '-' + args["output"] = "-" # Handle the CLI specific options here, removing them from `args` in the process. - quiet, verbose = args.pop('quiet'), args.pop('verbose') - suppress_stdout = args['output'] == '-' + quiet, verbose = args.pop("quiet"), args.pop("verbose") + suppress_stdout = args["output"] == "-" configure_logging(quiet=quiet, verbose=verbose, suppress_stdout=suppress_stdout) # Allow piping in additional files. if not sys.stdin.isatty(): - args['add'] += extract_paths(sys.stdin.read()) + args["add"] += extract_paths(sys.stdin.read()) # Create the bundle with all of the arguments. try: create_bundle(**args) except FatalError as fatal_error: - logger.error('Fatal error encountered, exiting.') + logger.error("Fatal error encountered, exiting.") logger.error(fatal_error, exc_info=verbose) sys.exit(1) diff --git a/src/exodus_bundler/dependency_detection.py b/src/exodus_bundler/dependency_detection.py index a90e336..8c1a735 100644 --- a/src/exodus_bundler/dependency_detection.py +++ b/src/exodus_bundler/dependency_detection.py @@ -6,7 +6,7 @@ from exodus_bundler.launchers import find_executable -class PackageManager(object): +class PackageManager: """Base class representing a package manager. The class level attributes can be overwritten in derived classes to customize the behavior. @@ -21,11 +21,12 @@ class PackageManager(object): package that owns a specific file. owner_regex (str): A regex to extract the package name from the output of the owner command. """ + cache_directory = None list_command = None - list_regex = '(.*)' + list_regex = "(.*)" owner_command = None - owner_regex = '(.*)' + owner_regex = "(.*)" def find_dependencies(self, path): """Finds a list of all of the files contained with the package containing a file.""" @@ -37,7 +38,7 @@ def find_dependencies(self, path): process = subprocess.Popen(args, stdout=subprocess.PIPE) stdout, stderr = process.communicate() dependencies = [] - for line in stdout.decode('utf-8').split('\n'): + for line in stdout.decode("utf-8").split("\n"): match = re.search(self.list_regex, line.strip()) if match: dependency_path = match.groups()[0] @@ -52,10 +53,10 @@ def find_owner(self, path): return None args = self.owner_command + [path] env = os.environ.copy() - env['LC_ALL'] = 'C' + env["LC_ALL"] = "C" process = subprocess.Popen(args, stdout=subprocess.PIPE, env=env) stdout, stderr = process.communicate() - output = stdout.decode('utf-8').strip() + output = stdout.decode("utf-8").strip() match = re.search(self.owner_regex, output) if match: return match.groups()[0].strip() @@ -73,27 +74,27 @@ def commands_exist(self): class Apt(PackageManager): - cache_directory = '/var/cache/apt' - list_command = ['dpkg-query', '-L'] - list_regex = '(.+)' - owner_command = ['dpkg', '-S'] - owner_regex = '(.+): ' + cache_directory = "/var/cache/apt" + list_command = ["dpkg-query", "-L"] + list_regex = "(.+)" + owner_command = ["dpkg", "-S"] + owner_regex = "(.+): " class Pacman(PackageManager): - cache_directory = '/var/cache/pacman' - list_command = ['pacman', '-Ql'] - list_regex = r'.*\s+(\/.+)' - owner_command = ['pacman', '-Qo'] - owner_regex = r' is owned by (.*)\s+.*' + cache_directory = "/var/cache/pacman" + list_command = ["pacman", "-Ql"] + list_regex = r".*\s+(\/.+)" + owner_command = ["pacman", "-Qo"] + owner_regex = r" is owned by (.*)\s+.*" class Yum(PackageManager): - cache_directory = '/var/cache/yum' - list_command = ['rpm', '-ql'] - list_regex = r'(.+)' - owner_command = ['rpm', '-qf'] - owner_regex = r'(.+)' + cache_directory = "/var/cache/yum" + list_command = ["rpm", "-ql"] + list_regex = r"(.+)" + owner_command = ["rpm", "-qf"] + owner_regex = r"(.+)" package_managers = [ diff --git a/src/exodus_bundler/errors.py b/src/exodus_bundler/errors.py index 4ae1301..1d4ae3f 100644 --- a/src/exodus_bundler/errors.py +++ b/src/exodus_bundler/errors.py @@ -1,29 +1,35 @@ # -*- coding: utf-8 -*- class FatalError(Exception): """Base class for exceptions that should terminate program execution.""" + pass class DependencyDetectionError(FatalError): """Signifies that the dependency detection process failed.""" + pass class InvalidElfBinaryError(FatalError): """Signifies that a file was expected to be an ELF binary, but wasn't.""" + pass class MissingFileError(FatalError): """Signifies that a file was not found.""" + pass class UnexpectedDirectoryError(FatalError): """Signifies that a path was unexpectedly a directory.""" + pass class UnsupportedArchitectureError(FatalError): """Signifies that a binary has an unexpected architecture.""" + pass diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index a9ee725..91b3354 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -5,23 +5,23 @@ # We don't actually want to include anything in these directories in bundles. blacklisted_directories = [ - '/dev/', - '/proc/', - '/run/', - '/sys/', + "/dev/", + "/proc/", + "/run/", + "/sys/", # This isn't a directory exactly, but it will filter out active bundling. - '/tmp/exodus-bundle-', + "/tmp/exodus-bundle-", ] exec_methods = [ - 'execve', - 'exec', - 'execl', - 'execlp', - 'execle', - 'execv', - 'execvp', - 'execvpe', + "execve", + "exec", + "execl", + "execlp", + "execle", + "execv", + "execvp", + "execvpe", ] @@ -31,7 +31,7 @@ def extract_exec_path(line): for method in exec_methods: prefix = method + '("' if line.startswith(prefix): - line = line[len(prefix):] + line = line[len(prefix) :] parts = line.split('", ') if len(parts) > 1: return parts[0] @@ -43,14 +43,14 @@ def extract_open_path(line): line = strip_pid_prefix(line) for prefix in ['openat(AT_FDCWD, "', 'open("']: if line.startswith(prefix): - parts = line[len(prefix):].split('", ') + parts = line[len(prefix) :].split('", ') if len(parts) != 2: continue - if 'ENOENT' in parts[1]: + if "ENOENT" in parts[1]: continue - if 'O_RDONLY' not in parts[1]: + if "O_RDONLY" not in parts[1]: continue - if 'O_DIRECTORY' in parts[1]: + if "O_DIRECTORY" in parts[1]: continue return parts[0] return None @@ -61,8 +61,8 @@ def extract_stat_path(line): line = strip_pid_prefix(line) prefix = 'stat("' if line.startswith(prefix): - parts = line[len(prefix):].split('", ') - if len(parts) == 2 and 'ENOENT' not in parts[1]: + parts = line[len(prefix) :].split('", ') + if len(parts) == 2 and "ENOENT" not in parts[1]: return parts[0] return None @@ -104,7 +104,7 @@ def extract_paths(content, existing_only=True): def strip_pid_prefix(line): """Strips out the `[pid XXX] ` prefix if present.""" - match = re.match(r'\[pid\s+\d+\]\s*', line) + match = re.match(r"\[pid\s+\d+\]\s*", line) if match: - return line[len(match.group()):] + return line[len(match.group()) :] return line diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 3d28434..0ace939 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- """Methods to produce launchers that will invoke the relocated executables with the proper linker and library paths.""" + import os import re +import shutil import tempfile -from distutils.spawn import find_executable as find_executable_original from subprocess import PIPE from subprocess import Popen @@ -21,9 +22,9 @@ class CompilerNotFoundError(Exception): # This is kind of a hack to find things in PATH inside of bundles. def find_executable(binary_name, skip_original_for_testing=False): # This won't be set on Alpine Linux, but it's required for the `find_executable()` calls. - if 'PATH' not in os.environ: - os.environ['PATH'] = '/bin/:/usr/bin/' - executable = find_executable_original(binary_name) + if "PATH" not in os.environ: + os.environ["PATH"] = "/bin/:/usr/bin/" + executable = shutil.which(binary_name) if executable and not skip_original_for_testing: return executable # Try to find it within the same bundle if it's not actually in the PATH. @@ -33,14 +34,23 @@ def find_executable(binary_name, skip_original_for_testing=False): if not len(basename): break # The bundle directory. - if re.match('[A-Fa-f0-9]{64}', basename): - for bin_directory in os.environ['PATH'].split(':'): + if re.match("[A-Fa-f0-9]{64}", basename): + for bin_directory in os.environ["PATH"].split(":"): if os.path.isabs(bin_directory): - bin_directory = os.path.relpath(bin_directory, '/') - candidate_executable = os.path.join(directory, basename, - bin_directory, binary_name) + bin_directory = os.path.relpath(bin_directory, "/") + candidate_executable = os.path.join(directory, basename, bin_directory, binary_name) + # Check for binary launcher version (.bin extension) + candidate_executable_bin = candidate_executable + ".bin" + if os.path.exists(candidate_executable_bin): + return candidate_executable_bin + # Also check for shell launcher version (.sh extension) + candidate_executable_sh = candidate_executable + ".sh" + if os.path.exists(candidate_executable_sh): + return candidate_executable_sh + # Finally check for exact match (could be a symlink) if os.path.exists(candidate_executable): return candidate_executable + return None def compile(code): @@ -50,33 +60,32 @@ def compile(code): try: return compile_diet(code) except CompilerNotFoundError: - raise CompilerNotFoundError('No suiteable C compiler was found.') + raise CompilerNotFoundError("No suiteable C compiler was found.") def compile_diet(code): - diet = find_executable('diet') - gcc = find_executable('gcc') + diet = find_executable("diet") + gcc = find_executable("gcc") if diet is None or gcc is None: - raise CompilerNotFoundError('The diet compiler was not found.') - return compile_helper(code, [diet, 'gcc']) + raise CompilerNotFoundError("The diet compiler was not found.") + return compile_helper(code, [diet, "gcc"]) def compile_helper(code, initial_args): - f, input_filename = tempfile.mkstemp(prefix='exodus-bundle-', suffix='.c') + f, input_filename = tempfile.mkstemp(prefix="exodus-bundle-", suffix=".c") os.close(f) - f, output_filename = tempfile.mkstemp(prefix='exodus-bundle-') + f, output_filename = tempfile.mkstemp(prefix="exodus-bundle-") os.close(f) try: - with open(input_filename, 'w') as input_file: + with open(input_filename, "w") as input_file: input_file.write(code) - args = initial_args + ['-static', '-O3', input_filename, '-o', output_filename] + args = initial_args + ["-static", "-O3", input_filename, "-o", output_filename] process = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert process.returncode == 0, \ - 'There was an error compiling: %s' % stderr.decode('utf-8') + assert process.returncode == 0, f"There was an error compiling: {stderr.decode('utf-8')}" - with open(output_filename, 'rb') as output_file: + with open(output_filename, "rb") as output_file: return output_file.read() finally: os.remove(input_filename) @@ -84,24 +93,34 @@ def compile_helper(code, initial_args): def compile_musl(code): - musl = find_executable('musl-gcc') + musl = find_executable("musl-gcc") if musl is None: - raise CompilerNotFoundError('The musl compiler was not found.') + raise CompilerNotFoundError("The musl compiler was not found.") return compile_helper(code, [musl]) def construct_bash_launcher(linker, library_path, executable, full_linker=True): linker_dirname, linker_basename = os.path.split(linker) - full_linker = 'true' if full_linker else 'false' - return render_template_file('launcher.sh', linker_basename=linker_basename, - linker_dirname=linker_dirname, library_path=library_path, - executable=executable, full_linker=full_linker) + full_linker = "true" if full_linker else "false" + return render_template_file( + "launcher.sh", + linker_basename=linker_basename, + linker_dirname=linker_dirname, + library_path=library_path, + executable=executable, + full_linker=full_linker, + ) def construct_binary_launcher(linker, library_path, executable, full_linker=True): linker_dirname, linker_basename = os.path.split(linker) - full_linker = '1' if full_linker else '0' - code = render_template_file('launcher.c', linker_basename=linker_basename, - linker_dirname=linker_dirname, library_path=library_path, - executable=executable, full_linker=full_linker) + full_linker = "1" if full_linker else "0" + code = render_template_file( + "launcher.c", + linker_basename=linker_basename, + linker_dirname=linker_dirname, + library_path=library_path, + executable=executable, + full_linker=full_linker, + ) return compile(code) diff --git a/src/exodus_bundler/templating.py b/src/exodus_bundler/templating.py index 506d997..59d330d 100644 --- a/src/exodus_bundler/templating.py +++ b/src/exodus_bundler/templating.py @@ -7,17 +7,17 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -template_directory = os.path.join(parent_directory, 'templates') +template_directory = os.path.join(parent_directory, "templates") def render_template(string, **context): for key, value in context.items(): - string = string.replace('{{%s}}' % key, value) + string = string.replace(f"{{{{{key}}}}}", value) return string def render_template_file(filename, **context): if not os.path.isabs(filename): filename = os.path.join(template_directory, filename) - with open(filename, 'r') as f: + with open(filename, "r") as f: return render_template(f.read(), **context) diff --git a/tests/test_bundling.py b/tests/test_bundling.py index 56dcbda..e86b5c5 100644 --- a/tests/test_bundling.py +++ b/tests/test_bundling.py @@ -20,55 +20,60 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -ldd_output_directory = os.path.join(parent_directory, 'data', 'ldd-output') -chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') -ldd = os.path.join(chroot, 'bin', 'ldd') -echo_args_glibc_32 = os.path.join(chroot, 'bin', 'echo-args-glibc-32') -echo_proc_self_exe_glibc_32 = os.path.join(chroot, 'bin', 'echo-proc-self-exe-glibc-32') -fizz_buzz_glibc_32 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32') -fizz_buzz_glibc_32_exe = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32-exe') -fizz_buzz_glibc_64 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-64') -fizz_buzz_musl_64 = os.path.join(chroot, 'bin', 'fizz-buzz-musl-64') - - -@pytest.mark.parametrize('path,expected_file_count', [ - (fizz_buzz_glibc_32, 3), - (fizz_buzz_glibc_64, 3), - (fizz_buzz_musl_64, 2), - (ldd, 1), -]) +ldd_output_directory = os.path.join(parent_directory, "data", "ldd-output") +chroot = os.path.join(parent_directory, "data", "binaries", "chroot") +ldd = os.path.join(chroot, "bin", "ldd") +echo_args_glibc_32 = os.path.join(chroot, "bin", "echo-args-glibc-32") +echo_proc_self_exe_glibc_32 = os.path.join(chroot, "bin", "echo-proc-self-exe-glibc-32") +fizz_buzz_glibc_32 = os.path.join(chroot, "bin", "fizz-buzz-glibc-32") +fizz_buzz_glibc_32_exe = os.path.join(chroot, "bin", "fizz-buzz-glibc-32-exe") +fizz_buzz_glibc_64 = os.path.join(chroot, "bin", "fizz-buzz-glibc-64") +fizz_buzz_musl_64 = os.path.join(chroot, "bin", "fizz-buzz-musl-64") + + +@pytest.mark.parametrize( + "path,expected_file_count", + [ + (fizz_buzz_glibc_32, 3), + (fizz_buzz_glibc_64, 3), + (fizz_buzz_musl_64, 2), + (ldd, 1), + ], +) def test_bundle_add_file(path, expected_file_count): bundle = Bundle(chroot=chroot) - assert len(bundle.files) == 0, 'The initial bundle should contain no files.' + assert len(bundle.files) == 0, "The initial bundle should contain no files." bundle.add_file(path) - assert len(bundle.files) == expected_file_count, \ - 'The bundle should include %d files.' % expected_file_count + assert len(bundle.files) == expected_file_count, ( + "The bundle should include %d files." % expected_file_count + ) def test_bundle_add_file_recursively(): bundle = Bundle(chroot=chroot) - assert len(bundle.files) == 0, 'The initial bundle should contain no files.' + assert len(bundle.files) == 0, "The initial bundle should contain no files." bundle.add_file(chroot) second_bundle = Bundle(chroot=chroot) for path in [ldd, fizz_buzz_glibc_32, fizz_buzz_glibc_32_exe, fizz_buzz_musl_64]: second_bundle.add_file(path) - assert second_bundle.files.issubset(bundle.files), \ - 'All of the executables and their dependencies should be in the first bundle.' + assert second_bundle.files.issubset(bundle.files), ( + "All of the executables and their dependencies should be in the first bundle." + ) def test_bundle_delete_working_directory(): bundle = Bundle() - assert bundle.working_directory is None, \ - 'A directory should only be created if passed `working_directory=True`.' + assert bundle.working_directory is None, ( + "A directory should only be created if passed `working_directory=True`." + ) bundle = Bundle(working_directory=True) working_directory = bundle.working_directory - assert os.path.exists(working_directory), \ - 'A working directory should have been created.' + assert os.path.exists(working_directory), "A working directory should have been created." bundle.delete_working_directory() - assert not os.path.exists(working_directory), \ - 'The working directory should have been deleted.' - assert bundle.working_directory is None, \ - 'The working directory should have been cleared after deletion.' + assert not os.path.exists(working_directory), "The working directory should have been deleted." + assert bundle.working_directory is None, ( + "The working directory should have been cleared after deletion." + ) def test_bundle_file_factory(): @@ -77,8 +82,7 @@ def test_bundle_file_factory(): # Note that `ldd` is a shell script, and should bring in no dependencies. [file] = bundle.files new_file = bundle.file_factory(ldd) - assert new_file is file, \ - 'The same file should be returned instead of making a new one.' + assert new_file is file, "The same file should be returned instead of making a new one." def test_bundle_hash(): @@ -87,40 +91,47 @@ def test_bundle_hash(): for filename in [fizz_buzz_glibc_32, fizz_buzz_glibc_64, fizz_buzz_musl_64]: bundle.add_file(filename) hashes.append(bundle.hash) - assert len(hashes) == len(set(hashes)), 'All of the hashes should be unique.' - assert all(len(hash) == 64 for hash in hashes), 'All of the hashes should have length 64.' + assert len(hashes) == len(set(hashes)), "All of the hashes should be unique." + assert all(len(hash) == 64 for hash in hashes), "All of the hashes should have length 64." def test_bundle_root(): try: bundle = Bundle(working_directory=True) - assert bundle.hash in bundle.bundle_root, 'Bundle path should include the hash.' - assert bundle.bundle_root.startswith(bundle.working_directory), \ - 'The bundle root should be a subdirectory of the working directory.' - except: # noqa: E722 + assert bundle.hash in bundle.bundle_root, "Bundle path should include the hash." + assert bundle.bundle_root.startswith(bundle.working_directory), ( + "The bundle root should be a subdirectory of the working directory." + ) + except Exception: raise finally: bundle.delete_working_directory() -@pytest.mark.parametrize('int,bytes,byteorder', [ - (1234567890, b'\xd2\x02\x96I\x00\x00\x00\x00', 'little'), - (1234567890, b'\x00\x00\x00\x00I\x96\x02\xd2', 'big'), - (9876543210, b'\xea\x16\xb0L\x02\x00\x00\x00', 'little'), - (9876543210, b'\x00\x00\x00\x02L\xb0\x16\xea', 'big'), -]) +@pytest.mark.parametrize( + "int,bytes,byteorder", + [ + (1234567890, b"\xd2\x02\x96I\x00\x00\x00\x00", "little"), + (1234567890, b"\x00\x00\x00\x00I\x96\x02\xd2", "big"), + (9876543210, b"\xea\x16\xb0L\x02\x00\x00\x00", "little"), + (9876543210, b"\x00\x00\x00\x02L\xb0\x16\xea", "big"), + ], +) def test_bytes_to_int(int, bytes, byteorder): - assert bytes_to_int(bytes, byteorder=byteorder) == int, 'Byte conversion should work.' - - -@pytest.mark.parametrize('fizz_buzz,shell_launchers', [ - (fizz_buzz_glibc_32, True), - (fizz_buzz_glibc_32, False), - (fizz_buzz_glibc_64, True), - (fizz_buzz_glibc_64, False), - (fizz_buzz_musl_64, True), - (fizz_buzz_musl_64, False), -]) + assert bytes_to_int(bytes, byteorder=byteorder) == int, "Byte conversion should work." + + +@pytest.mark.parametrize( + "fizz_buzz,shell_launchers", + [ + (fizz_buzz_glibc_32, True), + (fizz_buzz_glibc_32, False), + (fizz_buzz_glibc_64, True), + (fizz_buzz_glibc_64, False), + (fizz_buzz_musl_64, True), + (fizz_buzz_musl_64, False), + ], +) def test_create_unpackaged_bundle(fizz_buzz, shell_launchers): """This tests that the packaged executable runs as expected. At the very least, this tests that the symbolic links and launcher are functioning correctly. Unfortunately, @@ -128,197 +139,232 @@ def test_create_unpackaged_bundle(fizz_buzz, shell_launchers): present on the current system. FWIW, the CircleCI docker image being used is incompatible, so the continuous integration tests are more meaningful.""" root_directory = create_unpackaged_bundle( - rename=[], executables=[fizz_buzz], chroot=chroot, shell_launchers=shell_launchers) + rename=[], + executables=[fizz_buzz], + chroot=chroot, + shell_launchers=shell_launchers, + ) try: - binary_path = os.path.join(root_directory, 'bin', os.path.basename(fizz_buzz)) + binary_path = os.path.join(root_directory, "bin", os.path.basename(fizz_buzz)) process = Popen([binary_path], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert 'FIZZBUZZ' in stdout.decode('utf-8') - assert len(stderr.decode('utf-8')) == 0 + assert "FIZZBUZZ" in stdout.decode("utf-8") + assert len(stderr.decode("utf-8")) == 0 finally: - assert root_directory.startswith('/tmp/') + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) -@pytest.mark.parametrize('detect', [False, True]) +@pytest.mark.parametrize("detect", [False, True]) def test_create_unpackaged_bundle_detects_dependencies(detect): - binary_name = 'ls' - root_directory = create_unpackaged_bundle( - rename=[], executables=[binary_name], detect=detect) + binary_name = "ls" + root_directory = create_unpackaged_bundle(rename=[], executables=[binary_name], detect=detect) try: # Determine the bundle root. - binary_symlink = os.path.join(root_directory, 'bin', binary_name) + binary_symlink = os.path.join(root_directory, "bin", binary_name) binary_path = os.path.realpath(binary_symlink) dirname, basename = os.path.split(binary_path) while len(basename) != 64: dirname, basename = os.path.split(dirname) bundle_root = os.path.join(dirname, basename) - man_directory = os.path.join(bundle_root, 'usr', 'share', 'man') - assert os.path.exists(man_directory) == detect, \ - 'The man directory should only exist when `detect=True`.' + man_directory = os.path.join(bundle_root, "usr", "share", "man") + assert os.path.exists(man_directory) == detect, ( + "The man directory should only exist when `detect=True`." + ) finally: - assert root_directory.startswith('/tmp/') + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) def test_create_unpackaged_bundle_has_correct_args(): root_directory = create_unpackaged_bundle( - rename=[], executables=[echo_args_glibc_32], chroot=chroot) + rename=[], executables=[echo_args_glibc_32], chroot=chroot + ) try: - binary_path = os.path.join(root_directory, 'bin', os.path.basename(echo_args_glibc_32)) + binary_path = os.path.join(root_directory, "bin", os.path.basename(echo_args_glibc_32)) - process = Popen([binary_path, 'arg1', 'arg2'], stdout=PIPE, stderr=PIPE) + process = Popen([binary_path, "arg1", "arg2"], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert len(stderr.decode('utf-8')) == 0 - args = stdout.decode('utf-8').split('\n') - assert os.path.basename(args[0]) == '%s-x' % os.path.basename(echo_args_glibc_32), \ - 'The value of argv[0] should correspond to the local symlink.' - assert args[1] == 'arg1' and args[2] == 'arg2', \ - 'The other arguments should be passed through to the child process.' + assert len(stderr.decode("utf-8")) == 0 + args = stdout.decode("utf-8").split("\n") + assert os.path.basename(args[0]) == os.path.basename(echo_args_glibc_32), ( + "The value of argv[0] should correspond to the local symlink." + ) + assert args[1] == "arg1" and args[2] == "arg2", ( + "The other arguments should be passed through to the child process." + ) finally: - assert root_directory.startswith('/tmp/') + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) def test_create_unpackaged_bundle_has_correct_proc_self_exe(): root_directory = create_unpackaged_bundle( - rename=[], executables=[echo_proc_self_exe_glibc_32], chroot=chroot) + rename=[], executables=[echo_proc_self_exe_glibc_32], chroot=chroot + ) try: - binary_path = os.path.join(root_directory, 'bin', - os.path.basename(echo_proc_self_exe_glibc_32)) + binary_path = os.path.join( + root_directory, "bin", os.path.basename(echo_proc_self_exe_glibc_32) + ) process = Popen([binary_path], stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert len(stderr.decode('utf-8')) == 0 - proc_self_exe = stdout.decode('utf-8').strip() - assert os.path.basename(proc_self_exe).startswith('linker-'), \ - 'The linker should be the executing process.' + assert len(stderr.decode("utf-8")) == 0 + proc_self_exe = stdout.decode("utf-8").strip() + assert os.path.basename(proc_self_exe).startswith("linker-"), ( + "The linker should be the executing process." + ) relative_path = os.path.relpath(proc_self_exe, root_directory) - assert relative_path.startswith('bundles/'), \ - 'The process should be in the bundles directory.' + assert relative_path.startswith("bundles/"), ( + "The process should be in the bundles directory." + ) finally: - assert root_directory.startswith('/tmp/') + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) def test_detect_elf_binary(): - assert detect_elf_binary(fizz_buzz_glibc_32), 'The `fizz-buzz` file should be an ELF binary.' - assert not detect_elf_binary(ldd), 'The `ldd` file should be a shell script.' - - -@pytest.mark.parametrize('fizz_buzz,bits', [ - (fizz_buzz_glibc_32, 32), - (fizz_buzz_glibc_64, 64), - (fizz_buzz_musl_64, 64), -]) + assert detect_elf_binary(fizz_buzz_glibc_32), "The `fizz-buzz` file should be an ELF binary." + assert not detect_elf_binary(ldd), "The `ldd` file should be a shell script." + + +@pytest.mark.parametrize( + "fizz_buzz,bits", + [ + (fizz_buzz_glibc_32, 32), + (fizz_buzz_glibc_64, 64), + (fizz_buzz_musl_64, 64), + ], +) def test_elf_bits(fizz_buzz, bits): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) # Can be checked by running `file fizz-buzz`. - assert fizz_buzz_elf.bits == bits, \ - 'The fizz buzz executable should be %d-bit.' % bits + assert fizz_buzz_elf.bits == bits, "The fizz buzz executable should be %d-bit." % bits -@pytest.mark.parametrize('fizz_buzz', [ - (fizz_buzz_glibc_32), - (fizz_buzz_glibc_64), -]) +@pytest.mark.parametrize( + "fizz_buzz", + [ + (fizz_buzz_glibc_32), + (fizz_buzz_glibc_64), + ], +) def test_elf_dependencies(fizz_buzz): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) direct_dependencies = fizz_buzz_elf.direct_dependencies all_dependencies = fizz_buzz_elf.dependencies - assert set(direct_dependencies).issubset(all_dependencies), \ - 'The direct dependencies should be a subset of all dependencies.' - - -@pytest.mark.parametrize('fizz_buzz', [ - (fizz_buzz_glibc_32), - (fizz_buzz_glibc_64), - (fizz_buzz_musl_64), -]) + assert set(direct_dependencies).issubset(all_dependencies), ( + "The direct dependencies should be a subset of all dependencies." + ) + + +@pytest.mark.parametrize( + "fizz_buzz", + [ + (fizz_buzz_glibc_32), + (fizz_buzz_glibc_64), + (fizz_buzz_musl_64), + ], +) def test_elf_direct_dependencies(fizz_buzz): fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) dependencies = fizz_buzz_elf.direct_dependencies - assert all(file.path.startswith(chroot) for file in dependencies), \ - 'All dependencies should be located within the chroot.' - assert len(dependencies), 'There should be at least one dependency.' + assert all(file.path.startswith(chroot) for file in dependencies), ( + "All dependencies should be located within the chroot." + ) + assert len(dependencies), "There should be at least one dependency." # These don't apply to the musl binary. - if 'glib' in fizz_buzz: - assert len(dependencies) == 2, 'The linker and libc should be the only dependencies.' - assert any('libc.so' in file.path for file in dependencies), \ + if "glib" in fizz_buzz: + assert len(dependencies) == 2, "The linker and libc should be the only dependencies." + assert any("libc.so" in file.path for file in dependencies), ( '"libc" was not found as a direct dependency of the executable.' + ) -@pytest.mark.parametrize('fizz_buzz,expected_linker_path', [ - (fizz_buzz_glibc_32, '/lib/ld-linux.so.2'), - (fizz_buzz_glibc_64, '/lib64/ld-linux-x86-64.so.2'), - (fizz_buzz_musl_64, '/lib/ld-musl-x86_64.so.1'), -]) +@pytest.mark.parametrize( + "fizz_buzz,expected_linker_path", + [ + (fizz_buzz_glibc_32, "/lib/ld-linux.so.2"), + (fizz_buzz_glibc_64, "/lib64/ld-linux-x86-64.so.2"), + (fizz_buzz_musl_64, "/lib/ld-musl-x86_64.so.1"), + ], +) def test_elf_linker(fizz_buzz, expected_linker_path): # Found by running `readelf -l fizz-buzz`. fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) - expected_linker_path = os.path.join(chroot, os.path.relpath(expected_linker_path, '/')) - assert fizz_buzz_elf.linker_file.path == expected_linker_path, \ - 'The correct linker should be extracted from the ELF program header.' - - -@pytest.mark.parametrize('fizz_buzz, expected_type', [ - (fizz_buzz_glibc_32, 'shared'), - (fizz_buzz_glibc_32_exe, 'executable'), - (fizz_buzz_glibc_64, 'shared'), -]) + expected_linker_path = os.path.join(chroot, os.path.relpath(expected_linker_path, "/")) + assert fizz_buzz_elf.linker_file.path == expected_linker_path, ( + "The correct linker should be extracted from the ELF program header." + ) + + +@pytest.mark.parametrize( + "fizz_buzz, expected_type", + [ + (fizz_buzz_glibc_32, "shared"), + (fizz_buzz_glibc_32_exe, "executable"), + (fizz_buzz_glibc_64, "shared"), + ], +) def test_elf_type(fizz_buzz, expected_type): elf = Elf(fizz_buzz, chroot=chroot) - assert elf.type == expected_type, 'Fizz buzz should match the expected ELF binary type.' + assert elf.type == expected_type, "Fizz buzz should match the expected ELF binary type." def test_file_destination(): - arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) + arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) arch_directory = os.path.dirname(arch_file.destination) fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) fizz_buzz_directory = os.path.dirname(fizz_buzz_file.destination) - assert arch_directory == fizz_buzz_directory, \ - 'Executable and non-executable files should be written to the same directory.' + assert arch_directory == fizz_buzz_directory, ( + "Executable and non-executable files should be written to the same directory." + ) def test_file_executable(): fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) - arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) - assert fizz_buzz_file.executable, 'The fizz buzz executable should be executable.' - assert not arch_file.executable, 'The arch text file should not be executable.' + arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) + assert fizz_buzz_file.executable, "The fizz buzz executable should be executable." + assert not arch_file.executable, "The arch text file should not be executable." def test_file_elf(): fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) - arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) - assert fizz_buzz_file.elf, 'The fizz buzz executable should be an ELF binary.' - assert not arch_file.elf, 'The arch text file should not be an ELF binary.' + arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) + assert fizz_buzz_file.elf, "The fizz buzz executable should be an ELF binary." + assert not arch_file.elf, "The arch text file should not be an ELF binary." def test_file_hash(): - amazon_file = File(os.path.join(ldd_output_directory, 'htop-amazon-linux.txt')) - arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) - assert amazon_file.hash != arch_file.hash, 'The hashes should differ.' - assert len(amazon_file.hash) == len(arch_file.hash) == 64, \ - 'The hashes should have a consistent length of 64 characters.' + amazon_file = File(os.path.join(ldd_output_directory, "htop-amazon-linux.txt")) + arch_file = File(os.path.join(ldd_output_directory, "htop-arch.txt")) + assert amazon_file.hash != arch_file.hash, "The hashes should differ." + assert len(amazon_file.hash) == len(arch_file.hash) == 64, ( + "The hashes should have a consistent length of 64 characters." + ) # Found by executing `sha256sum fizz-buzz`. - expected_hash = 'd54ab4714215d7822bf490df5cdf49bc3f32b4c85a439b109fc7581355f9d9c5' - assert File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash, 'Hashes should match.' - - -@pytest.mark.parametrize('fizz_buzz', [ - (fizz_buzz_glibc_32), - (fizz_buzz_glibc_64), - (fizz_buzz_musl_64), -]) + expected_hash = "d54ab4714215d7822bf490df5cdf49bc3f32b4c85a439b109fc7581355f9d9c5" + assert File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash, "Hashes should match." + + +@pytest.mark.parametrize( + "fizz_buzz", + [ + (fizz_buzz_glibc_32), + (fizz_buzz_glibc_64), + (fizz_buzz_musl_64), + ], +) def test_file_requires_launcher(fizz_buzz): file = File(fizz_buzz, chroot=chroot) - assert file.requires_launcher, 'Fizz buzz should require a launcher.' - assert all(not dependency.requires_launcher for dependency in file.elf.dependencies), \ - 'All of the dependencies should not require launchers.' + assert file.requires_launcher, "Fizz buzz should require a launcher." + assert all(not dependency.requires_launcher for dependency in file.elf.dependencies), ( + "All of the dependencies should not require launchers." + ) def test_file_symlink(): @@ -328,62 +374,69 @@ def test_file_symlink(): file = next(iter(bundle.files)) file.copy(bundle.working_directory) symlink = file.symlink(bundle.working_directory, bundle.bundle_root) - assert os.path.islink(symlink), 'A symlink should have been created.' - assert os.path.exists(symlink), 'The symlink should point to the actual file.' - except: # noqa: E722 + assert os.path.islink(symlink), "A symlink should have been created." + assert os.path.exists(symlink), "The symlink should point to the actual file." + except Exception: raise finally: bundle.delete_working_directory() -@pytest.mark.parametrize('filename_prefix', [ - 'htop-amazon-linux', - 'htop-arch', - 'htop-ubuntu-14.04', -]) +@pytest.mark.parametrize( + "filename_prefix", + [ + "htop-amazon-linux", + "htop-arch", + "htop-ubuntu-14.04", + ], +) def test_parse_dependencies_from_ldd_output(filename_prefix): - ldd_output_filename = filename_prefix + '.txt' + ldd_output_filename = filename_prefix + ".txt" with open(os.path.join(ldd_output_directory, ldd_output_filename)) as f: ldd_output = f.read() dependencies = parse_dependencies_from_ldd_output(ldd_output) - ldd_results_filename = filename_prefix + '-dependencies.txt' + ldd_results_filename = filename_prefix + "-dependencies.txt" with open(os.path.join(ldd_output_directory, ldd_results_filename)) as f: - expected_dependencies = [line for line in f.read().split('\n') if len(line)] + expected_dependencies = [line for line in f.read().split("\n") if len(line)] - assert set(dependencies) == set(expected_dependencies), \ + assert set(dependencies) == set(expected_dependencies), ( 'The dependencies were not parsed correctly from ldd output for "%s"' % filename_prefix + ) def test_resolve_binary(): binary_directory = os.path.dirname(fizz_buzz_glibc_32) binary = os.path.basename(fizz_buzz_glibc_32) - old_path = os.getenv('PATH', '') + old_path = os.getenv("PATH", "") try: - os.environ['PATH'] = '%s%s%s' % (binary_directory, os.pathsep, old_path) + os.environ["PATH"] = "%s%s%s" % (binary_directory, os.pathsep, old_path) resolved_binary = resolve_binary(binary) - assert resolved_binary == os.path.normpath(fizz_buzz_glibc_32), \ - 'The full binary path was not resolved correctly.' + assert resolved_binary == os.path.normpath(fizz_buzz_glibc_32), ( + "The full binary path was not resolved correctly." + ) finally: - os.environ['PATH'] = old_path + os.environ["PATH"] = old_path def test_resolve_file_path(): with pytest.raises(Exception): resolve_file_path(chroot) with pytest.raises(Exception): - resolve_file_path(os.path.join(chroot, 'non-existent-file')) - assert os.path.isabs(resolve_file_path(fizz_buzz_glibc_32)), \ - 'The resolved path should be absolute.' + resolve_file_path(os.path.join(chroot, "non-existent-file")) + assert os.path.isabs(resolve_file_path(fizz_buzz_glibc_32)), ( + "The resolved path should be absolute." + ) def test_run_ldd(): - assert any('libc.so' in line for line in run_ldd(ldd, fizz_buzz_glibc_32)), \ + assert any("libc.so" in line for line in run_ldd(ldd, fizz_buzz_glibc_32)), ( '"libc" was not found in the output of "ldd" for the executable.' + ) def test_stored_property(): - class Incrementer(object): + class Incrementer: def __init__(self): self.i = 0 @@ -394,4 +447,4 @@ def next(self): incrementer = Incrementer() for i in range(10): - assert incrementer.next == 1, '`Incrementer.next` should not change.' + assert incrementer.next == 1, "`Incrementer.next` should not change." diff --git a/tests/test_cli.py b/tests/test_cli.py index 9fc5ac6..ad7bd82 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,136 +13,137 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') -fizz_buzz_glibc_32 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32') -fizz_buzz_glibc_32_exe = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32-exe') -fizz_buzz_glibc_64 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-64') -fizz_buzz_musl_64 = os.path.join(chroot, 'bin', 'fizz-buzz-musl-64') +chroot = os.path.join(parent_directory, "data", "binaries", "chroot") +fizz_buzz_glibc_32 = os.path.join(chroot, "bin", "fizz-buzz-glibc-32") +fizz_buzz_glibc_32_exe = os.path.join(chroot, "bin", "fizz-buzz-glibc-32-exe") +fizz_buzz_glibc_64 = os.path.join(chroot, "bin", "fizz-buzz-glibc-64") +fizz_buzz_musl_64 = os.path.join(chroot, "bin", "fizz-buzz-musl-64") def run_exodus(args, **options): - options['universal_newlines'] = options.get('universal_newlines', True) + options["universal_newlines"] = options.get("universal_newlines", True) # Allow specifying content to pipe into stdin, with options['stdin'] - if 'stdin' in options: - input = options['stdin'].encode('utf-8') - options['stdin'] = subprocess.PIPE + if "stdin" in options: + input = options["stdin"].encode("utf-8") + options["stdin"] = subprocess.PIPE else: input = None process = subprocess.Popen( - ['exodus'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **options) + ["exodus"] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **options + ) stdout, stderr = process.communicate(input=input) return process.returncode, stdout, stderr def test_adding_additional_files(capsys): - args = ['--chroot', chroot, '--output', '-', '--tarball', fizz_buzz_glibc_32] - stdin = '\n'.join((fizz_buzz_glibc_32_exe, fizz_buzz_glibc_64)) + args = ["--chroot", chroot, "--output", "-", "--tarball", fizz_buzz_glibc_32] + stdin = "\n".join((fizz_buzz_glibc_32_exe, fizz_buzz_glibc_64)) returncode, stdout, stderr = run_exodus(args, universal_newlines=False, stdin=stdin) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." stream = io.BytesIO(stdout) - with tarfile.open(fileobj=stream, mode='r:gz') as f: + with tarfile.open(fileobj=stream, mode="r:gz") as f: names = f.getnames() - assert 'exodus/bin/fizz-buzz-glibc-32' in names, stderr + assert "exodus/bin/fizz-buzz-glibc-32" in names, stderr # These shouldn't be entrypoints, but should be included - assert 'exodus/bin/fizz-buzz-glibc-32-exe' not in names, stderr - assert 'exodus/bin/fizz-buzz-glibc-64' not in names, stderr + assert "exodus/bin/fizz-buzz-glibc-32-exe" not in names, stderr + assert "exodus/bin/fizz-buzz-glibc-64" not in names, stderr assert any(fizz_buzz_glibc_32_exe in name for name in names), stderr assert any(fizz_buzz_glibc_64 in name for name in names), stderr def test_logging_outputs(capsys): # There should be no output before configuring the logger. - logger.error('error') + logger.error("error") out, err = capsys.readouterr() print(out, err) assert len(out) == len(err) == 0 # The different levels should be routed separately to stdout/stderr. configure_logging(verbose=True, quiet=False) - logger.debug('debug') - logger.warning('warn') - logger.info('info') - logger.error('error') + logger.debug("debug") + logger.warning("warn") + logger.info("info") + logger.error("error") out, err = capsys.readouterr() - assert all(output in out for output in ('info')) - assert all(output not in out for output in ('debug', 'warn', 'error')) - assert all(output in err for output in ('warn', 'error')) - assert all(output not in err for output in ('info', 'debug')) + assert all(output in out for output in ("info")) + assert all(output not in out for output in ("debug", "warn", "error")) + assert all(output in err for output in ("warn", "error")) + assert all(output not in err for output in ("info", "debug")) def test_missing_binary(capsys): # Without the --verbose flag. - command = 'this-is-almost-definitely-not-going-to-be-a-command-anywhere' + command = "this-is-almost-definitely-not-going-to-be-a-command-anywhere" returncode, stdout, stderr = run_exodus([command]) - assert returncode != 0, 'Running exodus should have failed.' - assert 'Traceback' not in stderr, 'Traceback should not be included without the --verbose flag.' + assert returncode != 0, "Running exodus should have failed." + assert "Traceback" not in stderr, "Traceback should not be included without the --verbose flag." # With the --verbose flag. - returncode, stdout, stderr = run_exodus(['--verbose', command]) - assert returncode != 0, 'Running exodus should have failed.' - assert 'Traceback' in stderr, 'Traceback should be included with the --verbose flag.' + returncode, stdout, stderr = run_exodus(["--verbose", command]) + assert returncode != 0, "Running exodus should have failed." + assert "Traceback" in stderr, "Traceback should be included with the --verbose flag." def test_required_argument(): with pytest.raises(SystemExit): parse_args([]) - parse_args(['/bin/bash']) + parse_args(["/bin/bash"]) def test_return_type_is_dict(): - assert type(parse_args(['/bin/bash'])) == dict + assert isinstance(parse_args(["/bin/bash"]), dict) def test_quiet_and_verbose_flags(): - result = parse_args(['--quiet', '/bin/bash']) - assert result['quiet'] and not result['verbose'] - result = parse_args(['--verbose', '/bin/bash']) - assert result['verbose'] and not result['quiet'] + result = parse_args(["--quiet", "/bin/bash"]) + assert result["quiet"] and not result["verbose"] + result = parse_args(["--verbose", "/bin/bash"]) + assert result["verbose"] and not result["quiet"] def test_writing_bundle_to_disk(): - f, filename = tempfile.mkstemp(suffix='.sh') + f, filename = tempfile.mkstemp(suffix=".sh") os.close(f) - args = ['--chroot', chroot, '--output', filename, fizz_buzz_glibc_32] + args = ["--chroot", chroot, "--output", filename, fizz_buzz_glibc_32] try: returncode, stdout, stderr = run_exodus(args) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." - with open(filename, 'rb') as f_in: + with open(filename, "rb") as f_in: first_line = f_in.readline().strip() - assert first_line == b'#! /bin/bash', stderr + assert first_line == b"#! /bin/bash", stderr finally: if os.path.exists(filename): os.unlink(filename) def test_writing_bundle_to_stdout(): - args = ['--chroot', chroot, '--output', '-', fizz_buzz_glibc_32] + args = ["--chroot", chroot, "--output", "-", fizz_buzz_glibc_32] returncode, stdout, stderr = run_exodus(args) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." - assert stdout.startswith('#! /bin/sh'), stderr + assert stdout.startswith("#! /bin/sh"), stderr def test_writing_tarball_to_disk(): - f, filename = tempfile.mkstemp(suffix='.tgz') + f, filename = tempfile.mkstemp(suffix=".tgz") os.close(f) - args = ['--chroot', chroot, '--output', filename, '--tarball', fizz_buzz_glibc_32] + args = ["--chroot", chroot, "--output", filename, "--tarball", fizz_buzz_glibc_32] try: returncode, stdout, stderr = run_exodus(args) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." assert tarfile.is_tarfile(filename), stderr - with tarfile.open(filename, mode='r:gz') as f_in: - assert 'exodus/bin/fizz-buzz-glibc-32' in f_in.getnames() + with tarfile.open(filename, mode="r:gz") as f_in: + assert "exodus/bin/fizz-buzz-glibc-32" in f_in.getnames() finally: if os.path.exists(filename): os.unlink(filename) def test_writing_tarball_to_stdout(): - args = ['--chroot', chroot, '--output', '-', '--tarball', fizz_buzz_glibc_32] + args = ["--chroot", chroot, "--output", "-", "--tarball", fizz_buzz_glibc_32] returncode, stdout, stderr = run_exodus(args, universal_newlines=False) assert returncode == 0, "Exodus should have exited with a success status code, but didn't." stream = io.BytesIO(stdout) - with tarfile.open(fileobj=stream, mode='r:gz') as f: - assert 'exodus/bin/fizz-buzz-glibc-32' in f.getnames(), stderr + with tarfile.open(fileobj=stream, mode="r:gz") as f: + assert "exodus/bin/fizz-buzz-glibc-32" in f.getnames(), stderr diff --git a/tests/test_dependency_detection.py b/tests/test_dependency_detection.py index ee11e9f..995e4bd 100644 --- a/tests/test_dependency_detection.py +++ b/tests/test_dependency_detection.py @@ -6,11 +6,12 @@ def test_detect_dependencies(): # This is a little janky, but the test suite won't run anywhere where it's not true. - ls = '/usr/bin/ls' + ls = "/usr/bin/ls" if not os.path.exists(ls): - ls = '/bin/ls' - assert os.path.exists(ls), 'This test assumes that `ls` is installed on the system.' + ls = "/bin/ls" + assert os.path.exists(ls), "This test assumes that `ls` is installed on the system." dependencies = detect_dependencies(ls) - assert any(ls in dependency for dependency in dependencies), \ - '`%s` should have been detected as a dependency for `ls`.' % ls + assert any(ls in dependency for dependency in dependencies), ( + "`%s` should have been detected as a dependency for `ls`." % ls + ) diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index 8eb06bb..e450ac9 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -9,91 +9,90 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -strace_output_directory = os.path.join(parent_directory, 'data', 'strace-output') -exodus_strace = os.path.join(strace_output_directory, 'exodus-output.txt') +strace_output_directory = os.path.join(parent_directory, "data", "strace-output") +exodus_strace = os.path.join(strace_output_directory, "exodus-output.txt") def test_extract_exec_path(): line = 'execve("/usr/bin/ls", ["ls"], 0x7ffea775ad70 /* 113 vars */) = 0' - assert extract_exec_path(line) == '/usr/bin/ls', \ - 'It should have extracted the path to the ls executable.' - assert extract_exec_path('blah') is None, \ - 'It should return `None` when there is no match.' + assert extract_exec_path(line) == "/usr/bin/ls", ( + "It should have extracted the path to the ls executable." + ) + assert extract_exec_path("blah") is None, "It should return `None` when there is no match." def test_extract_no_paths(): - input_paths = extract_paths('') - assert input_paths == [], 'It should return an empty list.' + input_paths = extract_paths("") + assert input_paths == [], "It should return an empty list." def test_extract_open_path(): line = ( 'openat(AT_FDCWD, "/usr/lib/root/tls/x86_64/libcap.so.2", O_RDONLY|O_CLOEXEC) ' - '= -1 ENOENT (No such file or directory)' + "= -1 ENOENT (No such file or directory)" ) - assert extract_open_path(line) is None, 'Missing files should not return paths.' + assert extract_open_path(line) is None, "Missing files should not return paths." line = 'open(".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4' - assert extract_open_path(line) is None, 'Opened directories should not return paths.' + assert extract_open_path(line) is None, "Opened directories should not return paths." line = 'open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 4' - assert extract_open_path(line) == '/usr/lib/locale/locale-archive', \ - 'An open() call should return a path.' + assert extract_open_path(line) == "/usr/lib/locale/locale-archive", ( + "An open() call should return a path." + ) line = 'openat(AT_FDCWD, "/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 4' - assert extract_open_path(line) == '/usr/lib/libc.so.6', \ - 'An openat() call relative to the current directory should return a path.' + assert extract_open_path(line) == "/usr/lib/libc.so.6", ( + "An openat() call relative to the current directory should return a path." + ) def test_extract_raw_paths(): input_paths = [ - '/absolute/path/to/file', - './relative/path', - '/another/absolute/path', + "/absolute/path/to/file", + "./relative/path", + "/another/absolute/path", ] - input_paths_with_whitespace = \ - [' ', ''] + [input_paths[0]] + [' '] + input_paths[1:] - input_content = '\n'.join(input_paths_with_whitespace) + input_paths_with_whitespace = [" ", ""] + [input_paths[0]] + [" "] + input_paths[1:] + input_content = "\n".join(input_paths_with_whitespace) extracted_paths = extract_paths(input_content) - assert set(input_paths) == set(extracted_paths), \ - 'The paths should have been extracted without the whitespace.' + assert set(input_paths) == set(extracted_paths), ( + "The paths should have been extracted without the whitespace." + ) def test_extract_stat_path(): line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.py", ' - '{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0' + "{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0" ) - expected_path = '/usr/local/lib/python3.6/encodings/__init__.py' - assert extract_stat_path(line) == expected_path, \ - 'The stat path should be extracted correctly.' + expected_path = "/usr/local/lib/python3.6/encodings/__init__.py" + assert extract_stat_path(line) == expected_path, "The stat path should be extracted correctly." line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.abi3.so", 0x7ffc9d6a0160) = -1 ' - 'ENOENT (No such file or directory)' + "ENOENT (No such file or directory)" ) - assert extract_stat_path(line) is None, \ - 'Non-existent files should not be extracted.' + assert extract_stat_path(line) is None, "Non-existent files should not be extracted." def test_extract_strace_paths(): - with open(exodus_strace, 'r') as f: + with open(exodus_strace, "r") as f: content = f.read() extracted_paths = extract_paths(content, existing_only=False) expected_paths = [ # `execve()` call - '/home/sangaline/projects/exodus/.env/bin/exodus', + "/home/sangaline/projects/exodus/.env/bin/exodus", # `openat()` call - '/usr/lib/libpthread.so.0', + "/usr/lib/libpthread.so.0", # `open()` call - '/usr/lib/gconv/gconv-modules', + "/usr/lib/gconv/gconv-modules", ] for path in expected_paths: - assert path in extracted_paths, \ - '"%s" should be present in the extracted paths.' % path + assert path in extracted_paths, '"%s" should be present in the extracted paths.' % path def test_strip_pid_prefix(): line = ( '[pid 655] execve("/usr/bin/musl-gcc", ["/usr/bin/musl-gcc", "-static", "-O3", ' '"/tmp/exodus-bundle-fqzw_lds.c", "-o", "/tmp/exodus-bundle-3p_c0osh"], [/* 45 vars */] ' - '' + "" ) - assert strip_pid_prefix(line).startswith('execve('), 'The PID prefix should be stripped.' + assert strip_pid_prefix(line).startswith("execve("), "The PID prefix should be stripped." diff --git a/tests/test_launchers.py b/tests/test_launchers.py index 9d1d0c2..9a62e21 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -18,26 +18,27 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') -echo_args_glibc_32 = os.path.join(chroot, 'bin', 'echo-args-glibc-32') -fizz_buzz_source_file = os.path.join(parent_directory, 'data', 'binaries', 'fizz-buzz.c') +chroot = os.path.join(parent_directory, "data", "binaries", "chroot") +echo_args_glibc_32 = os.path.join(chroot, "bin", "echo-args-glibc-32") +fizz_buzz_source_file = os.path.join(parent_directory, "data", "binaries", "fizz-buzz.c") def test_construct_bash_launcher(): - linker, library_path, executable = '../lib/ld-linux.so.2', '../lib/', 'grep' - script_content = construct_bash_launcher(linker=linker, library_path=library_path, - executable=executable) - assert script_content.startswith('#! /bin/bash\n') + linker, library_path, executable = "../lib/ld-linux.so.2", "../lib/", "grep" + script_content = construct_bash_launcher( + linker=linker, library_path=library_path, executable=executable + ) + assert script_content.startswith("#! /bin/bash\n") assert linker in script_content assert executable in script_content -@pytest.mark.parametrize('compiler', ['diet', 'musl']) +@pytest.mark.parametrize("compiler", ["diet", "musl"]) def test_compile(compiler): - with open(fizz_buzz_source_file, 'r') as f: + with open(fizz_buzz_source_file, "r") as f: code = f.read() - compile = compile_diet if compiler == 'diet' else None - compile = compile or (compile_musl if compiler == 'musl' else None) + compile = compile_diet if compiler == "diet" else None + compile = compile or (compile_musl if compiler == "musl" else None) try: content = compile(code) except CompilerNotFoundError: @@ -47,34 +48,36 @@ def test_compile(compiler): f, filename = tempfile.mkstemp() os.close(f) - with open(filename, 'wb') as f: + with open(filename, "wb") as f: f.write(content) st = os.stat(filename) os.chmod(f.name, st.st_mode | stat.S_IXUSR) process = Popen(f.name, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() - assert 'FIZZBUZZ' in stdout.decode('utf-8') - assert len(stderr.decode('utf-8')) == 0 + assert "FIZZBUZZ" in stdout.decode("utf-8") + assert len(stderr.decode("utf-8")) == 0 def test_find_executable(): - original_environment = os.environ.get('PATH') + original_environment = os.environ.get("PATH") original_parent_directory = launchers.parent_directory root_directory = create_unpackaged_bundle( - rename=[], executables=[echo_args_glibc_32], chroot=chroot) + rename=[], executables=[echo_args_glibc_32], chroot=chroot + ) try: binary_name = os.path.basename(echo_args_glibc_32) - binary_symlink = os.path.join(root_directory, 'bin', binary_name) + binary_symlink = os.path.join(root_directory, "bin", binary_name) binary_path = os.path.realpath(binary_symlink) # This is a pretend directory, but it doesn't check. - launchers.parent_directory = os.path.join(os.path.dirname(binary_path), 'somewhere', 'else') - os.environ['PATH'] = os.path.dirname(echo_args_glibc_32) - assert find_executable(binary_name, skip_original_for_testing=True) == binary_path, \ + launchers.parent_directory = os.path.join(os.path.dirname(binary_path), "somewhere", "else") + os.environ["PATH"] = os.path.dirname(echo_args_glibc_32) + assert find_executable(binary_name, skip_original_for_testing=True) == binary_path, ( 'It should have found the binary path "%s".' % binary_path + ) finally: launchers.parent_directory = original_parent_directory - os.environ['PATH'] = original_environment - assert root_directory.startswith('/tmp/') + os.environ["PATH"] = original_environment + assert root_directory.startswith("/tmp/") shutil.rmtree(root_directory) diff --git a/tests/test_pytest.py b/tests/test_pytest.py index 55c5967..547f985 100644 --- a/tests/test_pytest.py +++ b/tests/test_pytest.py @@ -5,4 +5,4 @@ def test_catching_a_value_error(): """Temporary test to make sure that we can run tests.""" with pytest.raises(KeyError): - {}['no-matching-key'] + {}["no-matching-key"] diff --git a/tests/test_templating.py b/tests/test_templating.py index 4a7d565..d4b0228 100644 --- a/tests/test_templating.py +++ b/tests/test_templating.py @@ -6,19 +6,19 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) -data_directory = os.path.join(parent_directory, 'data') +data_directory = os.path.join(parent_directory, "data") def test_render_template(): - template = '{{greeting}}, my name is {{name}}.' - expected = 'Hello, my name is Evan.' - result = render_template(template, greeting='Hello', name='Evan') + template = "{{greeting}}, my name is {{name}}." + expected = "Hello, my name is Evan." + result = render_template(template, greeting="Hello", name="Evan") assert expected == result def test_render_template_file(): - template_file = os.path.join(data_directory, 'template.txt') - result = render_template_file(template_file, noun='word', location='here') - with open(os.path.join(data_directory, 'template-result.txt'), 'r') as f: + template_file = os.path.join(data_directory, "template.txt") + result = render_template_file(template_file, noun="word", location="here") + with open(os.path.join(data_directory, "template-result.txt"), "r") as f: expected_result = f.read() assert result == expected_result diff --git a/tox.ini b/tox.ini index 5a2a422..630ee6c 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,16 @@ envlist = clean, check, - {py27,py39}, + {py310,py311,py312,py313}, report, [testenv] basepython = - py27: {env:TOXPYTHON:python2.7} - {clean,check,report,py39}: {env:TOXPYTHON:python3.9} + py310: {env:TOXPYTHON:python3.10} + py311: {env:TOXPYTHON:python3.11} + py312: {env:TOXPYTHON:python3.12} + py313: {env:TOXPYTHON:python3.13} + {clean,check,report}: {env:TOXPYTHON:python3.13} setenv = PYTHONPATH={toxinidir}/tests PYTHONUNBUFFERED=yes @@ -16,32 +19,28 @@ passenv = * usedevelop = false deps = - pytest - pytest-sugar - pytest-travis-fold - pytest-cov + pytest>=8.0 + pytest-sugar>=1.0 + pytest-cov>=5.0 commands = {posargs:py.test --cov --cov-report=term-missing -vv} [testenv:check] deps = - docutils - check-manifest - flake8 - flake8-commas - flake8-quotes - readme-renderer - pygments - isort + docutils>=0.20 + check-manifest>=0.49 + ruff>=0.4.0 + readme-renderer>=43.0 + pygments>=2.17 skip_install = true commands = - flake8 src tests setup.py - isort --verbose --check-only --diff --recursive src tests setup.py - python setup.py check --strict --metadata --restructuredtext + ruff check src tests setup.py + ruff format --check src tests setup.py + python setup.py check --strict --metadata check-manifest {toxinidir} [testenv:report] -deps = coverage +deps = coverage>=7.0 skip_install = true commands = coverage report @@ -50,7 +49,7 @@ commands = [testenv:clean] commands = coverage erase skip_install = true -deps = coverage +deps = coverage>=7.0 [flake8] ignore = E128,W504