diff --git a/pyproject.toml b/pyproject.toml index d9bd5eb..1fbcfc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ "PyYAML", "wheel>=0.38.1", "intbitset", - "fosslight_binary>=5.1.22", "scancode-toolkit>=32.0.2", "lxml>=6.0.1", # cryptography 49.x does not provide macOS x86_64 wheels, causing source builds to require OpenSSL/pkg-config. @@ -56,6 +55,9 @@ Download = "https://github.com/fosslight/fosslight_source_scanner" [project.scripts] fosslight_source = "fosslight_source.cli:main" +[project.entry-points."scancode_pre_scan"] +ignore-binaries = "fosslight_source._scancode_ignore_binaries:IgnoreBinaries" + [tool.setuptools] package-dir = {"" = "src"} diff --git a/src/fosslight_source/_scancode_ignore_binaries.py b/src/fosslight_source/_scancode_ignore_binaries.py new file mode 100644 index 0000000..140e112 --- /dev/null +++ b/src/fosslight_source/_scancode_ignore_binaries.py @@ -0,0 +1,69 @@ +# Copyright (c) 2018 nexB Inc. and others. +# Copyright (c) 2026 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# Vendored from scancode-ignore-binaries (aboutcode-org/scancode-plugins) +# so PyPI installs do not need a GitHub git dependency. +# SPDX-PackageDownloadLocation: https://github.com/aboutcode-org/scancode-plugins/tree/main/misc/scancode-ignore-binaries + +from plugincode.pre_scan import PreScanPlugin +from plugincode.pre_scan import pre_scan_impl +from commoncode.cliutils import PluggableCommandLineOption +from commoncode.cliutils import PRE_SCAN_GROUP +from typecode.contenttype import get_type + + +@pre_scan_impl +class IgnoreBinaries(PreScanPlugin): + """ + Ignore binary files. + """ + + options = [ + PluggableCommandLineOption( + ('--ignore-binaries',), + is_flag=True, + help='Ignore binary files.', + sort_order=10, + help_group=PRE_SCAN_GROUP, + ) + ] + + def is_enabled(self, ignore_binaries, **kwargs): + return ignore_binaries + + def process_codebase(self, codebase, ignore_binaries, **kwargs): + """ + Remove binary Resources from the resource tree. + """ + if not ignore_binaries: + return + + resources_to_remove = [] + for resource in codebase.walk(): + if not resource.is_file: + continue + if is_binary(resource.location): + resources_to_remove.append(resource) + + for resource in resources_to_remove: + resource.remove(codebase) + + +def is_binary(location): + """ + Return True if the resource at location is a binary file. + """ + t = get_type(location) + return ( + t.is_binary + or t.is_archive + or t.is_media + or t.is_office_doc + or t.is_compressed + or t.is_filesystem + or t.is_winexe + or t.is_elf + or t.is_java_class + or t.is_data + ) diff --git a/src/fosslight_source/run_scancode.py b/src/fosslight_source/run_scancode.py index d20f682..a64cb7e 100755 --- a/src/fosslight_source/run_scancode.py +++ b/src/fosslight_source/run_scancode.py @@ -7,6 +7,8 @@ import multiprocessing import warnings import logging +from typing import Tuple + from scancode import cli import fosslight_util.constant as constant from fosslight_util.set_log import init_log @@ -14,15 +16,12 @@ from ._parsing_scancode_file_item import parsing_file_item from ._parsing_scancode_file_item import get_error_from_header from fosslight_util.output_format import check_output_formats_v2 -from fosslight_binary.binary_analysis import check_binary from fosslight_util.exclude import ( EXCLUDE_DIRECTORY, EXCLUDE_FILE_EXTENSION, EXCLUDE_FILENAME, PACKAGE_DIRECTORY, ) -from commoncode.fileset import is_included -from typing import Callable, Tuple logger = logging.getLogger(constant.LOGGER_NAME) warnings.filterwarnings("ignore", category=FutureWarning) @@ -68,8 +67,6 @@ def _apply_scancode_unset_workaround(kwargs: dict) -> None: "zip", "tar", "tgz", "gz", "bmp", "webp", "ico", } | {ext.lower() for ext in EXCLUDE_FILE_EXTENSION} -_SKIP_DIR_NAMES = frozenset(name.lower() for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY) -_SKIP_EXTS = frozenset(ext.lower() for ext in EXCLUDE_FILE_EXTENSION) def _normalize_custom_pattern(pattern: str, abs_path_to_scan: str) -> set: @@ -160,14 +157,15 @@ def _directory_ignore_pattern(dir_name: str) -> str: return f"**/{normalized}/**" -def _default_scancode_coarse_ignore_patterns( +def _default_scancode_ignore_patterns( path_to_exclude: list = None, abs_path_to_scan: str = "" -) -> frozenset: +) -> tuple: """ Coarse ignore patterns aligned with fosslight_util.get_excluded_paths() rules. Directory names use path-based globs (e.g. **/tests/**) so they do not match the scan root directory name itself. + Binary files are excluded separately via scancode --ignore-binaries. """ patterns = {".*"} for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY: @@ -182,87 +180,6 @@ def _default_scancode_coarse_ignore_patterns( pattern = os.path.relpath(pattern, abs_path_to_scan) patterns.update(_expand_custom_exclude_pattern(pattern, abs_path_to_scan)) - return frozenset(patterns) - - -def _to_excludes_dict(patterns) -> dict: - return {pattern: "exclude" for pattern in patterns} - - -def _is_path_covered(rel_path: str, excludes: dict) -> bool: - return not is_included(rel_path, includes={}, excludes=excludes) - - -def _add_ignore_pattern( - patterns: set, - excludes: dict, - pattern: str, - *, - sample_path: str = None, -) -> bool: - if pattern in patterns: - return False - if sample_path and _is_path_covered(sample_path, excludes): - return False - patterns.add(pattern) - excludes[pattern] = "exclude" - return True - - -def _make_pre_scan_skip_filter( - coarse_patterns: frozenset, -) -> Tuple[dict, Callable[[str, str], bool], Callable[[str, str], bool]]: - excludes = _to_excludes_dict(coarse_patterns) - - def should_skip_dir(dir_name: str, rel_dir: str) -> bool: - if dir_name.startswith('.'): - return True - if dir_name.lower() in _SKIP_DIR_NAMES: - return True - return _is_path_covered(f"{rel_dir}/_", excludes) - - def should_skip_file(file_name: str, rel_path: str) -> bool: - if file_name.startswith('.'): - return True - ext = os.path.splitext(file_name)[1].lstrip('.').lower() - if ext in _SKIP_EXTS: - return True - return _is_path_covered(rel_path, excludes) - - return excludes, should_skip_dir, should_skip_file - - -def _add_binary_ignore_patterns( - patterns: set, - excludes: dict, - binary_paths: list, -) -> None: - extensions = set() - no_ext_paths = [] - - for rel_path in binary_paths: - ext = os.path.splitext(rel_path)[1].lstrip('.').lower() - if ext: - extensions.add(ext) - else: - no_ext_paths.append(rel_path) - - for ext in extensions: - _add_ignore_pattern(patterns, excludes, f"*.{ext}") - - for rel_path in no_ext_paths: - _add_ignore_pattern( - patterns, excludes, f"**/{rel_path}", sample_path=rel_path - ) - - -def _build_scancode_ignore_patterns( - coarse_patterns: frozenset, - binary_paths: list, -) -> tuple: - patterns = set(coarse_patterns) - excludes = _to_excludes_dict(coarse_patterns) - _add_binary_ignore_patterns(patterns, excludes, binary_paths) return tuple(sorted(patterns)) @@ -332,38 +249,9 @@ def run_scan( pretty_params["path_to_exclude"] = path_to_exclude pretty_params["output_file"] = output_file_name abs_path_to_scan = os.path.abspath(path_to_scan) - binary_paths = [] - coarse_patterns = _default_scancode_coarse_ignore_patterns( + ignore_tuple = _default_scancode_ignore_patterns( path_to_exclude, abs_path_to_scan ) - _, should_skip_dir, should_skip_file = _make_pre_scan_skip_filter( - coarse_patterns - ) - - for root, dirs, files in os.walk(path_to_scan): - rel_root = os.path.relpath(root, abs_path_to_scan).replace("\\", "/") - dirs[:] = [ - d for d in dirs - if not should_skip_dir( - d, d if rel_root == "." else f"{rel_root}/{d}" - ) - ] - for name in files: - rel_path = name if rel_root == "." else f"{rel_root}/{name}" - if should_skip_file(name, rel_path): - continue - full_path = os.path.join(root, name) - try: - if not check_binary(full_path, True): - continue - except Exception: - continue - binary_paths.append(rel_path) - logger.debug(f"Excluded binary from scancode: {rel_path}") - - ignore_tuple = _build_scancode_ignore_patterns( - coarse_patterns, binary_paths - ) logger.debug(f"Scancode ignore patterns: {len(ignore_tuple)}") kwargs = { @@ -382,6 +270,7 @@ def run_scan( "timeout": time_out, "include": (), "ignore": ignore_tuple, + "ignore_binaries": True, "quiet": hide_progress } _apply_scancode_unset_workaround(kwargs) @@ -407,12 +296,6 @@ def run_scan( success = True result_list = sorted( result_list, key=lambda row: (''.join(row.licenses))) - - for scan_item in result_list: - if os.path.isdir(scan_item.source_name_or_path): - continue - if check_binary(os.path.join(path_to_scan, scan_item.source_name_or_path), True): - scan_item.exclude = True except Exception as ex: success = False msg = str(ex)