From 198a09e6edac9c2f08e3c76cdb01498e2420592e Mon Sep 17 00:00:00 2001 From: Park Wonjae Date: Thu, 9 Jul 2026 15:14:53 +0900 Subject: [PATCH 1/4] refactor: optimize scancode ignore pattern size using wildcards Signed-off-by: Park Wonjae --- src/fosslight_source/cli.py | 6 +-- src/fosslight_source/run_scancode.py | 74 +++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 651917b1..c317fa5f 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -567,7 +567,7 @@ def run_scanners( excluded_path_without_dot, excluded_files, cnt_file_except_skipped) = get_excluded_paths(path_to_scan, path_to_exclude_with_filename) - logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}") + logger.debug(f"Skipped paths count: {len(excluded_path_with_default_exclusion)}") if not selected_scanner: selected_scanner = ALL_MODE @@ -575,8 +575,8 @@ def run_scanners( success, result_log[RESULT_KEY], scancode_result, license_list = run_scan( path_to_scan, output_file_name, write_json_file, num_cores, True, print_matched_text, formats, called_by_cli, time_out, correct_mode, - correct_filepath, excluded_path_with_default_exclusion, - excluded_files, hide_progress, + correct_filepath, path_to_exclude, + hide_progress=hide_progress, ) excluded_files = set(excluded_files) if excluded_files else set() if selected_scanner in ['scanoss', ALL_MODE]: diff --git a/src/fosslight_source/run_scancode.py b/src/fosslight_source/run_scancode.py index cf6f4e6f..d91e0c0e 100755 --- a/src/fosslight_source/run_scancode.py +++ b/src/fosslight_source/run_scancode.py @@ -63,6 +63,43 @@ def _apply_scancode_unset_workaround(kwargs: dict) -> None: logger.debug("scancode UNSET workaround skipped: %s", ex) +_WILDCARD_EXTENSIONS = { + "png", "mp3", "ogg", "comp", "bin", "o", "db", "tflite", + "ttf", "pyc", "exe", "dll", "jpg", "gif" +} + + +def _normalize_custom_pattern(pattern: str, abs_path_to_scan: str) -> set: + pat = pattern.replace('\\', '/').strip() + if not pat: + return set() + + patterns_to_add = {pat} + + if pat.endswith("/**"): + base = pat[:-3].rstrip("/") + if base: + patterns_to_add.add(base) + elif pat.endswith("/*"): + base = pat[:-2].rstrip("/") + if base: + patterns_to_add.add(base) + patterns_to_add.add(f"{base}/**") + elif pat.endswith("/"): + base = pat.rstrip("/") + if base: + patterns_to_add.add(base) + patterns_to_add.add(f"{base}/**") + patterns_to_add.add(f"{base}/*") + else: + full_path = os.path.join(abs_path_to_scan, pat) + if os.path.isdir(full_path): + patterns_to_add.add(f"{pat}/**") + patterns_to_add.add(f"{pat}/*") + + return patterns_to_add + + def _directory_ignore_pattern(dir_name: str) -> str: """Path-based glob for a directory name (avoids matching the scan root itself).""" normalized = dir_name.strip().strip("/").replace("\\", "/") @@ -71,7 +108,10 @@ def _directory_ignore_pattern(dir_name: str) -> str: return f"**/{normalized}/**" -def _default_scancode_coarse_ignore_patterns() -> frozenset: +def _default_scancode_coarse_ignore_patterns( + path_to_exclude: list = [], + abs_path_to_scan: str = "" +) -> frozenset: """ Coarse ignore patterns aligned with fosslight_util.get_excluded_paths() rules. Directory names use path-based globs (e.g. **/tests/**) so they do not match @@ -84,6 +124,10 @@ def _default_scancode_coarse_ignore_patterns() -> frozenset: patterns.add(f"*.{ext}") for name in EXCLUDE_FILENAME: patterns.add(name) + + for pattern in path_to_exclude or []: + patterns.update(_normalize_custom_pattern(pattern, abs_path_to_scan)) + return frozenset(patterns) @@ -127,10 +171,17 @@ def _add_path_to_exclude_pattern( else: patterns.add(exclude_path_normalized) elif os.path.isfile(full_exclude_path): - if not _is_covered_by_coarse_ignore(exclude_path_normalized, coarse_patterns): + ext = os.path.splitext(exclude_path_normalized)[1].lstrip('.').lower() + if ext in _WILDCARD_EXTENSIONS: + patterns.add(f"*.{ext}") + elif not _is_covered_by_coarse_ignore(exclude_path_normalized, coarse_patterns): patterns.add(f"**/{exclude_path_normalized}") else: - patterns.add(exclude_path_normalized) + ext = os.path.splitext(exclude_path_normalized)[1].lstrip('.').lower() + if ext in _WILDCARD_EXTENSIONS: + patterns.add(f"*.{ext}") + else: + patterns.add(exclude_path_normalized) def _build_scancode_ignore_patterns( @@ -138,7 +189,7 @@ def _build_scancode_ignore_patterns( abs_path_to_scan: str, binary_paths: list, ) -> tuple: - coarse_patterns = _default_scancode_coarse_ignore_patterns() + coarse_patterns = _default_scancode_coarse_ignore_patterns(path_to_exclude, abs_path_to_scan) patterns = set(coarse_patterns) for path in path_to_exclude or []: @@ -149,7 +200,11 @@ def _build_scancode_ignore_patterns( _add_path_to_exclude_pattern(patterns, exclude_path, abs_path_to_scan, coarse_patterns) for rel_path in binary_paths: - patterns.add(f"**/{rel_path}") + ext = os.path.splitext(rel_path)[1].lstrip('.').lower() + if ext in _WILDCARD_EXTENSIONS: + patterns.add(f"*.{ext}") + else: + patterns.add(f"**/{rel_path}") return tuple(sorted(patterns)) @@ -199,8 +254,13 @@ def run_scan( output_json_file = "" if not called_by_cli: - logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"), - True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude) + log_file_path = os.path.join( + output_path, f"fosslight_log_src_{_start_time}.txt" + ) + logger, _result_log = init_log( + log_file_path, True, logging.INFO, logging.DEBUG, + _PKG_NAME, path_to_scan, path_to_exclude + ) logger.info(f"Tool Info : {_result_log['Tool Info']}") From 9df5b7fe3a4aa1e17e5147e8ad991ccf9ef12ef5 Mon Sep 17 00:00:00 2001 From: Park Wonjae Date: Thu, 9 Jul 2026 17:06:37 +0900 Subject: [PATCH 2/4] refactor: prune hidden and custom-excluded paths during pre-scan walk Signed-off-by: Park Wonjae --- src/fosslight_source/run_scancode.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/fosslight_source/run_scancode.py b/src/fosslight_source/run_scancode.py index d91e0c0e..68a8811b 100755 --- a/src/fosslight_source/run_scancode.py +++ b/src/fosslight_source/run_scancode.py @@ -132,7 +132,7 @@ def _default_scancode_coarse_ignore_patterns( def _is_covered_by_coarse_ignore(rel_path: str, coarse_patterns: Iterable[str]) -> bool: - excludes = {pattern: "" for pattern in coarse_patterns} + excludes = {pattern: "exclude" for pattern in coarse_patterns} return not is_included(rel_path, includes={}, excludes=excludes) @@ -275,18 +275,30 @@ def run_scan( pretty_params["output_file"] = output_file_name abs_path_to_scan = os.path.abspath(path_to_scan) binary_paths = [] - for root, _, files in os.walk(path_to_scan): + coarse_patterns = _default_scancode_coarse_ignore_patterns(path_to_exclude, abs_path_to_scan) + + for root, dirs, files in os.walk(path_to_scan): + dirs[:] = [ + d for d in dirs + if not d.startswith('.') and not _is_covered_by_coarse_ignore( + os.path.relpath(os.path.join(root, d), abs_path_to_scan).replace("\\", "/") + "/a", + coarse_patterns + ) + ] for name in files: + if name.startswith('.'): + continue full_path = os.path.join(root, name) + rel_path = os.path.relpath(full_path, abs_path_to_scan).replace("\\", "/") + if _is_covered_by_coarse_ignore(rel_path, coarse_patterns): + continue try: if not check_binary(full_path, True): continue except Exception: continue - rel_path = os.path.relpath(full_path, abs_path_to_scan) - rel_norm = os.path.normpath(rel_path).replace("\\", "/") - binary_paths.append(rel_norm) - logger.debug(f"Excluded binary from scancode: {rel_norm}") + binary_paths.append(rel_path) + logger.debug(f"Excluded binary from scancode: {rel_path}") ignore_tuple = _build_scancode_ignore_patterns( path_to_exclude, abs_path_to_scan, binary_paths From a2a6f620fb2e0d072f23f5fea4e5465ad3a8d7ca Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Fri, 10 Jul 2026 14:57:36 +0900 Subject: [PATCH 3/4] refactor: deduplicate scancode ignore patterns and speed up pre-scan walk Consolidate custom exclude handling into coarse patterns once, compress binary paths by extension, and cache exclude filters during os.walk to keep ignore_tuple small and avoid redundant is_included calls. --- src/fosslight_source/run_scancode.py | 207 +++++++++++++++++---------- 1 file changed, 134 insertions(+), 73 deletions(-) diff --git a/src/fosslight_source/run_scancode.py b/src/fosslight_source/run_scancode.py index 44b40fdf..4c8c7595 100755 --- a/src/fosslight_source/run_scancode.py +++ b/src/fosslight_source/run_scancode.py @@ -22,7 +22,7 @@ PACKAGE_DIRECTORY, ) from commoncode.fileset import is_included -from typing import Tuple, Iterable +from typing import Callable, Tuple logger = logging.getLogger(constant.LOGGER_NAME) warnings.filterwarnings("ignore", category=FutureWarning) @@ -63,9 +63,13 @@ def _apply_scancode_unset_workaround(kwargs: dict) -> None: _WILDCARD_EXTENSIONS = { - "png", "mp3", "ogg", "comp", "bin", "o", "db", "tflite", - "ttf", "pyc", "exe", "dll", "jpg", "gif" + "png", "mp3", "wav", "comp", "bin", "o", "db", "tflite", + "ttf", "pyc", "exe", "dll", "jpg", "jpeg", "gif", + "zip", "tar", "tgz", "gz", + "bmp", "webp", "ico", } +_SKIP_DIR_NAMES = frozenset(name.lower() for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY) +_SKIP_EXTS = frozenset(ext.lower() for ext in EXCLUDE_FILE_EXTENSION) def _normalize_custom_pattern(pattern: str, abs_path_to_scan: str) -> set: @@ -99,6 +103,55 @@ def _normalize_custom_pattern(pattern: str, abs_path_to_scan: str) -> set: return patterns_to_add +def _expand_custom_exclude_pattern(pattern: str, abs_path_to_scan: str) -> set: + exclude_path_normalized = os.path.normpath( + pattern.replace('\\', '/').strip() + ).replace("\\", "/") + if not exclude_path_normalized: + return set() + + patterns = set(_normalize_custom_pattern(exclude_path_normalized, abs_path_to_scan)) + + if exclude_path_normalized.endswith("/**"): + base_dir = exclude_path_normalized[:-3].rstrip("/") + if base_dir: + full_exclude_path = os.path.join(abs_path_to_scan, base_dir) + if os.path.isdir(full_exclude_path): + patterns.add(base_dir) + patterns.add(exclude_path_normalized) + else: + patterns.add(exclude_path_normalized) + return patterns + + has_glob_chars = any(char in exclude_path_normalized for char in ['*', '?', '[']) + if has_glob_chars: + patterns.add(exclude_path_normalized) + return patterns + + full_exclude_path = os.path.join(abs_path_to_scan, exclude_path_normalized) + if os.path.isdir(full_exclude_path): + base_path = exclude_path_normalized.rstrip("/") + if base_path: + patterns.add(base_path) + patterns.add(f"{base_path}/**") + else: + patterns.add(exclude_path_normalized) + elif os.path.isfile(full_exclude_path): + ext = os.path.splitext(exclude_path_normalized)[1].lstrip('.').lower() + if ext in _WILDCARD_EXTENSIONS: + patterns.add(f"*.{ext}") + else: + patterns.add(f"**/{exclude_path_normalized}") + else: + ext = os.path.splitext(exclude_path_normalized)[1].lstrip('.').lower() + if ext in _WILDCARD_EXTENSIONS: + patterns.add(f"*.{ext}") + else: + patterns.add(exclude_path_normalized) + + return patterns + + def _directory_ignore_pattern(dir_name: str) -> str: """Path-based glob for a directory name (avoids matching the scan root itself).""" normalized = dir_name.strip().strip("/").replace("\\", "/") @@ -108,7 +161,7 @@ def _directory_ignore_pattern(dir_name: str) -> str: def _default_scancode_coarse_ignore_patterns( - path_to_exclude: list = [], + path_to_exclude: list = None, abs_path_to_scan: str = "" ) -> frozenset: """ @@ -125,86 +178,91 @@ def _default_scancode_coarse_ignore_patterns( patterns.add(name) for pattern in path_to_exclude or []: - patterns.update(_normalize_custom_pattern(pattern, abs_path_to_scan)) + if os.path.isabs(pattern): + pattern = os.path.relpath(pattern, abs_path_to_scan) + patterns.update(_expand_custom_exclude_pattern(pattern, abs_path_to_scan)) return frozenset(patterns) -def _is_covered_by_coarse_ignore(rel_path: str, coarse_patterns: Iterable[str]) -> bool: - excludes = {pattern: "exclude" for pattern in coarse_patterns} +def _to_excludes_dict(patterns) -> dict: + return {pattern: "exclude" for pattern in patterns} + + +def _is_path_covered(rel_path: str, excludes: dict) -> bool: return not is_included(rel_path, includes={}, excludes=excludes) -def _add_path_to_exclude_pattern( +def _add_ignore_pattern( patterns: set, - exclude_path: str, - abs_path_to_scan: str, + excludes: dict, + pattern: str, + *, + sample_path: str = None, +) -> bool: + if pattern in patterns: + return False + if sample_path and _is_path_covered(sample_path, excludes): + return False + patterns.add(pattern) + excludes[pattern] = "exclude" + return True + + +def _make_pre_scan_skip_filter( coarse_patterns: frozenset, -) -> None: - exclude_path_normalized = os.path.normpath(exclude_path).replace("\\", "/") +) -> Tuple[dict, Callable[[str, str], bool], Callable[[str, str], bool]]: + excludes = _to_excludes_dict(coarse_patterns) - if exclude_path_normalized.endswith("/**"): - base_dir = exclude_path_normalized[:-3].rstrip("/") - if base_dir: - full_exclude_path = os.path.join(abs_path_to_scan, base_dir) - if os.path.isdir(full_exclude_path): - patterns.add(base_dir) - patterns.add(exclude_path_normalized) - else: - patterns.add(exclude_path_normalized) - else: - patterns.add(exclude_path_normalized) - return + def should_skip_dir(dir_name: str, rel_dir: str) -> bool: + if dir_name.startswith('.'): + return True + if dir_name.lower() in _SKIP_DIR_NAMES: + return True + return _is_path_covered(f"{rel_dir}/_", excludes) - has_glob_chars = any(char in exclude_path_normalized for char in ['*', '?', '[']) - if has_glob_chars: - patterns.add(exclude_path_normalized) - return + def should_skip_file(file_name: str, rel_path: str) -> bool: + if file_name.startswith('.'): + return True + ext = os.path.splitext(file_name)[1].lstrip('.').lower() + if ext in _SKIP_EXTS: + return True + return _is_path_covered(rel_path, excludes) - full_exclude_path = os.path.join(abs_path_to_scan, exclude_path_normalized) - if os.path.isdir(full_exclude_path): - base_path = exclude_path_normalized.rstrip("/") - if base_path: - patterns.add(base_path) - patterns.add(f"{base_path}/**") - else: - patterns.add(exclude_path_normalized) - elif os.path.isfile(full_exclude_path): - ext = os.path.splitext(exclude_path_normalized)[1].lstrip('.').lower() - if ext in _WILDCARD_EXTENSIONS: - patterns.add(f"*.{ext}") - elif not _is_covered_by_coarse_ignore(exclude_path_normalized, coarse_patterns): - patterns.add(f"**/{exclude_path_normalized}") - else: - ext = os.path.splitext(exclude_path_normalized)[1].lstrip('.').lower() - if ext in _WILDCARD_EXTENSIONS: - patterns.add(f"*.{ext}") - else: - patterns.add(exclude_path_normalized) + return excludes, should_skip_dir, should_skip_file -def _build_scancode_ignore_patterns( - path_to_exclude: list, - abs_path_to_scan: str, +def _add_binary_ignore_patterns( + patterns: set, + excludes: dict, binary_paths: list, -) -> tuple: - coarse_patterns = _default_scancode_coarse_ignore_patterns(path_to_exclude, abs_path_to_scan) - patterns = set(coarse_patterns) - - for path in path_to_exclude or []: - if os.path.isabs(path): - exclude_path = os.path.relpath(path, abs_path_to_scan) - else: - exclude_path = path - _add_path_to_exclude_pattern(patterns, exclude_path, abs_path_to_scan, coarse_patterns) +) -> None: + extensions = set() + no_ext_paths = [] for rel_path in binary_paths: ext = os.path.splitext(rel_path)[1].lstrip('.').lower() - if ext in _WILDCARD_EXTENSIONS: - patterns.add(f"*.{ext}") + if ext: + extensions.add(ext) else: - patterns.add(f"**/{rel_path}") + no_ext_paths.append(rel_path) + for ext in extensions: + _add_ignore_pattern(patterns, excludes, f"*.{ext}") + + for rel_path in no_ext_paths: + _add_ignore_pattern( + patterns, excludes, f"**/{rel_path}", sample_path=rel_path + ) + + +def _build_scancode_ignore_patterns( + coarse_patterns: frozenset, + binary_paths: list, +) -> tuple: + patterns = set(coarse_patterns) + excludes = _to_excludes_dict(coarse_patterns) + _add_binary_ignore_patterns(patterns, excludes, binary_paths) return tuple(sorted(patterns)) @@ -274,23 +332,26 @@ def run_scan( pretty_params["output_file"] = output_file_name abs_path_to_scan = os.path.abspath(path_to_scan) binary_paths = [] - coarse_patterns = _default_scancode_coarse_ignore_patterns(path_to_exclude, abs_path_to_scan) + coarse_patterns = _default_scancode_coarse_ignore_patterns( + path_to_exclude, abs_path_to_scan + ) + _, should_skip_dir, should_skip_file = _make_pre_scan_skip_filter( + coarse_patterns + ) for root, dirs, files in os.walk(path_to_scan): + rel_root = os.path.relpath(root, abs_path_to_scan).replace("\\", "/") dirs[:] = [ d for d in dirs - if not d.startswith('.') and not _is_covered_by_coarse_ignore( - os.path.relpath(os.path.join(root, d), abs_path_to_scan).replace("\\", "/") + "/a", - coarse_patterns + if not should_skip_dir( + d, d if rel_root == "." else f"{rel_root}/{d}" ) ] for name in files: - if name.startswith('.'): + rel_path = name if rel_root == "." else f"{rel_root}/{name}" + if should_skip_file(name, rel_path): continue full_path = os.path.join(root, name) - rel_path = os.path.relpath(full_path, abs_path_to_scan).replace("\\", "/") - if _is_covered_by_coarse_ignore(rel_path, coarse_patterns): - continue try: if not check_binary(full_path, True): continue @@ -300,7 +361,7 @@ def run_scan( logger.debug(f"Excluded binary from scancode: {rel_path}") ignore_tuple = _build_scancode_ignore_patterns( - path_to_exclude, abs_path_to_scan, binary_paths + coarse_patterns, binary_paths ) logger.debug(f"Scancode ignore patterns: {len(ignore_tuple)}") From 4f1b6e47b561d1422c1d5bc89559304727c0cb1f Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Mon, 13 Jul 2026 06:38:04 +0900 Subject: [PATCH 4/4] refactor: include EXCLUDE_FILE_EXTENSION in wildcard ignore extensions Avoid adding redundant per-file ignore paths when a custom exclude already matches a default exclude extension covered by *.ext patterns. --- src/fosslight_source/run_scancode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fosslight_source/run_scancode.py b/src/fosslight_source/run_scancode.py index 4c8c7595..5c2b99e1 100755 --- a/src/fosslight_source/run_scancode.py +++ b/src/fosslight_source/run_scancode.py @@ -64,10 +64,10 @@ def _apply_scancode_unset_workaround(kwargs: dict) -> None: _WILDCARD_EXTENSIONS = { "png", "mp3", "wav", "comp", "bin", "o", "db", "tflite", - "ttf", "pyc", "exe", "dll", "jpg", "jpeg", "gif", + "ttf", "exe", "dll", "jpg", "jpeg", "gif", "zip", "tar", "tgz", "gz", "bmp", "webp", "ico", -} +} | {ext.lower() for ext in EXCLUDE_FILE_EXTENSION} _SKIP_DIR_NAMES = frozenset(name.lower() for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY) _SKIP_EXTS = frozenset(ext.lower() for ext in EXCLUDE_FILE_EXTENSION)