From d23b261669e08f5e1a09a2642abd18c11357df8f Mon Sep 17 00:00:00 2001 From: virtualian Date: Sun, 7 Dec 2025 12:49:18 +0000 Subject: [PATCH] Improve cache management (issue #8) - Move cache to ~/.npm-scanner/ (persistent across reboots) - Add 200MB size limit with automatic eviction of oldest files - Change expiration from 24h to 3 days - Minimize cached data to only needed fields (~97% smaller) - Add package list cache for faster counting (mtime invalidation) - Add `npm-scanner.sh cache` command (--status, --clear, --clear-expired) - Display cache stats after scans --- docs/npm-scanner.md | 30 ++++- npm-scanner.sh | 97 ++++++++++++++ scripts/audit-installed-packages.sh | 45 +++++-- scripts/generate-summary-report.sh | 5 +- scripts/npm-audit-lib.sh | 195 ++++++++++++++++++++++++++-- 5 files changed, 350 insertions(+), 22 deletions(-) diff --git a/docs/npm-scanner.md b/docs/npm-scanner.md index a8650f1..29c8d4c 100644 --- a/docs/npm-scanner.md +++ b/docs/npm-scanner.md @@ -54,6 +54,33 @@ Scan installed packages or project dependencies for security issues. ./npm-scanner.sh scan --project --yes . ``` +### cache + +Manage the npm metadata cache. + +```bash +./npm-scanner.sh cache [options] +``` + +**Options:** +- `--status` - Show cache statistics (default) +- `--clear` - Clear all cached data +- `--clear-expired` - Clear only expired cache entries + +**Example output:** +``` +npm-scanner Cache Status +======================== +Location: /Users/you/.npm-scanner/cache +Packages: 119 cached +Size: 4.5 MB / 200 MB limit +Expiration: 3 days +``` + +**Environment Variables:** +- `NPM_SCANNER_CACHE_MAX_SIZE_MB` - Maximum cache size in MB (default: 200) +- `NPM_SCANNER_CACHE_MAX_AGE_DAYS` - Cache expiration in days (default: 3) + ### list-iocs Display all current indicators of compromise (IOCs) without running a scan. @@ -150,7 +177,8 @@ Validation results are printed to stdout with: ## Environment Variables -- `TMPDIR` - Cache directory location (recommended: `/tmp`) +- `NPM_SCANNER_CACHE_MAX_SIZE_MB` - Maximum cache size in MB (default: 200) +- `NPM_SCANNER_CACHE_MAX_AGE_DAYS` - Cache expiration in days (default: 3) ## Exit Codes diff --git a/npm-scanner.sh b/npm-scanner.sh index e64e758..4104f43 100755 --- a/npm-scanner.sh +++ b/npm-scanner.sh @@ -24,6 +24,7 @@ Commands: scan Scan installed packages or project dependencies for security issues validate Validate a package before installation list-iocs Display all current indicators of compromise (IOCs) + cache Manage the npm metadata cache help Show this help message version Show version information @@ -33,6 +34,8 @@ Examples: npm-scanner.sh scan --local ~/projects # Scan node_modules directories npm-scanner.sh validate lodash # Check package before installing npm-scanner.sh list-iocs # Show all IOCs being checked + npm-scanner.sh cache --status # Show cache statistics + npm-scanner.sh cache --clear # Clear all cached data Run 'npm-scanner.sh --help' for command-specific options. @@ -114,6 +117,97 @@ cmd_list_iocs() { echo "" } +show_cache_help() { + cat << 'EOF' +Usage: npm-scanner.sh cache [options] + +Manage the npm metadata cache. + +Options: + --status Show cache statistics (default if no option given) + --clear Clear all cached data + --clear-expired Clear only expired cache entries + +Environment Variables: + NPM_SCANNER_CACHE_MAX_SIZE_MB Maximum cache size in MB (default: 200) + NPM_SCANNER_CACHE_MAX_AGE_DAYS Cache expiration in days (default: 3) + +Examples: + npm-scanner.sh cache # Show cache status + npm-scanner.sh cache --status # Show cache status + npm-scanner.sh cache --clear # Clear all cache + npm-scanner.sh cache --clear-expired # Clear only expired entries +EOF +} + +cmd_cache() { + source "$SCRIPTS_DIR/npm-audit-lib.sh" + + local action="status" + + while [ $# -gt 0 ]; do + case "$1" in + --help|-h) + show_cache_help + exit 0 + ;; + --status) + action="status" + shift + ;; + --clear) + action="clear" + shift + ;; + --clear-expired) + action="clear-expired" + shift + ;; + *) + echo "Unknown option: $1" + show_cache_help + exit 1 + ;; + esac + done + + case "$action" in + status) + local stats=$(lib_get_cache_stats_detailed) + local count=$(echo "$stats" | cut -d'|' -f1) + local size_mb=$(echo "$stats" | cut -d'|' -f2) + local location=$(echo "$stats" | cut -d'|' -f3) + local max_size=$(echo "$stats" | cut -d'|' -f4) + local max_age=$(echo "$stats" | cut -d'|' -f5) + local pkglist_count=$(lib_get_pkglist_cache_stats) + + echo "" + echo "npm-scanner Cache Status" + echo "========================" + echo "Location: ~/.npm-scanner/" + echo "" + echo "Package metadata (npm registry data):" + echo " Packages: $count cached" + echo " Size: ${size_mb} MB / ${max_size} MB limit" + echo " Expiration: ${max_age} days" + echo "" + echo "Package lists (node_modules indexes):" + echo " Directories: $pkglist_count cached" + echo " Invalidation: on directory change" + echo "" + ;; + clear) + local removed=$(lib_clear_cache) + local pkglist_removed=$(lib_clear_pkglist_cache) + echo "Cleared $removed cached packages, $pkglist_removed directory indexes" + ;; + clear-expired) + local removed=$(lib_clear_expired_cache) + echo "Cleared $removed expired cache entries" + ;; + esac +} + # Parse command if [ $# -eq 0 ]; then show_help @@ -133,6 +227,9 @@ case "$COMMAND" in list-iocs) cmd_list_iocs ;; + cache) + cmd_cache "$@" + ;; help|--help|-h) show_help ;; diff --git a/scripts/audit-installed-packages.sh b/scripts/audit-installed-packages.sh index d377cf0..799919a 100755 --- a/scripts/audit-installed-packages.sh +++ b/scripts/audit-installed-packages.sh @@ -380,7 +380,7 @@ is_valid_npm_package() { return 1 } -# Count valid npm packages in a directory +# Count valid npm packages in a directory (with caching) # Usage: count_valid_packages [show_progress] count_valid_packages() { local nm_dir=$1 @@ -388,16 +388,35 @@ count_valid_packages() { local show_progress=${3:-false} local count=0 - while IFS= read -r pkg_json_path; do - [ -z "$pkg_json_path" ] && continue - local pkg_dir=$(dirname "$pkg_json_path") - if is_valid_npm_package "$pkg_dir"; then - count=$((count + 1)) - if [ "$show_progress" = true ]; then - printf "\rCounting packages... %d" "$count" >&2 + # Try to get cached package list + local cached_list=$(lib_get_cached_pkglist "$nm_dir" "$max_depth") + + if [ -n "$cached_list" ]; then + # Use cached list - just count lines + count=$(echo "$cached_list" | wc -l | tr -d ' ') + if [ "$show_progress" = true ]; then + printf "\rCounting packages... %d (cached)" "$count" >&2 + fi + else + # Build package list and cache it + local pkg_list="" + while IFS= read -r pkg_json_path; do + [ -z "$pkg_json_path" ] && continue + local pkg_dir=$(dirname "$pkg_json_path") + if is_valid_npm_package "$pkg_dir"; then + count=$((count + 1)) + pkg_list="${pkg_list}${pkg_json_path}"$'\n' + if [ "$show_progress" = true ]; then + printf "\rCounting packages... %d" "$count" >&2 + fi fi + done < <(find "$nm_dir" -maxdepth "$max_depth" -name "package.json" -type f 2>/dev/null) + + # Save to cache (remove trailing newline) + if [ -n "$pkg_list" ]; then + echo -n "${pkg_list%$'\n'}" | lib_save_pkglist_cache "$nm_dir" "$max_depth" fi - done < <(find "$nm_dir" -maxdepth "$max_depth" -name "package.json" -type f 2>/dev/null) + fi echo "$count" } @@ -506,6 +525,14 @@ show_detection_summary() { echo " - Attacker patterns: ${#ATTACKER_PATTERNS[@]}" echo " - Typosquatting targets: ${#TYPOSQUATTING_TARGETS[@]}" echo " - Issues found: $issues_found" + + # Show cache status + local cache_stats=$(lib_get_cache_stats_detailed) + local cache_count=$(echo "$cache_stats" | cut -d'|' -f1) + local cache_size=$(echo "$cache_stats" | cut -d'|' -f2) + local cache_location=$(echo "$cache_stats" | cut -d'|' -f3) + echo "" + echo "Cache: $cache_count packages (${cache_size} MB) in $cache_location" } # Show what the scan checks for diff --git a/scripts/generate-summary-report.sh b/scripts/generate-summary-report.sh index 1765d68..9e86ab1 100755 --- a/scripts/generate-summary-report.sh +++ b/scripts/generate-summary-report.sh @@ -111,8 +111,9 @@ echo "" echo "================================================" echo "CACHE STATISTICS" echo "================================================" - echo "Packages cached: $(ls /tmp/.npm-audit-cache/ 2>/dev/null | wc -l)" - echo "Cache location: /tmp/.npm-audit-cache/" + CACHE_DIR="${HOME}/.npm-scanner/cache" + echo "Packages cached: $(ls "$CACHE_DIR" 2>/dev/null | wc -l | tr -d ' ')" + echo "Cache location: $CACHE_DIR" echo "" echo "================================================" diff --git a/scripts/npm-audit-lib.sh b/scripts/npm-audit-lib.sh index 9e8e85b..1b7908a 100755 --- a/scripts/npm-audit-lib.sh +++ b/scripts/npm-audit-lib.sh @@ -59,11 +59,17 @@ export TYPOSQUATTING_TARGETS=( # CACHE MANAGEMENT # ============================================================================ +# Cache configuration +NPM_SCANNER_CACHE_MAX_SIZE_MB="${NPM_SCANNER_CACHE_MAX_SIZE_MB:-200}" +NPM_SCANNER_CACHE_MAX_AGE_DAYS="${NPM_SCANNER_CACHE_MAX_AGE_DAYS:-3}" + # Initialize cache directory # Usage: lib_init_cache lib_init_cache() { - export NPM_AUDIT_CACHE_DIR="${TMPDIR:-/tmp}/.npm-audit-cache" + export NPM_AUDIT_CACHE_DIR="${HOME}/.npm-scanner/cache" + export NPM_AUDIT_PKGLIST_CACHE_DIR="${HOME}/.npm-scanner/pkglist" mkdir -p "$NPM_AUDIT_CACHE_DIR" + mkdir -p "$NPM_AUDIT_PKGLIST_CACHE_DIR" } # Get cache file path for a package @@ -74,41 +80,78 @@ lib_get_cache_file() { echo "$cache_file" } -# Check if cache is valid (less than 24 hours old) +# Check if cache is valid (less than configured max age) # Usage: lib_is_cache_valid "cache_file_path" # Returns: 0 if valid, 1 if invalid lib_is_cache_valid() { local cache_file="$1" - + if [ ! -f "$cache_file" ]; then return 1 fi - + # Get file modification time local file_time=$(stat -f %m "$cache_file" 2>/dev/null || stat -c %Y "$cache_file" 2>/dev/null || echo 0) local current_time=$(date +%s) local age=$((current_time - file_time)) - - # 24 hours = 86400 seconds - if [ $age -lt 86400 ]; then + + # Calculate max age in seconds (days * 24 * 60 * 60) + local max_age_seconds=$((NPM_SCANNER_CACHE_MAX_AGE_DAYS * 86400)) + + if [ $age -lt $max_age_seconds ]; then return 0 else return 1 fi } +# Get current cache size in bytes +# Usage: lib_get_cache_size_bytes +# Output: Size in bytes +lib_get_cache_size_bytes() { + if [ -d "$NPM_AUDIT_CACHE_DIR" ]; then + du -sk "$NPM_AUDIT_CACHE_DIR" 2>/dev/null | cut -f1 | awk '{print $1 * 1024}' + else + echo "0" + fi +} + +# Enforce cache size limit by removing oldest files +# Usage: lib_enforce_cache_limit +lib_enforce_cache_limit() { + local max_bytes=$((NPM_SCANNER_CACHE_MAX_SIZE_MB * 1024 * 1024)) + local current_bytes=$(lib_get_cache_size_bytes) + + if [ "$current_bytes" -gt "$max_bytes" ]; then + # Remove oldest files until under limit + ls -t "$NPM_AUDIT_CACHE_DIR"/*.json 2>/dev/null | tail -n +$(($(ls "$NPM_AUDIT_CACHE_DIR"/*.json 2>/dev/null | wc -l) / 2 + 1)) | while read -r file; do + rm -f "$file" + current_bytes=$(lib_get_cache_size_bytes) + if [ "$current_bytes" -le "$max_bytes" ]; then + break + fi + done + fi +} + # Get package info from npm (with caching) # Usage: lib_get_package_info "package-name" -# Output: JSON string +# Output: JSON string (minimized to only fields we need) lib_get_package_info() { local pkg="$1" local cache_file=$(lib_get_cache_file "$pkg") - + if lib_is_cache_valid "$cache_file"; then cat "$cache_file" else - local info=$(npm view "$pkg" --json 2>/dev/null || echo "{}") + # Fetch full data then extract only what we need: + # - maintainers (for solo maintainer check, attacker pattern matching) + # - time.created (for age check) + # - name (for reference) + local full_info=$(npm view "$pkg" --json 2>/dev/null || echo "{}") + local info=$(echo "$full_info" | jq '{name, maintainers, time: {created: .time.created}}' 2>/dev/null || echo "$full_info") echo "$info" > "$cache_file" + lib_enforce_cache_limit echo "$info" fi } @@ -124,6 +167,127 @@ lib_get_cache_stats() { fi } +# Get detailed cache statistics for display +# Usage: lib_get_cache_stats_detailed +# Output: "count|size_mb|location|max_size_mb|max_age_days" +lib_get_cache_stats_detailed() { + local count=$(lib_get_cache_stats) + local size_bytes=$(lib_get_cache_size_bytes) + local size_mb=$(echo "scale=1; $size_bytes / 1024 / 1024" | bc 2>/dev/null || echo "0") + echo "${count}|${size_mb}|${NPM_AUDIT_CACHE_DIR}|${NPM_SCANNER_CACHE_MAX_SIZE_MB}|${NPM_SCANNER_CACHE_MAX_AGE_DAYS}" +} + +# Clear all cache files +# Usage: lib_clear_cache +# Output: Number of files removed +lib_clear_cache() { + if [ -d "$NPM_AUDIT_CACHE_DIR" ]; then + local count=$(ls "$NPM_AUDIT_CACHE_DIR"/*.json 2>/dev/null | wc -l | tr -d ' ') + rm -f "$NPM_AUDIT_CACHE_DIR"/*.json 2>/dev/null + echo "$count" + else + echo "0" + fi +} + +# Clear expired cache files only +# Usage: lib_clear_expired_cache +# Output: Number of files removed +lib_clear_expired_cache() { + local removed=0 + if [ -d "$NPM_AUDIT_CACHE_DIR" ]; then + for cache_file in "$NPM_AUDIT_CACHE_DIR"/*.json; do + [ -f "$cache_file" ] || continue + if ! lib_is_cache_valid "$cache_file"; then + rm -f "$cache_file" + removed=$((removed + 1)) + fi + done + fi + echo "$removed" +} + +# ============================================================================ +# PACKAGE LIST CACHE (for faster counting) +# ============================================================================ + +# Get cache file path for a node_modules directory +# Usage: lib_get_pkglist_cache_file "/path/to/node_modules" +lib_get_pkglist_cache_file() { + local nm_dir="$1" + # Create a hash of the path for the filename + local path_hash=$(echo "$nm_dir" | md5 2>/dev/null || echo "$nm_dir" | md5sum 2>/dev/null | cut -d' ' -f1) + echo "$NPM_AUDIT_PKGLIST_CACHE_DIR/${path_hash}.list" +} + +# Get mtime of a directory +# Usage: lib_get_dir_mtime "/path/to/dir" +lib_get_dir_mtime() { + local dir="$1" + stat -f %m "$dir" 2>/dev/null || stat -c %Y "$dir" 2>/dev/null || echo 0 +} + +# Get cached package list for a node_modules directory +# Returns package paths if cache is valid, empty if cache miss +# Usage: lib_get_cached_pkglist "/path/to/node_modules" "max_depth" +# Output: One package.json path per line, or empty if cache miss +lib_get_cached_pkglist() { + local nm_dir="$1" + local max_depth="$2" + local cache_file=$(lib_get_pkglist_cache_file "$nm_dir") + + if [ ! -f "$cache_file" ]; then + return + fi + + # First line is mtime|max_depth, rest is package list + local cached_header=$(head -1 "$cache_file") + local cached_mtime=$(echo "$cached_header" | cut -d'|' -f1) + local cached_depth=$(echo "$cached_header" | cut -d'|' -f2) + local current_mtime=$(lib_get_dir_mtime "$nm_dir") + + # Validate cache: mtime must match and depth must be >= requested + if [ "$cached_mtime" = "$current_mtime" ] && [ "$cached_depth" -ge "$max_depth" ]; then + tail -n +2 "$cache_file" + fi +} + +# Save package list to cache +# Usage: lib_save_pkglist_cache "/path/to/node_modules" "max_depth" < package_list +lib_save_pkglist_cache() { + local nm_dir="$1" + local max_depth="$2" + local cache_file=$(lib_get_pkglist_cache_file "$nm_dir") + local mtime=$(lib_get_dir_mtime "$nm_dir") + + # Write header then package list + echo "${mtime}|${max_depth}" > "$cache_file" + cat >> "$cache_file" +} + +# Clear package list cache +# Usage: lib_clear_pkglist_cache +lib_clear_pkglist_cache() { + if [ -d "$NPM_AUDIT_PKGLIST_CACHE_DIR" ]; then + local count=$(ls "$NPM_AUDIT_PKGLIST_CACHE_DIR"/*.list 2>/dev/null | wc -l | tr -d ' ') + rm -f "$NPM_AUDIT_PKGLIST_CACHE_DIR"/*.list 2>/dev/null + echo "$count" + else + echo "0" + fi +} + +# Get package list cache stats +# Usage: lib_get_pkglist_cache_stats +# Output: count of cached directories +lib_get_pkglist_cache_stats() { + if [ -d "$NPM_AUDIT_PKGLIST_CACHE_DIR" ]; then + ls "$NPM_AUDIT_PKGLIST_CACHE_DIR"/*.list 2>/dev/null | wc -l | tr -d ' ' + else + echo "0" + fi +} + # ============================================================================ # MAINTAINER DATA PARSING # ============================================================================ @@ -694,8 +858,19 @@ lib_init_cache export -f lib_init_cache export -f lib_get_cache_file export -f lib_is_cache_valid +export -f lib_get_cache_size_bytes +export -f lib_enforce_cache_limit export -f lib_get_package_info export -f lib_get_cache_stats +export -f lib_get_cache_stats_detailed +export -f lib_clear_cache +export -f lib_clear_expired_cache +export -f lib_get_pkglist_cache_file +export -f lib_get_dir_mtime +export -f lib_get_cached_pkglist +export -f lib_save_pkglist_cache +export -f lib_clear_pkglist_cache +export -f lib_get_pkglist_cache_stats export -f lib_parse_maintainers export -f lib_output_maintainer_data export -f lib_get_first_maintainer