Enhance email risk assessment (issue #6)

npm-scanner.sh

@@ -14,13 +14,24 @@ show_version() {
     echo "npm-scanner $VERSION ($RELEASE_DATE)"
 }
 
+# Data directory for downloaded lists
+DATA_DIR="$HOME/.npm-scanner/data"
+DATA_REFRESH_FILE="$DATA_DIR/.last-refresh"
+DATA_STALE_DAYS=30
+
+# URLs for data files
+DISPOSABLE_DOMAINS_URL="https://raw.githubusercontent.com/disposable-email-domains/disposable-email-domains/main/disposable_email_blocklist.conf"
+TRANCO_LIST_URL="https://tranco-list.eu/top-1m.csv.zip"
+
 show_help() {
     cat << 'EOF'
 npm-scanner - Security toolkit for npm supply chain protection
 
 Usage: npm-scanner.sh <command> [options]
 
 Commands:
+  init       Download required data files (run once after install)
+  refresh    Update data files to latest versions
   scan       Scan installed packages or project dependencies for security issues
   validate   Validate a package before installation
   list-iocs  Display all current indicators of compromise (IOCs)
@@ -29,6 +40,8 @@ Commands:
   version    Show version information
 
 Examples:
+  npm-scanner.sh init                          # Initial setup
+  npm-scanner.sh refresh                       # Update data files
   npm-scanner.sh scan --global                 # Scan globally installed packages
   npm-scanner.sh scan --project ~/code         # Scan project dependencies
   npm-scanner.sh scan --local ~/projects       # Scan node_modules directories
@@ -43,11 +56,203 @@ Documentation: docs/README.md
 EOF
 }
 
+# Check if data files exist and are not stale
+# Returns: 0 if OK, 1 if missing, 2 if stale
+check_data_status() {
+    local disposable_file="$DATA_DIR/disposable-email-domains.txt"
+    local tranco_file="$DATA_DIR/tranco-top-100k.csv"
+
+    if [ ! -f "$disposable_file" ] || [ ! -f "$tranco_file" ]; then
+        return 1  # Missing
+    fi
+
+    if [ ! -f "$DATA_REFRESH_FILE" ]; then
+        return 2  # No refresh date, consider stale
+    fi
+
+    local last_refresh=$(cat "$DATA_REFRESH_FILE" 2>/dev/null)
+    local last_refresh_ts=$(date -j -f "%Y-%m-%d" "$last_refresh" +%s 2>/dev/null || date -d "$last_refresh" +%s 2>/dev/null || echo 0)
+    local now_ts=$(date +%s)
+    local age_days=$(( (now_ts - last_refresh_ts) / 86400 ))
+
+    if [ "$age_days" -gt "$DATA_STALE_DAYS" ]; then
+        return 2  # Stale
+    fi
+
+    return 0  # OK
+}
+
+# Get human-readable data status
+get_data_status_message() {
+    local disposable_file="$DATA_DIR/disposable-email-domains.txt"
+
+    if [ ! -f "$disposable_file" ]; then
+        echo "not initialized"
+        return
+    fi
+
+    if [ ! -f "$DATA_REFRESH_FILE" ]; then
+        echo "unknown (no refresh date)"
+        return
+    fi
+
+    local last_refresh=$(cat "$DATA_REFRESH_FILE" 2>/dev/null)
+    local last_refresh_ts=$(date -j -f "%Y-%m-%d" "$last_refresh" +%s 2>/dev/null || date -d "$last_refresh" +%s 2>/dev/null || echo 0)
+    local now_ts=$(date +%s)
+    local age_days=$(( (now_ts - last_refresh_ts) / 86400 ))
+
+    if [ "$age_days" -gt "$DATA_STALE_DAYS" ]; then
+        echo "stale (last refresh: $last_refresh, $age_days days ago)"
+    else
+        echo "current (last refresh: $last_refresh)"
+    fi
+}
+
+# Warn if data is missing or stale
+warn_data_status() {
+    check_data_status
+    local status=$?
+
+    if [ $status -eq 1 ]; then
+        echo ""
+        echo "WARNING: Data files not initialized."
+        echo "Run 'npm-scanner.sh init' to download required data files."
+        echo ""
+        return 1
+    elif [ $status -eq 2 ]; then
+        echo ""
+        echo "WARNING: Data files are stale ($(get_data_status_message))."
+        echo "Run 'npm-scanner.sh refresh' to update."
+        echo ""
+    fi
+    return 0
+}
+
+# Check if required tools are available
+check_download_prerequisites() {
+    local missing=""
+    if ! command -v curl >/dev/null 2>&1; then
+        missing="$missing curl"
+    fi
+    if ! command -v unzip >/dev/null 2>&1; then
+        missing="$missing unzip"
+    fi
+    if [ -n "$missing" ]; then
+        echo "ERROR: Missing required tools:$missing"
+        echo "Please install them and try again."
+        return 1
+    fi
+    return 0
+}
+
+# Download data files
+download_data_files() {
+    local verbose=${1:-true}
+
+    # Check prerequisites
+    if ! check_download_prerequisites; then
+        return 1
+    fi
+
+    mkdir -p "$DATA_DIR"
+
+    local disposable_file="$DATA_DIR/disposable-email-domains.txt"
+    local tranco_file="$DATA_DIR/tranco-top-100k.csv"
+
+    # Download disposable email domains
+    [ "$verbose" = true ] && echo "Downloading disposable email domains list..."
+    if curl -sL "$DISPOSABLE_DOMAINS_URL" -o "$disposable_file.tmp"; then
+        local count=$(wc -l < "$disposable_file.tmp" | tr -d ' ')
+        mv "$disposable_file.tmp" "$disposable_file"
+        [ "$verbose" = true ] && echo "  Downloaded $count domains"
+    else
+        [ "$verbose" = true ] && echo "  ERROR: Failed to download disposable domains list"
+        rm -f "$disposable_file.tmp"
+        return 1
+    fi
+
+    # Download Tranco top 1M domains (zip), extract and keep top 100k
+    [ "$verbose" = true ] && echo "Downloading Tranco domain rankings..."
+    local tranco_zip="$DATA_DIR/tranco.zip"
+    if curl -sL "$TRANCO_LIST_URL" -o "$tranco_zip"; then
+        # Unzip, convert CRLF to LF, and keep top 100k only
+        if unzip -p "$tranco_zip" top-1m.csv 2>/dev/null | tr -d '\r' | head -100000 > "$tranco_file.tmp"; then
+            local count=$(wc -l < "$tranco_file.tmp" | tr -d ' ')
+            mv "$tranco_file.tmp" "$tranco_file"
+            rm -f "$tranco_zip"
+            [ "$verbose" = true ] && echo "  Downloaded top $count domains"
+        else
+            [ "$verbose" = true ] && echo "  ERROR: Failed to extract Tranco list"
+            rm -f "$tranco_zip" "$tranco_file.tmp"
+            return 1
+        fi
+    else
+        [ "$verbose" = true ] && echo "  ERROR: Failed to download Tranco list"
+        rm -f "$tranco_zip"
+        return 1
+    fi
+
+    # Record refresh date
+    date +%Y-%m-%d > "$DATA_REFRESH_FILE"
+
+    [ "$verbose" = true ] && echo ""
+    [ "$verbose" = true ] && echo "Data files updated: $(date +%Y-%m-%d)"
+    [ "$verbose" = true ] && echo ""
+    [ "$verbose" = true ] && echo "Sources:"
+    [ "$verbose" = true ] && echo "  - Disposable domains: github.com/disposable-email-domains/disposable-email-domains"
+    [ "$verbose" = true ] && echo "  - Domain rankings: tranco-list.eu"
+
+    return 0
+}
+
+cmd_init() {
+    echo ""
+    echo "npm-scanner - Initializing data files"
+    echo ""
+
+    if check_data_status; then
+        echo "Data files already initialized ($(get_data_status_message))."
+        echo "Use 'npm-scanner.sh refresh' to update."
+        return 0
+    fi
+
+    download_data_files
+    local status=$?
+
+    if [ $status -eq 0 ]; then
+        echo ""
+        echo "Initialization complete. You can now use npm-scanner."
+    fi
+
+    return $status
+}
+
+cmd_refresh() {
+    echo ""
+    echo "npm-scanner - Refreshing data files"
+    echo ""
+
+    local current_status=$(get_data_status_message)
+    echo "Current status: $current_status"
+    echo ""
+
+    download_data_files
+    local status=$?
+
+    if [ $status -eq 0 ]; then
+        echo ""
+        echo "Refresh complete."
+    fi
+
+    return $status
+}
+
 show_scan_help() {
     "$SCRIPTS_DIR/audit-installed-packages.sh" --help
 }
 
 cmd_scan() {
+    warn_data_status || return 1
     "$SCRIPTS_DIR/audit-installed-packages.sh" "$@"
 }
 
@@ -75,6 +280,7 @@ cmd_validate() {
         show_validate_help
         exit 0
     fi
+    warn_data_status || return 1
     node "$SCRIPTS_DIR/package-validator.js" "$@"
 }
 
@@ -218,6 +424,12 @@ COMMAND="$1"
 shift
 
 case "$COMMAND" in
+    init)
+        cmd_init
+        ;;
+    refresh)
+        cmd_refresh
+        ;;
     scan)
         cmd_scan "$@"
         ;;