diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index bbbf9cf..9af7014 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -18,6 +18,6 @@ jobs: - name: Build Bins run: ./scripts/bins.sh - name: E2E - run: ./scripts/e2e.sh + run: ETE_DIR=$HOME ./scripts/e2e.sh - name: Print SHA run: cat scnnr_bins.zip.sha256 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dd5960f..c01eda9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,7 @@ release: - go run main.go -m fsf -s 1MB -d $HOME - go run main.go -m fnf -f main,DEFCON -p $HOME - go run main.go -k main,const,let,var,for -p $HOME + - ETE_DIR=$HOME ./scripts/e2e.sh - go run cmd/pack/main.go - go run main.go -m fff -k $(go run cmd/checksum/main.go) -p . - cat scnnr_bins.zip.sha256 @@ -26,4 +27,3 @@ release: - scnnr_bins.zip.sha256 - LICENSE - README.md - diff --git a/README.md b/README.md index 579bf61..a03b92b 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,14 @@ $ scnnr -h -s string REQUIRED SizeFinder MODE size: 1MB,10MB,100MB,1GB,10GB,100GB,1TB + -xd string + OPTIONAL Scnnr MODE + comma-delimited list of directory names to exclude from scanning + ex: -xd ".git,node_modules,.venv" + -xe string + OPTIONAL Scnnr MODE + comma-delimited list of file extensions to exclude from scanning + ex: -xe ".log,.tmp,.json" ``` ### No Keywords @@ -187,6 +195,37 @@ README.md:383:36:cache README.md:393:40:cache ``` +### Keywords while excluding dirs (-xd) and file extensions (-xe) + +```bash +go run main.go -k main,let,for -p . -c -xd ".git,scnnr_bins,pkg,cmd,.zip" -xe ".yml,.sh,.md" +.editorconfig:11:21:let +.gitignore:3:1:main +.gitignore:4:1:main +.dockerignore:3:1:main +.dockerignore:4:1:main +Dockerfile:14:21:main +Dockerfile:15:25:main +main.go:25:9:main +main.go:36:6:main +main.go:46:8:for +main.go:47:8:for +main.go:48:8:for +main.go:49:8:for +main.go:70:65:for +main.go:78:57:for +main.go:84:23:for +main.go:89:34:for +main.go:125:3:for +main.go:129:3:for +main.go:172:3:for +main.go:179:42:for +main.go:186:3:for +scnnr_bins.zip:2053:93:for +scnnr_bins.zip:26325:8:let +scnnr_bins.zip:32123:53:let +``` + # File Name Finder (NameFinder) (fnf) ``` diff --git a/main.go b/main.go index 6f71219..9a29896 100644 --- a/main.go +++ b/main.go @@ -101,6 +101,16 @@ func main() { flag.StringVar(&size, "s", "", `REQUIRED SizeFinder MODE size: 1MB,10MB,100MB,1GB,10GB,100GB,1TB`) + var excludeDirs string + flag.StringVar(&excludeDirs, "xd", "", `OPTIONAL Scnnr MODE + comma-delimited list of directory names to exclude from scanning + ex: -xd ".git,node_modules,.venv"`) + + var excludeExts string + flag.StringVar(&excludeExts, "xe", "", `OPTIONAL Scnnr MODE + comma-delimited list of file extensions to exclude from scanning + ex: -xe ".log,.tmp,.json"`) + flag.Parse() directory = dir @@ -127,6 +137,17 @@ func main() { log.Fatal("Position tracking flags (-l, -c) require keywords (-k)") } + var excludeDirsList []string + var excludeExtsList []string + + if excludeDirs != "" { + excludeDirsList = strings.Split(excludeDirs, ",") + } + + if excludeExts != "" { + excludeExtsList = strings.Split(excludeExts, ",") + } + scanner := scnnr.Scanner{ Regex: rgx, Keywords: keywords, @@ -134,6 +155,8 @@ func main() { FileExtensions: extensions, ShowLines: showLines, ShowCols: showCols, + ExcludeDirs: excludeDirsList, + ExcludeExts: excludeExtsList, } err := scanner.Scan() diff --git a/pkg/scanner.go b/pkg/scanner.go index 7e20199..3fc350b 100644 --- a/pkg/scanner.go +++ b/pkg/scanner.go @@ -21,6 +21,8 @@ type Scanner struct { FileExtensions []string Keywords []string KeywordMatches []string + ExcludeDirs []string + ExcludeExts []string AllMatches []Match MatchedFilePaths []FileData } @@ -112,13 +114,23 @@ func (s *Scanner) scan(path string, info os.FileInfo, err error) error { return err } + // Check if directory should be excluded + if info.IsDir() && s.shouldExcludeDir(filepath.Base(path)) { + return filepath.SkipDir + } + if !info.IsDir() { + fileExtension := filepath.Ext(path) + + // Check if file extension should be excluded + if s.shouldExcludeExt(fileExtension) { + return nil + } + if s.FileExtensions[0] == "" { s.MatchedFilePaths = append(s.MatchedFilePaths, FileData{path, info}) } else { for _, pattern := range s.FileExtensions { - fileExtension := filepath.Ext(path) - if fileExtension == pattern { s.MatchedFilePaths = append(s.MatchedFilePaths, FileData{path, info}) } @@ -208,6 +220,26 @@ func (s *Scanner) parse(match FileData) { file.Close() } +// shouldExcludeDir checks if a directory name should be excluded +func (s *Scanner) shouldExcludeDir(dirName string) bool { + for _, excludeDir := range s.ExcludeDirs { + if dirName == excludeDir { + return true + } + } + return false +} + +// shouldExcludeExt checks if a file extension should be excluded +func (s *Scanner) shouldExcludeExt(ext string) bool { + for _, excludeExt := range s.ExcludeExts { + if ext == excludeExt { + return true + } + } + return false +} + func eachSlice(files []FileData) [][]FileData { var chunks [][]FileData var chunk []FileData diff --git a/scripts/e2e.sh b/scripts/e2e.sh index f131c17..22ce0f5 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -2,9 +2,20 @@ set -eou pipefail +e2e_dir=$HOME + +if [[ $ETE_DIR != "" ]] +then + e2e_dir=$ETE_DIR +fi + +################################################## +################################################## +################################################## + echo "--- FILE SIZE FINDER DRY RUN: BEGIN ---" -go run main.go -m fsf -s 1MB -d $HOME +go run main.go -m fsf -s 1MB -d $e2e_dir echo "--- FILE SIZE FINDER DRY RUN: DONE ---" @@ -16,7 +27,7 @@ sleep 2 echo "--- FILE NAME FINDER DRY RUN: BEGIN---" -go run main.go -m fnf -f main,DEFCON -p $HOME +go run main.go -m fnf -f main,DEFCON -p $e2e_dir echo "--- FILE NAME FINDER DRY RUN: DONE ---" @@ -28,12 +39,16 @@ sleep 2 echo "--- SCANNER DRY RUN: BEGIN---" -go run main.go -k main,const,let,var,for -p $HOME +go run main.go -k main,const,let,var,for -p $e2e_dir echo "--- SCANNER DRY RUN: DONE ---" sleep 2 +################################################## +################################################## +################################################## + echo "--- FILE FINGERPRINT FINDER DRY RUN: BEGIN---" go run main.go -m fff -k $(go run cmd/checksum/main.go) -d . @@ -42,16 +57,62 @@ echo "--- FILE FINGERPRINT FINDER DRY RUN: DONE---" sleep 2 +################################################## +################################################## +################################################## + echo "--- SCANNER LINE RUN: BEGIN---" -go run main.go -k main,const,let,var,for -p $HOME -l +go run main.go -k main,const,let,var,for -p $e2e_dir -l echo "--- SCANNER LINE RUN: DONE ---" sleep 2 +################################################## +################################################## +################################################## + echo "--- SCANNER LINE AND COL RUN: BEGIN---" -go run main.go -k main,const,let,var,for -p $HOME -c +go run main.go -k main,const,let,var,for -p $e2e_dir -c echo "--- SCANNER LINE AND COL RUN: DONE ---" + +sleep 2 + +################################################## +################################################## +################################################## + +echo "--- SCANNER DIR EXCLUDE RUN: BEGIN---" + +sleep 2 + +go run main.go -k main,const,let,var,for -p $e2e_dir -c -xd ".git" + +echo "--- SCANNER DIR EXCLUDE RUN: DONE ---" + +sleep 2 + +################################################## +################################################## +################################################## + +echo "--- SCANNER EXTENSION EXCLUDE RUN: BEGIN---" + +go run main.go -k main,const,let,var,for -p $e2e_dir -c -xe ".yml,.sh,.md" + +echo "--- SCANNER EXTENSION RUN: DONE ---" + +sleep 2 + +################################################## +################################################## +################################################## + +echo "--- SCANNER DIRECTORY AND EXTENSION EXCLUDE RUN: BEGIN---" + +go run main.go -k main,const,let,var,for -p $e2e_dir -c -xd ".git" -xe ".yml,.sh,.md" + +echo "--- SCANNER DIRECTORY AND EXTENSION RUN: DONE ---"