Honor ignore files and exclude tests from scans (#5)

* feat: honor ignore files and test exclusions

* fix: bound score penalties by issue density

* feat: report aggregate comment line stats

* feat: classify comment and code line stats

* feat: report deterministic scan state

Author: aksOps

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+- Parse `.gitignore`, `.ignore`, and `.git/info/exclude` during working-tree scans.
+- Add `--exclude-tests` to skip well-known test-case paths from aggregate scan analysis.
+- Replace the linear score deduction with a bounded issue-density curve so noisy large repositories do not collapse to zero.
+- Add aggregate code/blank/comment-only/inline-comment line statistics to scan and diff reports, and use code lines for issue-density scoring when available.
+- Add explicit scan and diff status, regression, threshold, and incomplete-report metadata.
+- Count files skipped due to size or per-file analysis budget, and mark affected reports incomplete.
+- Stream blob analysis so scans retain aggregate facts instead of repository-wide raw blob contents.
+
 ## 0.0.1 - 2026-06-30
 
 - Initial public code-signal scanner.

README.md

@@ -52,6 +52,9 @@ scanner scan .
 # Emit aggregate JSON
 scanner scan . --json
 
+# Exclude well-known test files and test directories
+scanner scan . --exclude-tests
+
 # Detect languages and module roots
 scanner detect .
 
@@ -92,17 +95,35 @@ code-signal detects and analyzes these languages with built-in lexical/parser ch
 
 Detection uses extensions, shebangs, and common manifests such as `go.mod`, `package.json`, `pyproject.toml`, `Cargo.toml`, `pom.xml`, Gradle files, `.csproj`, `.sln`, `composer.json`, `Gemfile`, `Dockerfile`, `Containerfile`, and `*.dockerfile`.
 
+## Aggregate line statistics
+
+Scan JSON includes `lines_scanned`, `blank_lines`, `code_lines`, `comment_lines`, `comment_only_lines`, `inline_comment_lines`, and `comment_density_percent`. The same aggregate fields are reported for `totals`, `by_language`, and `by_module`; diff JSON includes before/after/delta values under `totals_delta`.
+
+`lines_scanned` is physical scanned lines. `code_lines` excludes blank lines and comment-only lines, but keeps lines that contain code plus an inline comment. The score model uses `code_lines` when available and falls back to physical lines for older reports.
+
+Comment detection is lexical and language-aware for supported languages: it recognizes `//`, `#`, `/* ... */`, Rust nested block comments, Ruby `=begin`/`=end`, and standalone Python triple-quoted doc/comment blocks where those forms are valid. It ignores common markers inside quoted strings/raw strings/template literals and JavaScript/TypeScript regex literals, and never emits per-file locations or source snippets.
+
+## Ignore files, test exclusion, and skip policy
+
+Working-tree scans parse `.gitignore`, `.ignore`, and `.git/info/exclude` using gitignore-style rules before reading candidate files.
+
+Pass `--exclude-tests` to `scan` or `diff` to skip well-known test-case paths from aggregate analysis. The built-in defaults cover common Go (`*_test.go`, `testdata`), Java (`src/test/**`, `*Test.java`, `*Tests.java`, `*IT.java`), Python (`tests/**`, `test_*.py`, `*_test.py`, `conftest.py`), TypeScript/JavaScript (`__tests__/**`, `*.test.*`, `*.spec.*`, `cypress/**`, `playwright/**`, `e2e/**`), and Rust (`tests/**`, `benches/**`, `*_test.rs`) conventions.
+
+Built-in skips still apply for common dependency, generated, build, coverage, and minified paths such as `node_modules`, `vendor`, `target`, `dist`, `build`, `.next`, `coverage`, `third_party`, `generated`, `gen`, `*.min.js`, `*.min.css`, `*.pb.go`, and `*_generated.go`.
+
 ## Configuration
 
 Configuration is optional. If present, `scanner.json` is loaded from the scanned repository root or from `--config`.
 
 ```json
 {
   "scan": {
-    "default_timeout_seconds": 30,
+    "default_timeout_seconds": 0,
+    "max_file_analysis_ms": 2000,
     "max_file_bytes": 1048576,
     "follow_symlinks": false,
-    "workers": 0
+    "workers": 0,
+    "exclude_tests": false
   },
   "score": {
     "fail_under": 75,
@@ -120,6 +141,26 @@ Configuration is optional. If present, `scanner.json` is loaded from the scanned
 
 Unknown config fields and trailing JSON values are rejected.
 
+`scan.default_timeout_seconds` is disabled by default; use `--timeout` or set a
+positive value when a whole-command emergency brake is needed. Files larger than
+`scan.max_file_bytes` are skipped before content is read. Files whose built-in
+analysis exceeds `scan.max_file_analysis_ms` are skipped after the bounded
+attempt. These are included in `totals.files_skipped_due_to_size` or
+`totals.files_skipped_due_to_timeout`. When this happens, scan and diff reports
+set `incomplete: true`, so the score is clearly a quick optimistic signal over
+the scanned subset.
+
+## Score model
+
+The score is a bounded weighted issue-density signal, not a SonarQube debt rating:
+
+```text
+weighted_density_per_kloc = round((errors*10 + warnings*4 + info*1) * 1000 / max(code_lines, 1000))
+score = round(100 * 100 / (100 + weighted_density_per_kloc))
+```
+
+Error caps still apply after density scoring: any error caps the score at `89`, and five or more errors cap it at `69`. Bands are `excellent` for `90+`, `good` for `75-89`, `needs work` for `50-74`, and `poor` below `50`.
+
 ## Privacy and output contract
 
 Reports are aggregate-only. Facts and report JSON do not include file paths, line/column coordinates, snippets, clone groups, per-file findings, or source text. The scanner reads local files and local git objects only; it performs no telemetry, update checks, remote rule downloads, package installs, or network calls.

internal/cli/cli.go

@@ -23,6 +23,7 @@ const (
 
 	flagBase             = "--base"
 	flagConfig           = "--config"
+	flagExcludeTests     = "--exclude-tests"
 	flagFailOnRegression = "--fail-on-regression"
 	flagFailUnder        = "--fail-under"
 	flagHead             = "--head"
@@ -89,17 +90,19 @@ type diffArgs struct {
 	configPath       string
 	json             bool
 	failOnRegression bool
+	excludeTests     bool
 	timeout          time.Duration
 	timeoutSet       bool
 }
 
 type scanArgs struct {
-	path       string
-	configPath string
-	json       bool
-	failUnder  *int
-	timeout    time.Duration
-	timeoutSet bool
+	path         string
+	configPath   string
+	json         bool
+	failUnder    *int
+	excludeTests bool
+	timeout      time.Duration
+	timeoutSet   bool
 }
 
 type detectArgs struct {
@@ -119,7 +122,7 @@ func runDiff(ctx context.Context, args []string, stdout io.Writer) (int, error)
 	}
 	if parsed.timeoutSet {
 		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, parsed.timeout)
+		ctx, cancel = commandContext(ctx, config.Config{}, parsed.timeout, true)
 		defer cancel()
 	}
 	if err := resolveDiffRepoAndScope(ctx, &parsed); err != nil {
@@ -129,6 +132,17 @@ func runDiff(ctx context.Context, args []string, stdout io.Writer) (int, error)
 	if err != nil {
 		return 2, err
 	}
+	if parsed.excludeTests {
+		cfg.Scan.ExcludeTests = true
+	}
+	if parsed.failOnRegression {
+		cfg.Diff.FailOnScoreDrop = true
+	}
+	if !parsed.timeoutSet {
+		var cancel context.CancelFunc
+		ctx, cancel = commandContext(ctx, cfg, 0, false)
+		defer cancel()
+	}
 	diff, err := scan.Diff(ctx, parsed.repo, parsed.base, parsed.head, parsed.scopePath, scan.DiffOptions{Config: cfg})
 	if err != nil {
 		return 1, err
@@ -151,15 +165,19 @@ func runScan(ctx context.Context, args []string, stdout io.Writer) (int, error)
 	if err != nil {
 		return 2, err
 	}
-	if parsed.timeoutSet {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, parsed.timeout)
-		defer cancel()
-	}
 	cfg, err := loadConfig(parsed.configPath, parsed.path)
 	if err != nil {
 		return 2, err
 	}
+	if parsed.excludeTests {
+		cfg.Scan.ExcludeTests = true
+	}
+	if parsed.failUnder != nil {
+		cfg.Score.FailUnder = *parsed.failUnder
+	}
+	var cancel context.CancelFunc
+	ctx, cancel = commandContext(ctx, cfg, parsed.timeout, parsed.timeoutSet)
+	defer cancel()
 	scanReport, err := scan.ScanPath(ctx, parsed.path, scan.ScanOptions{Config: cfg})
 	if err != nil {
 		return 1, err
@@ -186,6 +204,8 @@ func runDetect(ctx context.Context, args []string, stdout io.Writer) (int, error
 	if err != nil {
 		return 2, err
 	}
+	ctx, cancel := commandContext(ctx, cfg, 0, false)
+	defer cancel()
 	detected, err := scan.DetectPath(ctx, parsed.path, scan.DetectOptions{Config: cfg})
 	if err != nil {
 		return 1, err
@@ -260,6 +280,9 @@ func parseDiffFlag(args []string, index int, parsed *diffArgs) (int, bool, error
 	case flagConfig:
 		next, err := assignFlagValue(args, index, flagConfig, &parsed.configPath)
 		return next, true, err
+	case flagExcludeTests:
+		parsed.excludeTests = true
+		return index, true, nil
 	case flagTimeout:
 		next, err := assignTimeoutValue(args, index, &parsed.timeout, &parsed.timeoutSet)
 		return next, true, err
@@ -303,6 +326,9 @@ func parseScanFlag(args []string, index int, parsed *scanArgs) (int, bool, error
 	case flagConfig:
 		next, err := assignFlagValue(args, index, flagConfig, &parsed.configPath)
 		return next, true, err
+	case flagExcludeTests:
+		parsed.excludeTests = true
+		return index, true, nil
 	case flagTimeout:
 		next, err := assignTimeoutValue(args, index, &parsed.timeout, &parsed.timeoutSet)
 		return next, true, err
@@ -417,6 +443,16 @@ func parseTimeout(value string) (time.Duration, error) {
 	return time.Duration(seconds) * time.Second, nil
 }
 
+func commandContext(ctx context.Context, cfg config.Config, timeout time.Duration, timeoutSet bool) (context.Context, context.CancelFunc) {
+	if timeoutSet {
+		return context.WithTimeout(ctx, timeout)
+	}
+	if cfg.Scan.DefaultTimeoutSeconds <= 0 {
+		return context.WithCancel(ctx)
+	}
+	return context.WithTimeout(ctx, time.Duration(cfg.Scan.DefaultTimeoutSeconds)*time.Second)
+}
+
 func appendPositional(command string, positionals []string, arg string) ([]string, error) {
 	if strings.HasPrefix(arg, "-") {
 		return positionals, fmt.Errorf("unknown %s flag %q", command, arg)
@@ -539,10 +575,17 @@ func writeString(stdout io.Writer, value string) error {
 func printScan(stdout io.Writer, scanReport model.ScanReport) error {
 	var builder strings.Builder
 	fmt.Fprintf(&builder, "Score: %d (%s)\n", scanReport.Score, scanReport.Band)
+	fmt.Fprintf(&builder, "Status: %s (score %d, fail-under %d)\n", scanReport.Status, scanReport.Score, scanReport.Thresholds.FailUnder)
+	if scanReport.Incomplete {
+		fmt.Fprintf(&builder, "Incomplete: true (%s)\n", strings.Join(scanReport.IncompleteReasons, ", "))
+	}
 	fmt.Fprintf(&builder, "Issues: %d (errors: %d, warnings: %d, info: %d)\n",
 		scanReport.Totals.Issues, scanReport.Totals.Errors, scanReport.Totals.Warnings, scanReport.Totals.Info)
-	fmt.Fprintf(&builder, "Files scanned: %d | Lines scanned: %d | Files skipped: %d\n",
-		scanReport.Totals.FilesScanned, scanReport.Totals.LinesScanned, scanReport.Totals.FilesSkipped)
+	fmt.Fprintf(&builder, "Files scanned: %d | Lines scanned: %d | Code lines: %d | Files skipped: %d\n",
+		scanReport.Totals.FilesScanned, scanReport.Totals.LinesScanned, scanReport.Totals.CodeLines, scanReport.Totals.FilesSkipped)
+	fmt.Fprintf(&builder, "Comments: %d lines (%.1f%%) | Comment-only: %d | Inline: %d | Blank: %d\n",
+		scanReport.Totals.CommentLines, scanReport.Totals.CommentDensityPercent,
+		scanReport.Totals.CommentOnlyLines, scanReport.Totals.InlineCommentLines, scanReport.Totals.BlankLines)
 	fmt.Fprintf(&builder, "Languages: %s | Modules: %d (%s)\n",
 		joinOrNone(scanLanguages(scanReport)), scanModuleCount(scanReport), moduleProfileText(scanReport))
 	printDuplicationSummary(&builder, scanReport.Duplication)