feat(reporter): merge external SARIF runs

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - KDL Document Language syntax validation (`.kdl`) via [sblinch/kdl-go](https://github.com/sblinch/kdl-go) (closes #463)
 - Documentation website at https://boeing.github.io/config-file-validator
 - `--reporter=github` option that emits validation errors as GitHub Actions workflow commands so they appear as inline PR annotations, without requiring the separate `validate-configs-action` wrapper (closes #459)
+- `--merge-sarif` and `--merge-sarif-dir` options for appending external SARIF runs to the validator's SARIF report (closes #460)
 - Justfile syntax validation (`.just`, `justfile`, `Justfile`, `.justfile`) via embedded [go-just](https://github.com/Boeing/go-just) parser
 - Automatic file type detection from GitHub Linguist's `languages.yml` via `go generate`
 - ~90 known filenames auto-detected (`.babelrc`, `tsconfig.json`, `Pipfile`, `pom.xml`, `.gitconfig`, etc.)

cmd/validator/testdata/sarif_merge.txtar

@@ -0,0 +1,99 @@
+# Merge explicit SARIF file
+exec validator --reporter=sarif:- --merge-sarif=external.sarif good.json
+stdout '"name": "config-file-validator"'
+stdout '"name": "external-tool"'
+stdout '"version": "1.2.3"'
+stdout '"id": "external-rule"'
+stdout '"ruleId": "external-rule"'
+
+# Merge SARIF files from a directory
+exec validator --reporter=sarif:- --merge-sarif-dir=reports good.json
+stdout '"name": "dir-one"'
+stdout '"name": "dir-two"'
+
+# Merge SARIF file when the SARIF reporter comes from config
+exec validator --config=cfv/sarif.toml --merge-sarif=external.sarif good.json
+stdout '"name": "config-file-validator"'
+stdout '"name": "external-tool"'
+
+# Merge options only apply to SARIF output
+! exec validator --reporter=json:- --merge-sarif=external.sarif good.json
+
+# Merge options still require SARIF output after config is applied
+! exec validator --merge-sarif=external.sarif good.json
+
+# Invalid SARIF input fails the run
+! exec validator --reporter=sarif:- --merge-sarif=bad.sarif good.json
+
+-- good.json --
+{"key": "value"}
+
+-- cfv/sarif.toml --
+reporter = ["sarif:-"]
+
+-- bad.sarif --
+{"not": "sarif"}
+
+-- external.sarif --
+{
+  "version": "2.1.0",
+  "$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "external-tool",
+          "version": "1.2.3",
+          "rules": [
+            {
+              "id": "external-rule",
+              "shortDescription": {
+                "text": "Preserved external rule"
+              }
+            }
+          ]
+        }
+      },
+      "results": [
+        {
+          "ruleId": "external-rule",
+          "message": {
+            "text": "external result"
+          }
+        }
+      ]
+    }
+  ]
+}
+
+-- reports/one.sarif --
+{
+  "version": "2.1.0",
+  "$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "dir-one"
+        }
+      },
+      "results": []
+    }
+  ]
+}
+
+-- reports/two.sarif --
+{
+  "version": "2.1.0",
+  "$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+  "runs": [
+    {
+      "tool": {
+        "driver": {
+          "name": "dir-two"
+        }
+      },
+      "results": []
+    }
+  ]
+}

cmd/validator/validator.go

@@ -27,6 +27,10 @@ optional flags:
 		Multiple reporters can be specified: --reporter json:file_path.json --reporter junit:another_file_path.xml
 		Omit the file path to output to stdout: --reporter json or explicitly specify stdout using "-": --reporter json:-
 		Supported formats: standard, json, junit, sarif, and github (default: "standard")
+  -merge-sarif string
+		External SARIF file to merge into SARIF output. Repeatable and requires --reporter=sarif.
+  -merge-sarif-dir string
+		Directory tree containing SARIF files to merge into SARIF output. Requires --reporter=sarif.
   -version
     	Version prints the release version of validator
 */
@@ -80,6 +84,8 @@ type validatorConfig struct {
 	configPath       *string
 	noConfig         *bool
 	gitignore        *bool
+	mergeSarif       sarifMergeFlags
+	mergeSarifDir    *string
 }
 
 type reporterFlags []string
@@ -120,6 +126,17 @@ func (sf *schemaMapFlags) Set(value string) error {
 	return nil
 }
 
+type sarifMergeFlags []string
+
+func (smf *sarifMergeFlags) String() string {
+	return fmt.Sprint(*smf)
+}
+
+func (smf *sarifMergeFlags) Set(value string) error {
+	*smf = append(*smf, value)
+	return nil
+}
+
 // Custom Usage function to cover. Uses the current flagSet when available.
 func validatorUsage() {
 	fmt.Println("Usage: validator [OPTIONS] [<search_path>...]")
@@ -213,6 +230,8 @@ func getFlags(args []string) (validatorConfig, error) {
 			"Disable automatic discovery of .cfv.toml configuration files.")
 		gitignorePtr = flagSet.Bool("gitignore", false,
 			"Skip files and directories matched by .gitignore patterns.")
+		mergeSarifDirPtr = flagSet.String("merge-sarif-dir", "",
+			"Directory tree containing SARIF files to merge into SARIF output. Requires --reporter=sarif.")
 	)
 	flagSet.Var(
 		&reporterConfigFlags,
@@ -241,6 +260,13 @@ func getFlags(args []string) (validatorConfig, error) {
 			"  --schema-map=\"**/config.xml:schemas/config.xsd\"",
 	)
 
+	mergeSarifConfigFlags := sarifMergeFlags{}
+	flagSet.Var(
+		&mergeSarifConfigFlags,
+		"merge-sarif",
+		"External SARIF file to merge into SARIF output. Repeatable and requires --reporter=sarif.",
+	)
+
 	if err := flagSet.Parse(args); err != nil {
 		return validatorConfig{}, err
 	}
@@ -263,7 +289,7 @@ func getFlags(args []string) (validatorConfig, error) {
 		return validatorConfig{}, err
 	}
 
-	if err := validateFlagValues(excludeFileTypesPtr, fileTypesPtr, depthPtr, reporterConf, groupOutputPtr); err != nil {
+	if err := validateFlagValues(excludeFileTypesPtr, fileTypesPtr, depthPtr, reporterConf, groupOutputPtr, mergeSarifConfigFlags, mergeSarifDirPtr); err != nil {
 		return validatorConfig{}, err
 	}
 
@@ -287,12 +313,14 @@ func getFlags(args []string) (validatorConfig, error) {
 		configPathPtr,
 		noConfigPtr,
 		gitignorePtr,
+		mergeSarifConfigFlags,
+		mergeSarifDirPtr,
 	}
 
 	return config, nil
 }
 
-func validateFlagValues(excludeFileTypesPtr, fileTypesPtr *string, depthPtr *int, reporterConf []reporterConfig, groupOutputPtr *string) error {
+func validateFlagValues(excludeFileTypesPtr, fileTypesPtr *string, depthPtr *int, reporterConf []reporterConfig, groupOutputPtr *string, mergeSarif []string, mergeSarifDir *string) error {
 	if err := validateReporterConf(reporterConf, groupOutputPtr); err != nil {
 		return err
 	}
@@ -305,7 +333,11 @@ func validateFlagValues(excludeFileTypesPtr, fileTypesPtr *string, depthPtr *int
 		return err
 	}
 
-	return validateGroupByConf(groupOutputPtr)
+	if err := validateGroupByConf(groupOutputPtr); err != nil {
+		return err
+	}
+
+	return validateSARIFMergeConf(reporterConf, mergeSarif, mergeSarifDir)
 }
 
 func validateFileTypeFlags(excludeFileTypesPtr, fileTypesPtr *string) error {
@@ -345,6 +377,63 @@ func validateReporterConf(conf []reporterConfig, groupBy *string) error {
 	return nil
 }
 
+func validateSARIFMergeConf(conf []reporterConfig, mergeSarif []string, mergeSarifDir *string) error {
+	for _, path := range mergeSarif {
+		if strings.TrimSpace(path) == "" {
+			return errors.New("--merge-sarif requires a file path")
+		}
+	}
+	if mergeSarifDir != nil && isFlagSet("merge-sarif-dir") && strings.TrimSpace(*mergeSarifDir) == "" {
+		return errors.New("--merge-sarif-dir requires a directory path")
+	}
+
+	if isFlagSet("reporter") {
+		return validateSARIFMergeReporters(conf, mergeSarif, mergeSarifDir)
+	}
+	return nil
+}
+
+func validateSARIFMergeReporters(conf []reporterConfig, mergeSarif []string, mergeSarifDir *string) error {
+	if !sarifMergeRequested(mergeSarif, mergeSarifDir) {
+		return nil
+	}
+
+	for _, reporterConf := range conf {
+		if reporterConf.reportType == "sarif" {
+			return nil
+		}
+	}
+	return errors.New("--merge-sarif and --merge-sarif-dir require --reporter=sarif")
+}
+
+func sarifMergeRequested(mergeSarif []string, mergeSarifDir *string) bool {
+	return len(mergeSarif) > 0 || (mergeSarifDir != nil && isFlagSet("merge-sarif-dir"))
+}
+
+func mergeSarifDirectoryValue(mergeSarifDir *string) string {
+	if mergeSarifDir == nil {
+		return ""
+	}
+	return *mergeSarifDir
+}
+
+// isFlagSet verifies if a given flag has been set or not
+func isFlagSet(flagName string) bool {
+	if flagSet == nil {
+		return false
+	}
+
+	var isSet bool
+
+	flagSet.Visit(func(f *flag.Flag) {
+		if f.Name == flagName {
+			isSet = true
+		}
+	})
+
+	return isSet
+}
+
 func validateGroupByConf(groupBy *string) error {
 	groupByCleanString := cleanString("groupby")
 	groupByUserInput := strings.Split(groupByCleanString, ",")
@@ -447,19 +536,6 @@ func validateUniqueReporterOutputDestinations(conf []reporterConfig) error {
 	return nil
 }
 
-// isFlagSet verifies if a given flag has been set or not
-func isFlagSet(flagName string) bool {
-	var isSet bool
-
-	flagSet.Visit(func(f *flag.Flag) {
-		if f.Name == flagName {
-			isSet = true
-		}
-	})
-
-	return isSet
-}
-
 func applyDefaultFlagsFromEnv() error {
 	flagsEnvMap := map[string]string{
 		"depth":              "CFV_DEPTH",
@@ -597,7 +673,15 @@ func resolveConfig(cfg *validatorConfig) (*resolvedConfig, error) {
 		return nil, errors.New("--no-schema cannot be used with --require-schema, --schema-map, or --schemastore")
 	}
 
-	reporters, err := buildReporters(cfg.reportType)
+	if err := validateSARIFMergeReporters(cfg.reportType, cfg.mergeSarif, cfg.mergeSarifDir); err != nil {
+		return nil, err
+	}
+
+	sarifMergeConfig := reporter.SARIFMergeConfig{
+		Files:     []string(cfg.mergeSarif),
+		Directory: mergeSarifDirectoryValue(cfg.mergeSarifDir),
+	}
+	reporters, err := buildReporters(cfg.reportType, sarifMergeConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -667,9 +751,13 @@ func buildCLI(rc *resolvedConfig) *cli.CLI {
 	return cli.Init(opts...)
 }
 
-func buildReporters(reporterConfigs []reporterConfig) ([]reporter.Reporter, error) {
+func buildReporters(reporterConfigs []reporterConfig, sarifMergeConfig reporter.SARIFMergeConfig) ([]reporter.Reporter, error) {
 	reporters := make([]reporter.Reporter, 0, len(reporterConfigs))
 	for _, rc := range reporterConfigs {
+		if rc.reportType == "sarif" {
+			reporters = append(reporters, reporter.NewSARIFReporterWithMerge(rc.outputDest, sarifMergeConfig))
+			continue
+		}
 		reporters = append(reporters, getReporter(rc.reportType, rc.outputDest))
 	}
 	return reporters, nil

cmd/validator/validator_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"flag"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -29,10 +30,17 @@ func Test_getFlags(t *testing.T) {
 		{"type-map", []string{"--type-map=**/inventory:ini", "."}, false},
 		{"multiple type-maps", []string{"--type-map=**/inventory:ini", "--type-map=**/configs/*:properties", "."}, false},
 		{"require-schema", []string{"--require-schema", "."}, false},
+		{"sarif merge file", []string{"--reporter=sarif", "--merge-sarif=external.sarif", "."}, false},
+		{"sarif merge dir", []string{"--reporter=sarif", "--merge-sarif-dir=reports", "."}, false},
 
 		// Invalid flag combinations
 		{"negative depth", []string{"-depth=-1", "."}, true},
 		{"wrong reporter", []string{"--reporter=wrong", "."}, true},
+		{"merge sarif requires sarif reporter", []string{"--reporter=json", "--merge-sarif=external.sarif", "."}, true},
+		{"empty merge sarif file", []string{"--reporter=sarif", "--merge-sarif=", "."}, true},
+		{"empty merge sarif file without sarif reporter", []string{"--reporter=json", "--merge-sarif=", "."}, true},
+		{"empty merge sarif dir", []string{"--reporter=sarif", "--merge-sarif-dir=", "."}, true},
+		{"empty merge sarif dir without sarif reporter", []string{"--reporter=json", "--merge-sarif-dir=", "."}, true},
 		{"reporter output path with colon", []string{"--reporter", "json:/a:/b", "."}, false},
 		{"invalid groupby", []string{"-groupby=badgroup", "."}, true},
 		{"groupby duplicate", []string{"--groupby=directory,directory", "."}, true},
@@ -60,6 +68,40 @@ func Test_getFlags(t *testing.T) {
 	}
 }
 
+func Test_resolveConfigAllowsNilMergeSarifDir(t *testing.T) {
+	flagSet = flag.NewFlagSet("validator", flag.ContinueOnError)
+
+	empty := ""
+	depth := 0
+	falseValue := false
+	trueValue := true
+	cfg := &validatorConfig{
+		searchPaths:      []string{"."},
+		excludeDirs:      &empty,
+		excludeFileTypes: &empty,
+		fileTypes:        &empty,
+		reportType:       []reporterConfig{{reportType: "sarif"}},
+		depth:            &depth,
+		versionQuery:     &falseValue,
+		groupOutput:      &empty,
+		quiet:            &falseValue,
+		globbing:         &falseValue,
+		requireSchema:    &falseValue,
+		noSchema:         &falseValue,
+		schemaStore:      &falseValue,
+		schemaStorePath:  &empty,
+		configPath:       &empty,
+		noConfig:         &trueValue,
+		gitignore:        &falseValue,
+		mergeSarifDir:    nil,
+	}
+
+	require.NotPanics(t, func() {
+		_, err := resolveConfig(cfg)
+		require.NoError(t, err)
+	})
+}
+
 func Test_getFlagsValues(t *testing.T) {
 	cfg, err := getFlags([]string{"-depth=3", "--exclude-dirs=vendor,node_modules", "--require-schema", "."})
 	require.NoError(t, err)