refactor(metrics,tests): simplify testing workflow

- delegate metrics comparison logics to package metrics
- remove usage of metric getter
- use tests.Case as config.Global.Tests input
- adapt TestPredicate to new tests API
- several renamings:
  - metrics.Metric -> metrics.Source
  - tests.Input -> tests.Case
  - tests.SingleResult -> tests.CaseResult

Author: GregoryAlbouy

internal/configparse/parse.go

@@ -44,9 +44,9 @@ type unmarshaledConfig struct {
 
 	Tests []struct {
 		Name      *string `yaml:"name" json:"name"`
-		Metric    *string `yaml:"metric" json:"metric"`
+		Source    *string `yaml:"source" json:"source"`
 		Predicate *string `yaml:"predicate" json:"predicate"`
-		Value     *string `yaml:"value" json:"value"`
+		Target    *string `yaml:"target" json:"target"`
 	} `yaml:"tests" json:"tests"`
 }
 
@@ -259,17 +259,17 @@ func newParsedConfig(uconf unmarshaledConfig) (parsedConfig, error) { //nolint:g
 
 	// TODO: handle nil values (assumed non nil for now)
 	if tests := uconf.Tests; len(tests) > 0 {
-		adaptedTests := make([]runner.TestConfig, len(tests))
+		cases := make([]runner.TestCase, len(tests))
 		for i, t := range tests {
-			d, _ := parseOptionalDuration(*t.Value)
-			adaptedTests[i] = runner.TestConfig{
+			d, _ := parseOptionalDuration(*t.Target)
+			cases[i] = runner.TestCase{
 				Name:      *t.Name,
-				Metric:    runner.TestMetric(*t.Metric),
+				Source:    runner.MetricsSource(*t.Source),
 				Predicate: runner.TestPredicate(*t.Predicate),
-				Value:     runner.TestValue(d),
+				Target:    runner.MetricsValue(d),
 			}
 		}
-		cfg.Tests = adaptedTests
+		cfg.Tests = cases
 		pconf.add(runner.ConfigFieldTests)
 	}
 

internal/configparse/parse_test.go

@@ -209,18 +209,18 @@ func newExpConfig() runner.Config {
 			Silent:   true,
 			Template: "{{ .Metrics.Avg }}",
 		},
-		Tests: []runner.TestConfig{
+		Tests: []runner.TestCase{
 			{
 				Name:      "minimum response time",
-				Metric:    "MIN",
+				Source:    "MIN",
 				Predicate: "GT",
-				Value:     80 * time.Millisecond,
+				Target:    80 * time.Millisecond,
 			},
 			{
 				Name:      "maximum response time",
-				Metric:    "MAX",
+				Source:    "MAX",
 				Predicate: "LTE",
-				Value:     120 * time.Millisecond,
+				Target:    120 * time.Millisecond,
 			},
 		},
 	}

runner/internal/config/config.go

@@ -84,20 +84,13 @@ type Output struct {
 	Template string
 }
 
-type Test struct {
-	Name      string
-	Metric    tests.Metric
-	Predicate tests.Predicate
-	Value     tests.Value
-}
-
 // Global represents the global configuration of the runner.
 // It must be validated using Global.Validate before usage.
 type Global struct {
 	Request Request
 	Runner  Runner
 	Output  Output
-	Tests   []Test
+	Tests   []tests.Case
 }
 
 // String returns an indented JSON representation of Config

runner/internal/metrics/aggregate.go

@@ -0,0 +1,43 @@
+package metrics
+
+import (
+	"time"
+
+	"github.com/benchttp/engine/runner/internal/recorder"
+)
+
+// Aggregate is an aggregate of metrics resulting from
+// from recorded requests.
+type Aggregate struct {
+	Min, Max, Avg                          time.Duration
+	SuccessCount, FailureCount, TotalCount int
+	// Median, StdDev            time.Duration
+	// Deciles                   map[int]float64
+	// StatusCodeDistribution    map[string]int
+	// RequestEventsDistribution map[recorder.Event]int
+}
+
+// Compute computes and aggregates metrics from the given
+// requests records.
+func Compute(records []recorder.Record) (agg Aggregate) {
+	if len(records) == 0 {
+		return
+	}
+
+	agg.TotalCount = len(records)
+	agg.Min, agg.Max = records[0].Time, records[0].Time
+	for _, rec := range records {
+		if rec.Error != "" {
+			agg.FailureCount++
+		}
+		if rec.Time < agg.Min {
+			agg.Min = rec.Time
+		}
+		if rec.Time > agg.Max {
+			agg.Max = rec.Time
+		}
+		agg.Avg += rec.Time / time.Duration(len(records))
+	}
+	agg.SuccessCount = agg.TotalCount - agg.FailureCount
+	return
+}

runner/internal/metrics/compare.go

@@ -0,0 +1,60 @@
+package metrics
+
+import (
+	"fmt"
+	"time"
+)
+
+type ComparisonResult int
+
+const (
+	INF ComparisonResult = -1
+	EQ  ComparisonResult = 0
+	SUP ComparisonResult = 1
+)
+
+func compareMetrics(m, n Metric) ComparisonResult {
+	a, b := m.Value, n.Value
+	if a, b, isDuration := assertDurations(a, b); isDuration {
+		return compareDurations(a, b)
+	}
+	if a, b, isInt := assertInts(a, b); isInt {
+		return compareInts(a, b)
+	}
+	panic(fmt.Sprintf(
+		"metrics: unhandled comparison: %v (%T) and %v (%T)",
+		a, a, b, b,
+	))
+}
+
+func compareInts(a, b int) ComparisonResult {
+	if b < a {
+		return INF
+	}
+	if b > a {
+		return SUP
+	}
+	return EQ
+}
+
+func compareDurations(a, b time.Duration) ComparisonResult {
+	return compareInts(int(a), int(b))
+}
+
+func assertInts(a, b Value) (x, y int, ok bool) {
+	x, ok = a.(int)
+	if !ok {
+		return
+	}
+	y, ok = b.(int)
+	return
+}
+
+func assertDurations(a, b Value) (x, y time.Duration, ok bool) {
+	x, ok = a.(time.Duration)
+	if !ok {
+		return
+	}
+	y, ok = b.(time.Duration)
+	return
+}

runner/internal/metrics/metrics.go

@@ -1,43 +1,44 @@
 package metrics
 
-import (
-	"time"
+type Value interface{}
 
-	"github.com/benchttp/engine/runner/internal/recorder"
+type Source string
+
+const (
+	ResponseTimeAvg     Source = "AVG"
+	ResponseTimeMin     Source = "MIN"
+	ResponseTimeMax     Source = "MAX"
+	RequestFailCount    Source = "FAILURE_COUNT"
+	RequestSuccessCount Source = "SUCCESS_COUNT"
+	RequestCount        Source = "TOTAL_COUNT"
 )
 
-// Aggregate is an aggregate of metrics resulting from
-// from recorded requests.
-type Aggregate struct {
-	Min, Max, Avg                          time.Duration
-	SuccessCount, FailureCount, TotalCount int
-	// Median, StdDev            time.Duration
-	// Deciles                   map[int]float64
-	// StatusCodeDistribution    map[string]int
-	// RequestEventsDistribution map[recorder.Event]int
+func (agg Aggregate) MetricOf(src Source) Metric {
+	var v interface{}
+	switch src {
+	case ResponseTimeAvg:
+		v = agg.Avg
+	case ResponseTimeMin:
+		v = agg.Min
+	case ResponseTimeMax:
+		v = agg.Max
+	case RequestFailCount:
+		v = agg.FailureCount
+	case RequestSuccessCount:
+		v = agg.SuccessCount
+	case RequestCount:
+		v = agg.TotalCount
+	default:
+		panic("metrics.Aggregate.MetricOf: unknown Source: " + src)
+	}
+	return Metric{Source: src, Value: v}
 }
 
-// Compute computes and aggregates metrics from the given
-// requests records.
-func Compute(records []recorder.Record) (agg Aggregate) {
-	if len(records) == 0 {
-		return
-	}
+type Metric struct {
+	Source Source
+	Value  Value
+}
 
-	agg.TotalCount = len(records)
-	agg.Min, agg.Max = records[0].Time, records[0].Time
-	for _, rec := range records {
-		if rec.Error != "" {
-			agg.FailureCount++
-		}
-		if rec.Time < agg.Min {
-			agg.Min = rec.Time
-		}
-		if rec.Time > agg.Max {
-			agg.Max = rec.Time
-		}
-		agg.Avg += rec.Time / time.Duration(len(records))
-	}
-	agg.SuccessCount = agg.TotalCount - agg.FailureCount
-	return
+func (m Metric) Compare(to Metric) ComparisonResult {
+	return compareMetrics(m, to)
 }

runner/internal/report/report.go

@@ -138,12 +138,12 @@ func (rep *Report) writeTestsResult(w io.StringWriter) {
 		writeIndent(w, 1)
 		writeResultString(w, tr.Pass)
 		w.WriteString(": ")
-		w.WriteString(tr.Name)
+		w.WriteString(tr.Input.Name)
 
 		if !tr.Pass {
 			w.WriteString("\n")
 			writeIndent(w, 2)
-			w.WriteString(tr.Explain)
+			w.WriteString(tr.Summary)
 		}
 
 		w.WriteString("\n")

runner/internal/tests/predicate.go

@@ -1,6 +1,8 @@
 package tests
 
-import "fmt"
+import (
+	"github.com/benchttp/engine/runner/internal/metrics"
+)
 
 type Predicate string
 
@@ -13,54 +15,41 @@ const (
 	LTE Predicate = "LTE"
 )
 
-func (p Predicate) Apply(left, right Value) SingleResult {
-	pass := p.passFunc()(left, right)
-	return SingleResult{
-		Pass:    pass,
-		Explain: p.explain(left, right, pass),
-	}
-}
+func (p Predicate) match(comparisonResult metrics.ComparisonResult) bool {
+	sup := comparisonResult == metrics.SUP
+	inf := comparisonResult == metrics.INF
 
-func (p Predicate) passFunc() func(left, right Value) bool {
-	return func(left, right Value) bool {
-		switch p {
-		case EQ:
-			return left == right
-		case NEQ:
-			return left != right
-		case GT:
-			return left > right
-		case GTE:
-			return left >= right
-		case LT:
-			return left < right
-		case LTE:
-			return left <= right
-		default:
-			panic(fmt.Sprintf("%s: unknown predicate", p))
-		}
-	}
-}
-
-func (p Predicate) explain(metric, compared Value, pass bool) string {
-	return fmt.Sprintf("want %s %d, got %d", p.Symbol(), compared, metric)
-}
-
-func (p Predicate) Symbol() string {
 	switch p {
 	case EQ:
-		return "=="
+		return !sup && !inf
 	case NEQ:
-		return "!="
+		return sup || inf
 	case GT:
-		return ">"
+		return sup
 	case GTE:
-		return ">="
+		return !inf
 	case LT:
-		return "<"
+		return inf
 	case LTE:
-		return "<="
+		return !sup
 	default:
-		return "[unknown predicate]"
+		panic("tests: unknown predicate: " + string(p))
+	}
+}
+
+var predicateSymbols = map[Predicate]string{
+	EQ:  "==",
+	NEQ: "!=",
+	GT:  ">",
+	GTE: ">=",
+	LT:  "<",
+	LTE: "<=",
+}
+
+func (p Predicate) symbol() string {
+	s, ok := predicateSymbols[p]
+	if !ok {
+		return "unknown predicate"
 	}
+	return s
 }