diff --git a/metrics/point_test.go b/metrics/point_test.go deleted file mode 100644 index 6c042823..00000000 --- a/metrics/point_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -package metrics - -import ( - "path/filepath" - T "testing" - - "github.com/GuanceCloud/cliutils/point" - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMergePoint(t *T.T) { - t.Run(`basic`, func(t *T.T) { - pts := []*point.Point{ - point.NewPointV2("m1", - append(point.NewTags(map[string]string{"t1": "v1", "t2": "v2"}), - point.NewKVs(map[string]any{"f1": 123})...), - point.DefaultLoggingOptions()..., - ), - - point.NewPointV2("m1", - append(point.NewTags(map[string]string{"t1": "v1", "t2": "v2"}), - point.NewKVs(map[string]any{"f2": "hello"})...), - point.DefaultLoggingOptions()..., - ), - } - - mPts := mergePts(pts) - assert.Len(t, mPts, 1) - mfs := mPts[0].Fields() - assert.Equal(t, []byte(`hello`), mfs.Get(`f2`).GetD()) - - t.Logf("point: %s", mPts[0].LineProto()) - }) - - t.Run(`merge-multiple-time-series`, func(t *T.T) { - pts := []*point.Point{ - point.NewPointV2("m1", - append(point.NewTags(map[string]string{"t1": "v1", "t2": "v2"}), - point.NewKVs(map[string]any{"f1": 123})...), - point.DefaultLoggingOptions()..., - ), - - point.NewPointV2("m1", - append(point.NewTags(map[string]string{"t1": "v1", "t2": "v2"}), - point.NewKVs(map[string]any{"f2": "hello"})...), - point.DefaultLoggingOptions()..., - ), - - point.NewPointV2("m1", - append(point.NewTags(map[string]string{"tag1": "v1", "tag2": "v2"}), - point.NewKVs(map[string]any{"f1": 123})...), - point.DefaultLoggingOptions()..., - ), - - point.NewPointV2("m1", - append(point.NewTags(map[string]string{"tag1": "v1", "tag2": "v2"}), - point.NewKVs(map[string]any{"f2": "hello"})...), - point.DefaultLoggingOptions()..., - ), - } - - mPts := mergePts(pts) - require.Len(t, mPts, 2) - for _, pt := range mPts { - t.Logf("point: %s", pt.LineProto()) - } - - mfs := mPts[0].Fields() - assert.Equal(t, []byte(`hello`), mfs.Get(`f2`).GetD()) - - mfs = mPts[1].Fields() - assert.Equal(t, []byte(`hello`), mfs.Get(`f2`).GetD()) - }) -} - -func TestGatherPoint(t *T.T) { - ns := "testing" - - t.Run(`basic`, func(t *T.T) { - cnt := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: ns, - Name: filepath.Base(t.Name()), - }, []string{"tag1", "tag2"}, - ) - reg := prometheus.NewRegistry() - reg.MustRegister(cnt) - - cnt.WithLabelValues("v1", "v2").Add(1.0) - cnt.WithLabelValues("v1", "v2").Add(1.0) - cnt.WithLabelValues("v1", "v2").Add(1.0) - - cnt.WithLabelValues("v3", "v4").Add(1.0) - cnt.WithLabelValues("v3", "v4").Add(1.0) - cnt.WithLabelValues("v3", "v4").Add(1.0) - - pts, err := doGatherPoints(reg) - assert.NoError(t, err) - - require.Len(t, pts, 2) - - tags := pts[0].Tags() - fs := pts[0].Fields() - assert.Equal(t, []byte(ns), pts[0].Name()) - assert.Equal(t, []byte("v1"), tags.Get("tag1").GetD()) - assert.Equal(t, []byte("v2"), tags.Get("tag2").GetD()) - assert.Equal(t, 3.0, fs.Get(filepath.Base(t.Name())).GetF()) - - tags = pts[1].Tags() - fs = pts[1].Fields() - assert.Equal(t, []byte(ns), pts[0].Name()) - assert.Equal(t, []byte("v3"), tags.Get("tag1").GetD()) - assert.Equal(t, []byte("v4"), tags.Get("tag2").GetD()) - assert.Equal(t, 3.0, fs.Get(filepath.Base(t.Name())).GetF()) - - for _, pt := range pts { - t.Logf("point: %s", pt.LineProto()) - } - }) -} diff --git a/point/check.go b/point/check.go index 26ef9630..6afd5346 100644 --- a/point/check.go +++ b/point/check.go @@ -32,6 +32,10 @@ func (c *checker) check(pt *Point) *Point { } func (c *checker) addWarn(t, msg string) { + defer func() { + pointCheckWarnVec.WithLabelValues(t).Inc() + }() + c.warns = append(c.warns, &Warn{ Type: t, Msg: msg, }) diff --git a/point/metrics.go b/point/metrics.go new file mode 100644 index 00000000..dcd85b01 --- /dev/null +++ b/point/metrics.go @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. +package point + +import ( + "github.com/GuanceCloud/cliutils/metrics" + p8s "github.com/prometheus/client_golang/prometheus" +) + +var ( + pointCheckWarnVec = p8s.NewCounterVec( + p8s.CounterOpts{ + Namespace: "point", + Name: "check_point_warning_total", + Help: "Warnings among build new point", + }, + []string{"type"}, + ) +) + +func ResetMetrics() { + pointCheckWarnVec.Reset() +} + +func init() { + metrics.MustRegister(pointCheckWarnVec) +} diff --git a/point/metrics_test.go b/point/metrics_test.go new file mode 100644 index 00000000..40342164 --- /dev/null +++ b/point/metrics_test.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. +package point + +import ( + T "testing" + + "github.com/GuanceCloud/cliutils/metrics" + p8s "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetrics(t *T.T) { + t.Run(`basic`, func(t *T.T) { + + ResetMetrics() + + defer t.Cleanup(func() { + ResetMetrics() + }) + + reg := p8s.NewRegistry() + reg.MustRegister(PointCheckWarnVec) + + pt := NewPointV2("", nil) + + t.Logf("pt: %s", pt.Pretty()) + + mfs, err := reg.Gather() + require.NoError(t, err) + + assert.Equal(t, 1.0, metrics.GetMetricOnLabels(mfs, + "point_check_point_warning_total", + "empty measurement, use __default", + "invalid_measurement", + ).GetCounter().GetValue()) + + t.Logf("\n%s", metrics.MetricFamily2Text(mfs)) + }) +} diff --git a/metrics/point.go b/point/p8s.go similarity index 71% rename from metrics/point.go rename to point/p8s.go index 62efe560..2279f94d 100644 --- a/metrics/point.go +++ b/point/p8s.go @@ -3,14 +3,13 @@ // This product includes software developed at Guance Cloud (https://www.guance.com/). // Copyright 2021-present Guance, Inc. -package metrics +package point import ( "strings" "time" - "github.com/GuanceCloud/cliutils/point" - "github.com/prometheus/client_golang/prometheus" + p8s "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" ) @@ -26,15 +25,15 @@ import ( // covered the time field. For prometheus metrics, these time value are // the same. // -// When point.Point are logging, due to the lack of `time-series', +// When Point are logging, due to the lack of `time-series', // we hava to merge multiple points' fields together to build a single point. // // For time-series data, we don't need to do this, the storage // engine merged them automatically(grouped by time-series). -func mergePts(pts []*point.Point) []*point.Point { +func mergePts(pts []*Point) []*Point { // same-hash point put together - var res []*point.Point - ptMap := map[string][]*point.Point{} + var res []*Point + ptMap := map[string][]*Point{} for _, pt := range pts { hash := pt.MD5() ptMap[hash] = append(ptMap[hash], pt) @@ -58,7 +57,7 @@ func mergePts(pts []*point.Point) []*point.Point { return res } -func doGatherPoints(reg prometheus.Gatherer) ([]*point.Point, error) { +func doGatherPoints(reg p8s.Gatherer) ([]*Point, error) { mfs, err := reg.Gather() if err != nil { return nil, err @@ -67,7 +66,7 @@ func doGatherPoints(reg prometheus.Gatherer) ([]*point.Point, error) { // All gathered data should have the same timestamp, we enforce it. now := time.Now() - var pts []*point.Point + var pts []*Point for _, mf := range mfs { arr := strings.SplitN(*mf.Name, "_", 2) @@ -75,17 +74,17 @@ func doGatherPoints(reg prometheus.Gatherer) ([]*point.Point, error) { fieldName := arr[1] for _, m := range mf.Metric { - var kvs point.KVs + var kvs KVs for _, label := range m.GetLabel() { - kvs = append(kvs, point.NewKV(label.GetName(), label.GetValue(), point.WithKVTagSet(true))) + kvs = append(kvs, NewKV(label.GetName(), label.GetValue(), WithKVTagSet(true))) } switch *mf.Type { case dto.MetricType_COUNTER: - kvs = append(kvs, point.NewKV(fieldName, m.GetCounter().GetValue())) + kvs = append(kvs, NewKV(fieldName, m.GetCounter().GetValue())) case dto.MetricType_SUMMARY: avg := uint64(m.GetSummary().GetSampleSum()) / m.GetSummary().GetSampleCount() - kvs = append(kvs, point.NewKV(fieldName, avg)) + kvs = append(kvs, NewKV(fieldName, avg)) case dto.MetricType_GAUGE: continue // TODO @@ -103,16 +102,15 @@ func doGatherPoints(reg prometheus.Gatherer) ([]*point.Point, error) { ts = time.Unix(0, int64(time.Millisecond)**m.TimestampMs) } - opts := append(point.DefaultMetricOptions(), point.WithTime(ts)) - pts = append(pts, point.NewPointV2(name, kvs, opts...)) + opts := append(DefaultMetricOptions(), WithTime(ts)) + pts = append(pts, NewPointV2(name, kvs, opts...)) } } return pts, nil } -// GatherPoints gather all metrics in global registry, but convert these metrics -// to point.Point. -func GatherPoints() ([]*point.Point, error) { +// GatherPoints gather all metrics in reg, but convert these metrics to Point. +func GatherPoints(reg p8s.Gatherer) ([]*Point, error) { return doGatherPoints(reg) } diff --git a/point/p8s_test.go b/point/p8s_test.go new file mode 100644 index 00000000..5ad88d92 --- /dev/null +++ b/point/p8s_test.go @@ -0,0 +1,126 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package point + +import ( + "path/filepath" + T "testing" + + p8s "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMergePoint(t *T.T) { + t.Run(`basic`, func(t *T.T) { + pts := []*Point{ + NewPointV2("m1", + append(NewTags(map[string]string{"t1": "v1", "t2": "v2"}), + NewKVs(map[string]any{"f1": 123})...), + DefaultLoggingOptions()..., + ), + + NewPointV2("m1", + append(NewTags(map[string]string{"t1": "v1", "t2": "v2"}), + NewKVs(map[string]any{"f2": "hello"})...), + DefaultLoggingOptions()..., + ), + } + + mPts := mergePts(pts) + assert.Len(t, mPts, 1) + mfs := mPts[0].Fields() + assert.Equal(t, `hello`, mfs.Get(`f2`).GetS()) + + t.Logf("point: %s", mPts[0].LineProto()) + }) + + t.Run(`merge-multiple-time-series`, func(t *T.T) { + pts := []*Point{ + NewPointV2("m1", + append(NewTags(map[string]string{"t1": "v1", "t2": "v2"}), + NewKVs(map[string]any{"f1": 123})...), + DefaultLoggingOptions()..., + ), + + NewPointV2("m1", + append(NewTags(map[string]string{"t1": "v1", "t2": "v2"}), + NewKVs(map[string]any{"f2": "hello"})...), + DefaultLoggingOptions()..., + ), + + NewPointV2("m1", + append(NewTags(map[string]string{"tag1": "v1", "tag2": "v2"}), + NewKVs(map[string]any{"f1": 123})...), + DefaultLoggingOptions()..., + ), + + NewPointV2("m1", + append(NewTags(map[string]string{"tag1": "v1", "tag2": "v2"}), + NewKVs(map[string]any{"f2": "hello"})...), + DefaultLoggingOptions()..., + ), + } + + mPts := mergePts(pts) + require.Len(t, mPts, 2) + for _, pt := range mPts { + t.Logf("point: %s", pt.LineProto()) + } + + mfs := mPts[0].Fields() + assert.Equal(t, `hello`, mfs.Get(`f2`).GetS()) + + mfs = mPts[1].Fields() + assert.Equal(t, `hello`, mfs.Get(`f2`).GetS()) + }) +} + +func TestGatherPoint(t *T.T) { + ns := "testing" + + t.Run(`basic`, func(t *T.T) { + cnt := p8s.NewCounterVec( + p8s.CounterOpts{ + Namespace: ns, + Name: filepath.Base(t.Name()), + }, []string{"tag1", "tag2"}, + ) + reg := p8s.NewRegistry() + reg.MustRegister(cnt) + + cnt.WithLabelValues("v1", "v2").Add(1.0) + cnt.WithLabelValues("v1", "v2").Add(1.0) + cnt.WithLabelValues("v1", "v2").Add(1.0) + + cnt.WithLabelValues("v3", "v4").Add(1.0) + cnt.WithLabelValues("v3", "v4").Add(1.0) + cnt.WithLabelValues("v3", "v4").Add(1.0) + + pts, err := doGatherPoints(reg) + assert.NoError(t, err) + + require.Len(t, pts, 2) + + tags := pts[0].Tags() + fs := pts[0].Fields() + assert.Equal(t, ns, pts[0].Name()) + assert.Equal(t, "v1", tags.Get("tag1").GetS()) + assert.Equal(t, "v2", tags.Get("tag2").GetS()) + assert.Equal(t, 3.0, fs.Get(filepath.Base(t.Name())).GetF()) + + tags = pts[1].Tags() + fs = pts[1].Fields() + assert.Equal(t, ns, pts[0].Name()) + assert.Equal(t, "v3", tags.Get("tag1").GetS()) + assert.Equal(t, "v4", tags.Get("tag2").GetS()) + assert.Equal(t, 3.0, fs.Get(filepath.Base(t.Name())).GetF()) + + for _, pt := range pts { + t.Logf("point: %s", pt.LineProto()) + } + }) +} diff --git a/point/point.go b/point/point.go index fdeddeec..ed8c0b28 100644 --- a/point/point.go +++ b/point/point.go @@ -361,6 +361,9 @@ func (p *Point) Del(k string) { p.kvs = p.kvs.Del(k) } +// AddDebug used to add one or more debug info to the point. +// These debug info will not build into point's key-values, and +// only for debugging. func (p *Point) AddDebug(d *Debug) { p.debugs = append(p.debugs, d) }