From 87da8451c5a84a1af634d8bc51a097580ade4b3b Mon Sep 17 00:00:00 2001 From: Vandit Singh Date: Sun, 27 Sep 2026 04:08:57 +0530 Subject: [PATCH 1/6] vtselect: fix trace search when running with a non-UTC time zone --- app/vtselect/traces/query/query.go | 2 +- app/vtselect/traces/tempo/query.go | 2 +- docs/victoriatraces/changelog/CHANGELOG.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/vtselect/traces/query/query.go b/app/vtselect/traces/query/query.go index 4f056e87b..e6997ddc3 100644 --- a/app/vtselect/traces/query/query.go +++ b/app/vtselect/traces/query/query.go @@ -278,7 +278,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr var startTimeLock sync.Mutex traceIDList := make([]string, 0, limit) - maxStartTimeStr := endTime.Format(time.RFC3339) + maxStartTimeStr := endTime.UTC().Format(time.RFC3339) cp.Query = q qctx := cp.NewQueryContext(ctx) diff --git a/app/vtselect/traces/tempo/query.go b/app/vtselect/traces/tempo/query.go index 491d9a6d5..1d3268ac2 100644 --- a/app/vtselect/traces/tempo/query.go +++ b/app/vtselect/traces/tempo/query.go @@ -192,7 +192,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr var traceIDListLock sync.Mutex var startTimeLock sync.Mutex traceIDList := make([]string, 0, limit) - maxStartTimeStr := endTime.Format(time.RFC3339) + maxStartTimeStr := endTime.UTC().Format(time.RFC3339) cp.Query = q qctx := cp.NewQueryContext(ctx) diff --git a/docs/victoriatraces/changelog/CHANGELOG.md b/docs/victoriatraces/changelog/CHANGELOG.md index 7906db50a..1a786b786 100644 --- a/docs/victoriatraces/changelog/CHANGELOG.md +++ b/docs/victoriatraces/changelog/CHANGELOG.md @@ -26,6 +26,7 @@ Released at 2026-09-29 * BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): return `startTimeUnixNano` as a JSON string in the Tempo `/api/search` response. Previously it was a JSON number, which broke clients that decode the field as a string. Thank @clain23 for [the pull request #256](https://github.com/VictoriaMetrics/VictoriaTraces/pull/256). * BUGFIX: vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): apply `-search.allowPartialResponse` for Tempo and Jaeger query APIs. Previously the flag affected only LogsQL query handlers, so Tempo and Jaeger requests still failed when a queried `vtstorage` node was unavailable. See [this issue #157](https://github.com/VictoriaMetrics/VictoriaTraces/issues/157). +* BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): return all the matching traces from the Jaeger and Tempo search APIs when VictoriaTraces runs with a time zone west of UTC, such as `TZ=America/New_York`. Previously the search could return fewer traces or an empty result. See [this issue #260](https://github.com/VictoriaMetrics/VictoriaTraces/issues/260). ## [v0.11.1](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.11.1) From 1e6cdc8f85fa35ca8685b8e0d08b955b1b360c67 Mon Sep 17 00:00:00 2001 From: jiekun Date: Wed, 30 Sep 2026 12:04:02 +0800 Subject: [PATCH 2/6] unify the time string format to RFC3339Nano when handling trace queries --- app/vtselect/traces/query/query.go | 9 +++++---- app/vtselect/traces/tempo/query.go | 9 +++++---- lib/timeutil/timeutil.go | 8 ++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 lib/timeutil/timeutil.go diff --git a/app/vtselect/traces/query/query.go b/app/vtselect/traces/query/query.go index e6997ddc3..f6f335f38 100644 --- a/app/vtselect/traces/query/query.go +++ b/app/vtselect/traces/query/query.go @@ -18,6 +18,7 @@ import ( "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage" vtstoragecommon "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage/common" otelpb "github.com/VictoriaMetrics/VictoriaTraces/lib/protoparser/opentelemetry/pb" + "github.com/VictoriaMetrics/VictoriaTraces/lib/timeutil" ) // TraceQueryParam is the parameters for querying a batch of traces. @@ -278,7 +279,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr var startTimeLock sync.Mutex traceIDList := make([]string, 0, limit) - maxStartTimeStr := endTime.UTC().Format(time.RFC3339) + maxStartTimeStr := timeutil.FormatUTC(endTime, time.RFC3339Nano) cp.Query = q qctx := cp.NewQueryContext(ctx) @@ -322,7 +323,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr // found enough trace_id, return directly if len(traceIDList) == limit { - maxStartTime, err := time.Parse(time.RFC3339, maxStartTimeStr) + maxStartTime, err := time.Parse(time.RFC3339Nano, maxStartTimeStr) if err != nil { return nil, maxStartTime, err } @@ -346,7 +347,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr return nil, time.Time{}, err } - maxStartTime, err := time.Parse(time.RFC3339, maxStartTimeStr) + maxStartTime, err := time.Parse(time.RFC3339Nano, maxStartTimeStr) if err != nil { return nil, maxStartTime, err } @@ -438,7 +439,7 @@ func findTraceIDTimeSplitTimeRange(ctx context.Context, q *logstorage.Query, cp // this could be the old format index, which records trace ID and the approximate timestamp only. // to transform this into new format (start time & end time), use [t-traceWindow, t+traceWindow]. // this code should be deprecated in the future. - timestamp, _ := time.Parse(time.RFC3339, timeStr) + timestamp, _ := time.Parse(time.RFC3339Nano, timeStr) return timestamp.Add(-*tracecommon.TraceMaxDurationWindow), timestamp.Add(*tracecommon.TraceMaxDurationWindow), nil } diff --git a/app/vtselect/traces/tempo/query.go b/app/vtselect/traces/tempo/query.go index 1d3268ac2..4eda08581 100644 --- a/app/vtselect/traces/tempo/query.go +++ b/app/vtselect/traces/tempo/query.go @@ -18,6 +18,7 @@ import ( "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage" vtstoragecommon "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage/common" otelpb "github.com/VictoriaMetrics/VictoriaTraces/lib/protoparser/opentelemetry/pb" + "github.com/VictoriaMetrics/VictoriaTraces/lib/timeutil" "github.com/VictoriaMetrics/VictoriaTraces/lib/traceql" ) @@ -192,7 +193,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr var traceIDListLock sync.Mutex var startTimeLock sync.Mutex traceIDList := make([]string, 0, limit) - maxStartTimeStr := endTime.UTC().Format(time.RFC3339) + maxStartTimeStr := timeutil.FormatUTC(endTime, time.RFC3339Nano) cp.Query = q qctx := cp.NewQueryContext(ctx) @@ -236,7 +237,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr // found enough trace_id, return directly if len(traceIDList) == limit { - maxStartTime, err := time.Parse(time.RFC3339, maxStartTimeStr) + maxStartTime, err := time.Parse(time.RFC3339Nano, maxStartTimeStr) if err != nil { return nil, maxStartTime, err } @@ -260,7 +261,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr return nil, time.Time{}, err } - maxStartTime, err := time.Parse(time.RFC3339, maxStartTimeStr) + maxStartTime, err := time.Parse(time.RFC3339Nano, maxStartTimeStr) if err != nil { return nil, maxStartTime, err } @@ -365,7 +366,7 @@ func findTraceIDTimeSplitTimeRange(ctx context.Context, q *logstorage.Query, cp // this could be the old format index, which records trace ID and the approximate timestamp only. // to transform this into new format (start time & end time), use [t-traceWindow, t+traceWindow]. // this code should be deprecated in the future. - timestamp, _ := time.Parse(time.RFC3339, timeStr) + timestamp, _ := time.Parse(time.RFC3339Nano, timeStr) return timestamp.Add(-*tracecommon.TraceMaxDurationWindow), timestamp.Add(*tracecommon.TraceMaxDurationWindow), nil } diff --git a/lib/timeutil/timeutil.go b/lib/timeutil/timeutil.go new file mode 100644 index 000000000..9215d2aba --- /dev/null +++ b/lib/timeutil/timeutil.go @@ -0,0 +1,8 @@ +package timeutil + +import "time" + +// FormatUTC is a shortcut of t.UTC().Format() to set the timezone to UTC before formatting. +func FormatUTC(t time.Time, format string) string { + return t.UTC().Format(format) +} From 226fc37aa91bbce464613d063b0deb01f82439fa Mon Sep 17 00:00:00 2001 From: jiekun Date: Wed, 30 Sep 2026 12:09:46 +0800 Subject: [PATCH 3/6] docs: update changelog entry to the proper place and add credit to the author --- docs/victoriatraces/changelog/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/victoriatraces/changelog/CHANGELOG.md b/docs/victoriatraces/changelog/CHANGELOG.md index 1a786b786..4e1f22566 100644 --- a/docs/victoriatraces/changelog/CHANGELOG.md +++ b/docs/victoriatraces/changelog/CHANGELOG.md @@ -12,6 +12,8 @@ The following `tip` changes can be tested by building VictoriaTraces components ## tip +* BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): apply correct time‑range filter and return proper traces from Jaeger and Tempo search APIs when VictoriaTraces runs in a non‑UTC time zone. Previously, searches could return fewer traces or empty results due to incorrect time‑zone conversion for the time‑range filter, especially for time zones west of UTC such as `TZ=America/New_York`, which impacted the end‑time boundary. See [issue #260](https://github.com/VictoriaMetrics/VictoriaTraces/issues/260). Thank @Vandit1604 for [pull request #262](https://github.com/VictoriaMetrics/VictoriaTraces/pull/262). + ## [v0.12.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.12.0) Released at 2026-09-29 @@ -26,7 +28,6 @@ Released at 2026-09-29 * BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): return `startTimeUnixNano` as a JSON string in the Tempo `/api/search` response. Previously it was a JSON number, which broke clients that decode the field as a string. Thank @clain23 for [the pull request #256](https://github.com/VictoriaMetrics/VictoriaTraces/pull/256). * BUGFIX: vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): apply `-search.allowPartialResponse` for Tempo and Jaeger query APIs. Previously the flag affected only LogsQL query handlers, so Tempo and Jaeger requests still failed when a queried `vtstorage` node was unavailable. See [this issue #157](https://github.com/VictoriaMetrics/VictoriaTraces/issues/157). -* BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): return all the matching traces from the Jaeger and Tempo search APIs when VictoriaTraces runs with a time zone west of UTC, such as `TZ=America/New_York`. Previously the search could return fewer traces or an empty result. See [this issue #260](https://github.com/VictoriaMetrics/VictoriaTraces/issues/260). ## [v0.11.1](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.11.1) From 2c56fe8ba79f6702edd64cca2a5122cf81d6edcc Mon Sep 17 00:00:00 2001 From: jiekun Date: Wed, 30 Sep 2026 12:25:59 +0800 Subject: [PATCH 4/6] replace string comparison with unix nano timestamp comparison --- app/vtselect/traces/query/query.go | 20 +++++--------------- app/vtselect/traces/tempo/query.go | 20 +++++--------------- lib/timeutil/timeutil.go | 8 -------- 3 files changed, 10 insertions(+), 38 deletions(-) delete mode 100644 lib/timeutil/timeutil.go diff --git a/app/vtselect/traces/query/query.go b/app/vtselect/traces/query/query.go index f6f335f38..d2feb85fc 100644 --- a/app/vtselect/traces/query/query.go +++ b/app/vtselect/traces/query/query.go @@ -18,7 +18,6 @@ import ( "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage" vtstoragecommon "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage/common" otelpb "github.com/VictoriaMetrics/VictoriaTraces/lib/protoparser/opentelemetry/pb" - "github.com/VictoriaMetrics/VictoriaTraces/lib/timeutil" ) // TraceQueryParam is the parameters for querying a batch of traces. @@ -279,7 +278,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr var startTimeLock sync.Mutex traceIDList := make([]string, 0, limit) - maxStartTimeStr := timeutil.FormatUTC(endTime, time.RFC3339Nano) + maxStartTimeNs := endTime.UnixNano() cp.Query = q qctx := cp.NewQueryContext(ctx) @@ -302,8 +301,8 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr case "_time": startTimeLock.Lock() for _, v := range columns[i].Values { - if v < maxStartTimeStr { - maxStartTimeStr = strings.Clone(v) + if ns, ok := logstorage.TryParseTimestampRFC3339Nano(v); ok && ns < maxStartTimeNs { + maxStartTimeNs = ns } } startTimeLock.Unlock() @@ -323,11 +322,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr // found enough trace_id, return directly if len(traceIDList) == limit { - maxStartTime, err := time.Parse(time.RFC3339Nano, maxStartTimeStr) - if err != nil { - return nil, maxStartTime, err - } - return checkTraceIDList(traceIDList), maxStartTime, nil + return checkTraceIDList(traceIDList), time.Unix(0, maxStartTimeNs), nil } // not enough trace_id, clear the result, extend the time range and try again. @@ -347,12 +342,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr return nil, time.Time{}, err } - maxStartTime, err := time.Parse(time.RFC3339Nano, maxStartTimeStr) - if err != nil { - return nil, maxStartTime, err - } - - return checkTraceIDList(traceIDList), maxStartTime, nil + return checkTraceIDList(traceIDList), time.Unix(0, maxStartTimeNs), nil } // findTraceIDTimeSplitTimeRange try to search from {trace_id_idx_stream="xx"} stream, which contains diff --git a/app/vtselect/traces/tempo/query.go b/app/vtselect/traces/tempo/query.go index 4eda08581..d9a1dcd5a 100644 --- a/app/vtselect/traces/tempo/query.go +++ b/app/vtselect/traces/tempo/query.go @@ -18,7 +18,6 @@ import ( "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage" vtstoragecommon "github.com/VictoriaMetrics/VictoriaTraces/app/vtstorage/common" otelpb "github.com/VictoriaMetrics/VictoriaTraces/lib/protoparser/opentelemetry/pb" - "github.com/VictoriaMetrics/VictoriaTraces/lib/timeutil" "github.com/VictoriaMetrics/VictoriaTraces/lib/traceql" ) @@ -193,7 +192,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr var traceIDListLock sync.Mutex var startTimeLock sync.Mutex traceIDList := make([]string, 0, limit) - maxStartTimeStr := timeutil.FormatUTC(endTime, time.RFC3339Nano) + maxStartTimeNs := endTime.UnixNano() cp.Query = q qctx := cp.NewQueryContext(ctx) @@ -216,8 +215,8 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr case "_time": startTimeLock.Lock() for _, v := range columns[i].Values { - if v < maxStartTimeStr { - maxStartTimeStr = strings.Clone(v) + if ns, ok := logstorage.TryParseTimestampRFC3339Nano(v); ok && ns < maxStartTimeNs { + maxStartTimeNs = ns } } startTimeLock.Unlock() @@ -237,11 +236,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr // found enough trace_id, return directly if len(traceIDList) == limit { - maxStartTime, err := time.Parse(time.RFC3339Nano, maxStartTimeStr) - if err != nil { - return nil, maxStartTime, err - } - return checkTraceIDList(traceIDList), maxStartTime, nil + return checkTraceIDList(traceIDList), time.Unix(0, maxStartTimeNs), nil } // not enough trace_id, clear the result, extend the time range and try again. @@ -261,12 +256,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr return nil, time.Time{}, err } - maxStartTime, err := time.Parse(time.RFC3339Nano, maxStartTimeStr) - if err != nil { - return nil, maxStartTime, err - } - - return checkTraceIDList(traceIDList), maxStartTime, nil + return checkTraceIDList(traceIDList), time.Unix(0, maxStartTimeNs), nil } // findTraceIDTimeSplitTimeRange try to search from {trace_id_idx_stream="xx"} stream, which contains diff --git a/lib/timeutil/timeutil.go b/lib/timeutil/timeutil.go deleted file mode 100644 index 9215d2aba..000000000 --- a/lib/timeutil/timeutil.go +++ /dev/null @@ -1,8 +0,0 @@ -package timeutil - -import "time" - -// FormatUTC is a shortcut of t.UTC().Format() to set the timezone to UTC before formatting. -func FormatUTC(t time.Time, format string) string { - return t.UTC().Format(format) -} From 19f2ccbbbfa9c241aea44b1cd6ba7c9e7f509d35 Mon Sep 17 00:00:00 2001 From: Vandit Singh Date: Wed, 30 Sep 2026 18:36:12 +0530 Subject: [PATCH 5/6] apptest: run trace search with a non-UTC time zone --- apptest/app.go | 4 +- apptest/model.go | 24 ++++++++ apptest/testcase.go | 10 +++- apptest/tests/search_timezone_test.go | 81 +++++++++++++++++++++++++++ apptest/vtsingle.go | 29 ++++++++-- 5 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 apptest/tests/search_timezone_test.go diff --git a/apptest/app.go b/apptest/app.go index c3bfd6cef..0ec916036 100644 --- a/apptest/app.go +++ b/apptest/app.go @@ -31,9 +31,10 @@ type app struct { } // appOptions holds the optional configuration of an app, such as default flags -// to set and things to extract from the app's log. +// to set, extra environment variables and things to extract from the app's log. type appOptions struct { defaultFlags map[string]string + env []string extractREs []*regexp.Regexp wait bool } @@ -54,6 +55,7 @@ func startApp(instance string, binary string, flags []string, opts *appOptions) flags = setDefaultFlags(flags, opts.defaultFlags) cmd := exec.Command(binary, flags...) + cmd.Env = append(os.Environ(), opts.env...) stdout, err := cmd.StdoutPipe() if err != nil { return nil, nil, err diff --git a/apptest/model.go b/apptest/model.go index fe8d3fbbb..070345fd3 100644 --- a/apptest/model.go +++ b/apptest/model.go @@ -265,6 +265,30 @@ func NewJaegerAPITracesResponse(t *testing.T, s string) *JaegerAPITracesResponse return res } +// TempoAPISearchResponse is an in-memory representation of the +// /select/tempo/api/search response. +type TempoAPISearchResponse struct { + Traces []TempoTraceSummary `json:"traces"` +} + +// TempoTraceSummary is the structure of an item in the `traces` field of the +// /select/tempo/api/search response. +type TempoTraceSummary struct { + TraceID string `json:"traceID"` +} + +// NewTempoAPISearchResponse is a test helper function that creates a new +// instance of TempoAPISearchResponse by unmarshalling a json string. +func NewTempoAPISearchResponse(t *testing.T, s string) *TempoAPISearchResponse { + t.Helper() + + res := &TempoAPISearchResponse{} + if err := json.Unmarshal([]byte(s), res); err != nil { + t.Fatalf("could not unmarshal query response data=\n%s\n: %v", s, err) + } + return res +} + // NewJaegerAPITraceResponse is a test helper function that creates a new // instance of JaegerAPITraceResponse by unmarshalling a json string. func NewJaegerAPITraceResponse(t *testing.T, s string) *JaegerAPITraceResponse { diff --git a/apptest/testcase.go b/apptest/testcase.go index 618c8d6a4..3c6778594 100644 --- a/apptest/testcase.go +++ b/apptest/testcase.go @@ -167,7 +167,15 @@ func (tc *TestCase) MustStartDefaultVtsingle() *Vtsingle { func (tc *TestCase) MustStartVtsingle(instance string, flags []string) *Vtsingle { tc.t.Helper() - app, err := StartVtsingle(instance, flags, tc.cli) + return tc.MustStartVtsingleWithEnv(instance, flags, nil) +} + +// MustStartVtsingleWithEnv is like MustStartVtsingle, but also passes the given +// environment variables, such as TZ, to the vtsingle process. +func (tc *TestCase) MustStartVtsingleWithEnv(instance string, flags, env []string) *Vtsingle { + tc.t.Helper() + + app, err := StartVtsingle(instance, flags, env, tc.cli) if err != nil { tc.t.Fatalf("Could not start %s: %v", instance, err) } diff --git a/apptest/tests/search_timezone_test.go b/apptest/tests/search_timezone_test.go new file mode 100644 index 000000000..f72d03f87 --- /dev/null +++ b/apptest/tests/search_timezone_test.go @@ -0,0 +1,81 @@ +package tests + +import ( + "fmt" + "testing" + "time" + + "github.com/VictoriaMetrics/VictoriaTraces/app/vtselect/traces/query" + at "github.com/VictoriaMetrics/VictoriaTraces/apptest" + otelpb "github.com/VictoriaMetrics/VictoriaTraces/lib/protoparser/opentelemetry/pb" +) + +// TestSingleSearchTracesNonUTCTimezone verifies that Jaeger and Tempo trace search return traces older than +// -search.traceMaxDurationWindow when VictoriaTraces runs in a time zone west of UTC. +// +// See https://github.com/VictoriaMetrics/VictoriaTraces/issues/260 +func TestSingleSearchTracesNonUTCTimezone(t *testing.T) { + tc := at.NewTestCase(t) + defer tc.Stop() + + sut := tc.MustStartVtsingleWithEnv("vtsingle", []string{ + "-storageDataPath=" + tc.Dir() + "/vtsingle", + "-retentionPeriod=100y", + }, []string{"TZ=America/New_York"}) + + serviceName := "testSearchTimezoneService" + spanTime := time.Now().Add(-2 * time.Hour) + req := &otelpb.ExportTraceServiceRequest{ + ResourceSpans: []*otelpb.ResourceSpans{ + { + Resource: otelpb.Resource{ + Attributes: []*otelpb.KeyValue{ + {Key: "service.name", Value: &otelpb.AnyValue{StringValue: &serviceName}}, + }, + }, + ScopeSpans: []*otelpb.ScopeSpans{ + { + Spans: []*otelpb.Span{ + { + TraceID: "5b8efff798038103d269b633813fc60c", + SpanID: "eee19b7ec3c1b174", + Name: "testSearchTimezoneSpan", + StartTimeUnixNano: uint64(spanTime.UnixNano()), + EndTimeUnixNano: uint64(spanTime.Add(time.Millisecond).UnixNano()), + }, + }, + }, + }, + }, + }, + } + sut.OTLPHTTPExportTraces(t, req, at.QueryOpts{}) + sut.ForceFlush(t) + + startTime := spanTime.Add(-time.Hour) + endTime := time.Now() + + tc.Assert(&at.AssertOptions{ + Msg: "unexpected number of traces from /select/jaeger/api/traces", + Got: func() any { + res := sut.JaegerAPITraces(t, at.JaegerQueryParam{ + TraceQueryParam: query.TraceQueryParam{ + ServiceName: serviceName, + StartTimeMin: startTime, + StartTimeMax: endTime, + }, + }, at.QueryOpts{}) + return len(res.Data) + }, + Want: 1, + }) + + tc.Assert(&at.AssertOptions{ + Msg: "unexpected number of traces from /select/tempo/api/search", + Got: func() any { + res := sut.TempoAPISearch(t, fmt.Sprintf(`{resource.service.name=%q}`, serviceName), startTime, endTime) + return len(res.Traces) + }, + Want: 1, + }) +} diff --git a/apptest/vtsingle.go b/apptest/vtsingle.go index 528593584..f7be749b1 100644 --- a/apptest/vtsingle.go +++ b/apptest/vtsingle.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "regexp" + "strconv" "testing" "time" @@ -32,17 +33,21 @@ type Vtsingle struct { jaegerAPITraceURL string jaegerAPIDependenciesURL string + tempoAPISearchURL string + logsQLQueryURL string otlpTracesURL string otlpGRPCTracesURL string } -// StartVtsingle starts an instance of Vtsingle with the given flags. It also -// sets the default flags and populates the app instance state with runtime -// values extracted from the application log (such as httpListenAddr). -func StartVtsingle(instance string, flags []string, cli *Client) (*Vtsingle, error) { +// StartVtsingle starts an instance of Vtsingle with the given flags and extra +// environment variables. It also sets the default flags and populates the app +// instance state with runtime values extracted from the application log (such +// as httpListenAddr). +func StartVtsingle(instance string, flags, env []string, cli *Client) (*Vtsingle, error) { app, stderrExtracts, err := startApp(instance, "../../bin/victoria-traces-race", flags, &appOptions{ + env: env, defaultFlags: map[string]string{ "-storageDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()), "-httpListenAddr": "127.0.0.1:0", @@ -79,6 +84,8 @@ func StartVtsingle(instance string, flags []string, cli *Client) (*Vtsingle, err jaegerAPITraceURL: fmt.Sprintf("http://%s/select/jaeger/api/traces/%%s", stderrExtracts[1]), jaegerAPIDependenciesURL: fmt.Sprintf("http://%s/select/jaeger/api/dependencies", stderrExtracts[1]), + tempoAPISearchURL: fmt.Sprintf("http://%s/select/tempo/api/search", stderrExtracts[1]), + logsQLQueryURL: fmt.Sprintf("http://%s/select/logsql/query", stderrExtracts[1]), otlpTracesURL: fmt.Sprintf("http://%s/insert/opentelemetry/v1/traces", stderrExtracts[1]), @@ -157,6 +164,20 @@ func (app *Vtsingle) JaegerAPITrace(t *testing.T, traceID string, opts QueryOpts return NewJaegerAPITraceResponse(t, res) } +// TempoAPISearch is a test helper function that searches for traces with a TraceQL query +// by sending an HTTP GET request to /select/tempo/api/search Vtsingle endpoint. +func (app *Vtsingle) TempoAPISearch(t *testing.T, q string, start, end time.Time) *TempoAPISearchResponse { + t.Helper() + + uv := url.Values{ + "q": {q}, + "start": {strconv.FormatInt(start.Unix(), 10)}, + "end": {strconv.FormatInt(end.Unix(), 10)}, + } + res, _ := app.cli.Get(t, app.tempoAPISearchURL+"?"+uv.Encode()) + return NewTempoAPISearchResponse(t, res) +} + // JaegerAPIDependencies is a test helper function that queries for the dependencies. // This method is not implemented in Vtsingle and this test is no-op for now. func (app *Vtsingle) JaegerAPIDependencies(t *testing.T, param JaegerDependenciesParam, opts QueryOpts) *JaegerAPIDependenciesResponse { From a3fdccd2f74ca48ddf1277c16b5c4cd9f24c61f3 Mon Sep 17 00:00:00 2001 From: Jiekun Date: Thu, 1 Oct 2026 09:55:16 +0800 Subject: [PATCH 6/6] test: update apptest without indroducing extra function --- apptest/testcase.go | 14 +++----------- apptest/tests/search_timezone_test.go | 8 ++++---- apptest/vtagent.go | 3 ++- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/apptest/testcase.go b/apptest/testcase.go index 3c6778594..cfcaff54e 100644 --- a/apptest/testcase.go +++ b/apptest/testcase.go @@ -164,15 +164,7 @@ func (tc *TestCase) MustStartDefaultVtsingle() *Vtsingle { // MustStartVtsingle is a test helper function that starts an instance of // vtsingle and fails the test if the app fails to start. -func (tc *TestCase) MustStartVtsingle(instance string, flags []string) *Vtsingle { - tc.t.Helper() - - return tc.MustStartVtsingleWithEnv(instance, flags, nil) -} - -// MustStartVtsingleWithEnv is like MustStartVtsingle, but also passes the given -// environment variables, such as TZ, to the vtsingle process. -func (tc *TestCase) MustStartVtsingleWithEnv(instance string, flags, env []string) *Vtsingle { +func (tc *TestCase) MustStartVtsingle(instance string, flags []string, env ...string) *Vtsingle { tc.t.Helper() app, err := StartVtsingle(instance, flags, env, tc.cli) @@ -185,10 +177,10 @@ func (tc *TestCase) MustStartVtsingleWithEnv(instance string, flags, env []strin // MustStartVtagent is a test helper function that starts an instance of // vtagent and fails the test if the app fails to start. -func (tc *TestCase) MustStartVtagent(instance string, remoteWriteURLs, flags []string) *Vtagent { +func (tc *TestCase) MustStartVtagent(instance string, remoteWriteURLs, flags []string, env ...string) *Vtagent { tc.t.Helper() - app, err := StartVtagent(instance, remoteWriteURLs, flags, tc.cli) + app, err := StartVtagent(instance, remoteWriteURLs, flags, env, tc.cli) if err != nil { tc.t.Fatalf("Could not start %s: %v", instance, err) } diff --git a/apptest/tests/search_timezone_test.go b/apptest/tests/search_timezone_test.go index f72d03f87..7b6de0860 100644 --- a/apptest/tests/search_timezone_test.go +++ b/apptest/tests/search_timezone_test.go @@ -10,18 +10,18 @@ import ( otelpb "github.com/VictoriaMetrics/VictoriaTraces/lib/protoparser/opentelemetry/pb" ) -// TestSingleSearchTracesNonUTCTimezone verifies that Jaeger and Tempo trace search return traces older than -// -search.traceMaxDurationWindow when VictoriaTraces runs in a time zone west of UTC. +// TestSingleSearchTracesNonUTCTimezone verifies that Jaeger and Tempo trace search return correct traces +// when VictoriaTraces runs in a time zone other than UTC. // // See https://github.com/VictoriaMetrics/VictoriaTraces/issues/260 func TestSingleSearchTracesNonUTCTimezone(t *testing.T) { tc := at.NewTestCase(t) defer tc.Stop() - sut := tc.MustStartVtsingleWithEnv("vtsingle", []string{ + sut := tc.MustStartVtsingle("vtsingle", []string{ "-storageDataPath=" + tc.Dir() + "/vtsingle", "-retentionPeriod=100y", - }, []string{"TZ=America/New_York"}) + }, []string{"TZ=America/New_York"}...) serviceName := "testSearchTimezoneService" spanTime := time.Now().Add(-2 * time.Hour) diff --git a/apptest/vtagent.go b/apptest/vtagent.go index aaac48e6c..b5f1de1c6 100644 --- a/apptest/vtagent.go +++ b/apptest/vtagent.go @@ -29,8 +29,9 @@ type Vtagent struct { // StartVtagent starts an instance of Vtagent with the given flags. // It also sets the default flags and populates the app instance state with // values extracted from the application log (such as httpListenAddr). -func StartVtagent(instance string, remoteWriteURLs, flags []string, cli *Client) (*Vtagent, error) { +func StartVtagent(instance string, remoteWriteURLs, flags, env []string, cli *Client) (*Vtagent, error) { app, stderrExtracts, err := startApp(instance, "../../bin/vtagent-race", flags, &appOptions{ + env: env, defaultFlags: map[string]string{ "-httpListenAddr": "127.0.0.1:0", "-otlpGRPCListenAddr": "127.0.0.1:0",