diff --git a/app/vtselect/traces/query/query.go b/app/vtselect/traces/query/query.go index 4f056e87b..d2feb85fc 100644 --- a/app/vtselect/traces/query/query.go +++ b/app/vtselect/traces/query/query.go @@ -278,7 +278,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr var startTimeLock sync.Mutex traceIDList := make([]string, 0, limit) - maxStartTimeStr := endTime.Format(time.RFC3339) + maxStartTimeNs := endTime.UnixNano() cp.Query = q qctx := cp.NewQueryContext(ctx) @@ -301,8 +301,8 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr case "_time": startTimeLock.Lock() for _, v := range columns[i].Values { - if v < maxStartTimeStr { - maxStartTimeStr = strings.Clone(v) + if ns, ok := logstorage.TryParseTimestampRFC3339Nano(v); ok && ns < maxStartTimeNs { + maxStartTimeNs = ns } } startTimeLock.Unlock() @@ -322,11 +322,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr // found enough trace_id, return directly if len(traceIDList) == limit { - maxStartTime, err := time.Parse(time.RFC3339, maxStartTimeStr) - if err != nil { - return nil, maxStartTime, err - } - return checkTraceIDList(traceIDList), maxStartTime, nil + return checkTraceIDList(traceIDList), time.Unix(0, maxStartTimeNs), nil } // not enough trace_id, clear the result, extend the time range and try again. @@ -346,12 +342,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr return nil, time.Time{}, err } - maxStartTime, err := time.Parse(time.RFC3339, maxStartTimeStr) - if err != nil { - return nil, maxStartTime, err - } - - return checkTraceIDList(traceIDList), maxStartTime, nil + return checkTraceIDList(traceIDList), time.Unix(0, maxStartTimeNs), nil } // findTraceIDTimeSplitTimeRange try to search from {trace_id_idx_stream="xx"} stream, which contains @@ -438,7 +429,7 @@ func findTraceIDTimeSplitTimeRange(ctx context.Context, q *logstorage.Query, cp // this could be the old format index, which records trace ID and the approximate timestamp only. // to transform this into new format (start time & end time), use [t-traceWindow, t+traceWindow]. // this code should be deprecated in the future. - timestamp, _ := time.Parse(time.RFC3339, timeStr) + timestamp, _ := time.Parse(time.RFC3339Nano, timeStr) return timestamp.Add(-*tracecommon.TraceMaxDurationWindow), timestamp.Add(*tracecommon.TraceMaxDurationWindow), nil } diff --git a/app/vtselect/traces/tempo/query.go b/app/vtselect/traces/tempo/query.go index 491d9a6d5..d9a1dcd5a 100644 --- a/app/vtselect/traces/tempo/query.go +++ b/app/vtselect/traces/tempo/query.go @@ -192,7 +192,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr var traceIDListLock sync.Mutex var startTimeLock sync.Mutex traceIDList := make([]string, 0, limit) - maxStartTimeStr := endTime.Format(time.RFC3339) + maxStartTimeNs := endTime.UnixNano() cp.Query = q qctx := cp.NewQueryContext(ctx) @@ -215,8 +215,8 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr case "_time": startTimeLock.Lock() for _, v := range columns[i].Values { - if v < maxStartTimeStr { - maxStartTimeStr = strings.Clone(v) + if ns, ok := logstorage.TryParseTimestampRFC3339Nano(v); ok && ns < maxStartTimeNs { + maxStartTimeNs = ns } } startTimeLock.Unlock() @@ -236,11 +236,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr // found enough trace_id, return directly if len(traceIDList) == limit { - maxStartTime, err := time.Parse(time.RFC3339, maxStartTimeStr) - if err != nil { - return nil, maxStartTime, err - } - return checkTraceIDList(traceIDList), maxStartTime, nil + return checkTraceIDList(traceIDList), time.Unix(0, maxStartTimeNs), nil } // not enough trace_id, clear the result, extend the time range and try again. @@ -260,12 +256,7 @@ func findTraceIDsSplitTimeRange(ctx context.Context, q *logstorage.Query, cp *tr return nil, time.Time{}, err } - maxStartTime, err := time.Parse(time.RFC3339, maxStartTimeStr) - if err != nil { - return nil, maxStartTime, err - } - - return checkTraceIDList(traceIDList), maxStartTime, nil + return checkTraceIDList(traceIDList), time.Unix(0, maxStartTimeNs), nil } // findTraceIDTimeSplitTimeRange try to search from {trace_id_idx_stream="xx"} stream, which contains @@ -365,7 +356,7 @@ func findTraceIDTimeSplitTimeRange(ctx context.Context, q *logstorage.Query, cp // this could be the old format index, which records trace ID and the approximate timestamp only. // to transform this into new format (start time & end time), use [t-traceWindow, t+traceWindow]. // this code should be deprecated in the future. - timestamp, _ := time.Parse(time.RFC3339, timeStr) + timestamp, _ := time.Parse(time.RFC3339Nano, timeStr) return timestamp.Add(-*tracecommon.TraceMaxDurationWindow), timestamp.Add(*tracecommon.TraceMaxDurationWindow), nil } diff --git a/apptest/app.go b/apptest/app.go index c3bfd6cef..0ec916036 100644 --- a/apptest/app.go +++ b/apptest/app.go @@ -31,9 +31,10 @@ type app struct { } // appOptions holds the optional configuration of an app, such as default flags -// to set and things to extract from the app's log. +// to set, extra environment variables and things to extract from the app's log. type appOptions struct { defaultFlags map[string]string + env []string extractREs []*regexp.Regexp wait bool } @@ -54,6 +55,7 @@ func startApp(instance string, binary string, flags []string, opts *appOptions) flags = setDefaultFlags(flags, opts.defaultFlags) cmd := exec.Command(binary, flags...) + cmd.Env = append(os.Environ(), opts.env...) stdout, err := cmd.StdoutPipe() if err != nil { return nil, nil, err diff --git a/apptest/model.go b/apptest/model.go index fe8d3fbbb..070345fd3 100644 --- a/apptest/model.go +++ b/apptest/model.go @@ -265,6 +265,30 @@ func NewJaegerAPITracesResponse(t *testing.T, s string) *JaegerAPITracesResponse return res } +// TempoAPISearchResponse is an in-memory representation of the +// /select/tempo/api/search response. +type TempoAPISearchResponse struct { + Traces []TempoTraceSummary `json:"traces"` +} + +// TempoTraceSummary is the structure of an item in the `traces` field of the +// /select/tempo/api/search response. +type TempoTraceSummary struct { + TraceID string `json:"traceID"` +} + +// NewTempoAPISearchResponse is a test helper function that creates a new +// instance of TempoAPISearchResponse by unmarshalling a json string. +func NewTempoAPISearchResponse(t *testing.T, s string) *TempoAPISearchResponse { + t.Helper() + + res := &TempoAPISearchResponse{} + if err := json.Unmarshal([]byte(s), res); err != nil { + t.Fatalf("could not unmarshal query response data=\n%s\n: %v", s, err) + } + return res +} + // NewJaegerAPITraceResponse is a test helper function that creates a new // instance of JaegerAPITraceResponse by unmarshalling a json string. func NewJaegerAPITraceResponse(t *testing.T, s string) *JaegerAPITraceResponse { diff --git a/apptest/testcase.go b/apptest/testcase.go index 618c8d6a4..cfcaff54e 100644 --- a/apptest/testcase.go +++ b/apptest/testcase.go @@ -164,10 +164,10 @@ func (tc *TestCase) MustStartDefaultVtsingle() *Vtsingle { // MustStartVtsingle is a test helper function that starts an instance of // vtsingle and fails the test if the app fails to start. -func (tc *TestCase) MustStartVtsingle(instance string, flags []string) *Vtsingle { +func (tc *TestCase) MustStartVtsingle(instance string, flags []string, env ...string) *Vtsingle { tc.t.Helper() - app, err := StartVtsingle(instance, flags, tc.cli) + app, err := StartVtsingle(instance, flags, env, tc.cli) if err != nil { tc.t.Fatalf("Could not start %s: %v", instance, err) } @@ -177,10 +177,10 @@ func (tc *TestCase) MustStartVtsingle(instance string, flags []string) *Vtsingle // MustStartVtagent is a test helper function that starts an instance of // vtagent and fails the test if the app fails to start. -func (tc *TestCase) MustStartVtagent(instance string, remoteWriteURLs, flags []string) *Vtagent { +func (tc *TestCase) MustStartVtagent(instance string, remoteWriteURLs, flags []string, env ...string) *Vtagent { tc.t.Helper() - app, err := StartVtagent(instance, remoteWriteURLs, flags, tc.cli) + app, err := StartVtagent(instance, remoteWriteURLs, flags, env, tc.cli) if err != nil { tc.t.Fatalf("Could not start %s: %v", instance, err) } diff --git a/apptest/tests/search_timezone_test.go b/apptest/tests/search_timezone_test.go new file mode 100644 index 000000000..7b6de0860 --- /dev/null +++ b/apptest/tests/search_timezone_test.go @@ -0,0 +1,81 @@ +package tests + +import ( + "fmt" + "testing" + "time" + + "github.com/VictoriaMetrics/VictoriaTraces/app/vtselect/traces/query" + at "github.com/VictoriaMetrics/VictoriaTraces/apptest" + otelpb "github.com/VictoriaMetrics/VictoriaTraces/lib/protoparser/opentelemetry/pb" +) + +// TestSingleSearchTracesNonUTCTimezone verifies that Jaeger and Tempo trace search return correct traces +// when VictoriaTraces runs in a time zone other than UTC. +// +// See https://github.com/VictoriaMetrics/VictoriaTraces/issues/260 +func TestSingleSearchTracesNonUTCTimezone(t *testing.T) { + tc := at.NewTestCase(t) + defer tc.Stop() + + sut := tc.MustStartVtsingle("vtsingle", []string{ + "-storageDataPath=" + tc.Dir() + "/vtsingle", + "-retentionPeriod=100y", + }, []string{"TZ=America/New_York"}...) + + serviceName := "testSearchTimezoneService" + spanTime := time.Now().Add(-2 * time.Hour) + req := &otelpb.ExportTraceServiceRequest{ + ResourceSpans: []*otelpb.ResourceSpans{ + { + Resource: otelpb.Resource{ + Attributes: []*otelpb.KeyValue{ + {Key: "service.name", Value: &otelpb.AnyValue{StringValue: &serviceName}}, + }, + }, + ScopeSpans: []*otelpb.ScopeSpans{ + { + Spans: []*otelpb.Span{ + { + TraceID: "5b8efff798038103d269b633813fc60c", + SpanID: "eee19b7ec3c1b174", + Name: "testSearchTimezoneSpan", + StartTimeUnixNano: uint64(spanTime.UnixNano()), + EndTimeUnixNano: uint64(spanTime.Add(time.Millisecond).UnixNano()), + }, + }, + }, + }, + }, + }, + } + sut.OTLPHTTPExportTraces(t, req, at.QueryOpts{}) + sut.ForceFlush(t) + + startTime := spanTime.Add(-time.Hour) + endTime := time.Now() + + tc.Assert(&at.AssertOptions{ + Msg: "unexpected number of traces from /select/jaeger/api/traces", + Got: func() any { + res := sut.JaegerAPITraces(t, at.JaegerQueryParam{ + TraceQueryParam: query.TraceQueryParam{ + ServiceName: serviceName, + StartTimeMin: startTime, + StartTimeMax: endTime, + }, + }, at.QueryOpts{}) + return len(res.Data) + }, + Want: 1, + }) + + tc.Assert(&at.AssertOptions{ + Msg: "unexpected number of traces from /select/tempo/api/search", + Got: func() any { + res := sut.TempoAPISearch(t, fmt.Sprintf(`{resource.service.name=%q}`, serviceName), startTime, endTime) + return len(res.Traces) + }, + Want: 1, + }) +} diff --git a/apptest/vtagent.go b/apptest/vtagent.go index aaac48e6c..b5f1de1c6 100644 --- a/apptest/vtagent.go +++ b/apptest/vtagent.go @@ -29,8 +29,9 @@ type Vtagent struct { // StartVtagent starts an instance of Vtagent with the given flags. // It also sets the default flags and populates the app instance state with // values extracted from the application log (such as httpListenAddr). -func StartVtagent(instance string, remoteWriteURLs, flags []string, cli *Client) (*Vtagent, error) { +func StartVtagent(instance string, remoteWriteURLs, flags, env []string, cli *Client) (*Vtagent, error) { app, stderrExtracts, err := startApp(instance, "../../bin/vtagent-race", flags, &appOptions{ + env: env, defaultFlags: map[string]string{ "-httpListenAddr": "127.0.0.1:0", "-otlpGRPCListenAddr": "127.0.0.1:0", diff --git a/apptest/vtsingle.go b/apptest/vtsingle.go index 528593584..f7be749b1 100644 --- a/apptest/vtsingle.go +++ b/apptest/vtsingle.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "regexp" + "strconv" "testing" "time" @@ -32,17 +33,21 @@ type Vtsingle struct { jaegerAPITraceURL string jaegerAPIDependenciesURL string + tempoAPISearchURL string + logsQLQueryURL string otlpTracesURL string otlpGRPCTracesURL string } -// StartVtsingle starts an instance of Vtsingle with the given flags. It also -// sets the default flags and populates the app instance state with runtime -// values extracted from the application log (such as httpListenAddr). -func StartVtsingle(instance string, flags []string, cli *Client) (*Vtsingle, error) { +// StartVtsingle starts an instance of Vtsingle with the given flags and extra +// environment variables. It also sets the default flags and populates the app +// instance state with runtime values extracted from the application log (such +// as httpListenAddr). +func StartVtsingle(instance string, flags, env []string, cli *Client) (*Vtsingle, error) { app, stderrExtracts, err := startApp(instance, "../../bin/victoria-traces-race", flags, &appOptions{ + env: env, defaultFlags: map[string]string{ "-storageDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()), "-httpListenAddr": "127.0.0.1:0", @@ -79,6 +84,8 @@ func StartVtsingle(instance string, flags []string, cli *Client) (*Vtsingle, err jaegerAPITraceURL: fmt.Sprintf("http://%s/select/jaeger/api/traces/%%s", stderrExtracts[1]), jaegerAPIDependenciesURL: fmt.Sprintf("http://%s/select/jaeger/api/dependencies", stderrExtracts[1]), + tempoAPISearchURL: fmt.Sprintf("http://%s/select/tempo/api/search", stderrExtracts[1]), + logsQLQueryURL: fmt.Sprintf("http://%s/select/logsql/query", stderrExtracts[1]), otlpTracesURL: fmt.Sprintf("http://%s/insert/opentelemetry/v1/traces", stderrExtracts[1]), @@ -157,6 +164,20 @@ func (app *Vtsingle) JaegerAPITrace(t *testing.T, traceID string, opts QueryOpts return NewJaegerAPITraceResponse(t, res) } +// TempoAPISearch is a test helper function that searches for traces with a TraceQL query +// by sending an HTTP GET request to /select/tempo/api/search Vtsingle endpoint. +func (app *Vtsingle) TempoAPISearch(t *testing.T, q string, start, end time.Time) *TempoAPISearchResponse { + t.Helper() + + uv := url.Values{ + "q": {q}, + "start": {strconv.FormatInt(start.Unix(), 10)}, + "end": {strconv.FormatInt(end.Unix(), 10)}, + } + res, _ := app.cli.Get(t, app.tempoAPISearchURL+"?"+uv.Encode()) + return NewTempoAPISearchResponse(t, res) +} + // JaegerAPIDependencies is a test helper function that queries for the dependencies. // This method is not implemented in Vtsingle and this test is no-op for now. func (app *Vtsingle) JaegerAPIDependencies(t *testing.T, param JaegerDependenciesParam, opts QueryOpts) *JaegerAPIDependenciesResponse { diff --git a/docs/victoriatraces/changelog/CHANGELOG.md b/docs/victoriatraces/changelog/CHANGELOG.md index 7906db50a..4e1f22566 100644 --- a/docs/victoriatraces/changelog/CHANGELOG.md +++ b/docs/victoriatraces/changelog/CHANGELOG.md @@ -12,6 +12,8 @@ The following `tip` changes can be tested by building VictoriaTraces components ## tip +* BUGFIX: [Single-node VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) and vtselect in [VictoriaTraces cluster](https://docs.victoriametrics.com/victoriatraces/cluster/): apply correct time‑range filter and return proper traces from Jaeger and Tempo search APIs when VictoriaTraces runs in a non‑UTC time zone. Previously, searches could return fewer traces or empty results due to incorrect time‑zone conversion for the time‑range filter, especially for time zones west of UTC such as `TZ=America/New_York`, which impacted the end‑time boundary. See [issue #260](https://github.com/VictoriaMetrics/VictoriaTraces/issues/260). Thank @Vandit1604 for [pull request #262](https://github.com/VictoriaMetrics/VictoriaTraces/pull/262). + ## [v0.12.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.12.0) Released at 2026-09-29