From fa4d55da2338218138c2e63a5ba0dd4cf3c01f79 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Mon, 6 Apr 2026 16:30:53 +0300 Subject: [PATCH 1/8] add query length limit for async searches list --- .../seqapi/v1/http/get_async_searches_list.go | 25 +++++++++- .../v1/http/get_async_searches_list_test.go | 48 ++++++++++++++++++- metric/metric.go | 6 +++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/internal/api/seqapi/v1/http/get_async_searches_list.go b/internal/api/seqapi/v1/http/get_async_searches_list.go index 50e6161..59c5f14 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list.go @@ -11,6 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/metric" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -111,6 +112,8 @@ func (r getAsyncSearchesListRequest) toProto() (*seqapi.GetAsyncSearchesListRequ }, nil } +const asyncSearchListQueryLimit = 1000 + type getAsyncSearchesListResponse struct { Searches []asyncSearchesListItem `json:"searches"` Error apiError `json:"error"` @@ -142,7 +145,7 @@ func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResp searches = append(searches, asyncSearchesListItem{ SearchID: s.SearchId, Status: asyncSearchStatusFromProto(s.Status), - Request: startAsyncSearchRequestFromProto(s.Request), + Request: startAsyncSearchListRequestFromProto(s.Request), StartedAt: s.StartedAt.AsTime(), ExpiresAt: s.ExpiresAt.AsTime(), CanceledAt: canceledAt, @@ -159,6 +162,15 @@ func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResp } } +func startAsyncSearchListRequestFromProto(r *seqapi.StartAsyncSearchRequest) startAsyncSearchRequest { + req := startAsyncSearchRequestFromProto(r) + if trimmedQuery, ok := trimQueryToLimit(req.Query); ok { + metric.ServerRequestQueryTooLong.Inc() + req.Query = trimmedQuery + } + return req +} + func startAsyncSearchRequestFromProto(r *seqapi.StartAsyncSearchRequest) startAsyncSearchRequest { var hist *AsyncSearchRequestHistogram if r.Hist != nil { @@ -199,3 +211,14 @@ func aggregationTsQueriesFromProto(aggs []*seqapi.AggregationQuery) aggregationT return result } + +func trimQueryToLimit(query string) (string, bool) { + count := 0 + for i := range query { + if count == asyncSearchListQueryLimit { + return query[:i], true + } + count++ + } + return query, false +} diff --git a/internal/api/seqapi/v1/http/get_async_searches_list_test.go b/internal/api/seqapi/v1/http/get_async_searches_list_test.go index f82fd89..5f1e552 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list_test.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list_test.go @@ -28,8 +28,9 @@ func TestServeGetAsyncSearchesList(t *testing.T) { mockProfileID1 int64 = 1 mockProfileID2 int64 = 1 errorMsg = "some error" - - mockTime = time.Date(2025, 8, 6, 17, 52, 12, 123, time.UTC) + tooLongQuery = strings.Repeat("message:error and level:3", 41) + TruncatedQuery = strings.Repeat("message:error and level:3", 40) + mockTime = time.Date(2025, 8, 6, 17, 52, 12, 123, time.UTC) ) type mockArgs struct { @@ -237,6 +238,49 @@ func TestServeGetAsyncSearchesList(t *testing.T) { reqBody: "invalid-request", wantStatus: http.StatusBadRequest, }, + { + name: "query_too_long", + reqBody: "{}", + mockArgs: &mockArgs{ + proxyReq: &seqapi.GetAsyncSearchesListRequest{}, + proxyResp: &seqapi.GetAsyncSearchesListResponse{ + Searches: []*seqapi.GetAsyncSearchesListResponse_ListItem{ + { + SearchId: mockSearchID1, + Status: seqapi.AsyncSearchStatus_ASYNC_SEARCH_STATUS_DONE, + Request: &seqapi.StartAsyncSearchRequest{ + Retention: durationpb.New(60 * time.Second), + Query: tooLongQuery, + From: timestamppb.New(mockTime.Add(-15 * time.Minute)), + To: timestamppb.New(mockTime), + WithDocs: true, + Size: 100, + }, + StartedAt: timestamppb.New(mockTime.Add(-30 * time.Second)), + ExpiresAt: timestamppb.New(mockTime.Add(30 * time.Second)), + Progress: 1, + DiskUsage: 512, + OwnerName: mockUserName1, + }, + }, + Error: &seqapi.Error{ + Code: seqapi.ErrorCode_ERROR_CODE_PARTIAL_RESPONSE, + Message: "partial response", + }, + }, + repoReq: types.GetAsyncSearchesListRequest{}, + repoResp: []types.AsyncSearchInfo{ + { + SearchID: mockSearchID1, + OwnerID: mockProfileID1, + OwnerName: mockUserName1, + }, + }, + searchIDs: []string{mockSearchID1}, + }, + wantRespBody: `{"searches":[{"search_id":"c9a34cf8-4c66-484e-9cc2-42979d848656","status":"done","request":{"retention":"seconds:60","query":"` + TruncatedQuery + `","from":"2025-08-06T17:37:12.000000123Z","to":"2025-08-06T17:52:12.000000123Z","with_docs":true,"size":100},"started_at":"2025-08-06T17:51:42.000000123Z","expires_at":"2025-08-06T17:52:42.000000123Z","progress":1,"disk_usage":"512","owner_name":"some_user_1"}],"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`, + wantStatus: http.StatusOK, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/metric/metric.go b/metric/metric.go index 272a7d1..55c07ac 100644 --- a/metric/metric.go +++ b/metric/metric.go @@ -93,6 +93,12 @@ var ( Name: "cache_redis_misses_total", Help: "", }) + ServerRequestQueryTooLong = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: seqUINS, + Subsystem: serverSubsys, + Name: "requests_query_too_long_total", + Help: "", + }) // client metrics SeqDBClientRequestSent = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: seqUINS, From 9f56564e6728fd756fde09eacfc4a58d1167a6ca Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 7 Apr 2026 13:50:16 +0300 Subject: [PATCH 2/8] fix --- docs/en/02-configuration.md | 4 ++++ docs/ru/02-configuration.md | 4 ++++ .../seqapi/v1/http/get_async_searches_list.go | 20 +++++++++---------- .../v1/http/get_async_searches_list_test.go | 17 ++++++++++++++-- internal/app/config/config.go | 5 +++++ metric/metric.go | 15 ++++++++------ 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/docs/en/02-configuration.md b/docs/en/02-configuration.md index eb201e4..74fc93b 100644 --- a/docs/en/02-configuration.md +++ b/docs/en/02-configuration.md @@ -532,6 +532,10 @@ Config for `/seqapi` API handlers. Max value for `limit` field in export requests. ++ **`async_search_list_query_limit`** *`int`* *`default=1000`* + + Maximum length of `request.query` in async searches list responses. + + **`seq_cli_max_search_limit`** *`int`* *`default=0`* The maximum number of logs that can be processed by seq-cli in one search command run. diff --git a/docs/ru/02-configuration.md b/docs/ru/02-configuration.md index 0df64db..d5f353e 100644 --- a/docs/ru/02-configuration.md +++ b/docs/ru/02-configuration.md @@ -532,6 +532,10 @@ handlers: Максимальное значение для поля `limit` в запросах на экспорт. ++ **`async_search_list_query_limit`** *`int`* *`default=1000`* + + Максимальная длина `request.query` в ответе списка отложенных запросов. + + **`seq_cli_max_search_limit`** *`int`* *`default=0`* Максимальное количество событий, которое может быть получено за одну команду поиска в seq-cli. diff --git a/internal/api/seqapi/v1/http/get_async_searches_list.go b/internal/api/seqapi/v1/http/get_async_searches_list.go index 59c5f14..09d98b4 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list.go @@ -84,7 +84,7 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) return } - wr.WriteJson(getAsyncSearchesListResponseFromProto(resp)) + wr.WriteJson(getAsyncSearchesListResponseFromProto(resp, a.config.AsyncSearchListQueryLimit)) } type getAsyncSearchesListRequest struct { @@ -112,8 +112,6 @@ func (r getAsyncSearchesListRequest) toProto() (*seqapi.GetAsyncSearchesListRequ }, nil } -const asyncSearchListQueryLimit = 1000 - type getAsyncSearchesListResponse struct { Searches []asyncSearchesListItem `json:"searches"` Error apiError `json:"error"` @@ -132,7 +130,7 @@ type asyncSearchesListItem struct { Error *string `json:"error,omitempty"` } // @name seqapi.v1.AsyncSearchesListItem -func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResponse) getAsyncSearchesListResponse { +func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResponse, asyncSearchListQueryLimit int) getAsyncSearchesListResponse { searches := make([]asyncSearchesListItem, 0, len(resp.Searches)) for _, s := range resp.Searches { @@ -145,7 +143,7 @@ func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResp searches = append(searches, asyncSearchesListItem{ SearchID: s.SearchId, Status: asyncSearchStatusFromProto(s.Status), - Request: startAsyncSearchListRequestFromProto(s.Request), + Request: startAsyncSearchListRequestFromProto(s.Request, asyncSearchListQueryLimit), StartedAt: s.StartedAt.AsTime(), ExpiresAt: s.ExpiresAt.AsTime(), CanceledAt: canceledAt, @@ -162,11 +160,11 @@ func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResp } } -func startAsyncSearchListRequestFromProto(r *seqapi.StartAsyncSearchRequest) startAsyncSearchRequest { +func startAsyncSearchListRequestFromProto(r *seqapi.StartAsyncSearchRequest, queryLimit int) startAsyncSearchRequest { req := startAsyncSearchRequestFromProto(r) - if trimmedQuery, ok := trimQueryToLimit(req.Query); ok { - metric.ServerRequestQueryTooLong.Inc() - req.Query = trimmedQuery + if trimmedQuery, ok := trimQueryToLimit(req.Query, queryLimit); ok { + metric.AsyncSearchQueryTooLong.Inc() + req.Query = trimmedQuery + "..." } return req } @@ -212,10 +210,10 @@ func aggregationTsQueriesFromProto(aggs []*seqapi.AggregationQuery) aggregationT return result } -func trimQueryToLimit(query string) (string, bool) { +func trimQueryToLimit(query string, limit int) (string, bool) { count := 0 for i := range query { - if count == asyncSearchListQueryLimit { + if count == limit { return query[:i], true } count++ diff --git a/internal/api/seqapi/v1/http/get_async_searches_list_test.go b/internal/api/seqapi/v1/http/get_async_searches_list_test.go index 5f1e552..b04572c 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list_test.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list_test.go @@ -13,6 +13,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" + "github.com/ozontech/seq-ui/internal/app/config" "github.com/ozontech/seq-ui/internal/app/types" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" mock_repo "github.com/ozontech/seq-ui/internal/pkg/repository/mock" @@ -48,6 +49,7 @@ func TestServeGetAsyncSearchesList(t *testing.T) { name string reqBody string + cfg config.SeqAPI wantRespBody string wantStatus int @@ -241,6 +243,11 @@ func TestServeGetAsyncSearchesList(t *testing.T) { { name: "query_too_long", reqBody: "{}", + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + AsyncSearchListQueryLimit: 1000, + }, + }, mockArgs: &mockArgs{ proxyReq: &seqapi.GetAsyncSearchesListRequest{}, proxyResp: &seqapi.GetAsyncSearchesListResponse{ @@ -278,7 +285,7 @@ func TestServeGetAsyncSearchesList(t *testing.T) { }, searchIDs: []string{mockSearchID1}, }, - wantRespBody: `{"searches":[{"search_id":"c9a34cf8-4c66-484e-9cc2-42979d848656","status":"done","request":{"retention":"seconds:60","query":"` + TruncatedQuery + `","from":"2025-08-06T17:37:12.000000123Z","to":"2025-08-06T17:52:12.000000123Z","with_docs":true,"size":100},"started_at":"2025-08-06T17:51:42.000000123Z","expires_at":"2025-08-06T17:52:42.000000123Z","progress":1,"disk_usage":"512","owner_name":"some_user_1"}],"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`, + wantRespBody: `{"searches":[{"search_id":"c9a34cf8-4c66-484e-9cc2-42979d848656","status":"done","request":{"retention":"seconds:60","query":"` + TruncatedQuery + `...","from":"2025-08-06T17:37:12.000000123Z","to":"2025-08-06T17:52:12.000000123Z","with_docs":true,"size":100},"started_at":"2025-08-06T17:51:42.000000123Z","expires_at":"2025-08-06T17:52:42.000000123Z","progress":1,"disk_usage":"512","owner_name":"some_user_1"}],"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`, wantStatus: http.StatusOK, }, } @@ -286,7 +293,13 @@ func TestServeGetAsyncSearchesList(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - seqData := test.APITestData{} + seqData := test.APITestData{ + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + AsyncSearchListQueryLimit: 1000, + }, + }, + } if tt.mockArgs != nil { ctrl := gomock.NewController(t) diff --git a/internal/app/config/config.go b/internal/app/config/config.go index 954594b..62b1d36 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -32,6 +32,7 @@ const ( defaultMaxSearchTotalLimit = 1000000 defaultMaxSearchOffsetLimit = 1000000 + defaultAsyncSearchListQueryLimit = 1000 defaultMaxAggregationsPerRequest = 1 defaultMaxBucketsPerAggregationTs = 200 defaultMaxParallelExportRequests = 1 @@ -259,6 +260,7 @@ type SeqAPIOptions struct { MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` MaxExportLimit int32 `yaml:"max_export_limit"` + AsyncSearchListQueryLimit int `yaml:"async_search_list_query_limit"` SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` @@ -351,6 +353,9 @@ func FromFile(cfgPath string) (Config, error) { if cfg.Handlers.SeqAPI.MaxSearchOffsetLimit <= 0 { cfg.Handlers.SeqAPI.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit } + if cfg.Handlers.SeqAPI.AsyncSearchListQueryLimit <= 0 { + cfg.Handlers.SeqAPI.AsyncSearchListQueryLimit = defaultAsyncSearchListQueryLimit + } if cfg.Handlers.SeqAPI.EventsCacheTTL <= 0 { cfg.Handlers.SeqAPI.EventsCacheTTL = defaultEventsCacheTTL } diff --git a/metric/metric.go b/metric/metric.go index 55c07ac..c28790e 100644 --- a/metric/metric.go +++ b/metric/metric.go @@ -17,6 +17,7 @@ const ( repoSubsys = "repository" clickHouseSubsys = "clickhouse" massExportSubsys = "mass_export" + asyncSearchSubsys = "async_search" componentLabel = "component" methodLabel = "method" @@ -93,12 +94,6 @@ var ( Name: "cache_redis_misses_total", Help: "", }) - ServerRequestQueryTooLong = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: seqUINS, - Subsystem: serverSubsys, - Name: "requests_query_too_long_total", - Help: "", - }) // client metrics SeqDBClientRequestSent = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: seqUINS, @@ -215,6 +210,14 @@ var ( Help: "", Buckets: defaultBuckets, }, []string{sessionIDLabel}) + + // async search metrics + AsyncSearchQueryTooLong = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: seqUINS, + Subsystem: asyncSearchSubsys, + Name: "requests_query_too_long_total", + Help: "", + }) ) // HandledIncomingRequest handles metrics for processed incoming request. From fe6b6fe72eed74ae6bde5c18d79a2d010832b880 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 7 Apr 2026 19:29:54 +0300 Subject: [PATCH 3/8] fix --- cmd/seq-ui/main.go | 2 +- docs/en/02-configuration.md | 21 +++- docs/ru/02-configuration.md | 21 +++- .../api/seqapi/v1/grpc/aggregation_test.go | 34 +++-- internal/api/seqapi/v1/grpc/api.go | 24 ++-- .../api/seqapi/v1/grpc/cluster_status_test.go | 2 +- internal/api/seqapi/v1/grpc/events_test.go | 18 +-- internal/api/seqapi/v1/grpc/fields_test.go | 16 ++- internal/api/seqapi/v1/grpc/get_envs_test.go | 106 ++++++++-------- internal/api/seqapi/v1/grpc/limits_test.go | 20 +-- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 10 +- internal/api/seqapi/v1/grpc/search_test.go | 60 +++++---- internal/api/seqapi/v1/grpc/test_data.go | 10 +- .../api/seqapi/v1/http/aggregation_test.go | 50 +++++--- .../api/seqapi/v1/http/aggregation_ts_test.go | 60 +++++---- internal/api/seqapi/v1/http/api.go | 28 ++--- internal/api/seqapi/v1/http/events_test.go | 18 +-- internal/api/seqapi/v1/http/export_test.go | 60 +++++---- internal/api/seqapi/v1/http/fields_test.go | 16 ++- .../seqapi/v1/http/get_async_searches_list.go | 4 +- .../v1/http/get_async_searches_list_test.go | 13 +- internal/api/seqapi/v1/http/get_envs_test.go | 116 +++++++++--------- internal/api/seqapi/v1/http/limits_test.go | 18 +-- .../api/seqapi/v1/http/logs_lifespan_test.go | 10 +- internal/api/seqapi/v1/http/search_test.go | 108 +++++++++------- .../api/seqapi/v1/http/start_async_search.go | 8 +- internal/api/seqapi/v1/http/test_data.go | 10 +- internal/api/seqapi/v1/seqapi.go | 2 +- internal/api/seqapi/v1/test/data.go | 26 ++-- internal/app/config/config.go | 14 ++- 30 files changed, 522 insertions(+), 383 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 8d7a543..37c3097 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -166,7 +166,7 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { asyncSearchesService = asyncsearches.New(ctx, repo, defaultClient, cfg.Handlers.AsyncSearch.AdminUsers) } - seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService, p) + seqApiV1 := seqapi_v1.New(*cfg.Handlers, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService, p) logger.Info("initializing clickhouse") ch, err := initClickHouse(ctx, cfg.Server.CH) diff --git a/docs/en/02-configuration.md b/docs/en/02-configuration.md index 74fc93b..b9990ab 100644 --- a/docs/en/02-configuration.md +++ b/docs/en/02-configuration.md @@ -532,10 +532,6 @@ Config for `/seqapi` API handlers. Max value for `limit` field in export requests. -+ **`async_search_list_query_limit`** *`int`* *`default=1000`* - - Maximum length of `request.query` in async searches list responses. - + **`seq_cli_max_search_limit`** *`int`* *`default=0`* The maximum number of logs that can be processed by seq-cli in one search command run. @@ -810,6 +806,23 @@ Config for `/massexport` API handlers. > The value must be passed in the duration format: `(ms|s|m|h)`. +### Async Search + +Configuration for async request. + +**`async_search`** *`AsyncSearch`* *`optional`* + +Поля `AsyncSearch`: + ++ **`admin_users`** *`[]string`* *`optional`* + + List of users allowed to cancel or delete async requests created by other users. + ++ **`query_length_limit`** *`int`* *`default=1000`* + + Maximum length of `request.query` in async searches list responses. + + ## Tracing The tracing configuration is set through environment variables. diff --git a/docs/ru/02-configuration.md b/docs/ru/02-configuration.md index d5f353e..42faa8e 100644 --- a/docs/ru/02-configuration.md +++ b/docs/ru/02-configuration.md @@ -506,6 +506,7 @@ handlers: seq_api: error_groups: mass_export: + async_search: ``` ### SeqAPI @@ -532,10 +533,6 @@ handlers: Максимальное значение для поля `limit` в запросах на экспорт. -+ **`async_search_list_query_limit`** *`int`* *`default=1000`* - - Максимальная длина `request.query` в ответе списка отложенных запросов. - + **`seq_cli_max_search_limit`** *`int`* *`default=0`* Максимальное количество событий, которое может быть получено за одну команду поиска в seq-cli. @@ -810,6 +807,22 @@ handlers: > Значение должно быть передано в `duration`-формате: `<число>(ms|s|m|h)`. +### Async Search + +**`async_search`** *`AsyncSearch`* *`optional`* + +Настройка функциональности отложенных запросов. + +Поля `AsyncSearch`: + ++ **`admin_users`** *`[]string`* *`optional`* + + Список пользователей, которые могут отменять или удалять отложенные поиски других пользователей. + ++ **`query_length_limit`** *`int`* *`default=1000`* + + Максимальная длина `request.query` в ответе списка отложенных запросов. + ## Tracing Конфигурация трейсинга задается переменными окружения. diff --git a/internal/api/seqapi/v1/grpc/aggregation_test.go b/internal/api/seqapi/v1/grpc/aggregation_test.go index b7a44c1..61e017c 100644 --- a/internal/api/seqapi/v1/grpc/aggregation_test.go +++ b/internal/api/seqapi/v1/grpc/aggregation_test.go @@ -30,7 +30,7 @@ func TestGetAggregation(t *testing.T) { apiErr bool clientErr error - cfg config.SeqAPI + cfg config.Handlers }{ { name: "ok_single_agg", @@ -47,9 +47,11 @@ func TestGetAggregation(t *testing.T) { Code: seqapi.ErrorCode_ERROR_CODE_NO, }, }, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 4, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 4, + }, }, }, }, @@ -71,9 +73,11 @@ func TestGetAggregation(t *testing.T) { Code: seqapi.ErrorCode_ERROR_CODE_NO, }, }, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -86,9 +90,11 @@ func TestGetAggregation(t *testing.T) { {Field: "test3"}, }, }, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 2, + }, }, }, apiErr: true, @@ -101,9 +107,11 @@ func TestGetAggregation(t *testing.T) { To: timestamppb.New(to), AggField: "test2", }, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 1, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 1, + }, }, }, clientErr: errors.New("client error"), diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index 9760bb4..49b2513 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -32,7 +32,7 @@ type apiParams struct { type API struct { seqapi.UnimplementedSeqAPIServiceServer - config config.SeqAPI + config config.Handlers params apiParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache @@ -44,7 +44,7 @@ type API struct { } func New( - cfg config.SeqAPI, + cfg config.Handlers, seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, @@ -52,23 +52,23 @@ func New( p *profiles.Profiles, ) *API { var globalfCache *fieldsCache - if cfg.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.FieldsCacheTTL) + if cfg.SeqAPI.FieldsCacheTTL > 0 { + globalfCache = newFieldsCache(cfg.SeqAPI.FieldsCacheTTL) } - globalMasker, err := mask.New(cfg.Masking) + globalMasker, err := mask.New(cfg.SeqAPI.Masking) if err != nil { logger.Fatal("failed to init masking", zap.Error(err)) } - globalPinnedFields := parsePinnedFields(cfg.PinnedFields) + globalPinnedFields := parsePinnedFields(cfg.SeqAPI.PinnedFields) var params apiParams var paramsByEnv map[string]apiParams - if len(cfg.Envs) > 0 { + if len(cfg.SeqAPI.Envs) > 0 { paramsByEnv = make(map[string]apiParams) - for envName, envConfig := range cfg.Envs { + for envName, envConfig := range cfg.SeqAPI.Envs { client := seqDBСlients[envConfig.SeqDB] options := envConfig.Options @@ -101,7 +101,7 @@ func New( params = apiParams{ client: client, - options: cfg.SeqAPIOptions, + options: cfg.SeqAPI.SeqAPIOptions, fieldsCache: globalfCache, masker: globalMasker, pinnedFields: globalPinnedFields, @@ -117,7 +117,7 @@ func New( nowFn: time.Now, asyncSearches: asyncSearches, profiles: p, - envsResponse: parseEnvs(cfg), + envsResponse: parseEnvs(cfg.SeqAPI), } } @@ -191,12 +191,12 @@ func (a *API) GetEnvFromContext(ctx context.Context) string { } func (a *API) GetParams(env string) (apiParams, error) { - if len(a.config.Envs) == 0 { + if len(a.config.SeqAPI.Envs) == 0 { return a.params, nil } if env == "" { - env = a.config.DefaultEnv + env = a.config.SeqAPI.DefaultEnv } params, exists := a.paramsByEnv[env] diff --git a/internal/api/seqapi/v1/grpc/cluster_status_test.go b/internal/api/seqapi/v1/grpc/cluster_status_test.go index 5a0d73e..7093032 100644 --- a/internal/api/seqapi/v1/grpc/cluster_status_test.go +++ b/internal/api/seqapi/v1/grpc/cluster_status_test.go @@ -64,7 +64,7 @@ func TestStatus(t *testing.T) { seqDbMock.EXPECT().Status(gomock.Any(), nil). Return(proto.Clone(tt.resp), tt.clientErr).Times(1) - cfg := config.SeqAPI{} + cfg := config.Handlers{} seqData := test.APITestData{ Cfg: cfg, diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index d5d9cab..82195ea 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -102,9 +102,11 @@ func TestGetEvent(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + EventsCacheTTL: cacheTTL, + }, }, }, } @@ -364,10 +366,12 @@ func TestGetEventWithMasking(t *testing.T) { curEID := curEData.id seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, - Masking: tt.maskingCfg, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + EventsCacheTTL: cacheTTL, + Masking: tt.maskingCfg, + }, }, }, } diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index bfa23b7..acf47f9 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -106,9 +106,11 @@ func TestGetFieldsCached(t *testing.T) { const ttl = 20 * time.Millisecond seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - FieldsCacheTTL: ttl, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + FieldsCacheTTL: ttl, + }, }, }, Mocks: test.Mocks{ @@ -154,9 +156,11 @@ func TestGetPinnedFields(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - PinnedFields: tt.fields, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + PinnedFields: tt.fields, + }, }, }, } diff --git a/internal/api/seqapi/v1/grpc/get_envs_test.go b/internal/api/seqapi/v1/grpc/get_envs_test.go index 5dbbd8e..5a42a25 100644 --- a/internal/api/seqapi/v1/grpc/get_envs_test.go +++ b/internal/api/seqapi/v1/grpc/get_envs_test.go @@ -13,18 +13,20 @@ import ( func TestGetEnvs(t *testing.T) { tests := []struct { name string - cfg config.SeqAPI + cfg config.Handlers want *seqapi.GetEnvsResponse }{ { name: "single_env", - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: &seqapi.GetEnvsResponse{ @@ -42,55 +44,57 @@ func TestGetEnvs(t *testing.T) { }, { name: "multiple_envs", - cfg: config.SeqAPI{ - Envs: map[string]config.SeqAPIEnv{ - "test": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + Envs: map[string]config.SeqAPIEnv{ + "test": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, - }, - "prod": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 150, - MaxExportLimit: 250, - MaxParallelExportRequests: 3, - MaxAggregationsPerRequest: 6, - SeqCLIMaxSearchLimit: 15000, + "prod": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 150, + MaxExportLimit: 250, + MaxParallelExportRequests: 3, + MaxAggregationsPerRequest: 6, + SeqCLIMaxSearchLimit: 15000, + }, }, - }, - "cluster-10": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 150, - MaxExportLimit: 250, - MaxParallelExportRequests: 3, - MaxAggregationsPerRequest: 6, - SeqCLIMaxSearchLimit: 15000, + "cluster-10": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 150, + MaxExportLimit: 250, + MaxParallelExportRequests: 3, + MaxAggregationsPerRequest: 6, + SeqCLIMaxSearchLimit: 15000, + }, }, - }, - "cluster-102": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + "cluster-102": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, - }, - "cluster-220": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + "cluster-220": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, + DefaultEnv: "test", }, - DefaultEnv: "test", }, want: &seqapi.GetEnvsResponse{ Envs: []*seqapi.GetEnvsResponse_Env{ @@ -145,7 +149,7 @@ func TestGetEnvs(t *testing.T) { t.Parallel() api := API{ config: tt.cfg, - envsResponse: parseEnvs(tt.cfg), + envsResponse: parseEnvs(tt.cfg.SeqAPI), } resp, err := api.GetEnvs(context.TODO(), &seqapi.GetEnvsRequest{}) require.NoError(t, err) diff --git a/internal/api/seqapi/v1/grpc/limits_test.go b/internal/api/seqapi/v1/grpc/limits_test.go index 971198c..5b30453 100644 --- a/internal/api/seqapi/v1/grpc/limits_test.go +++ b/internal/api/seqapi/v1/grpc/limits_test.go @@ -14,18 +14,20 @@ import ( func TestGetLimits(t *testing.T) { tests := []struct { name string - cfg config.SeqAPI + cfg config.Handlers want *seqapi.GetLimitsResponse }{ { name: "ok", - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, want: &seqapi.GetLimitsResponse{ @@ -38,7 +40,7 @@ func TestGetLimits(t *testing.T) { }, { name: "empty", - cfg: config.SeqAPI{}, + cfg: config.Handlers{}, want: &seqapi.GetLimitsResponse{}, }, } diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index 3b884ff..1525e28 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -109,10 +109,12 @@ func TestGetLogsLifespan(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - LogsLifespanCacheKey: cacheKey, - LogsLifespanCacheTTL: cacheTTL, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + LogsLifespanCacheKey: cacheKey, + LogsLifespanCacheTTL: cacheTTL, + }, }, }, } diff --git a/internal/api/seqapi/v1/grpc/search_test.go b/internal/api/seqapi/v1/grpc/search_test.go index 6145aa3..3d59c7b 100644 --- a/internal/api/seqapi/v1/grpc/search_test.go +++ b/internal/api/seqapi/v1/grpc/search_test.go @@ -35,7 +35,7 @@ func TestSearch(t *testing.T) { apiErr bool clientErr error - cfg config.SeqAPI + cfg config.Handlers }{ { name: "ok", @@ -64,10 +64,12 @@ func TestSearch(t *testing.T) { Code: seqapi.ErrorCode_ERROR_CODE_NO, }, }, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 5, + }, }, }), }, @@ -83,11 +85,13 @@ func TestSearch(t *testing.T) { req: &seqapi.SearchRequest{ Limit: 10, }, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, - }, + }), apiErr: true, }, { @@ -100,10 +104,12 @@ func TestSearch(t *testing.T) { {Field: "test3"}, }, }, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 2, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 2, + }, }, }), apiErr: true, @@ -117,10 +123,12 @@ func TestSearch(t *testing.T) { Limit: limit, Offset: 11, }, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxSearchOffsetLimit: 10, + }, }, }), apiErr: true, @@ -147,10 +155,12 @@ func TestSearch(t *testing.T) { Message: api_error.ErrQueryTooHeavy.Error(), }, }, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchTotalLimit: int64(limit), + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxSearchTotalLimit: int64(limit), + }, }, }), }, @@ -163,9 +173,11 @@ func TestSearch(t *testing.T) { Limit: limit, Offset: 0, }, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), clientErr: errors.New("client error"), diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index 778d702..3d76905 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -14,13 +14,13 @@ import ( func initTestAPI(data test.APITestData) *API { // when test cases don't explicitly provide configuration - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} + if data.Cfg.SeqAPI.SeqAPIOptions == nil { + data.Cfg.SeqAPI.SeqAPIOptions = &config.SeqAPIOptions{} } seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB - for _, envConfig := range data.Cfg.Envs { + for _, envConfig := range data.Cfg.SeqAPI.Envs { seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB } @@ -28,8 +28,8 @@ func initTestAPI(data test.APITestData) *API { } func initTestAPIWithAsyncSearches(data test.APITestData) *API { - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} + if data.Cfg.SeqAPI.SeqAPIOptions == nil { + data.Cfg.SeqAPI.SeqAPIOptions = &config.SeqAPIOptions{} } seqDBClients := map[string]seqdb.Client{ config.DefaultSeqDBClientID: data.Mocks.SeqDB, diff --git a/internal/api/seqapi/v1/http/aggregation_test.go b/internal/api/seqapi/v1/http/aggregation_test.go index f3fcdf5..1b650c3 100644 --- a/internal/api/seqapi/v1/http/aggregation_test.go +++ b/internal/api/seqapi/v1/http/aggregation_test.go @@ -50,7 +50,7 @@ func TestServeGetAggregation(t *testing.T) { wantStatus int mockArgs *mockArgs - cfg config.SeqAPI + cfg config.Handlers }{ { name: "ok_single_agg", @@ -72,9 +72,11 @@ func TestServeGetAggregation(t *testing.T) { }, wantRespBody: `{"aggregation":{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2}]},"aggregations":[{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2}]}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -106,9 +108,11 @@ func TestServeGetAggregation(t *testing.T) { }, wantRespBody: `{"aggregation":{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2},{"key":"test3","value":3}]},"aggregations":[{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2},{"key":"test3","value":3}]},{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2},{"key":"test3","value":3}]}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -153,9 +157,11 @@ func TestServeGetAggregation(t *testing.T) { }, wantRespBody: `{"aggregation":{"buckets":[{"key":"test1","value":1,"not_exists":10,"quantiles":[100,150]},{"key":"test2","value":2,"not_exists":10,"quantiles":[100,150]},{"key":"test3","value":3,"not_exists":10,"quantiles":[100,150]}]},"aggregations":[{"buckets":[{"key":"test1","value":1,"not_exists":10,"quantiles":[100,150]},{"key":"test2","value":2,"not_exists":10,"quantiles":[100,150]},{"key":"test3","value":3,"not_exists":10,"quantiles":[100,150]}]},{"buckets":[{"key":"test1","value":1,"not_exists":10,"quantiles":[100,150]},{"key":"test2","value":2,"not_exists":10,"quantiles":[100,150]},{"key":"test3","value":3,"not_exists":10,"quantiles":[100,150]}]}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -179,9 +185,11 @@ func TestServeGetAggregation(t *testing.T) { }, wantRespBody: `{"aggregation":{"buckets":[]},"aggregations":[],"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"},"partialResponse":true}`, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + }, }, }, }, @@ -194,9 +202,11 @@ func TestServeGetAggregation(t *testing.T) { name: "err_aggs_limit_max", reqBody: formatReqBody("", aggregationQueries{{}, {}, {}}), wantStatus: http.StatusBadRequest, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 2, + }, }, }, }, @@ -213,9 +223,11 @@ func TestServeGetAggregation(t *testing.T) { err: errors.New("client error"), }, wantStatus: http.StatusInternalServerError, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/aggregation_ts_test.go b/internal/api/seqapi/v1/http/aggregation_ts_test.go index 3df29bf..601e59f 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts_test.go +++ b/internal/api/seqapi/v1/http/aggregation_ts_test.go @@ -48,7 +48,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus int mockArgs *mockArgs - cfg config.SeqAPI + cfg config.Handlers }{ { name: "ok_count", @@ -100,10 +100,12 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"test_count1":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count1":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count1":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}},{"data":{"result":[{"metric":{"test_count2":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count2":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count2":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -159,10 +161,12 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"quantile":"p95","service":"test1"},"values":[{"timestamp":1695637231,"value":100}]},{"metric":{"quantile":"p99","service":"test1"},"values":[{"timestamp":1695637231,"value":150}]},{"metric":{"quantile":"p95","service":"test2"},"values":[{"timestamp":1695637232,"value":100}]},{"metric":{"quantile":"p99","service":"test2"},"values":[{"timestamp":1695637232,"value":150}]},{"metric":{"quantile":"p95","service":"test3"},"values":[{"timestamp":1695637233,"value":100}]},{"metric":{"quantile":"p99","service":"test3"},"values":[{"timestamp":1695637233,"value":150}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -185,10 +189,12 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantRespBody: `{"aggregations":null,"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -201,10 +207,12 @@ func TestServeGetAggregationTs(t *testing.T) { name: "err_aggs_limit_max", reqBody: formatReqBody(aggregationTsQueries{{}, {}, {}}), wantStatus: http.StatusBadRequest, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, - MaxBucketsPerAggregationTs: 100, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 2, + MaxBucketsPerAggregationTs: 100, + }, }, }, }, @@ -216,10 +224,12 @@ func TestServeGetAggregationTs(t *testing.T) { }, }), wantStatus: http.StatusBadRequest, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 8, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 8, + }, }, }, }, @@ -235,9 +245,11 @@ func TestServeGetAggregationTs(t *testing.T) { err: errors.New("client error"), }, wantStatus: http.StatusInternalServerError, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index 16c603e..34206ac 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -32,7 +32,7 @@ type apiParams struct { } type API struct { - config config.SeqAPI + config config.Handlers params apiParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache @@ -44,7 +44,7 @@ type API struct { } func New( - cfg config.SeqAPI, + cfg config.Handlers, seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, @@ -52,24 +52,24 @@ func New( p *profiles.Profiles, ) *API { var globalfCache *fieldsCache - if cfg.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.FieldsCacheTTL) + if cfg.SeqAPI.FieldsCacheTTL > 0 { + globalfCache = newFieldsCache(cfg.SeqAPI.FieldsCacheTTL) } - globalMasker, err := mask.New(cfg.Masking) + globalMasker, err := mask.New(cfg.SeqAPI.Masking) if err != nil { logger.Fatal("failed to init masking", zap.Error(err)) } - globalPinnedFields := parsePinnedFields(cfg.PinnedFields) - globalExportLimiter := tokenlimiter.New(cfg.MaxParallelExportRequests) + globalPinnedFields := parsePinnedFields(cfg.SeqAPI.PinnedFields) + globalExportLimiter := tokenlimiter.New(cfg.SeqAPI.MaxParallelExportRequests) var params apiParams var paramsByEnv map[string]apiParams - if len(cfg.Envs) > 0 { + if len(cfg.SeqAPI.Envs) > 0 { paramsByEnv = make(map[string]apiParams) - for envName, envConfig := range cfg.Envs { + for envName, envConfig := range cfg.SeqAPI.Envs { client := seqDBСlients[envConfig.SeqDB] options := envConfig.Options @@ -104,7 +104,7 @@ func New( params = apiParams{ client: client, - options: cfg.SeqAPIOptions, + options: cfg.SeqAPI.SeqAPIOptions, fieldsCache: globalfCache, masker: globalMasker, pinnedFields: globalPinnedFields, @@ -112,7 +112,7 @@ func New( } } // for export - if len(cfg.Envs) > 0 { + if len(cfg.SeqAPI.Envs) > 0 { for _, param := range paramsByEnv { if param.masker != nil { param.client.WithMasking(param.masker) @@ -131,7 +131,7 @@ func New( nowFn: time.Now, asyncSearches: asyncSearches, profiles: p, - envsResponse: parseEnvs(cfg), + envsResponse: parseEnvs(cfg.SeqAPI), } } @@ -234,12 +234,12 @@ const ( ) func (a *API) GetEnvParams(env string) (apiParams, error) { - if len(a.config.Envs) == 0 { + if len(a.config.SeqAPI.Envs) == 0 { return a.params, nil } if env == "" { - env = a.config.DefaultEnv + env = a.config.SeqAPI.DefaultEnv } params, exists := a.paramsByEnv[env] diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index f4dd625..cae25a4 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -125,9 +125,11 @@ func TestServeGetEvent(t *testing.T) { ctrl := gomock.NewController(t) seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + EventsCacheTTL: cacheTTL, + }, }, }, } @@ -393,10 +395,12 @@ func TestGetEventWithMasking(t *testing.T) { curEID := curEData.id seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, - Masking: tt.maskingCfg, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + EventsCacheTTL: cacheTTL, + Masking: tt.maskingCfg, + }, }, }, } diff --git a/internal/api/seqapi/v1/http/export_test.go b/internal/api/seqapi/v1/http/export_test.go index ad69b9e..46fb88a 100644 --- a/internal/api/seqapi/v1/http/export_test.go +++ b/internal/api/seqapi/v1/http/export_test.go @@ -56,7 +56,7 @@ func TestServeExport(t *testing.T) { wantStatus int mockArgs *mockArgs - cfg config.SeqAPI + cfg config.Handlers }{ { name: "ok_jsonl", @@ -71,10 +71,12 @@ func TestServeExport(t *testing.T) { }, }, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, }, @@ -93,10 +95,12 @@ func TestServeExport(t *testing.T) { }, }, wantStatus: http.StatusOK, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, }, @@ -109,9 +113,11 @@ func TestServeExport(t *testing.T) { name: "err_parallel_limited", reqBody: formatReqBody(0, "", nil), wantStatus: http.StatusTooManyRequests, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxParallelExportRequests: 0, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxParallelExportRequests: 0, + }, }, }, }, @@ -119,10 +125,12 @@ func TestServeExport(t *testing.T) { name: "err_export_limit_max", reqBody: formatReqBody(10, "", nil), wantStatus: http.StatusBadRequest, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 5, - MaxParallelExportRequests: 1, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 5, + MaxParallelExportRequests: 1, + }, }, }, }, @@ -130,10 +138,12 @@ func TestServeExport(t *testing.T) { name: "err_csv_empty_fields", reqBody: formatReqBody(10, efCSV, nil), wantStatus: http.StatusBadRequest, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, }, @@ -151,10 +161,12 @@ func TestServeExport(t *testing.T) { err: errors.New("client error"), }, wantStatus: http.StatusInternalServerError, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, + }, }, }, }, diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index eda72cb..86cf2da 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -132,9 +132,11 @@ func TestServeGetFieldsCached(t *testing.T) { const ttl = 20 * time.Millisecond seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - FieldsCacheTTL: ttl, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + FieldsCacheTTL: ttl, + }, }, }, Mocks: test.Mocks{ @@ -191,9 +193,11 @@ func TestServeGetPinnedFields(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - PinnedFields: tt.fields, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + PinnedFields: tt.fields, + }, }, }, } diff --git a/internal/api/seqapi/v1/http/get_async_searches_list.go b/internal/api/seqapi/v1/http/get_async_searches_list.go index 09d98b4..6e3fe8d 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list.go @@ -11,7 +11,6 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/metric" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -84,7 +83,7 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) return } - wr.WriteJson(getAsyncSearchesListResponseFromProto(resp, a.config.AsyncSearchListQueryLimit)) + wr.WriteJson(getAsyncSearchesListResponseFromProto(resp, a.config.AsyncSearch.QueryLengthLimit)) } type getAsyncSearchesListRequest struct { @@ -163,7 +162,6 @@ func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResp func startAsyncSearchListRequestFromProto(r *seqapi.StartAsyncSearchRequest, queryLimit int) startAsyncSearchRequest { req := startAsyncSearchRequestFromProto(r) if trimmedQuery, ok := trimQueryToLimit(req.Query, queryLimit); ok { - metric.AsyncSearchQueryTooLong.Inc() req.Query = trimmedQuery + "..." } return req diff --git a/internal/api/seqapi/v1/http/get_async_searches_list_test.go b/internal/api/seqapi/v1/http/get_async_searches_list_test.go index b04572c..cf5c2c5 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list_test.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list_test.go @@ -49,7 +49,7 @@ func TestServeGetAsyncSearchesList(t *testing.T) { name string reqBody string - cfg config.SeqAPI + cfg config.Handlers wantRespBody string wantStatus int @@ -243,11 +243,6 @@ func TestServeGetAsyncSearchesList(t *testing.T) { { name: "query_too_long", reqBody: "{}", - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - AsyncSearchListQueryLimit: 1000, - }, - }, mockArgs: &mockArgs{ proxyReq: &seqapi.GetAsyncSearchesListRequest{}, proxyResp: &seqapi.GetAsyncSearchesListResponse{ @@ -294,9 +289,9 @@ func TestServeGetAsyncSearchesList(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - AsyncSearchListQueryLimit: 1000, + Cfg: config.Handlers{ + AsyncSearch: config.AsyncSearch{ + QueryLengthLimit: 1000, }, }, } diff --git a/internal/api/seqapi/v1/http/get_envs_test.go b/internal/api/seqapi/v1/http/get_envs_test.go index 6916208..d649e1c 100644 --- a/internal/api/seqapi/v1/http/get_envs_test.go +++ b/internal/api/seqapi/v1/http/get_envs_test.go @@ -14,18 +14,20 @@ import ( func TestServeGetEnvs(t *testing.T) { tests := []struct { name string - cfg config.SeqAPI + cfg config.Handlers wantEnvs []envInfo }{ { name: "single_env", - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, wantEnvs: []envInfo{ @@ -41,61 +43,63 @@ func TestServeGetEnvs(t *testing.T) { }, { name: "ok_multiple_envs", - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{}, - Envs: map[string]config.SeqAPIEnv{ - "cluster-220": { - SeqDB: "pro-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 1000, - MaxExportLimit: 500, - MaxParallelExportRequests: 10, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 2000, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{}, + Envs: map[string]config.SeqAPIEnv{ + "cluster-220": { + SeqDB: "pro-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 1000, + MaxExportLimit: 500, + MaxParallelExportRequests: 10, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 2000, + }, }, - }, - "cluster-10": { - SeqDB: "prod-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 1000, - MaxExportLimit: 500, - MaxParallelExportRequests: 10, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 2000, + "cluster-10": { + SeqDB: "prod-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 1000, + MaxExportLimit: 500, + MaxParallelExportRequests: 10, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 2000, + }, }, - }, - "cluster-102": { - SeqDB: "staging-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + "cluster-102": { + SeqDB: "staging-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, - }, - "prod": { - SeqDB: "stag-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + "prod": { + SeqDB: "stag-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, - }, - "wyanki": { - SeqDB: "sta-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, + "wyanki": { + SeqDB: "sta-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, + }, }, }, + DefaultEnv: "cluster-10", }, - DefaultEnv: "cluster-10", }, wantEnvs: []envInfo{ { diff --git a/internal/api/seqapi/v1/http/limits_test.go b/internal/api/seqapi/v1/http/limits_test.go index 28d3748..026dadb 100644 --- a/internal/api/seqapi/v1/http/limits_test.go +++ b/internal/api/seqapi/v1/http/limits_test.go @@ -14,19 +14,21 @@ func TestServeGetLimits(t *testing.T) { tests := []struct { name string env string - cfg config.SeqAPI + cfg config.Handlers wantRespBody string }{ { name: "ok", env: "default", - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, + cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, + }, }, }, wantRespBody: `{"maxSearchLimit":100,"maxExportLimit":200,"maxParallelExportRequests":2,"maxAggregationsPerRequest":5,"seqCliMaxSearchLimit":10000}`, diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index a272aac..883d48e 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -110,10 +110,12 @@ func TestServeGetLogsLifespan(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - LogsLifespanCacheKey: cacheKey, - LogsLifespanCacheTTL: cacheTTL, + Cfg: config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + LogsLifespanCacheKey: cacheKey, + LogsLifespanCacheTTL: cacheTTL, + }, }, }, } diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index 19a6a8a..f13099a 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -65,7 +65,7 @@ func TestServeSearch(t *testing.T) { wantStatus int mockArgs *mockArgs - cfg config.SeqAPI + cfg config.Handlers }{ { name: "ok_simple", @@ -87,9 +87,11 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), }, @@ -115,9 +117,11 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"total":"10","error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), }, @@ -142,9 +146,11 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), }, @@ -172,9 +178,11 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"histogram":{"buckets":[{"key":"0","docCount":"1"},{"key":"100","docCount":"2"}]},"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), }, @@ -214,9 +222,11 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"aggregations":[{"buckets":[{"key":"test1","value":1,"not_exists":5},{"key":"test2","value":2,"not_exists":5},{"key":"test3","value":3,"not_exists":5}]},{"buckets":[{"key":"test1","value":1,"not_exists":5},{"key":"test2","value":2,"not_exists":5},{"key":"test3","value":3,"not_exists":5}]}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), }, @@ -240,9 +250,11 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), }, @@ -268,9 +280,11 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"}],"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"},"partialResponse":true}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), }, @@ -288,30 +302,36 @@ func TestServeSearch(t *testing.T) { name: "err_search_limit_max", reqBody: formatReqBody(10, 0, false, "", nil, ""), wantStatus: http.StatusBadRequest, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, - }, + }), }, { name: "err_aggs_limit_max", reqBody: formatReqBody(3, 0, false, "", aggregationQueries{{}, {}, {}}, ""), wantStatus: http.StatusBadRequest, - cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 2, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 2, + }, }, - }, + }), }, { name: "err_offset_too_high", reqBody: formatReqBody(3, 11, false, "", nil, ""), wantStatus: http.StatusBadRequest, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxSearchOffsetLimit: 10, + }, }, }), }, @@ -338,10 +358,12 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: fmt.Sprintf(`{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"}],"total":"11","error":{"code":"ERROR_CODE_QUERY_TOO_HEAVY","message":%q},"partialResponse":false}`, api_error.ErrQueryTooHeavy.Error()), wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxSearchOffsetLimit: 10, + }, }, }), }, @@ -359,9 +381,11 @@ func TestServeSearch(t *testing.T) { err: errors.New("client error"), }, wantStatus: http.StatusInternalServerError, - cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, + cfg: test.SetCfgDefaults(config.Handlers{ + SeqAPI: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + }, }, }), }, diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index 0904cc1..ca7aabe 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "time" + "unicode/utf8" "go.opentelemetry.io/otel/attribute" "google.golang.org/protobuf/types/known/durationpb" @@ -13,6 +14,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/app/types" + "github.com/ozontech/seq-ui/metric" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -42,7 +44,9 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) return } - + if utf8.RuneCountInString(httpReq.Query) > a.config.AsyncSearch.QueryLengthLimit { + metric.AsyncSearchQueryTooLong.Inc() + } parsedRetention, err := time.ParseDuration(httpReq.Retention) if err != nil { wr.Error(fmt.Errorf("failed to parse retention: %w", err), http.StatusBadRequest) @@ -54,7 +58,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { continue } if err := api_error.CheckAggregationTsInterval(agg.Interval, httpReq.From, httpReq.To, - a.config.MaxBucketsPerAggregationTs, + a.config.SeqAPI.MaxBucketsPerAggregationTs, ); err != nil { wr.Error(err, http.StatusBadRequest) return diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 06a7a97..bf1b236 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -14,13 +14,13 @@ import ( func initTestAPI(data test.APITestData) *API { // when test cases don't explicitly provide configuration. - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} + if data.Cfg.SeqAPI.SeqAPIOptions == nil { + data.Cfg.SeqAPI.SeqAPIOptions = &config.SeqAPIOptions{} } seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB - for _, envConfig := range data.Cfg.Envs { + for _, envConfig := range data.Cfg.SeqAPI.Envs { seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB } @@ -28,8 +28,8 @@ func initTestAPI(data test.APITestData) *API { } func initTestAPIWithAsyncSearches(data test.APITestData) *API { - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} + if data.Cfg.SeqAPI.SeqAPIOptions == nil { + data.Cfg.SeqAPI.SeqAPIOptions = &config.SeqAPIOptions{} } seqDBClients := map[string]seqdb.Client{ config.DefaultSeqDBClientID: data.Mocks.SeqDB, diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index 9cdae47..fbc91df 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -18,7 +18,7 @@ type SeqAPI struct { } func New( - cfg config.SeqAPI, + cfg config.Handlers, seqDB map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, diff --git a/internal/api/seqapi/v1/test/data.go b/internal/api/seqapi/v1/test/data.go index 9b18e2a..6a923cb 100644 --- a/internal/api/seqapi/v1/test/data.go +++ b/internal/api/seqapi/v1/test/data.go @@ -27,7 +27,7 @@ type Mocks struct { } type APITestData struct { - Cfg config.SeqAPI + Cfg config.Handlers Mocks Mocks } @@ -103,24 +103,24 @@ func MakeAggregations(aggCount, bucketCount int, opts *MakeAggOpts) []*seqapi.Ag return aggs } -func SetCfgDefaults(cfg config.SeqAPI) config.SeqAPI { - if cfg.MaxAggregationsPerRequest <= 0 { - cfg.MaxAggregationsPerRequest = 1 +func SetCfgDefaults(cfg config.Handlers) config.Handlers { + if cfg.SeqAPI.MaxAggregationsPerRequest <= 0 { + cfg.SeqAPI.MaxAggregationsPerRequest = 1 } - if cfg.MaxParallelExportRequests <= 0 { - cfg.MaxParallelExportRequests = 1 + if cfg.SeqAPI.MaxParallelExportRequests <= 0 { + cfg.SeqAPI.MaxParallelExportRequests = 1 } - if cfg.MaxSearchTotalLimit <= 0 { - cfg.MaxSearchTotalLimit = 1000000 + if cfg.SeqAPI.MaxSearchTotalLimit <= 0 { + cfg.SeqAPI.MaxSearchTotalLimit = 1000000 } - if cfg.MaxSearchOffsetLimit <= 0 { - cfg.MaxSearchOffsetLimit = 1000000 + if cfg.SeqAPI.MaxSearchOffsetLimit <= 0 { + cfg.SeqAPI.MaxSearchOffsetLimit = 1000000 } - for envName, envConfig := range cfg.Envs { + for envName, envConfig := range cfg.SeqAPI.Envs { if envConfig.Options == nil { - envConfig.Options = cfg.SeqAPIOptions - cfg.Envs[envName] = envConfig + envConfig.Options = cfg.SeqAPI.SeqAPIOptions + cfg.SeqAPI.Envs[envName] = envConfig } } diff --git a/internal/app/config/config.go b/internal/app/config/config.go index 62b1d36..d296e6d 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -30,9 +30,10 @@ const ( minGRPCKeepaliveTime = 10 * time.Second minGRPCKeepaliveTimeout = 1 * time.Second + defaultAsyncSearchQueryLengthLimit = 1000 + defaultMaxSearchTotalLimit = 1000000 defaultMaxSearchOffsetLimit = 1000000 - defaultAsyncSearchListQueryLimit = 1000 defaultMaxAggregationsPerRequest = 1 defaultMaxBucketsPerAggregationTs = 200 defaultMaxParallelExportRequests = 1 @@ -260,7 +261,6 @@ type SeqAPIOptions struct { MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` MaxExportLimit int32 `yaml:"max_export_limit"` - AsyncSearchListQueryLimit int `yaml:"async_search_list_query_limit"` SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` @@ -313,7 +313,8 @@ type ErrorGroups struct { } type AsyncSearch struct { - AdminUsers []string `yaml:"admin_users"` + AdminUsers []string `yaml:"admin_users"` + QueryLengthLimit int `yaml:"query_length_limit"` } // FromFile parse config from config path. @@ -338,6 +339,9 @@ func FromFile(cfgPath string) (Config, error) { ) } + if cfg.Handlers == nil { + return Config{}, fmt.Errorf("invalid config: handlers must be not nil") + } if cfg.Handlers.SeqAPI.MaxAggregationsPerRequest <= 0 { cfg.Handlers.SeqAPI.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest } @@ -353,8 +357,8 @@ func FromFile(cfgPath string) (Config, error) { if cfg.Handlers.SeqAPI.MaxSearchOffsetLimit <= 0 { cfg.Handlers.SeqAPI.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit } - if cfg.Handlers.SeqAPI.AsyncSearchListQueryLimit <= 0 { - cfg.Handlers.SeqAPI.AsyncSearchListQueryLimit = defaultAsyncSearchListQueryLimit + if cfg.Handlers.AsyncSearch.QueryLengthLimit <= 0 { + cfg.Handlers.AsyncSearch.QueryLengthLimit = defaultAsyncSearchQueryLengthLimit } if cfg.Handlers.SeqAPI.EventsCacheTTL <= 0 { cfg.Handlers.SeqAPI.EventsCacheTTL = defaultEventsCacheTTL From f1337a40acf5333ccafb6bd5712c65a1b1864658 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 7 Apr 2026 19:31:23 +0300 Subject: [PATCH 4/8] fix --- docs/en/02-configuration.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/02-configuration.md b/docs/en/02-configuration.md index b9990ab..3bede01 100644 --- a/docs/en/02-configuration.md +++ b/docs/en/02-configuration.md @@ -822,7 +822,6 @@ Configuration for async request. Maximum length of `request.query` in async searches list responses. - ## Tracing The tracing configuration is set through environment variables. From d86903253bc83171880b9e6601930ed2816e8567 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 7 Apr 2026 19:33:49 +0300 Subject: [PATCH 5/8] fix --- docs/en/02-configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/02-configuration.md b/docs/en/02-configuration.md index 3bede01..e86e27f 100644 --- a/docs/en/02-configuration.md +++ b/docs/en/02-configuration.md @@ -506,6 +506,7 @@ handlers: seq_api: error_groups: mass_export: + async_search: ``` ### SeqAPI From 8015da1adf422c88551c987d2a13b881e40000f5 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 8 Apr 2026 14:10:46 +0300 Subject: [PATCH 6/8] fix --- cmd/seq-ui/main.go | 4 +- docs/en/02-configuration.md | 6 +- docs/ru/02-configuration.md | 6 +- .../api/seqapi/v1/grpc/aggregation_test.go | 34 ++--- internal/api/seqapi/v1/grpc/api.go | 24 ++-- .../api/seqapi/v1/grpc/cluster_status_test.go | 2 +- internal/api/seqapi/v1/grpc/events_test.go | 18 ++- internal/api/seqapi/v1/grpc/fields_test.go | 16 +-- internal/api/seqapi/v1/grpc/get_envs_test.go | 106 ++++++++-------- internal/api/seqapi/v1/grpc/limits_test.go | 20 ++- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 10 +- internal/api/seqapi/v1/grpc/search_test.go | 58 ++++----- internal/api/seqapi/v1/grpc/test_data.go | 12 +- .../api/seqapi/v1/http/aggregation_test.go | 50 +++----- .../api/seqapi/v1/http/aggregation_ts_test.go | 60 ++++----- internal/api/seqapi/v1/http/api.go | 28 ++--- internal/api/seqapi/v1/http/events_test.go | 18 ++- internal/api/seqapi/v1/http/export_test.go | 60 ++++----- internal/api/seqapi/v1/http/fields_test.go | 16 +-- .../seqapi/v1/http/get_async_searches_list.go | 25 +--- .../v1/http/get_async_searches_list_test.go | 6 +- internal/api/seqapi/v1/http/get_envs_test.go | 116 +++++++++--------- internal/api/seqapi/v1/http/limits_test.go | 18 ++- .../api/seqapi/v1/http/logs_lifespan_test.go | 10 +- internal/api/seqapi/v1/http/search_test.go | 104 ++++++---------- .../api/seqapi/v1/http/start_async_search.go | 7 +- internal/api/seqapi/v1/http/test_data.go | 12 +- internal/api/seqapi/v1/seqapi.go | 2 +- internal/api/seqapi/v1/test/data.go | 29 ++--- internal/app/config/config.go | 13 +- .../pkg/service/async_searches/service.go | 32 ++++- 31 files changed, 401 insertions(+), 521 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 37c3097..d661fcd 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -163,10 +163,10 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { userProfileV1 = userprofile_v1.New(svc, p) dashboardsV1 = dashboards_v1.New(svc, p) - asyncSearchesService = asyncsearches.New(ctx, repo, defaultClient, cfg.Handlers.AsyncSearch.AdminUsers) + asyncSearchesService = asyncsearches.New(ctx, repo, defaultClient, cfg.Handlers.AsyncSearch) } - seqApiV1 := seqapi_v1.New(*cfg.Handlers, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService, p) + seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService, p) logger.Info("initializing clickhouse") ch, err := initClickHouse(ctx, cfg.Server.CH) diff --git a/docs/en/02-configuration.md b/docs/en/02-configuration.md index e86e27f..5167467 100644 --- a/docs/en/02-configuration.md +++ b/docs/en/02-configuration.md @@ -809,11 +809,11 @@ Config for `/massexport` API handlers. ### Async Search -Configuration for async request. +Configuration for async search request. **`async_search`** *`AsyncSearch`* *`optional`* -Поля `AsyncSearch`: +Fields `AsyncSearch`: + **`admin_users`** *`[]string`* *`optional`* @@ -821,7 +821,7 @@ Configuration for async request. + **`query_length_limit`** *`int`* *`default=1000`* - Maximum length of `request.query` in async searches list responses. + Maximum length of `request.query` in async searches list responses. Requests exceeding the limit will be truncated to it ## Tracing diff --git a/docs/ru/02-configuration.md b/docs/ru/02-configuration.md index 42faa8e..8598f2a 100644 --- a/docs/ru/02-configuration.md +++ b/docs/ru/02-configuration.md @@ -811,7 +811,7 @@ handlers: **`async_search`** *`AsyncSearch`* *`optional`* -Настройка функциональности отложенных запросов. +Настройка функциональности отложенных поисковых запросов. Поля `AsyncSearch`: @@ -819,9 +819,9 @@ handlers: Список пользователей, которые могут отменять или удалять отложенные поиски других пользователей. -+ **`query_length_limit`** *`int`* *`default=1000`* ++ **`list_query_length_limit`** *`int`* *`default=1000`* - Максимальная длина `request.query` в ответе списка отложенных запросов. + Максимальная длина `request.query` в ответе списка отложенных запросов. Запросы, превышающие лимит, будут обрезаны до этого значения. ## Tracing diff --git a/internal/api/seqapi/v1/grpc/aggregation_test.go b/internal/api/seqapi/v1/grpc/aggregation_test.go index 61e017c..b7a44c1 100644 --- a/internal/api/seqapi/v1/grpc/aggregation_test.go +++ b/internal/api/seqapi/v1/grpc/aggregation_test.go @@ -30,7 +30,7 @@ func TestGetAggregation(t *testing.T) { apiErr bool clientErr error - cfg config.Handlers + cfg config.SeqAPI }{ { name: "ok_single_agg", @@ -47,11 +47,9 @@ func TestGetAggregation(t *testing.T) { Code: seqapi.ErrorCode_ERROR_CODE_NO, }, }, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 4, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 4, }, }, }, @@ -73,11 +71,9 @@ func TestGetAggregation(t *testing.T) { Code: seqapi.ErrorCode_ERROR_CODE_NO, }, }, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, }, }, }, @@ -90,11 +86,9 @@ func TestGetAggregation(t *testing.T) { {Field: "test3"}, }, }, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 2, }, }, apiErr: true, @@ -107,11 +101,9 @@ func TestGetAggregation(t *testing.T) { To: timestamppb.New(to), AggField: "test2", }, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 1, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 1, }, }, clientErr: errors.New("client error"), diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index 49b2513..9760bb4 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -32,7 +32,7 @@ type apiParams struct { type API struct { seqapi.UnimplementedSeqAPIServiceServer - config config.Handlers + config config.SeqAPI params apiParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache @@ -44,7 +44,7 @@ type API struct { } func New( - cfg config.Handlers, + cfg config.SeqAPI, seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, @@ -52,23 +52,23 @@ func New( p *profiles.Profiles, ) *API { var globalfCache *fieldsCache - if cfg.SeqAPI.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.SeqAPI.FieldsCacheTTL) + if cfg.FieldsCacheTTL > 0 { + globalfCache = newFieldsCache(cfg.FieldsCacheTTL) } - globalMasker, err := mask.New(cfg.SeqAPI.Masking) + globalMasker, err := mask.New(cfg.Masking) if err != nil { logger.Fatal("failed to init masking", zap.Error(err)) } - globalPinnedFields := parsePinnedFields(cfg.SeqAPI.PinnedFields) + globalPinnedFields := parsePinnedFields(cfg.PinnedFields) var params apiParams var paramsByEnv map[string]apiParams - if len(cfg.SeqAPI.Envs) > 0 { + if len(cfg.Envs) > 0 { paramsByEnv = make(map[string]apiParams) - for envName, envConfig := range cfg.SeqAPI.Envs { + for envName, envConfig := range cfg.Envs { client := seqDBСlients[envConfig.SeqDB] options := envConfig.Options @@ -101,7 +101,7 @@ func New( params = apiParams{ client: client, - options: cfg.SeqAPI.SeqAPIOptions, + options: cfg.SeqAPIOptions, fieldsCache: globalfCache, masker: globalMasker, pinnedFields: globalPinnedFields, @@ -117,7 +117,7 @@ func New( nowFn: time.Now, asyncSearches: asyncSearches, profiles: p, - envsResponse: parseEnvs(cfg.SeqAPI), + envsResponse: parseEnvs(cfg), } } @@ -191,12 +191,12 @@ func (a *API) GetEnvFromContext(ctx context.Context) string { } func (a *API) GetParams(env string) (apiParams, error) { - if len(a.config.SeqAPI.Envs) == 0 { + if len(a.config.Envs) == 0 { return a.params, nil } if env == "" { - env = a.config.SeqAPI.DefaultEnv + env = a.config.DefaultEnv } params, exists := a.paramsByEnv[env] diff --git a/internal/api/seqapi/v1/grpc/cluster_status_test.go b/internal/api/seqapi/v1/grpc/cluster_status_test.go index 7093032..5a0d73e 100644 --- a/internal/api/seqapi/v1/grpc/cluster_status_test.go +++ b/internal/api/seqapi/v1/grpc/cluster_status_test.go @@ -64,7 +64,7 @@ func TestStatus(t *testing.T) { seqDbMock.EXPECT().Status(gomock.Any(), nil). Return(proto.Clone(tt.resp), tt.clientErr).Times(1) - cfg := config.Handlers{} + cfg := config.SeqAPI{} seqData := test.APITestData{ Cfg: cfg, diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index 82195ea..d5d9cab 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -102,11 +102,9 @@ func TestGetEvent(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + EventsCacheTTL: cacheTTL, }, }, } @@ -366,12 +364,10 @@ func TestGetEventWithMasking(t *testing.T) { curEID := curEData.id seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, - Masking: tt.maskingCfg, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + EventsCacheTTL: cacheTTL, + Masking: tt.maskingCfg, }, }, } diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index acf47f9..bfa23b7 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -106,11 +106,9 @@ func TestGetFieldsCached(t *testing.T) { const ttl = 20 * time.Millisecond seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - FieldsCacheTTL: ttl, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + FieldsCacheTTL: ttl, }, }, Mocks: test.Mocks{ @@ -156,11 +154,9 @@ func TestGetPinnedFields(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - PinnedFields: tt.fields, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + PinnedFields: tt.fields, }, }, } diff --git a/internal/api/seqapi/v1/grpc/get_envs_test.go b/internal/api/seqapi/v1/grpc/get_envs_test.go index 5a42a25..5dbbd8e 100644 --- a/internal/api/seqapi/v1/grpc/get_envs_test.go +++ b/internal/api/seqapi/v1/grpc/get_envs_test.go @@ -13,20 +13,18 @@ import ( func TestGetEnvs(t *testing.T) { tests := []struct { name string - cfg config.Handlers + cfg config.SeqAPI want *seqapi.GetEnvsResponse }{ { name: "single_env", - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, }, }, want: &seqapi.GetEnvsResponse{ @@ -44,57 +42,55 @@ func TestGetEnvs(t *testing.T) { }, { name: "multiple_envs", - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - Envs: map[string]config.SeqAPIEnv{ - "test": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, - }, + cfg: config.SeqAPI{ + Envs: map[string]config.SeqAPIEnv{ + "test": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, }, - "prod": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 150, - MaxExportLimit: 250, - MaxParallelExportRequests: 3, - MaxAggregationsPerRequest: 6, - SeqCLIMaxSearchLimit: 15000, - }, + }, + "prod": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 150, + MaxExportLimit: 250, + MaxParallelExportRequests: 3, + MaxAggregationsPerRequest: 6, + SeqCLIMaxSearchLimit: 15000, }, - "cluster-10": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 150, - MaxExportLimit: 250, - MaxParallelExportRequests: 3, - MaxAggregationsPerRequest: 6, - SeqCLIMaxSearchLimit: 15000, - }, + }, + "cluster-10": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 150, + MaxExportLimit: 250, + MaxParallelExportRequests: 3, + MaxAggregationsPerRequest: 6, + SeqCLIMaxSearchLimit: 15000, }, - "cluster-102": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, - }, + }, + "cluster-102": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, }, - "cluster-220": { - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, - }, + }, + "cluster-220": { + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, }, }, - DefaultEnv: "test", }, + DefaultEnv: "test", }, want: &seqapi.GetEnvsResponse{ Envs: []*seqapi.GetEnvsResponse_Env{ @@ -149,7 +145,7 @@ func TestGetEnvs(t *testing.T) { t.Parallel() api := API{ config: tt.cfg, - envsResponse: parseEnvs(tt.cfg.SeqAPI), + envsResponse: parseEnvs(tt.cfg), } resp, err := api.GetEnvs(context.TODO(), &seqapi.GetEnvsRequest{}) require.NoError(t, err) diff --git a/internal/api/seqapi/v1/grpc/limits_test.go b/internal/api/seqapi/v1/grpc/limits_test.go index 5b30453..971198c 100644 --- a/internal/api/seqapi/v1/grpc/limits_test.go +++ b/internal/api/seqapi/v1/grpc/limits_test.go @@ -14,20 +14,18 @@ import ( func TestGetLimits(t *testing.T) { tests := []struct { name string - cfg config.Handlers + cfg config.SeqAPI want *seqapi.GetLimitsResponse }{ { name: "ok", - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, }, }, want: &seqapi.GetLimitsResponse{ @@ -40,7 +38,7 @@ func TestGetLimits(t *testing.T) { }, { name: "empty", - cfg: config.Handlers{}, + cfg: config.SeqAPI{}, want: &seqapi.GetLimitsResponse{}, }, } diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index 1525e28..3b884ff 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -109,12 +109,10 @@ func TestGetLogsLifespan(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - LogsLifespanCacheKey: cacheKey, - LogsLifespanCacheTTL: cacheTTL, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + LogsLifespanCacheKey: cacheKey, + LogsLifespanCacheTTL: cacheTTL, }, }, } diff --git a/internal/api/seqapi/v1/grpc/search_test.go b/internal/api/seqapi/v1/grpc/search_test.go index 3d59c7b..8d49b7f 100644 --- a/internal/api/seqapi/v1/grpc/search_test.go +++ b/internal/api/seqapi/v1/grpc/search_test.go @@ -35,7 +35,7 @@ func TestSearch(t *testing.T) { apiErr bool clientErr error - cfg config.Handlers + cfg config.SeqAPI }{ { name: "ok", @@ -64,12 +64,10 @@ func TestSearch(t *testing.T) { Code: seqapi.ErrorCode_ERROR_CODE_NO, }, }, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 5, }, }), }, @@ -85,11 +83,9 @@ func TestSearch(t *testing.T) { req: &seqapi.SearchRequest{ Limit: 10, }, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), apiErr: true, @@ -104,12 +100,10 @@ func TestSearch(t *testing.T) { {Field: "test3"}, }, }, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 2, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 2, }, }), apiErr: true, @@ -123,12 +117,10 @@ func TestSearch(t *testing.T) { Limit: limit, Offset: 11, }, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxSearchOffsetLimit: 10, }, }), apiErr: true, @@ -155,12 +147,10 @@ func TestSearch(t *testing.T) { Message: api_error.ErrQueryTooHeavy.Error(), }, }, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchTotalLimit: int64(limit), - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxSearchTotalLimit: int64(limit), }, }), }, @@ -173,11 +163,9 @@ func TestSearch(t *testing.T) { Limit: limit, Offset: 0, }, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), clientErr: errors.New("client error"), diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index 3d76905..4b605ea 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -14,13 +14,13 @@ import ( func initTestAPI(data test.APITestData) *API { // when test cases don't explicitly provide configuration - if data.Cfg.SeqAPI.SeqAPIOptions == nil { - data.Cfg.SeqAPI.SeqAPIOptions = &config.SeqAPIOptions{} + if data.Cfg.SeqAPIOptions == nil { + data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} } seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB - for _, envConfig := range data.Cfg.SeqAPI.Envs { + for _, envConfig := range data.Cfg.Envs { seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB } @@ -28,13 +28,13 @@ func initTestAPI(data test.APITestData) *API { } func initTestAPIWithAsyncSearches(data test.APITestData) *API { - if data.Cfg.SeqAPI.SeqAPIOptions == nil { - data.Cfg.SeqAPI.SeqAPIOptions = &config.SeqAPIOptions{} + if data.Cfg.SeqAPIOptions == nil { + data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} } seqDBClients := map[string]seqdb.Client{ config.DefaultSeqDBClientID: data.Mocks.SeqDB, } - as := asyncsearches.New(context.Background(), data.Mocks.AsyncSearchesRepo, data.Mocks.SeqDB, []string{}) + as := asyncsearches.New(context.Background(), data.Mocks.AsyncSearchesRepo, data.Mocks.SeqDB, data.AsyncCfg) s := service.New(&repository.Repository{ UserProfiles: data.Mocks.ProfilesRepo, }) diff --git a/internal/api/seqapi/v1/http/aggregation_test.go b/internal/api/seqapi/v1/http/aggregation_test.go index 1b650c3..f3fcdf5 100644 --- a/internal/api/seqapi/v1/http/aggregation_test.go +++ b/internal/api/seqapi/v1/http/aggregation_test.go @@ -50,7 +50,7 @@ func TestServeGetAggregation(t *testing.T) { wantStatus int mockArgs *mockArgs - cfg config.Handlers + cfg config.SeqAPI }{ { name: "ok_single_agg", @@ -72,11 +72,9 @@ func TestServeGetAggregation(t *testing.T) { }, wantRespBody: `{"aggregation":{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2}]},"aggregations":[{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2}]}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, }, }, }, @@ -108,11 +106,9 @@ func TestServeGetAggregation(t *testing.T) { }, wantRespBody: `{"aggregation":{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2},{"key":"test3","value":3}]},"aggregations":[{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2},{"key":"test3","value":3}]},{"buckets":[{"key":"test1","value":1},{"key":"test2","value":2},{"key":"test3","value":3}]}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, }, }, }, @@ -157,11 +153,9 @@ func TestServeGetAggregation(t *testing.T) { }, wantRespBody: `{"aggregation":{"buckets":[{"key":"test1","value":1,"not_exists":10,"quantiles":[100,150]},{"key":"test2","value":2,"not_exists":10,"quantiles":[100,150]},{"key":"test3","value":3,"not_exists":10,"quantiles":[100,150]}]},"aggregations":[{"buckets":[{"key":"test1","value":1,"not_exists":10,"quantiles":[100,150]},{"key":"test2","value":2,"not_exists":10,"quantiles":[100,150]},{"key":"test3","value":3,"not_exists":10,"quantiles":[100,150]}]},{"buckets":[{"key":"test1","value":1,"not_exists":10,"quantiles":[100,150]},{"key":"test2","value":2,"not_exists":10,"quantiles":[100,150]},{"key":"test3","value":3,"not_exists":10,"quantiles":[100,150]}]}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, }, }, }, @@ -185,11 +179,9 @@ func TestServeGetAggregation(t *testing.T) { }, wantRespBody: `{"aggregation":{"buckets":[]},"aggregations":[],"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"},"partialResponse":true}`, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, }, }, }, @@ -202,11 +194,9 @@ func TestServeGetAggregation(t *testing.T) { name: "err_aggs_limit_max", reqBody: formatReqBody("", aggregationQueries{{}, {}, {}}), wantStatus: http.StatusBadRequest, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 2, }, }, }, @@ -223,11 +213,9 @@ func TestServeGetAggregation(t *testing.T) { err: errors.New("client error"), }, wantStatus: http.StatusInternalServerError, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, }, }, }, diff --git a/internal/api/seqapi/v1/http/aggregation_ts_test.go b/internal/api/seqapi/v1/http/aggregation_ts_test.go index 601e59f..3df29bf 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts_test.go +++ b/internal/api/seqapi/v1/http/aggregation_ts_test.go @@ -48,7 +48,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantStatus int mockArgs *mockArgs - cfg config.Handlers + cfg config.SeqAPI }{ { name: "ok_count", @@ -100,12 +100,10 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"test_count1":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count1":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count1":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}},{"data":{"result":[{"metric":{"test_count2":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count2":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count2":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, }, }, }, @@ -161,12 +159,10 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"quantile":"p95","service":"test1"},"values":[{"timestamp":1695637231,"value":100}]},{"metric":{"quantile":"p99","service":"test1"},"values":[{"timestamp":1695637231,"value":150}]},{"metric":{"quantile":"p95","service":"test2"},"values":[{"timestamp":1695637232,"value":100}]},{"metric":{"quantile":"p99","service":"test2"},"values":[{"timestamp":1695637232,"value":150}]},{"metric":{"quantile":"p95","service":"test3"},"values":[{"timestamp":1695637233,"value":100}]},{"metric":{"quantile":"p99","service":"test3"},"values":[{"timestamp":1695637233,"value":150}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, }, }, }, @@ -189,12 +185,10 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantRespBody: `{"aggregations":null,"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 100, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 100, }, }, }, @@ -207,12 +201,10 @@ func TestServeGetAggregationTs(t *testing.T) { name: "err_aggs_limit_max", reqBody: formatReqBody(aggregationTsQueries{{}, {}, {}}), wantStatus: http.StatusBadRequest, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 2, - MaxBucketsPerAggregationTs: 100, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 2, + MaxBucketsPerAggregationTs: 100, }, }, }, @@ -224,12 +216,10 @@ func TestServeGetAggregationTs(t *testing.T) { }, }), wantStatus: http.StatusBadRequest, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - MaxBucketsPerAggregationTs: 8, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, + MaxBucketsPerAggregationTs: 8, }, }, }, @@ -245,11 +235,9 @@ func TestServeGetAggregationTs(t *testing.T) { err: errors.New("client error"), }, wantStatus: http.StatusInternalServerError, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxAggregationsPerRequest: 3, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxAggregationsPerRequest: 3, }, }, }, diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index 34206ac..16c603e 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -32,7 +32,7 @@ type apiParams struct { } type API struct { - config config.Handlers + config config.SeqAPI params apiParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache @@ -44,7 +44,7 @@ type API struct { } func New( - cfg config.Handlers, + cfg config.SeqAPI, seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, @@ -52,24 +52,24 @@ func New( p *profiles.Profiles, ) *API { var globalfCache *fieldsCache - if cfg.SeqAPI.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.SeqAPI.FieldsCacheTTL) + if cfg.FieldsCacheTTL > 0 { + globalfCache = newFieldsCache(cfg.FieldsCacheTTL) } - globalMasker, err := mask.New(cfg.SeqAPI.Masking) + globalMasker, err := mask.New(cfg.Masking) if err != nil { logger.Fatal("failed to init masking", zap.Error(err)) } - globalPinnedFields := parsePinnedFields(cfg.SeqAPI.PinnedFields) - globalExportLimiter := tokenlimiter.New(cfg.SeqAPI.MaxParallelExportRequests) + globalPinnedFields := parsePinnedFields(cfg.PinnedFields) + globalExportLimiter := tokenlimiter.New(cfg.MaxParallelExportRequests) var params apiParams var paramsByEnv map[string]apiParams - if len(cfg.SeqAPI.Envs) > 0 { + if len(cfg.Envs) > 0 { paramsByEnv = make(map[string]apiParams) - for envName, envConfig := range cfg.SeqAPI.Envs { + for envName, envConfig := range cfg.Envs { client := seqDBСlients[envConfig.SeqDB] options := envConfig.Options @@ -104,7 +104,7 @@ func New( params = apiParams{ client: client, - options: cfg.SeqAPI.SeqAPIOptions, + options: cfg.SeqAPIOptions, fieldsCache: globalfCache, masker: globalMasker, pinnedFields: globalPinnedFields, @@ -112,7 +112,7 @@ func New( } } // for export - if len(cfg.SeqAPI.Envs) > 0 { + if len(cfg.Envs) > 0 { for _, param := range paramsByEnv { if param.masker != nil { param.client.WithMasking(param.masker) @@ -131,7 +131,7 @@ func New( nowFn: time.Now, asyncSearches: asyncSearches, profiles: p, - envsResponse: parseEnvs(cfg.SeqAPI), + envsResponse: parseEnvs(cfg), } } @@ -234,12 +234,12 @@ const ( ) func (a *API) GetEnvParams(env string) (apiParams, error) { - if len(a.config.SeqAPI.Envs) == 0 { + if len(a.config.Envs) == 0 { return a.params, nil } if env == "" { - env = a.config.SeqAPI.DefaultEnv + env = a.config.DefaultEnv } params, exists := a.paramsByEnv[env] diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index cae25a4..f4dd625 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -125,11 +125,9 @@ func TestServeGetEvent(t *testing.T) { ctrl := gomock.NewController(t) seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + EventsCacheTTL: cacheTTL, }, }, } @@ -395,12 +393,10 @@ func TestGetEventWithMasking(t *testing.T) { curEID := curEData.id seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - EventsCacheTTL: cacheTTL, - Masking: tt.maskingCfg, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + EventsCacheTTL: cacheTTL, + Masking: tt.maskingCfg, }, }, } diff --git a/internal/api/seqapi/v1/http/export_test.go b/internal/api/seqapi/v1/http/export_test.go index 46fb88a..ad69b9e 100644 --- a/internal/api/seqapi/v1/http/export_test.go +++ b/internal/api/seqapi/v1/http/export_test.go @@ -56,7 +56,7 @@ func TestServeExport(t *testing.T) { wantStatus int mockArgs *mockArgs - cfg config.Handlers + cfg config.SeqAPI }{ { name: "ok_jsonl", @@ -71,12 +71,10 @@ func TestServeExport(t *testing.T) { }, }, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, }, }, }, @@ -95,12 +93,10 @@ func TestServeExport(t *testing.T) { }, }, wantStatus: http.StatusOK, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, }, }, }, @@ -113,11 +109,9 @@ func TestServeExport(t *testing.T) { name: "err_parallel_limited", reqBody: formatReqBody(0, "", nil), wantStatus: http.StatusTooManyRequests, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxParallelExportRequests: 0, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxParallelExportRequests: 0, }, }, }, @@ -125,12 +119,10 @@ func TestServeExport(t *testing.T) { name: "err_export_limit_max", reqBody: formatReqBody(10, "", nil), wantStatus: http.StatusBadRequest, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 5, - MaxParallelExportRequests: 1, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 5, + MaxParallelExportRequests: 1, }, }, }, @@ -138,12 +130,10 @@ func TestServeExport(t *testing.T) { name: "err_csv_empty_fields", reqBody: formatReqBody(10, efCSV, nil), wantStatus: http.StatusBadRequest, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, }, }, }, @@ -161,12 +151,10 @@ func TestServeExport(t *testing.T) { err: errors.New("client error"), }, wantStatus: http.StatusInternalServerError, - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxExportLimit: 100, - MaxParallelExportRequests: 1, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxExportLimit: 100, + MaxParallelExportRequests: 1, }, }, }, diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 86cf2da..eda72cb 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -132,11 +132,9 @@ func TestServeGetFieldsCached(t *testing.T) { const ttl = 20 * time.Millisecond seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - FieldsCacheTTL: ttl, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + FieldsCacheTTL: ttl, }, }, Mocks: test.Mocks{ @@ -193,11 +191,9 @@ func TestServeGetPinnedFields(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - PinnedFields: tt.fields, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + PinnedFields: tt.fields, }, }, } diff --git a/internal/api/seqapi/v1/http/get_async_searches_list.go b/internal/api/seqapi/v1/http/get_async_searches_list.go index 6e3fe8d..50e6161 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list.go @@ -83,7 +83,7 @@ func (a *API) serveGetAsyncSearchesList(w http.ResponseWriter, r *http.Request) return } - wr.WriteJson(getAsyncSearchesListResponseFromProto(resp, a.config.AsyncSearch.QueryLengthLimit)) + wr.WriteJson(getAsyncSearchesListResponseFromProto(resp)) } type getAsyncSearchesListRequest struct { @@ -129,7 +129,7 @@ type asyncSearchesListItem struct { Error *string `json:"error,omitempty"` } // @name seqapi.v1.AsyncSearchesListItem -func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResponse, asyncSearchListQueryLimit int) getAsyncSearchesListResponse { +func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResponse) getAsyncSearchesListResponse { searches := make([]asyncSearchesListItem, 0, len(resp.Searches)) for _, s := range resp.Searches { @@ -142,7 +142,7 @@ func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResp searches = append(searches, asyncSearchesListItem{ SearchID: s.SearchId, Status: asyncSearchStatusFromProto(s.Status), - Request: startAsyncSearchListRequestFromProto(s.Request, asyncSearchListQueryLimit), + Request: startAsyncSearchRequestFromProto(s.Request), StartedAt: s.StartedAt.AsTime(), ExpiresAt: s.ExpiresAt.AsTime(), CanceledAt: canceledAt, @@ -159,14 +159,6 @@ func getAsyncSearchesListResponseFromProto(resp *seqapi.GetAsyncSearchesListResp } } -func startAsyncSearchListRequestFromProto(r *seqapi.StartAsyncSearchRequest, queryLimit int) startAsyncSearchRequest { - req := startAsyncSearchRequestFromProto(r) - if trimmedQuery, ok := trimQueryToLimit(req.Query, queryLimit); ok { - req.Query = trimmedQuery + "..." - } - return req -} - func startAsyncSearchRequestFromProto(r *seqapi.StartAsyncSearchRequest) startAsyncSearchRequest { var hist *AsyncSearchRequestHistogram if r.Hist != nil { @@ -207,14 +199,3 @@ func aggregationTsQueriesFromProto(aggs []*seqapi.AggregationQuery) aggregationT return result } - -func trimQueryToLimit(query string, limit int) (string, bool) { - count := 0 - for i := range query { - if count == limit { - return query[:i], true - } - count++ - } - return query, false -} diff --git a/internal/api/seqapi/v1/http/get_async_searches_list_test.go b/internal/api/seqapi/v1/http/get_async_searches_list_test.go index cf5c2c5..1811ae6 100644 --- a/internal/api/seqapi/v1/http/get_async_searches_list_test.go +++ b/internal/api/seqapi/v1/http/get_async_searches_list_test.go @@ -289,10 +289,8 @@ func TestServeGetAsyncSearchesList(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.Handlers{ - AsyncSearch: config.AsyncSearch{ - QueryLengthLimit: 1000, - }, + AsyncCfg: config.AsyncSearch{ + ListQueryLengthLimit: 1000, }, } diff --git a/internal/api/seqapi/v1/http/get_envs_test.go b/internal/api/seqapi/v1/http/get_envs_test.go index d649e1c..6916208 100644 --- a/internal/api/seqapi/v1/http/get_envs_test.go +++ b/internal/api/seqapi/v1/http/get_envs_test.go @@ -14,20 +14,18 @@ import ( func TestServeGetEnvs(t *testing.T) { tests := []struct { name string - cfg config.Handlers + cfg config.SeqAPI wantEnvs []envInfo }{ { name: "single_env", - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, }, }, wantEnvs: []envInfo{ @@ -43,63 +41,61 @@ func TestServeGetEnvs(t *testing.T) { }, { name: "ok_multiple_envs", - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{}, - Envs: map[string]config.SeqAPIEnv{ - "cluster-220": { - SeqDB: "pro-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 1000, - MaxExportLimit: 500, - MaxParallelExportRequests: 10, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 2000, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{}, + Envs: map[string]config.SeqAPIEnv{ + "cluster-220": { + SeqDB: "pro-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 1000, + MaxExportLimit: 500, + MaxParallelExportRequests: 10, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 2000, }, - "cluster-10": { - SeqDB: "prod-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 1000, - MaxExportLimit: 500, - MaxParallelExportRequests: 10, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 2000, - }, + }, + "cluster-10": { + SeqDB: "prod-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 1000, + MaxExportLimit: 500, + MaxParallelExportRequests: 10, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 2000, }, - "cluster-102": { - SeqDB: "staging-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, - }, + }, + "cluster-102": { + SeqDB: "staging-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, }, - "prod": { - SeqDB: "stag-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, - }, + }, + "prod": { + SeqDB: "stag-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, }, - "wyanki": { - SeqDB: "sta-seqdb", - Options: &config.SeqAPIOptions{ - MaxSearchLimit: 500, - MaxExportLimit: 250, - MaxParallelExportRequests: 5, - MaxAggregationsPerRequest: 3, - SeqCLIMaxSearchLimit: 1000, - }, + }, + "wyanki": { + SeqDB: "sta-seqdb", + Options: &config.SeqAPIOptions{ + MaxSearchLimit: 500, + MaxExportLimit: 250, + MaxParallelExportRequests: 5, + MaxAggregationsPerRequest: 3, + SeqCLIMaxSearchLimit: 1000, }, }, - DefaultEnv: "cluster-10", }, + DefaultEnv: "cluster-10", }, wantEnvs: []envInfo{ { diff --git a/internal/api/seqapi/v1/http/limits_test.go b/internal/api/seqapi/v1/http/limits_test.go index 026dadb..28d3748 100644 --- a/internal/api/seqapi/v1/http/limits_test.go +++ b/internal/api/seqapi/v1/http/limits_test.go @@ -14,21 +14,19 @@ func TestServeGetLimits(t *testing.T) { tests := []struct { name string env string - cfg config.Handlers + cfg config.SeqAPI wantRespBody string }{ { name: "ok", env: "default", - cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 100, - MaxExportLimit: 200, - MaxParallelExportRequests: 2, - MaxAggregationsPerRequest: 5, - SeqCLIMaxSearchLimit: 10000, - }, + cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 100, + MaxExportLimit: 200, + MaxParallelExportRequests: 2, + MaxAggregationsPerRequest: 5, + SeqCLIMaxSearchLimit: 10000, }, }, wantRespBody: `{"maxSearchLimit":100,"maxExportLimit":200,"maxParallelExportRequests":2,"maxAggregationsPerRequest":5,"seqCliMaxSearchLimit":10000}`, diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index 883d48e..a272aac 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -110,12 +110,10 @@ func TestServeGetLogsLifespan(t *testing.T) { t.Parallel() seqData := test.APITestData{ - Cfg: config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - LogsLifespanCacheKey: cacheKey, - LogsLifespanCacheTTL: cacheTTL, - }, + Cfg: config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + LogsLifespanCacheKey: cacheKey, + LogsLifespanCacheTTL: cacheTTL, }, }, } diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index f13099a..b2d0952 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -65,7 +65,7 @@ func TestServeSearch(t *testing.T) { wantStatus int mockArgs *mockArgs - cfg config.Handlers + cfg config.SeqAPI }{ { name: "ok_simple", @@ -87,11 +87,9 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, @@ -117,11 +115,9 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"total":"10","error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, @@ -146,11 +142,9 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, @@ -178,11 +172,9 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"histogram":{"buckets":[{"key":"0","docCount":"1"},{"key":"100","docCount":"2"}]},"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, @@ -222,11 +214,9 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test2","data":{"field1":"val1","field2":"val2"},"time":"2023-09-25T10:20:30.001Z"},{"id":"test3","data":{"field1":"val1","field2":"val2","field3":"val3"},"time":"2023-09-25T10:20:30.001Z"}],"aggregations":[{"buckets":[{"key":"test1","value":1,"not_exists":5},{"key":"test2","value":2,"not_exists":5},{"key":"test3","value":3,"not_exists":5}]},{"buckets":[{"key":"test1","value":1,"not_exists":5},{"key":"test2","value":2,"not_exists":5},{"key":"test3","value":3,"not_exists":5}]}],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, @@ -250,11 +240,9 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[],"error":{"code":"ERROR_CODE_NO"},"partialResponse":false}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, @@ -280,11 +268,9 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: `{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"}],"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"},"partialResponse":true}`, wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, @@ -302,11 +288,9 @@ func TestServeSearch(t *testing.T) { name: "err_search_limit_max", reqBody: formatReqBody(10, 0, false, "", nil, ""), wantStatus: http.StatusBadRequest, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, @@ -314,24 +298,20 @@ func TestServeSearch(t *testing.T) { name: "err_aggs_limit_max", reqBody: formatReqBody(3, 0, false, "", aggregationQueries{{}, {}, {}}, ""), wantStatus: http.StatusBadRequest, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxAggregationsPerRequest: 2, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxAggregationsPerRequest: 2, }, }), }, { name: "err_offset_too_high", reqBody: formatReqBody(3, 11, false, "", nil, ""), wantStatus: http.StatusBadRequest, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxSearchOffsetLimit: 10, }, }), }, @@ -358,12 +338,10 @@ func TestServeSearch(t *testing.T) { }, wantRespBody: fmt.Sprintf(`{"events":[{"id":"test1","data":{"field1":"val1"},"time":"2023-09-25T10:20:30.001Z"}],"total":"11","error":{"code":"ERROR_CODE_QUERY_TOO_HEAVY","message":%q},"partialResponse":false}`, api_error.ErrQueryTooHeavy.Error()), wantStatus: http.StatusOK, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - MaxSearchOffsetLimit: 10, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, + MaxSearchOffsetLimit: 10, }, }), }, @@ -381,11 +359,9 @@ func TestServeSearch(t *testing.T) { err: errors.New("client error"), }, wantStatus: http.StatusInternalServerError, - cfg: test.SetCfgDefaults(config.Handlers{ - SeqAPI: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ - MaxSearchLimit: 5, - }, + cfg: test.SetCfgDefaults(config.SeqAPI{ + SeqAPIOptions: &config.SeqAPIOptions{ + MaxSearchLimit: 5, }, }), }, diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index ca7aabe..b0384e1 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "time" - "unicode/utf8" "go.opentelemetry.io/otel/attribute" "google.golang.org/protobuf/types/known/durationpb" @@ -14,7 +13,6 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/app/types" - "github.com/ozontech/seq-ui/metric" "github.com/ozontech/seq-ui/pkg/seqapi/v1" "github.com/ozontech/seq-ui/tracing" ) @@ -44,9 +42,6 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) return } - if utf8.RuneCountInString(httpReq.Query) > a.config.AsyncSearch.QueryLengthLimit { - metric.AsyncSearchQueryTooLong.Inc() - } parsedRetention, err := time.ParseDuration(httpReq.Retention) if err != nil { wr.Error(fmt.Errorf("failed to parse retention: %w", err), http.StatusBadRequest) @@ -58,7 +53,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { continue } if err := api_error.CheckAggregationTsInterval(agg.Interval, httpReq.From, httpReq.To, - a.config.SeqAPI.MaxBucketsPerAggregationTs, + a.config.MaxBucketsPerAggregationTs, ); err != nil { wr.Error(err, http.StatusBadRequest) return diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index bf1b236..455b472 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -14,13 +14,13 @@ import ( func initTestAPI(data test.APITestData) *API { // when test cases don't explicitly provide configuration. - if data.Cfg.SeqAPI.SeqAPIOptions == nil { - data.Cfg.SeqAPI.SeqAPIOptions = &config.SeqAPIOptions{} + if data.Cfg.SeqAPIOptions == nil { + data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} } seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB - for _, envConfig := range data.Cfg.SeqAPI.Envs { + for _, envConfig := range data.Cfg.Envs { seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB } @@ -28,13 +28,13 @@ func initTestAPI(data test.APITestData) *API { } func initTestAPIWithAsyncSearches(data test.APITestData) *API { - if data.Cfg.SeqAPI.SeqAPIOptions == nil { - data.Cfg.SeqAPI.SeqAPIOptions = &config.SeqAPIOptions{} + if data.Cfg.SeqAPIOptions == nil { + data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} } seqDBClients := map[string]seqdb.Client{ config.DefaultSeqDBClientID: data.Mocks.SeqDB, } - as := asyncsearches.New(context.Background(), data.Mocks.AsyncSearchesRepo, data.Mocks.SeqDB, []string{}) + as := asyncsearches.New(context.Background(), data.Mocks.AsyncSearchesRepo, data.Mocks.SeqDB, data.AsyncCfg) s := service.New(&repository.Repository{ UserProfiles: data.Mocks.ProfilesRepo, }) diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index fbc91df..9cdae47 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -18,7 +18,7 @@ type SeqAPI struct { } func New( - cfg config.Handlers, + cfg config.SeqAPI, seqDB map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, diff --git a/internal/api/seqapi/v1/test/data.go b/internal/api/seqapi/v1/test/data.go index 6a923cb..1bf1a24 100644 --- a/internal/api/seqapi/v1/test/data.go +++ b/internal/api/seqapi/v1/test/data.go @@ -27,8 +27,9 @@ type Mocks struct { } type APITestData struct { - Cfg config.Handlers - Mocks Mocks + Cfg config.SeqAPI + AsyncCfg config.AsyncSearch + Mocks Mocks } func MakeEvent(id string, countData int, t time.Time) *seqapi.Event { @@ -103,24 +104,24 @@ func MakeAggregations(aggCount, bucketCount int, opts *MakeAggOpts) []*seqapi.Ag return aggs } -func SetCfgDefaults(cfg config.Handlers) config.Handlers { - if cfg.SeqAPI.MaxAggregationsPerRequest <= 0 { - cfg.SeqAPI.MaxAggregationsPerRequest = 1 +func SetCfgDefaults(cfg config.SeqAPI) config.SeqAPI { + if cfg.MaxAggregationsPerRequest <= 0 { + cfg.MaxAggregationsPerRequest = 1 } - if cfg.SeqAPI.MaxParallelExportRequests <= 0 { - cfg.SeqAPI.MaxParallelExportRequests = 1 + if cfg.MaxParallelExportRequests <= 0 { + cfg.MaxParallelExportRequests = 1 } - if cfg.SeqAPI.MaxSearchTotalLimit <= 0 { - cfg.SeqAPI.MaxSearchTotalLimit = 1000000 + if cfg.MaxSearchTotalLimit <= 0 { + cfg.MaxSearchTotalLimit = 1000000 } - if cfg.SeqAPI.MaxSearchOffsetLimit <= 0 { - cfg.SeqAPI.MaxSearchOffsetLimit = 1000000 + if cfg.MaxSearchOffsetLimit <= 0 { + cfg.MaxSearchOffsetLimit = 1000000 } - for envName, envConfig := range cfg.SeqAPI.Envs { + for envName, envConfig := range cfg.Envs { if envConfig.Options == nil { - envConfig.Options = cfg.SeqAPI.SeqAPIOptions - cfg.SeqAPI.Envs[envName] = envConfig + envConfig.Options = cfg.SeqAPIOptions + cfg.Envs[envName] = envConfig } } diff --git a/internal/app/config/config.go b/internal/app/config/config.go index d296e6d..e751e32 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -30,7 +30,7 @@ const ( minGRPCKeepaliveTime = 10 * time.Second minGRPCKeepaliveTimeout = 1 * time.Second - defaultAsyncSearchQueryLengthLimit = 1000 + defaultAsyncSearchListQueryLengthLimit = 1000 defaultMaxSearchTotalLimit = 1000000 defaultMaxSearchOffsetLimit = 1000000 @@ -313,8 +313,8 @@ type ErrorGroups struct { } type AsyncSearch struct { - AdminUsers []string `yaml:"admin_users"` - QueryLengthLimit int `yaml:"query_length_limit"` + AdminUsers []string `yaml:"admin_users"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` } // FromFile parse config from config path. @@ -339,9 +339,6 @@ func FromFile(cfgPath string) (Config, error) { ) } - if cfg.Handlers == nil { - return Config{}, fmt.Errorf("invalid config: handlers must be not nil") - } if cfg.Handlers.SeqAPI.MaxAggregationsPerRequest <= 0 { cfg.Handlers.SeqAPI.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest } @@ -357,8 +354,8 @@ func FromFile(cfgPath string) (Config, error) { if cfg.Handlers.SeqAPI.MaxSearchOffsetLimit <= 0 { cfg.Handlers.SeqAPI.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit } - if cfg.Handlers.AsyncSearch.QueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.QueryLengthLimit = defaultAsyncSearchQueryLengthLimit + if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { + cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } if cfg.Handlers.SeqAPI.EventsCacheTTL <= 0 { cfg.Handlers.SeqAPI.EventsCacheTTL = defaultEventsCacheTTL diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index a91102b..017adf9 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -5,14 +5,17 @@ import ( "fmt" "slices" "time" + "unicode/utf8" "github.com/cenkalti/backoff/v4" "go.uber.org/zap" + "github.com/ozontech/seq-ui/internal/app/config" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" "github.com/ozontech/seq-ui/logger" + "github.com/ozontech/seq-ui/metric" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -27,19 +30,19 @@ type Service struct { repo repository.AsyncSearches seqDB seqdb.Client - admin_users []string + cfg config.AsyncSearch } func New( ctx context.Context, repo repository.AsyncSearches, seqDB seqdb.Client, - admins []string, + cfg config.AsyncSearch, ) *Service { s := &Service{ - repo: repo, - seqDB: seqDB, - admin_users: admins, + repo: repo, + seqDB: seqDB, + cfg: cfg, } go s.deleteExpiredAsyncSearches(ctx) @@ -52,6 +55,9 @@ func (s *Service) StartAsyncSearch( ownerID int64, req *seqapi.StartAsyncSearchRequest, ) (*seqapi.StartAsyncSearchResponse, error) { + if utf8.RuneCountInString(req.Query) > s.cfg.ListQueryLengthLimit { + metric.AsyncSearchQueryTooLong.Inc() + } resp, err := s.seqDB.StartAsyncSearch(ctx, req) if err != nil { return nil, fmt.Errorf("failed to start async search: %w", err) @@ -179,6 +185,9 @@ func (s *Service) GetAsyncSearchesList( for _, as := range resp.Searches { as.OwnerName = ownerNameByID[as.SearchId] + if trimmedQuery, ok := s.trimQueryToLimit(as.Request.Query, s.cfg.ListQueryLengthLimit); ok { + as.Request.Query = trimmedQuery + "..." + } } return resp, nil @@ -190,7 +199,7 @@ func (s *Service) isAdmin(ctx context.Context) bool { return false } - return slices.Index(s.admin_users, userName) >= 0 + return slices.Index(s.cfg.AdminUsers, userName) >= 0 } func (s *Service) deleteExpiredAsyncSearches(ctx context.Context) { @@ -209,3 +218,14 @@ func (s *Service) deleteExpiredAsyncSearches(ctx context.Context) { } } } + +func (s *Service) trimQueryToLimit(query string, limit int) (string, bool) { + count := 0 + for i := range query { + if count == limit { + return query[:i], true + } + count++ + } + return query, false +} From 5a3832c5abe1293488ac376e51daa8fe3879d235 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 8 Apr 2026 14:20:35 +0300 Subject: [PATCH 7/8] fix --- internal/api/seqapi/v1/grpc/search_test.go | 4 ++-- internal/api/seqapi/v1/http/search_test.go | 8 ++++---- internal/api/seqapi/v1/http/start_async_search.go | 1 + internal/pkg/service/async_searches/service.go | 10 ++++------ 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/internal/api/seqapi/v1/grpc/search_test.go b/internal/api/seqapi/v1/grpc/search_test.go index 8d49b7f..6145aa3 100644 --- a/internal/api/seqapi/v1/grpc/search_test.go +++ b/internal/api/seqapi/v1/grpc/search_test.go @@ -83,11 +83,11 @@ func TestSearch(t *testing.T) { req: &seqapi.SearchRequest{ Limit: 10, }, - cfg: test.SetCfgDefaults(config.SeqAPI{ + cfg: config.SeqAPI{ SeqAPIOptions: &config.SeqAPIOptions{ MaxSearchLimit: 5, }, - }), + }, apiErr: true, }, { diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index b2d0952..19a6a8a 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -288,22 +288,22 @@ func TestServeSearch(t *testing.T) { name: "err_search_limit_max", reqBody: formatReqBody(10, 0, false, "", nil, ""), wantStatus: http.StatusBadRequest, - cfg: test.SetCfgDefaults(config.SeqAPI{ + cfg: config.SeqAPI{ SeqAPIOptions: &config.SeqAPIOptions{ MaxSearchLimit: 5, }, - }), + }, }, { name: "err_aggs_limit_max", reqBody: formatReqBody(3, 0, false, "", aggregationQueries{{}, {}, {}}, ""), wantStatus: http.StatusBadRequest, - cfg: test.SetCfgDefaults(config.SeqAPI{ + cfg: config.SeqAPI{ SeqAPIOptions: &config.SeqAPIOptions{ MaxSearchLimit: 5, MaxAggregationsPerRequest: 2, }, - }), + }, }, { name: "err_offset_too_high", reqBody: formatReqBody(3, 11, false, "", nil, ""), diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index b0384e1..0904cc1 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -42,6 +42,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) return } + parsedRetention, err := time.ParseDuration(httpReq.Retention) if err != nil { wr.Error(fmt.Errorf("failed to parse retention: %w", err), http.StatusBadRequest) diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index 017adf9..0960785 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -185,9 +185,7 @@ func (s *Service) GetAsyncSearchesList( for _, as := range resp.Searches { as.OwnerName = ownerNameByID[as.SearchId] - if trimmedQuery, ok := s.trimQueryToLimit(as.Request.Query, s.cfg.ListQueryLengthLimit); ok { - as.Request.Query = trimmedQuery + "..." - } + as.Request.Query = s.trimQueryToLimit(as.Request.Query, s.cfg.ListQueryLengthLimit) } return resp, nil @@ -219,13 +217,13 @@ func (s *Service) deleteExpiredAsyncSearches(ctx context.Context) { } } -func (s *Service) trimQueryToLimit(query string, limit int) (string, bool) { +func (s *Service) trimQueryToLimit(query string, limit int) string { count := 0 for i := range query { if count == limit { - return query[:i], true + return query[:i] + "..." } count++ } - return query, false + return query } From 92f6a628d42582dfbc8b610d7da4adf9df1735db Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 8 Apr 2026 15:53:06 +0300 Subject: [PATCH 8/8] fix --- docs/en/02-configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/02-configuration.md b/docs/en/02-configuration.md index 5167467..c713932 100644 --- a/docs/en/02-configuration.md +++ b/docs/en/02-configuration.md @@ -813,13 +813,13 @@ Configuration for async search request. **`async_search`** *`AsyncSearch`* *`optional`* -Fields `AsyncSearch`: +`AsyncSearch` fields: + **`admin_users`** *`[]string`* *`optional`* List of users allowed to cancel or delete async requests created by other users. -+ **`query_length_limit`** *`int`* *`default=1000`* ++ **`list_query_length_limit`** *`int`* *`default=1000`* Maximum length of `request.query` in async searches list responses. Requests exceeding the limit will be truncated to it