diff --git a/asyncsearcher/async_searcher.go b/asyncsearcher/async_searcher.go index 05a70437..f1f0967a 100644 --- a/asyncsearcher/async_searcher.go +++ b/asyncsearcher/async_searcher.go @@ -18,6 +18,7 @@ import ( "github.com/google/uuid" "go.uber.org/zap" + "golang.org/x/sync/errgroup" "github.com/ozontech/seq-db/bytespool" "github.com/ozontech/seq-db/frac" @@ -38,6 +39,8 @@ const ( minRetention = 5 * time.Minute maxRetention = 30 * 24 * time.Hour // 30 days + + defaultSearchInterval = 5 * time.Minute ) type infoVersion uint8 @@ -45,8 +48,11 @@ type infoVersion uint8 const ( infoVersion1 infoVersion = iota + 1 // MIDs stored in milliseconds infoVersion2 // MIDs stored in nanoseconds + infoVersion3 // one file per N-minute interval instead of per fraction ) +type searchInterval [2]seq.MID + type MappingProvider interface { GetMapping() seq.Mapping } @@ -74,8 +80,7 @@ type AsyncSearcherConfig struct { } type fractionAcquirer interface { - AcquireFractions() (_ fracmanager.List, release func()) - AcquireFraction(name string) (_ frac.Fraction, release func(), ok bool) + AcquireFractionsInRange(from, to seq.MID) (_ fracmanager.List, release func()) } func MustStartAsync(config AsyncSearcherConfig, mp MappingProvider, fracProvider fractionAcquirer) *AsyncSearcher { @@ -159,11 +164,7 @@ type asyncSearchInfo struct { infoSize *atomic.Int64 } -func newAsyncSearchInfo(r AsyncSearchRequest, fracNames []string) asyncSearchInfo { - fracsToSearch := make([]fracSearchState, 0, len(fracNames)) - for _, name := range fracNames { - fracsToSearch = append(fracsToSearch, fracSearchState{Name: name}) - } +func newAsyncSearchInfo(r AsyncSearchRequest) asyncSearchInfo { ctx, cancel := context.WithCancel(context.Background()) return asyncSearchInfo{ Version: infoVersion2, @@ -173,7 +174,6 @@ func newAsyncSearchInfo(r AsyncSearchRequest, fracNames []string) asyncSearchInf ctx: ctx, cancel: cancel, Request: r, - Fractions: fracsToSearch, StartedAt: time.Now(), merged: &atomic.Bool{}, qprsSize: &atomic.Int64{}, @@ -213,6 +213,7 @@ func (as *AsyncSearcher) StartSearch(r AsyncSearchRequest, fracProvider fraction if as.readOnly.Load() { return fmt.Errorf("cannot start search on read-only mode") } + as.requestsMu.RLock() _, ok := as.requests[r.ID] as.requestsMu.RUnlock() @@ -220,6 +221,7 @@ func (as *AsyncSearcher) StartSearch(r AsyncSearchRequest, fracProvider fraction logger.Warn("async search already started", zap.String("id", r.ID)) return nil } + if err := uuid.Validate(r.ID); err != nil { return fmt.Errorf("invalid id %q: %s", r.ID, err) } @@ -240,27 +242,25 @@ func (as *AsyncSearcher) StartSearch(r AsyncSearchRequest, fracProvider fraction return fmt.Errorf("retention time should be less than %s, got %s", maxRetention, r.Retention) } - fracs, release := fracProvider.AcquireFractions() - defer release() - - fracNames := fracs.FilterInRange(r.Params.From, r.Params.To).Names() - if ok := as.saveSearchInfo(r, fracNames); !ok { + if ok := as.saveSearchInfo(r); !ok { // Request was saved previously, skip it return nil } + asyncSearchActiveSearches.Add(1) as.processWg.Add(1) go as.processRequest(r.ID, fracProvider) + return nil } -func (as *AsyncSearcher) saveSearchInfo(r AsyncSearchRequest, fracNames []string) bool { +func (as *AsyncSearcher) saveSearchInfo(r AsyncSearchRequest) bool { as.requestsMu.Lock() defer as.requestsMu.Unlock() if _, ok := as.requests[r.ID]; ok { return false } - info := newAsyncSearchInfo(r, fracNames) + info := newAsyncSearchInfo(r) as.storeSearchInfoLocked(r.ID, info) return true } @@ -307,26 +307,53 @@ func (as *AsyncSearcher) createDataDir() { func (as *AsyncSearcher) processRequest(asyncSearchID string, fracProvider fractionAcquirer) { defer as.processWg.Done() - as.rateLimit <- struct{}{} - defer func() { <-as.rateLimit }() - as.doSearch(asyncSearchID, fracProvider) asyncSearchActiveSearches.Add(-1) } +func buildIntervals(from, to seq.MID) []searchInterval { + splitInterval := seq.DurationToMID(defaultSearchInterval) + + var intervals []searchInterval + current := from + + for current < to { + end := min(current+splitInterval, to) + intervals = append(intervals, searchInterval{current, end - seq.MID(1)}) + current = end + } + + if len(intervals) > 0 { + // close last interval + intervals[len(intervals)-1][1] = intervals[len(intervals)-1][1] + seq.MID(1) + } + + return intervals +} + +func countIntervals(from, to seq.MID) int { + splitInterval := seq.DurationToMID(defaultSearchInterval) + diff := to - from + return int((diff + splitInterval - 1) / splitInterval) +} + +func intervalName(interval searchInterval) string { + return fmt.Sprintf("%d-%d", interval[0], interval[1]) +} + func (as *AsyncSearcher) doSearch(id string, fracProvider fractionAcquirer) { qprPaths, err := as.findQPRs(id) if err != nil { panic(fmt.Errorf("can't find QPRs for id %q: %s", id, err)) } - processedFracs := make(map[string]struct{}) + processedIntervals := make(map[string]struct{}) for _, qprPath := range qprPaths { - fracName, err := fracNameFromQPRPath(qprPath) + intervalName, err := intervalNameFromQPRPath(qprPath) if err != nil { logger.Fatal("cannot find previous QPRs", zap.Error(err)) } - processedFracs[fracName] = struct{}{} + processedIntervals[intervalName] = struct{}{} } info, ok := as.getSearchInfo(id) @@ -349,20 +376,43 @@ func (as *AsyncSearcher) doSearch(id string, fracProvider fractionAcquirer) { info.Request.Params.AST = ast.Root } - for _, fracInfo := range info.Fractions { - if _, ok := processedFracs[fracInfo.Name]; ok { + intervals := buildIntervals(info.Request.Params.From, info.Request.Params.To) + + eg, ctx := errgroup.WithContext(context.Background()) + +intervalsLoop: + for _, interval := range intervals { + + if _, ok := processedIntervals[intervalName(interval)]; ok { continue } - if as.shouldStopSearch(id) { - break - } - if err := as.acquireAndProcessFrac(fracInfo, info, fracProvider); err != nil { - as.updateSearchInfo(id, func(info *asyncSearchInfo) { - info.Error = err.Error() + + select { + case <-ctx.Done(): + break intervalsLoop + case as.rateLimit <- struct{}{}: + eg.Go(func() error { + defer func() { <-as.rateLimit }() + + if as.shouldStopSearch(id) { + return nil + } + if err := as.acquireAndProcessFracsInInterval(interval, info, fracProvider); err != nil { + return err + } + return nil }) - break } } + + if err := eg.Wait(); err != nil { + as.updateSearchInfo(id, func(info *asyncSearchInfo) { + info.Error = err.Error() + info.Finished = true + }) + return + } + as.updateSearchInfo(id, func(info *asyncSearchInfo) { info.Finished = true }) @@ -398,69 +448,102 @@ func compressQPR(qpr *seq.QPR, cb func(compressed []byte) error) error { return nil } -func (as *AsyncSearcher) acquireAndProcessFrac(fracInfo fracSearchState, searchInfo asyncSearchInfo, fracProvider fractionAcquirer) (err error) { - f, release, ok := fracProvider.AcquireFraction(fracInfo.Name) - if !ok { // oldest fracs may already be removed - logger.Info( - "async search: skip missing fraction", - zap.String("id", searchInfo.Request.ID), - zap.String("frac", fracInfo.Name), - ) - return - } +func (as *AsyncSearcher) acquireAndProcessFracsInInterval( + interval searchInterval, + searchInfo asyncSearchInfo, + fracProvider fractionAcquirer, +) (err error) { + params := searchInfo.Request.Params + intervalName := intervalName(interval) + + fracs, release := fracProvider.AcquireFractionsInRange(interval[0], interval[1]) defer release() - return as.processFrac(f, searchInfo) -} -func (as *AsyncSearcher) processFrac(f frac.Fraction, info asyncSearchInfo) (err error) { - defer func() { - if panicData := util.RecoverToError(recover(), asyncSearchPanics); panicData != nil { - err = fmt.Errorf("async search panic during processFrac: %w", panicData) + qpr := &seq.QPR{} + for _, f := range fracs { + fracQPR, err := as.processFrac(f, searchInfo, interval) + if err != nil { + return err } - }() + if fracQPR.Empty() { + continue + } + seq.MergeQPRs(qpr, []*seq.QPR{fracQPR}, params.Limit, seq.MillisToMID(params.HistInterval), params.Order) + } - qpr, err := f.Search(info.ctx, info.Request.Params) - if err != nil { - return err + if qpr.Empty() { + return nil } storeQPR := func(rawQPR []byte) error { - du := int(info.qprsSize.Load() + info.infoSize.Load()) + du := int(searchInfo.qprsSize.Load() + searchInfo.infoSize.Load()) if as.config.MaxSizePerRequest != 0 && du+len(rawQPR) > as.config.MaxSizePerRequest { return fmt.Errorf("cannot complete async search request since it requires more than %dMiB of memory", as.config.MaxSizePerRequest) } - name := getQPRFilename(info.Request.ID, f.Info().Name()) + name := getQPRFilename(searchInfo.Request.ID, intervalName) fpath := path.Join(as.config.DataDir, name) util.MustWriteFileAtomic(fpath, rawQPR, 0o666, asyncSearchTmpFile) - info.qprsSize.Add(int64(len(rawQPR))) + searchInfo.qprsSize.Add(int64(len(rawQPR))) return nil } if err := compressQPR(qpr, storeQPR); err != nil { return err } + return nil } -func getQPRFilename(id, fracName string) string { - // ..qpr - return id + "." + fracName + asyncSearchExtQPR +func (as *AsyncSearcher) processFrac( + f frac.Fraction, + info asyncSearchInfo, + interval searchInterval, +) (qpr *seq.QPR, err error) { + defer func() { + if panicData := util.RecoverToError(recover(), asyncSearchPanics); panicData != nil { + err = fmt.Errorf("async search panic during processFrac: %w", panicData) + } + }() + + // use interval's from and to + info.Request.Params.From = interval[0] + info.Request.Params.To = interval[1] + + qpr, err = f.Search(info.ctx, info.Request.Params) + if err != nil { + return nil, err + } + return qpr, nil +} + +func getQPRFilename(id, intervalName string) string { + // ..qpr + return id + "." + intervalName + asyncSearchExtQPR } -func fracNameFromQPRPath(qprPath string) (string, error) { +func intervalNameFromQPRPath(qprPath string) (string, error) { filename := path.Base(qprPath) parts := strings.Split(filename, ".") if len(parts) != 3 { - return "", fmt.Errorf("unknown qpr filename format") + return "", fmt.Errorf("unknown qpr filename format: %s", qprPath) } - // example path: a087f43f-40f6-49ae-8743-2fd7bef1dfd5.seq-db-01JQRFHKSBZTY5NSZ937WV987B.qpr + // example path: a087f43f-40f6-49ae-8743-2fd7bef1dfd5.1769594748228000000-1769595048228000000.qpr // parts[0] is request ID - // parts[1] is fraction name + // parts[1] is interval name // parts[2] is extension return parts[1], nil } +func searchIdFromQPRPath(qprPath string) (string, error) { + filename := path.Base(qprPath) + parts := strings.Split(filename, ".") + if len(parts) != 3 { + return "", fmt.Errorf("unknown qpr filename format: %s", qprPath) + } + return parts[0], nil +} + func (as *AsyncSearcher) findQPRs(id string) ([]string, error) { des, err := os.ReadDir(as.config.DataDir) if err != nil { @@ -583,7 +666,7 @@ func loadAsyncRequests(dataDir string) (map[string]asyncSearchInfo, error) { if err != nil { return err } - info := newAsyncSearchInfo(AsyncSearchRequest{}, nil) + info := newAsyncSearchInfo(AsyncSearchRequest{}) if err := json.Unmarshal(b, &info); err != nil { return fmt.Errorf("malformed async search info %q: %s", name, err) } @@ -607,6 +690,37 @@ func loadAsyncRequests(dataDir string) (map[string]asyncSearchInfo, error) { if err := util.VisitFilesWithExt(des, asyncSearchExtInfo, loadInfos); err != nil { return nil, err } + + // remove old not finished searches' qprs + oldSearchIDsToRemove := make(map[string]struct{}) + for id := range requests { + if !requests[id].Finished && len(requests[id].Fractions) > 0 { + oldSearchIDsToRemove[id] = struct{}{} + logger.Info("mark old per-frac async search to delete", zap.String("id", id)) + } + } + qprsToRemove := make([]string, 0) + findQPRsToRemove := func(name string) error { + searchID, err := searchIdFromQPRPath(name) + if err != nil { + return nil + } + if _, ok := oldSearchIDsToRemove[searchID]; !ok { + return nil + } + qprsToRemove = append(qprsToRemove, path.Join(dataDir, name)) + return nil + } + if err := util.VisitFilesWithExt(des, asyncSearchExtQPR, findQPRsToRemove); err != nil { + return nil, err + } + for _, qprPath := range qprsToRemove { + util.RemoveFile(qprPath) + } + if len(qprsToRemove) > 0 { + util.MustFsyncFile(dataDir) + } + return requests, nil } @@ -682,21 +796,21 @@ func (as *AsyncSearcher) FetchSearchResult(r FetchSearchResultRequest) (FetchSea } var qpr seq.QPR - var fracsDone, fracsInQueue int + var intervalsDone, intervalsInQueue int histInterval := seq.MillisToMID(info.Request.Params.HistInterval) if info.merged.Load() { p := path.Join(as.config.DataDir, r.ID+asyncSearchExtMergedQPR) qpr, _ = as.loadSearchResult([]string{p}, r.Limit, r.Order, histInterval) - fracsDone = len(info.Fractions) - fracsInQueue = 0 + intervalsDone = countIntervals(info.Request.Params.From, info.Request.Params.To) + intervalsInQueue = 0 } else { p, err := as.findQPRs(r.ID) if err != nil { logger.Fatal("can't load async search result", zap.String("id", r.ID), zap.Error(err)) } qpr, _ = as.loadSearchResult(p, r.Limit, r.Order, histInterval) - fracsDone = len(p) - fracsInQueue = len(info.Fractions) - fracsDone + intervalsDone = len(p) + intervalsInQueue = countIntervals(info.Request.Params.From, info.Request.Params.To) - intervalsDone } if info.Error != "" { @@ -711,8 +825,8 @@ func (as *AsyncSearcher) FetchSearchResult(r FetchSearchResultRequest) (FetchSea StartedAt: info.StartedAt, ExpiresAt: info.Expiration(), CanceledAt: info.CanceledAt, - FracsDone: fracsDone, - FracsInQueue: fracsInQueue, + FracsDone: intervalsDone, + FracsInQueue: intervalsInQueue, DiskUsage: int(info.infoSize.Load() + info.qprsSize.Load()), Error: info.Error, AggQueries: info.Request.Params.AggQ, @@ -782,7 +896,7 @@ func (as *AsyncSearcher) merge() { if !info.Finished || info.merged.Load() { continue } - if len(info.Fractions) < 2 { + if countIntervals(info.Request.Params.From, info.Request.Params.To) < 2 { // Nothing to merge continue } @@ -812,12 +926,15 @@ type mergeJob struct { func (as *AsyncSearcher) mergeQPRs(job mergeJob) { start := time.Now() var qprs []string - for _, f := range job.Info.Fractions { - qprFilename := getQPRFilename(job.ID, f.Name) + + params := job.Info.Request.Params + intervals := buildIntervals(params.From, params.To) + + for _, i := range intervals { + qprFilename := getQPRFilename(job.ID, intervalName(i)) qprPath := path.Join(as.config.DataDir, qprFilename) qprs = append(qprs, qprPath) } - params := job.Info.Request.Params qpr, sizeBefore := as.loadSearchResult(qprs, params.Limit, params.Order, seq.MillisToMID(params.HistInterval)) var sizeAfter int @@ -844,7 +961,7 @@ func (as *AsyncSearcher) mergeQPRs(job mergeJob) { logger.Info("QPRs have been merged", zap.String("id", job.ID), zap.Float64("ratio", float64(sizeBefore)/float64(sizeAfter)), - zap.Int("fracs", len(qprs)), + zap.Int("intervals", len(qprs)), zap.Duration("took", time.Since(start)), ) } @@ -1002,17 +1119,17 @@ func (as *AsyncSearcher) GetAsyncSearchesList(r GetAsyncSearchesListRequest) []* continue } - var fracsDone, fracsInQueue int + var intervalsDone, intervalsInQueue int if info.merged.Load() { - fracsDone = len(info.Fractions) - fracsInQueue = 0 + intervalsDone = countIntervals(info.Request.Params.From, info.Request.Params.To) + intervalsInQueue = 0 } else { p, err := as.findQPRs(id) if err != nil { logger.Fatal("can't load async search result", zap.String("id", id), zap.Error(err)) } - fracsDone = len(p) - fracsInQueue = len(info.Fractions) - fracsDone + intervalsDone = len(p) + intervalsInQueue = countIntervals(info.Request.Params.From, info.Request.Params.To) - intervalsDone } items = append(items, &AsyncSearchesListItem{ @@ -1021,8 +1138,8 @@ func (as *AsyncSearcher) GetAsyncSearchesList(r GetAsyncSearchesListRequest) []* StartedAt: info.StartedAt, ExpiresAt: info.Expiration(), CanceledAt: info.CanceledAt, - FracsDone: fracsDone, - FracsInQueue: fracsInQueue, + FracsDone: intervalsDone, + FracsInQueue: intervalsInQueue, DiskUsage: int(info.infoSize.Load() + info.qprsSize.Load()), AggQueries: info.Request.Params.AggQ, HistInterval: info.Request.Params.HistInterval, diff --git a/asyncsearcher/async_searcher_test.go b/asyncsearcher/async_searcher_test.go index df325ab8..838a2e49 100644 --- a/asyncsearcher/async_searcher_test.go +++ b/asyncsearcher/async_searcher_test.go @@ -40,16 +40,7 @@ type fakeDP struct { type fakeFractionProvider fracmanager.List -func (fp fakeFractionProvider) AcquireFraction(name string) (frac.Fraction, func(), bool) { - for _, f := range fp { - if f.Info().Name() == name { - return f, func() {}, true - } - } - return nil, func() {}, false -} - -func (fp fakeFractionProvider) AcquireFractions() (fracmanager.List, func()) { +func (fp fakeFractionProvider) AcquireFractionsInRange(from, to seq.MID) (fracmanager.List, func()) { return fracmanager.List(fp), func() {} } @@ -79,34 +70,9 @@ func TestAsyncSearcherMaintain(t *testing.T) { as.processWg.Wait() } -// partialProvider lists both fractions (so info.Fractions gets two entries), -// but only "present" is acquirable. "missing" simulates a fraction that was -// removed by the time the search reached it: it stays listed in info.Fractions -// yet never produces a .qpr file. -type partialProvider struct { - list fracmanager.List -} - -func (p *partialProvider) AcquireFractions() (fracmanager.List, func()) { - return p.list, func() {} -} - -func (p *partialProvider) AcquireFraction(name string) (frac.Fraction, func(), bool) { - for _, f := range p.list { - if f.Info().Name() == name && name == "present" { - return f, func() {}, true - } - } - return nil, func() {}, false -} - -// TestMergeSkipsMissingFrac is a regression test: when a fraction listed in -// info.Fractions was skipped (already removed) and produced no .qpr, merge used -// to build its path from info.Fractions, hit a missing file in loadSearchResult, -// discard the whole accumulated result, write an empty .mqpr and delete the -// real .qpr — losing the only matching document. -func TestMergeSkipsMissingFrac(t *testing.T) { +func TestMerge(t *testing.T) { r := require.New(t) + now := time.Now() cfg := AsyncSearcherConfig{DataDir: t.TempDir()} mp, err := mappingprovider.New("", mappingprovider.WithMapping(seq.Mapping{})) @@ -115,29 +81,104 @@ func TestMergeSkipsMissingFrac(t *testing.T) { as := MustStartAsync(cfg, mp, nil) t.Cleanup(func() { as.readOnly.Store(false) }) - presentFrac := &fakeFrac{ - info: common.Info{Path: "present"}, - dp: fakeDP{qpr: seq.QPR{IDs: []seq.IDSource{{ID: seq.ID{MID: 42}}}, Total: 1}}, + frac1 := &fakeFrac{ + info: common.Info{Path: "1", From: seq.TimeToMID(now.Add(-time.Minute * 11)), To: seq.TimeToMID(now.Add(-time.Minute * 6))}, + dp: fakeDP{qpr: seq.QPR{IDs: []seq.IDSource{{ID: seq.ID{MID: 1}}}, Total: 1}}, + } + frac2 := &fakeFrac{ + info: common.Info{Path: "2", From: seq.TimeToMID(now.Add(-time.Minute * 6)), To: seq.TimeToMID(now.Add(-time.Minute * 1))}, + dp: fakeDP{qpr: seq.QPR{IDs: []seq.IDSource{{ID: seq.ID{MID: 2}}}, Total: 1}}, } - missingFrac := &fakeFrac{info: common.Info{Path: "missing"}} - provider := &partialProvider{list: fracmanager.List{presentFrac, missingFrac}} + provider := &fakeFractionProvider{frac1, frac2} req := AsyncSearchRequest{ - ID: uuid.New().String(), - Params: processor.SearchParams{Limit: 1000, Order: seq.DocsOrderDesc}, + ID: uuid.New().String(), + Params: processor.SearchParams{ + Limit: 1000, + Order: seq.DocsOrderDesc, + From: seq.TimeToMID(now.UTC().Add(-time.Minute * 30).Truncate(time.Millisecond)), + To: seq.TimeToMID(now.UTC().Truncate(time.Millisecond)), + }, Query: "*", Retention: time.Hour, } r.NoError(as.StartSearch(req, provider)) as.processWg.Wait() - // "missing" produced no .qpr; "present" did. Merge must not drop the - // present result while collapsing the request into a single .mqpr. as.merge() resp, ok := as.FetchSearchResult(FetchSearchResultRequest{ID: req.ID, Limit: 1000, Order: seq.DocsOrderDesc}) r.True(ok) r.Equal(AsyncSearchStatusDone, resp.Status) - r.Len(resp.QPR.IDs, 1) - r.Equal(seq.MID(42), resp.QPR.IDs[0].ID.MID) + r.Len(resp.QPR.IDs, 2) + r.Equal(seq.MID(2), resp.QPR.IDs[0].ID.MID) + r.Equal(seq.MID(1), resp.QPR.IDs[1].ID.MID) +} + +func TestBuildIntervals(t *testing.T) { + tests := []struct { + name string + from seq.MID + to seq.MID + expected []searchInterval + }{ + { + name: "empty_range_from_equals_to", + from: 100, + to: 100, + expected: nil, + }, + { + name: "single_interval_small_range", + from: 0, + to: 100, + expected: []searchInterval{ + {0, 100}, + }, + }, + { + name: "single_interval_exact_split", + from: 0, + to: seq.DurationToMID(defaultSearchInterval), + expected: []searchInterval{ + {0, 300_000_000_000}, + }, + }, + { + name: "two_intervals", + from: 0, + to: seq.DurationToMID(defaultSearchInterval) * 2, + expected: []searchInterval{ + {0, 299_999_999_999}, + {300_000_000_000, 600_000_000_000}, + }, + }, + { + name: "three_intervals_with_remainder", + from: 0, + to: seq.DurationToMID(defaultSearchInterval)*3 + 50, + expected: []searchInterval{ + {0, 299_999_999_999}, + {300_000_000_000, 599_999_999_999}, + {600_000_000_000, 899_999_999_999}, + {900_000_000_000, 900_000_000_050}, + }, + }, + { + name: "minimal_range", + from: 5, + to: 6, + expected: []searchInterval{ + {5, 6}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := require.New(t) + result := buildIntervals(tt.from, tt.to) + r.Equal(tt.expected, result) + }) + } } diff --git a/fracmanager/fracmanager.go b/fracmanager/fracmanager.go index abd72846..5df309f6 100644 --- a/fracmanager/fracmanager.go +++ b/fracmanager/fracmanager.go @@ -15,6 +15,7 @@ import ( "github.com/ozontech/seq-db/frac" "github.com/ozontech/seq-db/frac/sealed" "github.com/ozontech/seq-db/logger" + "github.com/ozontech/seq-db/seq" "github.com/ozontech/seq-db/storage" "github.com/ozontech/seq-db/storage/s3" "github.com/ozontech/seq-db/util" @@ -157,6 +158,10 @@ func (fm *FracManager) AcquireFractions() (List, func()) { return fm.lc.registry.acquireAllFractions() } +func (fm *FracManager) AcquireFractionsInRange(from, to seq.MID) (List, func()) { + return fm.lc.registry.acquireFractionsInRange(from, to) +} + func (fm *FracManager) Oldest() uint64 { return fm.lc.registry.oldestTotal() } diff --git a/fracmanager/fraction_registry.go b/fracmanager/fraction_registry.go index e39f7562..cf2be275 100644 --- a/fracmanager/fraction_registry.go +++ b/fracmanager/fraction_registry.go @@ -10,6 +10,7 @@ import ( "github.com/ozontech/seq-db/frac" "github.com/ozontech/seq-db/logger" + "github.com/ozontech/seq-db/seq" "github.com/ozontech/seq-db/util" ) @@ -100,6 +101,14 @@ func (r *fractionRegistry) acquireAllFractions() ([]frac.Fraction, func()) { return r.snapshot.AcquireAll() } +// acquireFractionsInRange returns a read-only subset of fractions within the range +func (r *fractionRegistry) acquireFractionsInRange(from, to seq.MID) ([]frac.Fraction, func()) { + r.muSnapshot.RLock() + defer r.muSnapshot.RUnlock() + + return r.snapshot.AcquireInRange(from, to) +} + // statistics returns current size statistics of the registry. func (r *fractionRegistry) statistics() registryStats { r.mu.RLock() diff --git a/fracmanager/fractions_snapshot.go b/fracmanager/fractions_snapshot.go index 9d561f63..360db812 100644 --- a/fracmanager/fractions_snapshot.go +++ b/fracmanager/fractions_snapshot.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/ozontech/seq-db/frac" + "github.com/ozontech/seq-db/seq" ) // RefCounter provides reference counting capability. @@ -77,6 +78,33 @@ func (fs *fractionsSnapshot) AcquireAll() ([]frac.Fraction, func()) { } } +func (fs *fractionsSnapshot) AcquireInRange(from, to seq.MID) ([]frac.Fraction, func()) { + fracs := make(List, 0) + counters := make([]RefCounter, 0) + + for i := range len(fs.fractions) { + f := fs.fractions[i] + c := fs.counters[i] + + if f.Info().To < from { + continue + } + if f.Info().From > to { + break + } + + fracs = append(fracs, f) + c.Inc() + counters = append(counters, c) + } + + return fracs, func() { + for _, c := range counters { + c.Dec() + } + } +} + func (fs *fractionsSnapshot) AcquireOne(name string) (frac.Fraction, func(), bool) { i, ok := fs.names[name] if !ok { diff --git a/tests/integration_tests/integration_test.go b/tests/integration_tests/integration_test.go index c5dc49e9..59078c19 100644 --- a/tests/integration_tests/integration_test.go +++ b/tests/integration_tests/integration_test.go @@ -1579,6 +1579,7 @@ func (s *IntegrationTestSuite) TestTimeField() { func (s *IntegrationTestSuite) TestAsyncSearch() { t := s.T() r := require.New(t) + now := time.Now() cfg := *s.Config cfg.Mapping = map[string]seq.MappingTypes{ @@ -1618,8 +1619,8 @@ func (s *IntegrationTestSuite) TestAsyncSearch() { startReq := search.AsyncRequest{ Query: "* | fields ip, method, uri", - From: time.UnixMilli(0).UTC(), - To: time.Now().UTC().Add(time.Hour).Truncate(time.Millisecond), + From: now.UTC().Truncate(time.Millisecond), + To: now.UTC().Add(time.Minute).Truncate(time.Millisecond), Retention: time.Minute * 5, Aggregations: []search.AggQuery{ {