From 862caf8b48e92dda0776d278d9aeabb69ee37a66 Mon Sep 17 00:00:00 2001 From: Andrei Cheboksarov <37665782+cheb0@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:40:30 +0400 Subject: [PATCH] make token block configurable --- cmd/seq-db/seq-db.go | 1 + config/config.go | 5 +++++ config/validation.go | 1 + config/validation_test.go | 15 +++++++++++++++ docs/en/02-configuration.md | 6 ++++++ docs/ru/02-configuration.md | 6 ++++++ frac/common/seal_params.go | 5 +++-- frac/fraction_test.go | 5 +++-- fracmanager/config.go | 3 +++ indexwriter/index.go | 12 ++++++++---- sealing/sealer.go | 4 ++++ 11 files changed, 55 insertions(+), 8 deletions(-) diff --git a/cmd/seq-db/seq-db.go b/cmd/seq-db/seq-db.go index 25b65f32..a205b1ad 100644 --- a/cmd/seq-db/seq-db.go +++ b/cmd/seq-db/seq-db.go @@ -268,6 +268,7 @@ func startStore( IDsZstdLevel: cfg.Compression.SealedZstdCompressionLevel, LIDsZstdLevel: cfg.Compression.SealedZstdCompressionLevel, LIDBlockSize: cfg.Sealing.Lids.BlockSize, + TokenBlockSize: int(cfg.Sealing.Tokens.BlockSize), TokenListZstdLevel: cfg.Compression.SealedZstdCompressionLevel, DocsPositionsZstdLevel: cfg.Compression.SealedZstdCompressionLevel, TokenTableZstdLevel: cfg.Compression.SealedZstdCompressionLevel, diff --git a/config/config.go b/config/config.go index 176d51a3..7357023d 100644 --- a/config/config.go +++ b/config/config.go @@ -72,6 +72,11 @@ type Config struct { } `config:"storage"` Sealing struct { + Tokens struct { + // BlockSize sets max token block size in bytes. + BlockSize Bytes `config:"block_size" default:"16KiB"` + } `config:"tokens"` + Lids struct { // BlockSize sets max lids (postings) saved per LIDs block. BlockSize int `config:"block_size" default:"65536"` diff --git a/config/validation.go b/config/validation.go index a09cb23e..804212df 100644 --- a/config/validation.go +++ b/config/validation.go @@ -70,6 +70,7 @@ func (c *Config) storeValidations() []validateFn { inRange("compression.doc_block_zstd_compression_level", -7, 22, c.Compression.DocBlockZstdCompressionLevel), greaterThan("sealing.lids.block_size", 0, c.Sealing.Lids.BlockSize), lessOrEqThan("sealing.lids.block_size", 65536, c.Sealing.Lids.BlockSize), + greaterThan("sealing.tokens.block_size", 0, c.Sealing.Tokens.BlockSize), inRange("offloading.queue_size_percent", 0, 100, c.Offloading.QueueSizePercent), greaterThan("experimental.max_regex_tokens_check", -1, c.Experimental.MaxRegexTokensCheck), diff --git a/config/validation_test.go b/config/validation_test.go index b0813697..885c20ff 100644 --- a/config/validation_test.go +++ b/config/validation_test.go @@ -98,6 +98,21 @@ limits: env: map[string]string{"SEQDB_SEALING_LIDS_BLOCK_SIZE": "8192"}, expectErr: false, }, + { + name: "Invalid sealing.tokens.block_size", + cfg: createCfgFile(t, base+` +sealing: + tokens: + block_size: -1B +`), + expectErr: true, + }, + { + name: "Valid sealing.tokens.block_size", + cfg: baseCfg, + env: map[string]string{"SEQDB_SEALING_TOKENS_BLOCK_SIZE": "32KiB"}, + expectErr: false, + }, } for _, tt := range tests { diff --git a/docs/en/02-configuration.md b/docs/en/02-configuration.md index 11a98773..fe273671 100644 --- a/docs/en/02-configuration.md +++ b/docs/en/02-configuration.md @@ -115,6 +115,12 @@ Compression level settings for various data types. Settings for fraction sealing. +### Tokens + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `sealing.tokens.block_size` | Bytes | `16KiB` | Max token block size in bytes | + ### Lids | Field | Type | Default | Description | diff --git a/docs/ru/02-configuration.md b/docs/ru/02-configuration.md index 8042a057..20466bf4 100644 --- a/docs/ru/02-configuration.md +++ b/docs/ru/02-configuration.md @@ -115,6 +115,12 @@ id: configuration Настройки запечатывания фракций. +### Tokens + +| Параметр | Тип | Значение по умолчанию | Описание | +|----------|-----|----------------------|-----------| +| `sealing.tokens.block_size` | Bytes | `16KiB` | Максимальный размер блока токенов в байтах | + ### Lids | Параметр | Тип | Значение по умолчанию | Описание | diff --git a/frac/common/seal_params.go b/frac/common/seal_params.go index 5a73de4e..faf13c9c 100644 --- a/frac/common/seal_params.go +++ b/frac/common/seal_params.go @@ -12,6 +12,7 @@ type SealParams struct { TokenTableZstdLevel int DocBlocksZstdLevel int // DocBlocksZstdLevel is the zstd compress level of each document block. - LIDBlockSize int - DocBlockSize int // DocBlockSize is decompressed payload size of document block. + LIDBlockSize int + TokenBlockSize int + DocBlockSize int // DocBlockSize is decompressed payload size of document block. } diff --git a/frac/fraction_test.go b/frac/fraction_test.go index 180ec4e2..195cccf6 100644 --- a/frac/fraction_test.go +++ b/frac/fraction_test.go @@ -103,6 +103,7 @@ func (s *FractionTestSuite) SetupTestCommon() { TokenTableZstdLevel: 1, DocBlocksZstdLevel: 1, LIDBlockSize: 512, + TokenBlockSize: 128, DocBlockSize: 128 * int(units.KiB), } @@ -1911,11 +1912,11 @@ func (s *FractionTestSuite) TestFractionInfo() { s.Require().Equal(uint64(0), info.IndexOnDisk, "index on disk doesn't match") case *frac.Sealed: s.Require().Equal(uint64(0), info.MetaOnDisk, "meta on disk doesn't match. actual value") - s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1450), + s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1500), "index on disk doesn't match. actual value: %d", info.IndexOnDisk) case *frac.Remote: s.Require().Equal(uint64(0), info.MetaOnDisk, "meta on disk doesn't match. actual value") - s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1450), + s.Require().True(info.IndexOnDisk > uint64(1300) && info.IndexOnDisk < uint64(1500), "index on disk doesn't match. actual value: %d", info.IndexOnDisk) default: s.Require().Fail("unsupported fraction type") diff --git a/fracmanager/config.go b/fracmanager/config.go index 6bc8acf2..08f56cfd 100644 --- a/fracmanager/config.go +++ b/fracmanager/config.go @@ -62,6 +62,9 @@ func FillConfigWithDefault(config *Config) *Config { if config.SealParams.LIDBlockSize == 0 { config.SealParams.LIDBlockSize = consts.DefaultLIDBlockCap } + if config.SealParams.TokenBlockSize == 0 { + config.SealParams.TokenBlockSize = consts.RegularBlockSize + } if config.SealParams.TokenListZstdLevel == 0 { config.SealParams.TokenListZstdLevel = zstdDefaultLevel } diff --git a/indexwriter/index.go b/indexwriter/index.go index cdf6d1c5..ff1c6119 100644 --- a/indexwriter/index.go +++ b/indexwriter/index.go @@ -67,12 +67,16 @@ type IndexWriter struct { } func New(params common.SealParams) *IndexWriter { + if params.TokenBlockSize == 0 { + params.TokenBlockSize = consts.RegularBlockSize + } + return &IndexWriter{ params: params, - buf1: make([]byte, 0, consts.RegularBlockSize), - buf2: make([]byte, 0, consts.RegularBlockSize), + buf1: make([]byte, 0, params.TokenBlockSize), + buf2: make([]byte, 0, params.TokenBlockSize), buf32: make([]uint32, 0, consts.DefaultLIDBlockCap), - buf64: make([]uint64, 0, consts.RegularBlockSize), + buf64: make([]uint64, 0, params.TokenBlockSize), } } @@ -153,7 +157,7 @@ func (s *IndexWriter) WriteTokenTriplet(tws, lws io.WriteSeeker, src Source) err ) var allFieldsTables []token.FieldTable - for pair, err := range tokenBlock(src.TokenTriplets(), lidAccumulator.add, consts.RegularBlockSize) { + for pair, err := range tokenBlock(src.TokenTriplets(), lidAccumulator.add, s.params.TokenBlockSize) { if err != nil { return err } diff --git a/sealing/sealer.go b/sealing/sealer.go index 0c21ffc4..fd90c1d8 100644 --- a/sealing/sealer.go +++ b/sealing/sealer.go @@ -20,6 +20,10 @@ type Source = indexwriter.Source // and returns PreloadedData for fast initialization of the sealed fraction. func Seal(src Source, params common.SealParams) (*sealed.PreloadedData, error) { info := src.Info() + info.ConstRegularBlockSize = params.TokenBlockSize + if info.ConstRegularBlockSize == 0 { + info.ConstRegularBlockSize = consts.RegularBlockSize + } if info.To == 0 { return nil, errors.New("sealing of an empty active fraction is not supported")