services: add default values for NeoFSBlockFetcher configuration

The minimum sufficient configuration is Addresses and ContainerID,
example:
```
  NeoFSBlockFetcher:
    Enabled: true
    Addresses:
      - st1.storage.fs.neo.org:8080
      - st2.storage.fs.neo.org:8080
      - st3.storage.fs.neo.org:8080
      - st4.storage.fs.neo.org:8080
    ContainerID: "87JRc7vyWcjW8uS32LMoLTAj4ckCzFZWfKbacjU3sAob"
```

Close #3718

Signed-off-by: Ekaterina Pavlova <ekt@morphbits.io>
This commit is contained in:
Ekaterina Pavlova 2024-12-12 18:37:44 +03:00
parent 283ca5cb6c
commit c79ffa967f
5 changed files with 26 additions and 27 deletions

View file

@ -206,20 +206,6 @@ func TestNeoFSBlockFetcherValidation(t *testing.T) {
shouldFail: true,
errMsg: "BQueueSize (5) is lower than OIDBatchSize (10)",
},
{
cfg: NeoFSBlockFetcher{
InternalService: InternalService{Enabled: true},
Timeout: time.Second,
ContainerID: validContainerID,
Addresses: []string{"127.0.0.1"},
OIDBatchSize: 10,
BQueueSize: 20,
SkipIndexFilesSearch: false,
IndexFileSize: 0,
},
shouldFail: true,
errMsg: "IndexFileSize is not set",
},
}
for _, c := range cases {

View file

@ -38,14 +38,11 @@ func (cfg *NeoFSBlockFetcher) Validate() error {
if err != nil {
return fmt.Errorf("invalid container ID: %w", err)
}
if cfg.BQueueSize < cfg.OIDBatchSize {
if cfg.BQueueSize > 0 && cfg.BQueueSize < cfg.OIDBatchSize {
return fmt.Errorf("BQueueSize (%d) is lower than OIDBatchSize (%d)", cfg.BQueueSize, cfg.OIDBatchSize)
}
if len(cfg.Addresses) == 0 {
return errors.New("addresses are not set")
}
if !cfg.SkipIndexFilesSearch && cfg.IndexFileSize == 0 {
return errors.New("IndexFileSize is not set")
}
return nil
}

View file

@ -224,6 +224,9 @@ func newServerFromConstructors(config ServerConfig, chain Ledger, stSync StateSy
}, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking)
s.bSyncQueue = bqueue.New(s.stateSync, log, nil, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking)
if s.NeoFSBlockFetcherCfg.BQueueSize <= 0 {
s.NeoFSBlockFetcherCfg.BQueueSize = blockfetcher.DefaultQueueCacheSize
}
s.bFetcherQueue = bqueue.New(chain, log, nil, s.NeoFSBlockFetcherCfg.BQueueSize, updateBlockQueueLenMetric, bqueue.Blocking)
var err error
s.blockFetcher, err = blockfetcher.New(chain, s.NeoFSBlockFetcherCfg, log, s.bFetcherQueue.PutBlock,

View file

@ -32,11 +32,17 @@ const (
// oidSize is the size of the object ID in NeoFS.
oidSize = sha256.Size
// defaultTimeout is the default timeout for NeoFS requests.
defaultTimeout = 5 * time.Minute
// defaultOIDBatchSize is the default number of OIDs to search and fetch at once.
defaultOIDBatchSize = 8000
defaultTimeout = 10 * time.Minute
// DefaultQueueCacheSize is the default size of the queue cache.
DefaultQueueCacheSize = 16000
// defaultDownloaderWorkersCount is the default number of workers downloading blocks.
defaultDownloaderWorkersCount = 100
defaultDownloaderWorkersCount = 500
// defaultIndexFileSize is the default size of the index file.
defaultIndexFileSize = 128000
// DefaultBlockAttribute is the default attribute name for block objects.
defaultBlockAttribute = "Block"
// defaultIndexFileAttribute is the default attribute name for index file objects.
defaultIndexFileAttribute = "Index"
)
// Constants related to NeoFS pool request timeouts.
@ -146,13 +152,19 @@ func New(chain Ledger, cfg config.NeoFSBlockFetcher, logger *zap.Logger, putBloc
cfg.Timeout = defaultTimeout
}
if cfg.OIDBatchSize <= 0 {
cfg.OIDBatchSize = defaultOIDBatchSize
cfg.OIDBatchSize = cfg.BQueueSize / 2
}
if cfg.DownloaderWorkersCount <= 0 {
cfg.DownloaderWorkersCount = defaultDownloaderWorkersCount
}
if len(cfg.Addresses) == 0 {
return nil, errors.New("no addresses provided")
if cfg.IndexFileSize <= 0 {
cfg.IndexFileSize = defaultIndexFileSize
}
if cfg.BlockAttribute == "" {
cfg.BlockAttribute = defaultBlockAttribute
}
if cfg.IndexFileAttribute == "" {
cfg.IndexFileAttribute = defaultIndexFileAttribute
}
params := pool.DefaultOptions()

View file

@ -66,6 +66,7 @@ func TestServiceConstructor(t *testing.T) {
Enabled: true,
},
Addresses: []string{"localhost:8080"},
BQueueSize: DefaultQueueCacheSize,
}
service, err := New(ledger, cfg, logger, mockPut.putBlock, shutdownCallback)
require.NoError(t, err)
@ -73,7 +74,7 @@ func TestServiceConstructor(t *testing.T) {
require.Equal(t, service.IsActive(), false)
require.Equal(t, service.cfg.Timeout, defaultTimeout)
require.Equal(t, service.cfg.OIDBatchSize, defaultOIDBatchSize)
require.Equal(t, service.cfg.OIDBatchSize, DefaultQueueCacheSize/2)
require.Equal(t, service.cfg.DownloaderWorkersCount, defaultDownloaderWorkersCount)
require.Equal(t, service.IsActive(), false)
})