services: add default values for NeoFSBlockFetcher configuration

The minimum sufficient configuration is Addresses and ContainerID,
example:
```
  NeoFSBlockFetcher:
    Enabled: true
    Addresses:
      - st1.storage.fs.neo.org:8080
      - st2.storage.fs.neo.org:8080
      - st3.storage.fs.neo.org:8080
      - st4.storage.fs.neo.org:8080
    ContainerID: "87JRc7vyWcjW8uS32LMoLTAj4ckCzFZWfKbacjU3sAob"
```

Close #3718

Signed-off-by: Ekaterina Pavlova <ekt@morphbits.io>
This commit is contained in:
Ekaterina Pavlova 2024-12-12 18:37:44 +03:00
parent 283ca5cb6c
commit c79ffa967f
5 changed files with 26 additions and 27 deletions

View file

@ -206,20 +206,6 @@ func TestNeoFSBlockFetcherValidation(t *testing.T) {
shouldFail: true, shouldFail: true,
errMsg: "BQueueSize (5) is lower than OIDBatchSize (10)", errMsg: "BQueueSize (5) is lower than OIDBatchSize (10)",
}, },
{
cfg: NeoFSBlockFetcher{
InternalService: InternalService{Enabled: true},
Timeout: time.Second,
ContainerID: validContainerID,
Addresses: []string{"127.0.0.1"},
OIDBatchSize: 10,
BQueueSize: 20,
SkipIndexFilesSearch: false,
IndexFileSize: 0,
},
shouldFail: true,
errMsg: "IndexFileSize is not set",
},
} }
for _, c := range cases { for _, c := range cases {

View file

@ -38,14 +38,11 @@ func (cfg *NeoFSBlockFetcher) Validate() error {
if err != nil { if err != nil {
return fmt.Errorf("invalid container ID: %w", err) return fmt.Errorf("invalid container ID: %w", err)
} }
if cfg.BQueueSize < cfg.OIDBatchSize { if cfg.BQueueSize > 0 && cfg.BQueueSize < cfg.OIDBatchSize {
return fmt.Errorf("BQueueSize (%d) is lower than OIDBatchSize (%d)", cfg.BQueueSize, cfg.OIDBatchSize) return fmt.Errorf("BQueueSize (%d) is lower than OIDBatchSize (%d)", cfg.BQueueSize, cfg.OIDBatchSize)
} }
if len(cfg.Addresses) == 0 { if len(cfg.Addresses) == 0 {
return errors.New("addresses are not set") return errors.New("addresses are not set")
} }
if !cfg.SkipIndexFilesSearch && cfg.IndexFileSize == 0 {
return errors.New("IndexFileSize is not set")
}
return nil return nil
} }

View file

@ -224,6 +224,9 @@ func newServerFromConstructors(config ServerConfig, chain Ledger, stSync StateSy
}, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking) }, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking)
s.bSyncQueue = bqueue.New(s.stateSync, log, nil, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking) s.bSyncQueue = bqueue.New(s.stateSync, log, nil, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking)
if s.NeoFSBlockFetcherCfg.BQueueSize <= 0 {
s.NeoFSBlockFetcherCfg.BQueueSize = blockfetcher.DefaultQueueCacheSize
}
s.bFetcherQueue = bqueue.New(chain, log, nil, s.NeoFSBlockFetcherCfg.BQueueSize, updateBlockQueueLenMetric, bqueue.Blocking) s.bFetcherQueue = bqueue.New(chain, log, nil, s.NeoFSBlockFetcherCfg.BQueueSize, updateBlockQueueLenMetric, bqueue.Blocking)
var err error var err error
s.blockFetcher, err = blockfetcher.New(chain, s.NeoFSBlockFetcherCfg, log, s.bFetcherQueue.PutBlock, s.blockFetcher, err = blockfetcher.New(chain, s.NeoFSBlockFetcherCfg, log, s.bFetcherQueue.PutBlock,

View file

@ -32,11 +32,17 @@ const (
// oidSize is the size of the object ID in NeoFS. // oidSize is the size of the object ID in NeoFS.
oidSize = sha256.Size oidSize = sha256.Size
// defaultTimeout is the default timeout for NeoFS requests. // defaultTimeout is the default timeout for NeoFS requests.
defaultTimeout = 5 * time.Minute defaultTimeout = 10 * time.Minute
// defaultOIDBatchSize is the default number of OIDs to search and fetch at once. // DefaultQueueCacheSize is the default size of the queue cache.
defaultOIDBatchSize = 8000 DefaultQueueCacheSize = 16000
// defaultDownloaderWorkersCount is the default number of workers downloading blocks. // defaultDownloaderWorkersCount is the default number of workers downloading blocks.
defaultDownloaderWorkersCount = 100 defaultDownloaderWorkersCount = 500
// defaultIndexFileSize is the default size of the index file.
defaultIndexFileSize = 128000
// DefaultBlockAttribute is the default attribute name for block objects.
defaultBlockAttribute = "Block"
// defaultIndexFileAttribute is the default attribute name for index file objects.
defaultIndexFileAttribute = "Index"
) )
// Constants related to NeoFS pool request timeouts. // Constants related to NeoFS pool request timeouts.
@ -146,13 +152,19 @@ func New(chain Ledger, cfg config.NeoFSBlockFetcher, logger *zap.Logger, putBloc
cfg.Timeout = defaultTimeout cfg.Timeout = defaultTimeout
} }
if cfg.OIDBatchSize <= 0 { if cfg.OIDBatchSize <= 0 {
cfg.OIDBatchSize = defaultOIDBatchSize cfg.OIDBatchSize = cfg.BQueueSize / 2
} }
if cfg.DownloaderWorkersCount <= 0 { if cfg.DownloaderWorkersCount <= 0 {
cfg.DownloaderWorkersCount = defaultDownloaderWorkersCount cfg.DownloaderWorkersCount = defaultDownloaderWorkersCount
} }
if len(cfg.Addresses) == 0 { if cfg.IndexFileSize <= 0 {
return nil, errors.New("no addresses provided") cfg.IndexFileSize = defaultIndexFileSize
}
if cfg.BlockAttribute == "" {
cfg.BlockAttribute = defaultBlockAttribute
}
if cfg.IndexFileAttribute == "" {
cfg.IndexFileAttribute = defaultIndexFileAttribute
} }
params := pool.DefaultOptions() params := pool.DefaultOptions()

View file

@ -65,7 +65,8 @@ func TestServiceConstructor(t *testing.T) {
InternalService: config.InternalService{ InternalService: config.InternalService{
Enabled: true, Enabled: true,
}, },
Addresses: []string{"localhost:8080"}, Addresses: []string{"localhost:8080"},
BQueueSize: DefaultQueueCacheSize,
} }
service, err := New(ledger, cfg, logger, mockPut.putBlock, shutdownCallback) service, err := New(ledger, cfg, logger, mockPut.putBlock, shutdownCallback)
require.NoError(t, err) require.NoError(t, err)
@ -73,7 +74,7 @@ func TestServiceConstructor(t *testing.T) {
require.Equal(t, service.IsActive(), false) require.Equal(t, service.IsActive(), false)
require.Equal(t, service.cfg.Timeout, defaultTimeout) require.Equal(t, service.cfg.Timeout, defaultTimeout)
require.Equal(t, service.cfg.OIDBatchSize, defaultOIDBatchSize) require.Equal(t, service.cfg.OIDBatchSize, DefaultQueueCacheSize/2)
require.Equal(t, service.cfg.DownloaderWorkersCount, defaultDownloaderWorkersCount) require.Equal(t, service.cfg.DownloaderWorkersCount, defaultDownloaderWorkersCount)
require.Equal(t, service.IsActive(), false) require.Equal(t, service.IsActive(), false)
}) })