mirror of
https://github.com/nspcc-dev/neo-go.git
synced 2024-12-22 19:19:09 +00:00
services: add default values for NeoFSBlockFetcher configuration
The minimum sufficient configuration is Addresses and ContainerID, example: ``` NeoFSBlockFetcher: Enabled: true Addresses: - st1.storage.fs.neo.org:8080 - st2.storage.fs.neo.org:8080 - st3.storage.fs.neo.org:8080 - st4.storage.fs.neo.org:8080 ContainerID: "87JRc7vyWcjW8uS32LMoLTAj4ckCzFZWfKbacjU3sAob" ``` Close #3718 Signed-off-by: Ekaterina Pavlova <ekt@morphbits.io>
This commit is contained in:
parent
283ca5cb6c
commit
c79ffa967f
5 changed files with 26 additions and 27 deletions
|
@ -206,20 +206,6 @@ func TestNeoFSBlockFetcherValidation(t *testing.T) {
|
||||||
shouldFail: true,
|
shouldFail: true,
|
||||||
errMsg: "BQueueSize (5) is lower than OIDBatchSize (10)",
|
errMsg: "BQueueSize (5) is lower than OIDBatchSize (10)",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
cfg: NeoFSBlockFetcher{
|
|
||||||
InternalService: InternalService{Enabled: true},
|
|
||||||
Timeout: time.Second,
|
|
||||||
ContainerID: validContainerID,
|
|
||||||
Addresses: []string{"127.0.0.1"},
|
|
||||||
OIDBatchSize: 10,
|
|
||||||
BQueueSize: 20,
|
|
||||||
SkipIndexFilesSearch: false,
|
|
||||||
IndexFileSize: 0,
|
|
||||||
},
|
|
||||||
shouldFail: true,
|
|
||||||
errMsg: "IndexFileSize is not set",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
|
|
|
@ -38,14 +38,11 @@ func (cfg *NeoFSBlockFetcher) Validate() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("invalid container ID: %w", err)
|
return fmt.Errorf("invalid container ID: %w", err)
|
||||||
}
|
}
|
||||||
if cfg.BQueueSize < cfg.OIDBatchSize {
|
if cfg.BQueueSize > 0 && cfg.BQueueSize < cfg.OIDBatchSize {
|
||||||
return fmt.Errorf("BQueueSize (%d) is lower than OIDBatchSize (%d)", cfg.BQueueSize, cfg.OIDBatchSize)
|
return fmt.Errorf("BQueueSize (%d) is lower than OIDBatchSize (%d)", cfg.BQueueSize, cfg.OIDBatchSize)
|
||||||
}
|
}
|
||||||
if len(cfg.Addresses) == 0 {
|
if len(cfg.Addresses) == 0 {
|
||||||
return errors.New("addresses are not set")
|
return errors.New("addresses are not set")
|
||||||
}
|
}
|
||||||
if !cfg.SkipIndexFilesSearch && cfg.IndexFileSize == 0 {
|
|
||||||
return errors.New("IndexFileSize is not set")
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -224,6 +224,9 @@ func newServerFromConstructors(config ServerConfig, chain Ledger, stSync StateSy
|
||||||
}, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking)
|
}, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking)
|
||||||
|
|
||||||
s.bSyncQueue = bqueue.New(s.stateSync, log, nil, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking)
|
s.bSyncQueue = bqueue.New(s.stateSync, log, nil, bqueue.DefaultCacheSize, updateBlockQueueLenMetric, bqueue.NonBlocking)
|
||||||
|
if s.NeoFSBlockFetcherCfg.BQueueSize <= 0 {
|
||||||
|
s.NeoFSBlockFetcherCfg.BQueueSize = blockfetcher.DefaultQueueCacheSize
|
||||||
|
}
|
||||||
s.bFetcherQueue = bqueue.New(chain, log, nil, s.NeoFSBlockFetcherCfg.BQueueSize, updateBlockQueueLenMetric, bqueue.Blocking)
|
s.bFetcherQueue = bqueue.New(chain, log, nil, s.NeoFSBlockFetcherCfg.BQueueSize, updateBlockQueueLenMetric, bqueue.Blocking)
|
||||||
var err error
|
var err error
|
||||||
s.blockFetcher, err = blockfetcher.New(chain, s.NeoFSBlockFetcherCfg, log, s.bFetcherQueue.PutBlock,
|
s.blockFetcher, err = blockfetcher.New(chain, s.NeoFSBlockFetcherCfg, log, s.bFetcherQueue.PutBlock,
|
||||||
|
|
|
@ -32,11 +32,17 @@ const (
|
||||||
// oidSize is the size of the object ID in NeoFS.
|
// oidSize is the size of the object ID in NeoFS.
|
||||||
oidSize = sha256.Size
|
oidSize = sha256.Size
|
||||||
// defaultTimeout is the default timeout for NeoFS requests.
|
// defaultTimeout is the default timeout for NeoFS requests.
|
||||||
defaultTimeout = 5 * time.Minute
|
defaultTimeout = 10 * time.Minute
|
||||||
// defaultOIDBatchSize is the default number of OIDs to search and fetch at once.
|
// DefaultQueueCacheSize is the default size of the queue cache.
|
||||||
defaultOIDBatchSize = 8000
|
DefaultQueueCacheSize = 16000
|
||||||
// defaultDownloaderWorkersCount is the default number of workers downloading blocks.
|
// defaultDownloaderWorkersCount is the default number of workers downloading blocks.
|
||||||
defaultDownloaderWorkersCount = 100
|
defaultDownloaderWorkersCount = 500
|
||||||
|
// defaultIndexFileSize is the default size of the index file.
|
||||||
|
defaultIndexFileSize = 128000
|
||||||
|
// DefaultBlockAttribute is the default attribute name for block objects.
|
||||||
|
defaultBlockAttribute = "Block"
|
||||||
|
// defaultIndexFileAttribute is the default attribute name for index file objects.
|
||||||
|
defaultIndexFileAttribute = "Index"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Constants related to NeoFS pool request timeouts.
|
// Constants related to NeoFS pool request timeouts.
|
||||||
|
@ -146,13 +152,19 @@ func New(chain Ledger, cfg config.NeoFSBlockFetcher, logger *zap.Logger, putBloc
|
||||||
cfg.Timeout = defaultTimeout
|
cfg.Timeout = defaultTimeout
|
||||||
}
|
}
|
||||||
if cfg.OIDBatchSize <= 0 {
|
if cfg.OIDBatchSize <= 0 {
|
||||||
cfg.OIDBatchSize = defaultOIDBatchSize
|
cfg.OIDBatchSize = cfg.BQueueSize / 2
|
||||||
}
|
}
|
||||||
if cfg.DownloaderWorkersCount <= 0 {
|
if cfg.DownloaderWorkersCount <= 0 {
|
||||||
cfg.DownloaderWorkersCount = defaultDownloaderWorkersCount
|
cfg.DownloaderWorkersCount = defaultDownloaderWorkersCount
|
||||||
}
|
}
|
||||||
if len(cfg.Addresses) == 0 {
|
if cfg.IndexFileSize <= 0 {
|
||||||
return nil, errors.New("no addresses provided")
|
cfg.IndexFileSize = defaultIndexFileSize
|
||||||
|
}
|
||||||
|
if cfg.BlockAttribute == "" {
|
||||||
|
cfg.BlockAttribute = defaultBlockAttribute
|
||||||
|
}
|
||||||
|
if cfg.IndexFileAttribute == "" {
|
||||||
|
cfg.IndexFileAttribute = defaultIndexFileAttribute
|
||||||
}
|
}
|
||||||
|
|
||||||
params := pool.DefaultOptions()
|
params := pool.DefaultOptions()
|
||||||
|
|
|
@ -66,6 +66,7 @@ func TestServiceConstructor(t *testing.T) {
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
},
|
},
|
||||||
Addresses: []string{"localhost:8080"},
|
Addresses: []string{"localhost:8080"},
|
||||||
|
BQueueSize: DefaultQueueCacheSize,
|
||||||
}
|
}
|
||||||
service, err := New(ledger, cfg, logger, mockPut.putBlock, shutdownCallback)
|
service, err := New(ledger, cfg, logger, mockPut.putBlock, shutdownCallback)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -73,7 +74,7 @@ func TestServiceConstructor(t *testing.T) {
|
||||||
|
|
||||||
require.Equal(t, service.IsActive(), false)
|
require.Equal(t, service.IsActive(), false)
|
||||||
require.Equal(t, service.cfg.Timeout, defaultTimeout)
|
require.Equal(t, service.cfg.Timeout, defaultTimeout)
|
||||||
require.Equal(t, service.cfg.OIDBatchSize, defaultOIDBatchSize)
|
require.Equal(t, service.cfg.OIDBatchSize, DefaultQueueCacheSize/2)
|
||||||
require.Equal(t, service.cfg.DownloaderWorkersCount, defaultDownloaderWorkersCount)
|
require.Equal(t, service.cfg.DownloaderWorkersCount, defaultDownloaderWorkersCount)
|
||||||
require.Equal(t, service.IsActive(), false)
|
require.Equal(t, service.IsActive(), false)
|
||||||
})
|
})
|
||||||
|
|
Loading…
Reference in a new issue