From 2d43892fc9bfc0067689b5ef1370873c9bb5a7b1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Oct 2022 15:53:42 +0300 Subject: [PATCH] [#1840] neofs-node: Use blobstor paths to identify shard Signed-off-by: Evgenii Stratonikov --- cmd/neofs-node/config.go | 30 +++++++++++---- pkg/local_object_storage/engine/control.go | 38 +++++++++++++------ .../engine/control_test.go | 9 ++--- 3 files changed, 52 insertions(+), 25 deletions(-) diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go index e3c1148bf7..91f89a0132 100644 --- a/cmd/neofs-node/config.go +++ b/cmd/neofs-node/config.go @@ -8,6 +8,8 @@ import ( "net" "os" "os/signal" + "path/filepath" + "strings" "sync" atomicstd "sync/atomic" "syscall" @@ -135,6 +137,18 @@ type shardCfg struct { } } +// id returns persistent id of a shard. It is different from the ID used in runtime +// and is primarily used to identify shards in the configuration. +func (c *shardCfg) id() string { + // This calculation should be kept in sync with + // pkg/local_object_storage/engine/control.go file. + var sb strings.Builder + for i := range c.subStorages { + sb.WriteString(filepath.Clean(c.subStorages[i].path)) + } + return sb.String() +} + type subStorageCfg struct { // common for all storages typ string @@ -594,13 +608,13 @@ func (c *cfg) engineOpts() []engine.Option { return opts } -type shardOptsWithMetaPath struct { - metaPath string +type shardOptsWithID struct { + configID string shOpts []shard.Option } -func (c *cfg) shardOpts() []shardOptsWithMetaPath { - shards := make([]shardOptsWithMetaPath, 0, len(c.EngineCfg.shards)) +func (c *cfg) shardOpts() []shardOptsWithID { + shards := make([]shardOptsWithID, 0, len(c.EngineCfg.shards)) for _, shCfg := range c.EngineCfg.shards { var writeCacheOpts []writecache.Option @@ -663,8 +677,8 @@ func (c *cfg) shardOpts() []shardOptsWithMetaPath { } } - var sh shardOptsWithMetaPath - sh.metaPath = shCfg.metaCfg.path + var sh shardOptsWithID + sh.configID = shCfg.id() sh.shOpts = []shard.Option{ shard.WithLogger(c.log), shard.WithRefillMetabase(shCfg.refillMetabase), @@ -831,8 +845,8 @@ func (c *cfg) configWatcher(ctx context.Context) { } var rcfg engine.ReConfiguration - for _, optsWithMeta := range c.shardOpts() { - rcfg.AddShard(optsWithMeta.metaPath, optsWithMeta.shOpts) + for _, optsWithID := range c.shardOpts() { + rcfg.AddShard(optsWithID.configID, optsWithID.shOpts) } err = c.cfgObject.cfgLocalStorage.localStorage.Reload(rcfg) diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index e1f632d824..216267cb9e 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -3,6 +3,8 @@ package engine import ( "errors" "fmt" + "path/filepath" + "strings" "sync" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" @@ -217,18 +219,18 @@ func (rCfg *ReConfiguration) SetShardPoolSize(shardPoolSize uint32) { rCfg.shardPoolSize = shardPoolSize } -// AddShard adds a shard for the reconfiguration. Path to a metabase is used as -// an identifier of the shard in configuration. -func (rCfg *ReConfiguration) AddShard(metaPath string, opts []shard.Option) { +// AddShard adds a shard for the reconfiguration. +// Shard identifier is calculated from paths used in blobstor. +func (rCfg *ReConfiguration) AddShard(id string, opts []shard.Option) { if rCfg.shards == nil { rCfg.shards = make(map[string][]shard.Option) } - if _, found := rCfg.shards[metaPath]; found { + if _, found := rCfg.shards[id]; found { return } - rCfg.shards[metaPath] = opts + rCfg.shards[id] = opts } // Reload reloads StorageEngine's configuration in runtime. @@ -240,24 +242,26 @@ func (e *StorageEngine) Reload(rcfg ReConfiguration) error { // mark removed shards for removal for id, sh := range e.shards { - _, ok := rcfg.shards[sh.Shard.DumpInfo().MetaBaseInfo.Path] + _, ok := rcfg.shards[calculateShardID(sh.DumpInfo())] if !ok { shardsToRemove = append(shardsToRemove, id) } } // mark new shards for addition - for newPath := range rcfg.shards { + for newID := range rcfg.shards { addShard := true for _, sh := range e.shards { - if newPath == sh.Shard.DumpInfo().MetaBaseInfo.Path { + // This calculation should be kept in sync with node + // configuration parsing during SIGHUP. + if newID == calculateShardID(sh.DumpInfo()) { addShard = false break } } if addShard { - shardsToAdd = append(shardsToAdd, newPath) + shardsToAdd = append(shardsToAdd, newID) } } @@ -265,10 +269,10 @@ func (e *StorageEngine) Reload(rcfg ReConfiguration) error { e.removeShards(shardsToRemove...) - for _, newPath := range shardsToAdd { - sh, err := e.createShard(rcfg.shards[newPath]) + for _, newID := range shardsToAdd { + sh, err := e.createShard(rcfg.shards[newID]) if err != nil { - return fmt.Errorf("could not add new shard with '%s' metabase path: %w", newPath, err) + return fmt.Errorf("could not add new shard with '%s' metabase path: %w", newID, err) } idStr := sh.ID().String() @@ -293,3 +297,13 @@ func (e *StorageEngine) Reload(rcfg ReConfiguration) error { return nil } + +func calculateShardID(info shard.Info) string { + // This calculation should be kept in sync with node + // configuration parsing during SIGHUP. + var sb strings.Builder + for _, sub := range info.BlobStorInfo.SubStorages { + sb.WriteString(filepath.Clean(sub.Path)) + } + return sb.String() +} diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 59750424b1..1d2dee6c03 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -147,19 +147,18 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str e := New() for i := 0; i < num; i++ { - metaPath := filepath.Join(addPath, fmt.Sprintf("%d.metabase", i)) - currShards = append(currShards, metaPath) - - _, err := e.AddShard( + id, err := e.AddShard( shard.WithBlobStorOptions( blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(i)), errSmallSize))), shard.WithMetaBaseOptions( - meta.WithPath(metaPath), + meta.WithPath(filepath.Join(addPath, fmt.Sprintf("%d.metabase", i))), meta.WithPermissions(0700), meta.WithEpochState(epochState{}), ), ) require.NoError(t, err) + + currShards = append(currShards, calculateShardID(e.shards[id.String()].DumpInfo())) } require.Equal(t, num, len(e.shards))