forked from TrueCloudLab/frostfs-node
[#1840] neofs-node: Use blobstor paths to identify shard
Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
parent
4b005d3178
commit
2d43892fc9
3 changed files with 52 additions and 25 deletions
|
@ -8,6 +8,8 @@ import (
|
|||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
atomicstd "sync/atomic"
|
||||
"syscall"
|
||||
|
@ -135,6 +137,18 @@ type shardCfg struct {
|
|||
}
|
||||
}
|
||||
|
||||
// id returns persistent id of a shard. It is different from the ID used in runtime
|
||||
// and is primarily used to identify shards in the configuration.
|
||||
func (c *shardCfg) id() string {
|
||||
// This calculation should be kept in sync with
|
||||
// pkg/local_object_storage/engine/control.go file.
|
||||
var sb strings.Builder
|
||||
for i := range c.subStorages {
|
||||
sb.WriteString(filepath.Clean(c.subStorages[i].path))
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
type subStorageCfg struct {
|
||||
// common for all storages
|
||||
typ string
|
||||
|
@ -594,13 +608,13 @@ func (c *cfg) engineOpts() []engine.Option {
|
|||
return opts
|
||||
}
|
||||
|
||||
type shardOptsWithMetaPath struct {
|
||||
metaPath string
|
||||
type shardOptsWithID struct {
|
||||
configID string
|
||||
shOpts []shard.Option
|
||||
}
|
||||
|
||||
func (c *cfg) shardOpts() []shardOptsWithMetaPath {
|
||||
shards := make([]shardOptsWithMetaPath, 0, len(c.EngineCfg.shards))
|
||||
func (c *cfg) shardOpts() []shardOptsWithID {
|
||||
shards := make([]shardOptsWithID, 0, len(c.EngineCfg.shards))
|
||||
|
||||
for _, shCfg := range c.EngineCfg.shards {
|
||||
var writeCacheOpts []writecache.Option
|
||||
|
@ -663,8 +677,8 @@ func (c *cfg) shardOpts() []shardOptsWithMetaPath {
|
|||
}
|
||||
}
|
||||
|
||||
var sh shardOptsWithMetaPath
|
||||
sh.metaPath = shCfg.metaCfg.path
|
||||
var sh shardOptsWithID
|
||||
sh.configID = shCfg.id()
|
||||
sh.shOpts = []shard.Option{
|
||||
shard.WithLogger(c.log),
|
||||
shard.WithRefillMetabase(shCfg.refillMetabase),
|
||||
|
@ -831,8 +845,8 @@ func (c *cfg) configWatcher(ctx context.Context) {
|
|||
}
|
||||
|
||||
var rcfg engine.ReConfiguration
|
||||
for _, optsWithMeta := range c.shardOpts() {
|
||||
rcfg.AddShard(optsWithMeta.metaPath, optsWithMeta.shOpts)
|
||||
for _, optsWithID := range c.shardOpts() {
|
||||
rcfg.AddShard(optsWithID.configID, optsWithID.shOpts)
|
||||
}
|
||||
|
||||
err = c.cfgObject.cfgLocalStorage.localStorage.Reload(rcfg)
|
||||
|
|
|
@ -3,6 +3,8 @@ package engine
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
|
||||
|
@ -217,18 +219,18 @@ func (rCfg *ReConfiguration) SetShardPoolSize(shardPoolSize uint32) {
|
|||
rCfg.shardPoolSize = shardPoolSize
|
||||
}
|
||||
|
||||
// AddShard adds a shard for the reconfiguration. Path to a metabase is used as
|
||||
// an identifier of the shard in configuration.
|
||||
func (rCfg *ReConfiguration) AddShard(metaPath string, opts []shard.Option) {
|
||||
// AddShard adds a shard for the reconfiguration.
|
||||
// Shard identifier is calculated from paths used in blobstor.
|
||||
func (rCfg *ReConfiguration) AddShard(id string, opts []shard.Option) {
|
||||
if rCfg.shards == nil {
|
||||
rCfg.shards = make(map[string][]shard.Option)
|
||||
}
|
||||
|
||||
if _, found := rCfg.shards[metaPath]; found {
|
||||
if _, found := rCfg.shards[id]; found {
|
||||
return
|
||||
}
|
||||
|
||||
rCfg.shards[metaPath] = opts
|
||||
rCfg.shards[id] = opts
|
||||
}
|
||||
|
||||
// Reload reloads StorageEngine's configuration in runtime.
|
||||
|
@ -240,24 +242,26 @@ func (e *StorageEngine) Reload(rcfg ReConfiguration) error {
|
|||
|
||||
// mark removed shards for removal
|
||||
for id, sh := range e.shards {
|
||||
_, ok := rcfg.shards[sh.Shard.DumpInfo().MetaBaseInfo.Path]
|
||||
_, ok := rcfg.shards[calculateShardID(sh.DumpInfo())]
|
||||
if !ok {
|
||||
shardsToRemove = append(shardsToRemove, id)
|
||||
}
|
||||
}
|
||||
|
||||
// mark new shards for addition
|
||||
for newPath := range rcfg.shards {
|
||||
for newID := range rcfg.shards {
|
||||
addShard := true
|
||||
for _, sh := range e.shards {
|
||||
if newPath == sh.Shard.DumpInfo().MetaBaseInfo.Path {
|
||||
// This calculation should be kept in sync with node
|
||||
// configuration parsing during SIGHUP.
|
||||
if newID == calculateShardID(sh.DumpInfo()) {
|
||||
addShard = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if addShard {
|
||||
shardsToAdd = append(shardsToAdd, newPath)
|
||||
shardsToAdd = append(shardsToAdd, newID)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -265,10 +269,10 @@ func (e *StorageEngine) Reload(rcfg ReConfiguration) error {
|
|||
|
||||
e.removeShards(shardsToRemove...)
|
||||
|
||||
for _, newPath := range shardsToAdd {
|
||||
sh, err := e.createShard(rcfg.shards[newPath])
|
||||
for _, newID := range shardsToAdd {
|
||||
sh, err := e.createShard(rcfg.shards[newID])
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not add new shard with '%s' metabase path: %w", newPath, err)
|
||||
return fmt.Errorf("could not add new shard with '%s' metabase path: %w", newID, err)
|
||||
}
|
||||
|
||||
idStr := sh.ID().String()
|
||||
|
@ -293,3 +297,13 @@ func (e *StorageEngine) Reload(rcfg ReConfiguration) error {
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
func calculateShardID(info shard.Info) string {
|
||||
// This calculation should be kept in sync with node
|
||||
// configuration parsing during SIGHUP.
|
||||
var sb strings.Builder
|
||||
for _, sub := range info.BlobStorInfo.SubStorages {
|
||||
sb.WriteString(filepath.Clean(sub.Path))
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
|
|
@ -147,19 +147,18 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str
|
|||
|
||||
e := New()
|
||||
for i := 0; i < num; i++ {
|
||||
metaPath := filepath.Join(addPath, fmt.Sprintf("%d.metabase", i))
|
||||
currShards = append(currShards, metaPath)
|
||||
|
||||
_, err := e.AddShard(
|
||||
id, err := e.AddShard(
|
||||
shard.WithBlobStorOptions(
|
||||
blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(i)), errSmallSize))),
|
||||
shard.WithMetaBaseOptions(
|
||||
meta.WithPath(metaPath),
|
||||
meta.WithPath(filepath.Join(addPath, fmt.Sprintf("%d.metabase", i))),
|
||||
meta.WithPermissions(0700),
|
||||
meta.WithEpochState(epochState{}),
|
||||
),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
currShards = append(currShards, calculateShardID(e.shards[id.String()].DumpInfo()))
|
||||
}
|
||||
|
||||
require.Equal(t, num, len(e.shards))
|
||||
|
|
Loading…
Reference in a new issue