From 92e44a45e560e20f55e4303d720ab01ce67b99e3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Jan 2023 13:25:45 +0300 Subject: [PATCH 1/2] [#82] pilorama: Allow to store last sync height Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/tree.go | 38 ++++++++++++++ pkg/local_object_storage/pilorama/boltdb.go | 40 +++++++++++++++ pkg/local_object_storage/pilorama/forest.go | 21 ++++++++ .../pilorama/forest_test.go | 49 +++++++++++++++++++ pkg/local_object_storage/pilorama/inmemory.go | 5 +- .../pilorama/interface.go | 4 ++ pkg/local_object_storage/shard/tree.go | 16 ++++++ 7 files changed, 171 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index c52e345f7..692de98ee 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -213,6 +213,44 @@ func (e *StorageEngine) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { return err == nil, err } +// TreeUpdateLastSyncHeight implements the pilorama.Forest interface. +func (e *StorageEngine) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error { + index, lst, err := e.getTreeShard(cid, treeID) + if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { + return err + } + + err = lst[index].TreeUpdateLastSyncHeight(cid, treeID, height) + if err != nil && !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { + e.reportShardError(lst[index], "can't update tree synchronization height", err, + zap.Stringer("cid", cid), + zap.String("tree", treeID)) + } + return err +} + +// TreeLastSyncHeight implements the pilorama.Forest interface. +func (e *StorageEngine) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) { + var err error + var height uint64 + for _, sh := range e.sortShardsByWeight(cid) { + height, err = sh.TreeLastSyncHeight(cid, treeID) + if err != nil { + if err == shard.ErrPiloramaDisabled { + break + } + if !errors.Is(err, pilorama.ErrTreeNotFound) { + e.reportShardError(sh, "can't read tree synchronization height", err, + zap.Stringer("cid", cid), + zap.String("tree", treeID)) + } + continue + } + return height, err + } + return height, err +} + func (e *StorageEngine) getTreeShard(cid cidSDK.ID, treeID string) (int, []hashedShard, error) { lst := e.sortShardsByWeight(cid) for i, sh := range lst { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 247d07d28..9cf4b0b07 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -192,6 +192,46 @@ func (t *boltForest) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { return exists, err } +var syncHeightKey = []byte{'h'} + +// TreeUpdateLastSyncHeight implements the pilorama.Forest interface. +func (t *boltForest) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error { + rawHeight := make([]byte, 8) + binary.LittleEndian.PutUint64(rawHeight, height) + + buck := bucketName(cid, treeID) + return t.db.Batch(func(tx *bbolt.Tx) error { + treeRoot := tx.Bucket(buck) + if treeRoot == nil { + return ErrTreeNotFound + } + + b := treeRoot.Bucket(dataBucket) + return b.Put(syncHeightKey, rawHeight) + }) +} + +// TreeLastSyncHeight implements the pilorama.Forest interface. +func (t *boltForest) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) { + var height uint64 + + buck := bucketName(cid, treeID) + err := t.db.View(func(tx *bbolt.Tx) error { + treeRoot := tx.Bucket(buck) + if treeRoot == nil { + return ErrTreeNotFound + } + + b := treeRoot.Bucket(dataBucket) + data := b.Get(syncHeightKey) + if len(data) == 8 { + height = binary.LittleEndian.Uint64(data) + } + return nil + }) + return height, err +} + // TreeAddByPath implements the Forest interface. func (t *boltForest) TreeAddByPath(d CIDDescriptor, treeID string, attr string, path []string, meta []KeyValue) ([]Move, error) { if !d.checkValid() { diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 5b1896e20..c8e6a3caf 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -226,3 +226,24 @@ func (f *memoryForest) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { _, ok := f.treeMap[fullID] return ok, nil } + +// TreeUpdateLastSyncHeight implements the pilorama.Forest interface. +func (f *memoryForest) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error { + fullID := cid.EncodeToString() + "/" + treeID + t, ok := f.treeMap[fullID] + if !ok { + return ErrTreeNotFound + } + t.syncHeight = height + return nil +} + +// TreeLastSyncHeight implements the pilorama.Forest interface. +func (f *memoryForest) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) { + fullID := cid.EncodeToString() + "/" + treeID + t, ok := f.treeMap[fullID] + if !ok { + return 0, ErrTreeNotFound + } + return t.syncHeight, nil +} diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index cbd7f5143..323699139 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -1030,3 +1030,52 @@ func testTreeGetTrees(t *testing.T, s Forest) { require.ElementsMatch(t, treeIDs[cid], trees) } } + +func TestTreeLastSyncHeight(t *testing.T) { + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + testTreeLastSyncHeight(t, providers[i].construct(t)) + }) + } +} + +func testTreeLastSyncHeight(t *testing.T, f Forest) { + cnr := cidtest.ID() + treeID := "someTree" + + t.Run("ErrNotFound if no log operations are stored for a tree", func(t *testing.T) { + _, err := f.TreeLastSyncHeight(cnr, treeID) + require.ErrorIs(t, err, ErrTreeNotFound) + + err = f.TreeUpdateLastSyncHeight(cnr, treeID, 1) + require.ErrorIs(t, err, ErrTreeNotFound) + }) + + _, err := f.TreeMove(CIDDescriptor{CID: cnr, Size: 1}, treeID, &Move{ + Parent: RootID, + Child: 1, + }) + require.NoError(t, err) + + h, err := f.TreeLastSyncHeight(cnr, treeID) + require.NoError(t, err) + require.EqualValues(t, 0, h) + + t.Run("separate storages for separate containers", func(t *testing.T) { + _, err := f.TreeLastSyncHeight(cidtest.ID(), treeID) + require.ErrorIs(t, err, ErrTreeNotFound) + }) + + require.NoError(t, f.TreeUpdateLastSyncHeight(cnr, treeID, 10)) + + h, err = f.TreeLastSyncHeight(cnr, treeID) + require.NoError(t, err) + require.EqualValues(t, 10, h) + + t.Run("removed correctly", func(t *testing.T) { + require.NoError(t, f.TreeDrop(cnr, treeID)) + + _, err := f.TreeLastSyncHeight(cnr, treeID) + require.ErrorIs(t, err, ErrTreeNotFound) + }) +} diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index fbd84162c..92dc9b6aa 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -143,8 +143,9 @@ func (s *state) findSpareID() Node { // tree is a mapping from the child nodes to their parent and metadata. type tree struct { - infoMap map[Node]nodeInfo - childMap map[Node][]Node + syncHeight uint64 + infoMap map[Node]nodeInfo + childMap map[Node][]Node } func newTree() *tree { diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 238843dd5..d1a2f7d1b 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -44,6 +44,10 @@ type Forest interface { // TreeExists checks if a tree exists locally. // If the tree is not found, false and a nil error should be returned. TreeExists(cid cidSDK.ID, treeID string) (bool, error) + // TreeUpdateLastSyncHeight updates last log height synchronized with _all_ container nodes. + TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error + // TreeLastSyncHeight returns last log height synchronized with _all_ container nodes. + TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) } type ForestStorage interface { diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 124988479..2194b56d2 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -111,3 +111,19 @@ func (s *Shard) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { } return s.pilorama.TreeExists(cid, treeID) } + +// TreeUpdateLastSyncHeight implements the pilorama.Forest interface. +func (s *Shard) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error { + if s.pilorama == nil { + return ErrPiloramaDisabled + } + return s.pilorama.TreeUpdateLastSyncHeight(cid, treeID, height) +} + +// TreeLastSyncHeight implements the pilorama.Forest interface. +func (s *Shard) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) { + if s.pilorama == nil { + return 0, ErrPiloramaDisabled + } + return s.pilorama.TreeLastSyncHeight(cid, treeID) +} -- 2.45.2 From f6f56113757189a23e8173612de49a883413d29b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Jan 2023 15:44:44 +0300 Subject: [PATCH 2/2] [#82] services/tree: Save last synchronized height in a persistent storage Remember the last synchronized height and use it after service restart. Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + pkg/services/tree/service.go | 8 +++----- pkg/services/tree/sync.go | 35 ++++++++++++++--------------------- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a6324756..6111dda4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Changelog for FrostFS Node - Reload config for pprof and metrics on SIGHUP in `neofs-node` (#1868) - Multiple configs support (#44) - Parameters `nns-name` and `nns-zone` for command `frostfs-cli container create` (#37) +- Tree service now saves the last synchronization height which persists across restarts (#82) ### Changed - Change `frostfs_node_engine_container_size` to counting sizes of logical objects diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index acec01f62..bb3841dfc 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -31,10 +31,8 @@ type Service struct { syncChan chan struct{} syncPool *ants.Pool - // cnrMap maps contrainer and tree ID to the minimum height which was fetched from _each_ client. - // This allows us to better handle split-brain scenario, because we always synchronize - // from the last seen height. The inner map is read-only and should not be modified in-place. - cnrMap map[cidSDK.ID]map[string]uint64 + // cnrMap contains existing (used) container IDs. + cnrMap map[cidSDK.ID]struct{} // cnrMapMtx protects cnrMap cnrMapMtx sync.Mutex } @@ -63,7 +61,7 @@ func New(opts ...Option) *Service { s.replicateLocalCh = make(chan applyOp) s.replicationTasks = make(chan replicationTask, s.replicatorWorkerCount) s.containerCache.init(s.containerCacheSize) - s.cnrMap = make(map[cidSDK.ID]map[string]uint64) + s.cnrMap = make(map[cidSDK.ID]struct{}) s.syncChan = make(chan struct{}) s.syncPool, _ = ants.NewPool(defaultSyncWorkerCount) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 88dd37b7e..327d97434 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -86,31 +86,24 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { return fmt.Errorf("could not fetch tree ID list: %w", outErr) } - s.cnrMapMtx.Lock() - oldStatus := s.cnrMap[cid] - s.cnrMapMtx.Unlock() - - syncStatus := map[string]uint64{} - for i := range treesToSync { - syncStatus[treesToSync[i]] = 0 - } - for tid := range oldStatus { - if _, ok := syncStatus[tid]; ok { - syncStatus[tid] = oldStatus[tid] - } - } - for _, tid := range treesToSync { - h := s.synchronizeTree(ctx, d, syncStatus[tid], tid, nodes) - if syncStatus[tid] < h { - syncStatus[tid] = h + h, err := s.forest.TreeLastSyncHeight(d.CID, tid) + if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { + s.log.Warn("could not get last synchronized height for a tree", + zap.Stringer("cid", d.CID), + zap.String("tree", tid)) + continue + } + newHeight := s.synchronizeTree(ctx, d, h, tid, nodes) + if h < newHeight { + if err := s.forest.TreeUpdateLastSyncHeight(d.CID, tid, newHeight); err != nil { + s.log.Warn("could not update last synchronized height for a tree", + zap.Stringer("cid", d.CID), + zap.String("tree", tid)) + } } } - s.cnrMapMtx.Lock() - s.cnrMap[cid] = syncStatus - s.cnrMapMtx.Unlock() - return nil } -- 2.45.2