a-savchuk/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Pull requests Activity

[#1621] treesvc: Cancel background sync on failure

Browse source 
If applyOperationStream() exits prematurely, other goroutines will block
on send and errgroup will never finish waiting. In this commit we also
check whether context is cancelled.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2025-01-30 10:36:02 +03:00 • committed by Evgenii Stratonikov
parent 1df64c5cab
commit 6fcae9f75a
2 changed files with 26 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
pkg/services/tree/sync.go
View file

@ -131,7 +131,7 @@ func (s *Service) SynchronizeTree(ctx context.Context, cid cid.ID, treeID string
} }
// mergeOperationStreams performs merge sort for node operation streams to one stream. // mergeOperationStreams performs merge sort for node operation streams to one stream.
func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *pilorama.Move) uint64 { func mergeOperationStreams(ctx context.Context, streams []chan *pilorama.Move, merged chan<- *pilorama.Move) uint64 {
defer close(merged) defer close(merged)
// Merging different node streams shuffles incoming operations like that: // Merging different node streams shuffles incoming operations like that:
@ -147,7 +147,11 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram
ms := make([]*pilorama.Move, len(streams)) ms := make([]*pilorama.Move, len(streams))
for i := range streams { for i := range streams {
ms[i] = <-streams[i] select {
case ms[i] = <-streams[i]:
case <-ctx.Done():
return minStreamedLastHeight
}
} }
for { for {
@ -164,7 +168,11 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram
break break
} }
merged <- ms[minTimeMoveIndex] select {
case merged <- ms[minTimeMoveIndex]:
case <-ctx.Done():
return minStreamedLastHeight
}
height := ms[minTimeMoveIndex].Time height := ms[minTimeMoveIndex].Time
if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil { if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil {
minStreamedLastHeight = min(minStreamedLastHeight, height) minStreamedLastHeight = min(minStreamedLastHeight, height)
@ -176,7 +184,7 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram
func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID string, func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID string,
operationStream <-chan *pilorama.Move, operationStream <-chan *pilorama.Move,
) uint64 { ) (uint64, error) {
var prev *pilorama.Move var prev *pilorama.Move
var batch []*pilorama.Move var batch []*pilorama.Move
for m := range operationStream { for m := range operationStream {
@ -189,17 +197,17 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s
if len(batch) == s.syncBatchSize { if len(batch) == s.syncBatchSize {
if err := s.forest.TreeApplyBatch(ctx, cid, treeID, batch); err != nil { if err := s.forest.TreeApplyBatch(ctx, cid, treeID, batch); err != nil {
return batch[0].Time return batch[0].Time, err
} }
batch = batch[:0] batch = batch[:0]
} }
} }
if len(batch) > 0 { if len(batch) > 0 {
if err := s.forest.TreeApplyBatch(ctx, cid, treeID, batch); err != nil { if err := s.forest.TreeApplyBatch(ctx, cid, treeID, batch); err != nil {
return batch[0].Time return batch[0].Time, err
} }
} }
return math.MaxUint64 return math.MaxUint64, nil
} }
func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string,
@ -235,7 +243,11 @@ func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string,
if err := m.Meta.FromBytes(lm.GetMeta()); err != nil { if err := m.Meta.FromBytes(lm.GetMeta()); err != nil {
return err return err
} }
opsCh <- m select {
case opsCh <- m:
case <-ctx.Done():
return ctx.Err()
}
} }
if !errors.Is(err, io.EOF) { if !errors.Is(err, io.EOF) {
return err return err
@ -264,13 +276,14 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64,
merged := make(chan *pilorama.Move) merged := make(chan *pilorama.Move)
var minStreamedLastHeight uint64 var minStreamedLastHeight uint64
errGroup.Go(func() error { errGroup.Go(func() error {
minStreamedLastHeight = mergeOperationStreams(nodeOperationStreams, merged) minStreamedLastHeight = mergeOperationStreams(egCtx, nodeOperationStreams, merged)
return nil return nil
}) })
var minUnappliedHeight uint64 var minUnappliedHeight uint64
errGroup.Go(func() error { errGroup.Go(func() error {
minUnappliedHeight = s.applyOperationStream(ctx, cid, treeID, merged) var err error
return nil minUnappliedHeight, err = s.applyOperationStream(egCtx, cid, treeID, merged)
return err
}) })
var allNodesSynced atomic.Bool var allNodesSynced atomic.Bool

3
pkg/services/tree/sync_test.go
View file

@ -1,6 +1,7 @@
package tree package tree
import ( import (
"context"
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
@ -64,7 +65,7 @@ func Test_mergeOperationStreams(t *testing.T) {
merged := make(chan *pilorama.Move, 1) merged := make(chan *pilorama.Move, 1)
min := make(chan uint64) min := make(chan uint64)
go func() { go func() {
min <- mergeOperationStreams(nodeOpChans, merged) min <- mergeOperationStreams(context.Background(), nodeOpChans, merged)
}() }()
var res []uint64 var res []uint64