From a531eaf8bc246ee77780ac7accee241d265fc14b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 21 Sep 2023 18:18:20 +0300 Subject: [PATCH] [#661] blobstor: Add Rebuild implementation Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 2 ++ .../blobstor/blobovniczatree/rebuild.go | 11 ++++++++ .../blobstor/common/rebuild.go | 20 +++++++++++++ .../blobstor/common/storage.go | 1 + .../blobstor/fstree/fstree.go | 4 +++ .../blobstor/memstore/memstore.go | 4 +++ pkg/local_object_storage/blobstor/rebuild.go | 28 +++++++++++++++++-- .../blobstor/teststore/teststore.go | 4 +++ 8 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go create mode 100644 pkg/local_object_storage/blobstor/common/rebuild.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 0c5e162b3..c1afdadf7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -529,4 +529,6 @@ const ( BlobovniczaTreeFixingFileExtensionForFile = "fixing blobovnicza file extension..." BlobovniczaTreeFixingFileExtensionCompletedSuccessfully = "fixing blobovnicza file extension completed successfully" BlobovniczaTreeFixingFileExtensionFailed = "failed to fix blobovnicza file extension" + BlobstorRebuildFailedToRebuildStorages = "failed to rebuild storages" + BlobstorRebuildRebuildStoragesCompleted = "storages rebuild completed" ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go new file mode 100644 index 000000000..27c645abb --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -0,0 +1,11 @@ +package blobovniczatree + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" +) + +func (b *Blobovniczas) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +} diff --git a/pkg/local_object_storage/blobstor/common/rebuild.go b/pkg/local_object_storage/blobstor/common/rebuild.go new file mode 100644 index 000000000..1cbc2cbd7 --- /dev/null +++ b/pkg/local_object_storage/blobstor/common/rebuild.go @@ -0,0 +1,20 @@ +package common + +import ( + "context" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +type RebuildRes struct { + ObjectsMoved uint64 + FilesRemoved uint64 +} + +type RebuildPrm struct { + MetaStorage MetaStorage +} + +type MetaStorage interface { + UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error +} diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 852f020cc..e552fafea 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -30,4 +30,5 @@ type Storage interface { Put(context.Context, PutPrm) (PutRes, error) Delete(context.Context, DeletePrm) (DeleteRes, error) Iterate(context.Context, IteratePrm) (IterateRes, error) + Rebuild(context.Context, RebuildPrm) (RebuildRes, error) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index d1fe85901..948872fd2 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -574,3 +574,7 @@ func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { func (t *FSTree) SetParentID(parentID string) { t.metrics.SetParentID(parentID) } + +func (t *FSTree) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +} diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index cc4f6921a..39bed00bf 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -159,3 +159,7 @@ func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common } return common.IterateRes{}, nil } + +func (s *memstoreImpl) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +} diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go index 110735fad..29396da7b 100644 --- a/pkg/local_object_storage/blobstor/rebuild.go +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -3,13 +3,37 @@ package blobstor import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.uber.org/zap" ) type StorageIDUpdate interface { UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error } -func (b *BlobStor) Rebuild(_ context.Context, _ StorageIDUpdate) error { - return nil +func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate) error { + var summary common.RebuildRes + var rErr error + for _, storage := range b.storage { + res, err := storage.Storage.Rebuild(ctx, common.RebuildPrm{ + MetaStorage: upd, + }) + summary.FilesRemoved += res.FilesRemoved + summary.ObjectsMoved += res.ObjectsMoved + if err != nil { + b.log.Error(logs.BlobstorRebuildFailedToRebuildStorages, + zap.String("failed_storage_path", storage.Storage.Path()), + zap.String("failed_storage_type", storage.Storage.Type()), + zap.Error(err)) + rErr = err + break + } + } + b.log.Info(logs.BlobstorRebuildRebuildStoragesCompleted, + zap.Bool("success", rErr == nil), + zap.Uint64("total_files_removed", summary.FilesRemoved), + zap.Uint64("total_objects_moved", summary.ObjectsMoved)) + return rErr } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 3e5b21251..c0cdfacf8 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -229,3 +229,7 @@ func (s *TestStore) Iterate(ctx context.Context, req common.IteratePrm) (common. } func (s *TestStore) SetParentID(string) {} + +func (s *TestStore) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +}