From d62c6e4ce68f661ef7b677098a3734392e88686a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Apr 2023 17:01:29 +0300 Subject: [PATCH] [#242] node: Add tracing spans Add tracing spans for PUT requests. Add tracing spans for DELETE requests. Add tracing spans for SELECT requests. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/meta/inspect.go | 4 +- cmd/frostfs-node/notificator.go | 2 +- cmd/frostfs-node/object.go | 24 ++++++------ pkg/core/object/fmt.go | 13 ++++--- pkg/core/object/fmt_test.go | 23 +++++------ .../blobovnicza/blobovnicza_test.go | 2 +- .../blobovnicza/delete.go | 13 ++++++- .../blobstor/blobovniczatree/delete.go | 30 +++++++++----- .../blobstor/blobovniczatree/exists_test.go | 2 +- .../blobstor/blobovniczatree/put.go | 13 ++++++- .../blobstor/blobstor_test.go | 4 +- .../blobstor/common/storage.go | 4 +- pkg/local_object_storage/blobstor/delete.go | 18 +++++++-- .../blobstor/exists_test.go | 2 +- .../blobstor/fstree/fstree.go | 17 +++++++- .../blobstor/internal/blobstortest/common.go | 3 +- .../blobstor/internal/blobstortest/control.go | 4 +- .../blobstor/internal/blobstortest/delete.go | 10 ++--- .../blobstor/internal/blobstortest/iterate.go | 3 +- .../blobstor/iterate_test.go | 3 +- .../blobstor/memstore/memstore.go | 4 +- .../blobstor/memstore/memstore_test.go | 4 +- .../blobstor/perf_test.go | 6 +-- pkg/local_object_storage/blobstor/put.go | 15 ++++++- .../blobstor/teststore/teststore.go | 8 ++-- pkg/local_object_storage/engine/control.go | 2 +- .../engine/control_test.go | 2 +- pkg/local_object_storage/engine/delete.go | 12 +++++- .../engine/delete_test.go | 4 +- .../engine/engine_test.go | 4 +- pkg/local_object_storage/engine/error_test.go | 6 +-- .../engine/evacuate_test.go | 4 +- pkg/local_object_storage/engine/exists.go | 4 +- pkg/local_object_storage/engine/get.go | 12 +++--- pkg/local_object_storage/engine/head_test.go | 4 +- pkg/local_object_storage/engine/inhume.go | 20 ++++++++-- .../engine/inhume_test.go | 10 ++--- pkg/local_object_storage/engine/list_test.go | 3 +- pkg/local_object_storage/engine/lock.go | 27 +++++++++---- pkg/local_object_storage/engine/lock_test.go | 20 +++++----- pkg/local_object_storage/engine/put.go | 36 +++++++++-------- .../engine/remove_copies.go | 2 +- .../engine/remove_copies_test.go | 12 +++--- pkg/local_object_storage/engine/restore.go | 19 +++++++-- pkg/local_object_storage/engine/select.go | 23 ++++++++--- pkg/local_object_storage/engine/tree_test.go | 5 ++- pkg/local_object_storage/engine/writecache.go | 16 +++++++- .../metabase/control_test.go | 3 +- .../metabase/counter_test.go | 17 ++++---- pkg/local_object_storage/metabase/delete.go | 12 +++++- .../metabase/delete_test.go | 3 +- pkg/local_object_storage/metabase/exists.go | 12 +++++- pkg/local_object_storage/metabase/get.go | 13 ++++++- pkg/local_object_storage/metabase/get_test.go | 9 +++-- .../metabase/graveyard_test.go | 13 ++++--- pkg/local_object_storage/metabase/inhume.go | 7 +++- .../metabase/inhume_test.go | 21 +++++----- .../metabase/iterators_test.go | 9 +++-- pkg/local_object_storage/metabase/lock.go | 22 ++++++++++- .../metabase/lock_test.go | 39 ++++++++++--------- pkg/local_object_storage/metabase/movable.go | 12 +++++- .../metabase/movable_test.go | 3 +- pkg/local_object_storage/metabase/put.go | 12 +++++- pkg/local_object_storage/metabase/put_test.go | 3 +- pkg/local_object_storage/metabase/select.go | 12 +++++- .../metabase/select_test.go | 5 ++- .../metabase/storage_id.go | 12 +++++- .../metabase/storage_id_test.go | 3 +- pkg/local_object_storage/shard/control.go | 29 ++++++++------ .../shard/control_test.go | 15 +++---- pkg/local_object_storage/shard/delete.go | 25 ++++++++---- pkg/local_object_storage/shard/delete_test.go | 8 ++-- pkg/local_object_storage/shard/dump_test.go | 24 ++++++------ pkg/local_object_storage/shard/exists.go | 12 +++++- pkg/local_object_storage/shard/gc.go | 10 ++--- pkg/local_object_storage/shard/gc_test.go | 6 +-- pkg/local_object_storage/shard/get.go | 8 ++-- pkg/local_object_storage/shard/get_test.go | 6 +-- pkg/local_object_storage/shard/head.go | 2 +- pkg/local_object_storage/shard/head_test.go | 4 +- pkg/local_object_storage/shard/inhume.go | 13 ++++++- pkg/local_object_storage/shard/inhume_test.go | 2 +- pkg/local_object_storage/shard/list.go | 3 +- pkg/local_object_storage/shard/list_test.go | 3 +- pkg/local_object_storage/shard/lock.go | 28 +++++++++++-- pkg/local_object_storage/shard/lock_test.go | 14 +++---- .../shard/metrics_test.go | 5 ++- pkg/local_object_storage/shard/move.go | 16 +++++++- pkg/local_object_storage/shard/put.go | 19 +++++++-- pkg/local_object_storage/shard/range.go | 2 +- pkg/local_object_storage/shard/range_test.go | 2 +- pkg/local_object_storage/shard/reload_test.go | 13 ++++--- pkg/local_object_storage/shard/restore.go | 17 ++++++-- pkg/local_object_storage/shard/select.go | 15 ++++++- .../shard/shutdown_test.go | 2 +- pkg/local_object_storage/shard/writecache.go | 16 +++++++- pkg/local_object_storage/writecache/delete.go | 15 ++++++- pkg/local_object_storage/writecache/flush.go | 36 ++++++++++------- .../writecache/flush_test.go | 24 ++++++------ pkg/local_object_storage/writecache/init.go | 24 ++++++------ pkg/local_object_storage/writecache/mode.go | 18 +++++++-- .../writecache/options.go | 6 +-- pkg/local_object_storage/writecache/put.go | 19 +++++++-- .../writecache/storage.go | 3 +- .../writecache/writecache.go | 14 ++++--- pkg/services/control/server/flush_cache.go | 4 +- pkg/services/control/server/restore.go | 4 +- pkg/services/object/delete/util.go | 2 +- pkg/services/object/internal/client/client.go | 4 ++ pkg/services/object/put/distributed.go | 2 +- pkg/services/object/put/local.go | 10 ++--- pkg/services/object/put/streamer.go | 4 +- pkg/services/object/put/v2/streamer.go | 20 +++++++++- pkg/services/object/put/validation.go | 6 +-- pkg/services/object/search/local.go | 6 ++- pkg/services/object/search/search.go | 2 +- pkg/services/object/search/search_test.go | 2 +- pkg/services/object/search/service.go | 2 +- pkg/services/object/search/util.go | 4 +- .../object_manager/transformer/fmt.go | 4 +- .../object_manager/transformer/transformer.go | 4 +- .../object_manager/transformer/types.go | 2 +- 122 files changed, 863 insertions(+), 417 deletions(-) diff --git a/cmd/frostfs-lens/internal/meta/inspect.go b/cmd/frostfs-lens/internal/meta/inspect.go index fb0065a6..bc7f28a3 100644 --- a/cmd/frostfs-lens/internal/meta/inspect.go +++ b/cmd/frostfs-lens/internal/meta/inspect.go @@ -36,7 +36,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { storageID := meta.StorageIDPrm{} storageID.SetAddress(addr) - resStorageID, err := db.StorageID(storageID) + resStorageID, err := db.StorageID(cmd.Context(), storageID) common.ExitOnErr(cmd, common.Errf("could not check if the obj is small: %w", err)) if id := resStorageID.StorageID(); id != nil { @@ -51,7 +51,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { siErr := new(object.SplitInfoError) - res, err := db.Get(prm) + res, err := db.Get(cmd.Context(), prm) if errors.As(err, &siErr) { link, linkSet := siErr.SplitInfo().Link() last, lastSet := siErr.SplitInfo().LastPart() diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index 9c90e052..358b39a7 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -42,7 +42,7 @@ func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler for _, c := range listRes.Containers() { selectPrm.WithContainerID(c) - selectRes, err := n.e.Select(selectPrm) + selectRes, err := n.e.Select(ctx, selectPrm) if err != nil { log.Error(logs.FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer, zap.Stringer("cid", c), diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 8f5a83eb..08a202df 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -617,20 +617,20 @@ type engineWithNotifications struct { defaultTopic string } -func (e engineWithNotifications) IsLocked(address oid.Address) (bool, error) { - return e.base.IsLocked(address) +func (e engineWithNotifications) IsLocked(ctx context.Context, address oid.Address) (bool, error) { + return e.base.IsLocked(ctx, address) } func (e engineWithNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error { return e.base.Delete(ctx, tombstone, toDelete) } -func (e engineWithNotifications) Lock(locker oid.Address, toLock []oid.ID) error { - return e.base.Lock(locker, toLock) +func (e engineWithNotifications) Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error { + return e.base.Lock(ctx, locker, toLock) } -func (e engineWithNotifications) Put(o *objectSDK.Object) error { - if err := e.base.Put(o); err != nil { +func (e engineWithNotifications) Put(ctx context.Context, o *objectSDK.Object) error { + if err := e.base.Put(ctx, o); err != nil { return err } @@ -654,8 +654,8 @@ type engineWithoutNotifications struct { engine *engine.StorageEngine } -func (e engineWithoutNotifications) IsLocked(address oid.Address) (bool, error) { - return e.engine.IsLocked(address) +func (e engineWithoutNotifications) IsLocked(ctx context.Context, address oid.Address) (bool, error) { + return e.engine.IsLocked(ctx, address) } func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error { @@ -673,10 +673,10 @@ func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Ad return err } -func (e engineWithoutNotifications) Lock(locker oid.Address, toLock []oid.ID) error { - return e.engine.Lock(locker.Container(), locker.Object(), toLock) +func (e engineWithoutNotifications) Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error { + return e.engine.Lock(ctx, locker.Container(), locker.Object(), toLock) } -func (e engineWithoutNotifications) Put(o *objectSDK.Object) error { - return engine.Put(e.engine, o) +func (e engineWithoutNotifications) Put(ctx context.Context, o *objectSDK.Object) error { + return engine.Put(ctx, e.engine, o) } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 33373b7c..ef99f305 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -1,6 +1,7 @@ package object import ( + "context" "crypto/ecdsa" "errors" "fmt" @@ -42,7 +43,7 @@ type DeleteHandler interface { // LockSource is a source of lock relations between the objects. type LockSource interface { // IsLocked must clarify object's lock status. - IsLocked(address oid.Address) (bool, error) + IsLocked(ctx context.Context, address oid.Address) (bool, error) } // Locker is an object lock storage interface. @@ -89,7 +90,7 @@ func NewFormatValidator(opts ...FormatValidatorOption) *FormatValidator { // If unprepared is true, only fields set by user are validated. // // Returns nil error if the object has valid structure. -func (v *FormatValidator) Validate(obj *object.Object, unprepared bool) error { +func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unprepared bool) error { if obj == nil { return errNilObject } @@ -117,7 +118,7 @@ func (v *FormatValidator) Validate(obj *object.Object, unprepared bool) error { return fmt.Errorf("(%T) could not validate signature key: %w", v, err) } - if err := v.checkExpiration(obj); err != nil { + if err := v.checkExpiration(ctx, obj); err != nil { return fmt.Errorf("object did not pass expiration check: %w", err) } @@ -128,7 +129,7 @@ func (v *FormatValidator) Validate(obj *object.Object, unprepared bool) error { if obj = obj.Parent(); obj != nil { // Parent object already exists. - return v.Validate(obj, false) + return v.Validate(ctx, obj, false) } return nil @@ -327,7 +328,7 @@ func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *C var errExpired = errors.New("object has expired") -func (v *FormatValidator) checkExpiration(obj *object.Object) error { +func (v *FormatValidator) checkExpiration(ctx context.Context, obj *object.Object) error { exp, err := expirationEpochAttribute(obj) if err != nil { if errors.Is(err, errNoExpirationEpoch) { @@ -348,7 +349,7 @@ func (v *FormatValidator) checkExpiration(obj *object.Object) error { addr.SetContainer(cID) addr.SetObject(oID) - locked, err := v.e.IsLocked(addr) + locked, err := v.e.IsLocked(ctx, addr) if err != nil { return fmt.Errorf("locking status check for an expired object: %w", err) } diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 563c7827..be060254 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -1,6 +1,7 @@ package object import ( + "context" "crypto/ecdsa" "strconv" "testing" @@ -40,7 +41,7 @@ type testLockSource struct { m map[oid.Address]bool } -func (t testLockSource) IsLocked(address oid.Address) (bool, error) { +func (t testLockSource) IsLocked(_ context.Context, address oid.Address) (bool, error) { return t.m[address], nil } @@ -62,20 +63,20 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, err) t.Run("nil input", func(t *testing.T) { - require.Error(t, v.Validate(nil, true)) + require.Error(t, v.Validate(context.Background(), nil, true)) }) t.Run("nil identifier", func(t *testing.T) { obj := object.New() - require.ErrorIs(t, v.Validate(obj, false), errNilID) + require.ErrorIs(t, v.Validate(context.Background(), obj, false), errNilID) }) t.Run("nil container identifier", func(t *testing.T) { obj := object.New() obj.SetID(oidtest.ID()) - require.ErrorIs(t, v.Validate(obj, true), errNilCID) + require.ErrorIs(t, v.Validate(context.Background(), obj, true), errNilCID) }) t.Run("unsigned object", func(t *testing.T) { @@ -83,7 +84,7 @@ func TestFormatValidator_Validate(t *testing.T) { obj.SetContainerID(cidtest.ID()) obj.SetID(oidtest.ID()) - require.Error(t, v.Validate(obj, false)) + require.Error(t, v.Validate(context.Background(), obj, false)) }) t.Run("correct w/ session token", func(t *testing.T) { @@ -101,7 +102,7 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) - require.NoError(t, v.Validate(obj, false)) + require.NoError(t, v.Validate(context.Background(), obj, false)) }) t.Run("correct w/o session token", func(t *testing.T) { @@ -109,7 +110,7 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) - require.NoError(t, v.Validate(obj, false)) + require.NoError(t, v.Validate(context.Background(), obj, false)) }) t.Run("tombstone content", func(t *testing.T) { @@ -236,7 +237,7 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("invalid attribute value", func(t *testing.T) { val := "text" - err := v.Validate(fn(val), false) + err := v.Validate(context.Background(), fn(val), false) require.Error(t, err) }) @@ -245,7 +246,7 @@ func TestFormatValidator_Validate(t *testing.T) { obj := fn(val) t.Run("non-locked", func(t *testing.T) { - err := v.Validate(obj, false) + err := v.Validate(context.Background(), obj, false) require.ErrorIs(t, err, errExpired) }) @@ -258,14 +259,14 @@ func TestFormatValidator_Validate(t *testing.T) { addr.SetObject(oID) ls.m[addr] = true - err := v.Validate(obj, false) + err := v.Validate(context.Background(), obj, false) require.NoError(t, err) }) }) t.Run("alive object", func(t *testing.T) { val := strconv.FormatUint(curEpoch, 10) - err := v.Validate(fn(val), true) + err := v.Validate(context.Background(), fn(val), true) require.NoError(t, err) }) }) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 853628fb..5deaf5e4 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -88,7 +88,7 @@ func TestBlobovnicza(t *testing.T) { var dPrm DeletePrm dPrm.SetAddress(addr) - _, err := blz.Delete(dPrm) + _, err := blz.Delete(context.Background(), dPrm) require.NoError(t, err) // should return 404 diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 6ce6f349..29a587cc 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -1,10 +1,15 @@ package blobovnicza import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -30,7 +35,13 @@ func (p *DeletePrm) SetAddress(addr oid.Address) { // Returns an error of type apistatus.ObjectNotFound if the object to be deleted is not in blobovnicza. // // Should not be called in read-only configuration. -func (b *Blobovnicza) Delete(prm DeletePrm) (DeleteRes, error) { +func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Delete", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + addrKey := addressKey(prm.addr) removed := false diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 20280765..073e69b5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -1,13 +1,18 @@ package blobovniczatree import ( + "context" + "encoding/hex" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -15,7 +20,14 @@ import ( // // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. -func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err error) { +func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res common.DeleteRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Delete", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + )) + defer span.End() + if b.readOnly { return common.DeleteRes{}, common.ErrReadOnly } @@ -30,7 +42,7 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e return res, err } - return b.deleteObject(blz, bPrm, prm) + return b.deleteObject(ctx, blz, bPrm, prm) } activeCache := make(map[string]struct{}) @@ -42,7 +54,7 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e // don't process active blobovnicza of the level twice _, ok := activeCache[dirPath] - res, err = b.deleteObjectFromLevel(bPrm, p, !ok, prm) + res, err = b.deleteObjectFromLevel(ctx, bPrm, p, !ok, prm) if err != nil { if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, @@ -73,7 +85,7 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e // tries to delete object from particular blobovnicza. // // returns no error if object was removed from some blobovnicza of the same level. -func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath string, tryActive bool, dp common.DeletePrm) (common.DeleteRes, error) { +func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string, tryActive bool, dp common.DeletePrm) (common.DeleteRes, error) { lvlPath := filepath.Dir(blzPath) // try to remove from blobovnicza if it is opened @@ -81,7 +93,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath v, ok := b.opened.Get(blzPath) b.lruMtx.Unlock() if ok { - if res, err := b.deleteObject(v, prm, dp); err == nil { + if res, err := b.deleteObject(ctx, v, prm, dp); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza, @@ -100,7 +112,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath b.activeMtx.RUnlock() if ok && tryActive { - if res, err := b.deleteObject(active.blz, prm, dp); err == nil { + if res, err := b.deleteObject(ctx, active.blz, prm, dp); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza, @@ -125,11 +137,11 @@ func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath return common.DeleteRes{}, err } - return b.deleteObject(blz, prm, dp) + return b.deleteObject(ctx, blz, prm, dp) } // removes object from blobovnicza and returns common.DeleteRes. -func (b *Blobovniczas) deleteObject(blz *blobovnicza.Blobovnicza, prm blobovnicza.DeletePrm, dp common.DeletePrm) (common.DeleteRes, error) { - _, err := blz.Delete(prm) +func (b *Blobovniczas) deleteObject(ctx context.Context, blz *blobovnicza.Blobovnicza, prm blobovnicza.DeletePrm, dp common.DeletePrm) (common.DeleteRes, error) { + _, err := blz.Delete(ctx, prm) return common.DeleteRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 0c7c61d7..ff927ccb 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -33,7 +33,7 @@ func TestExistsInvalidStorageID(t *testing.T) { d, err := obj.Marshal() require.NoError(t, err) - putRes, err := b.Put(common.PutPrm{Address: addr, RawData: d, DontCompress: true}) + putRes, err := b.Put(context.Background(), common.PutPrm{Address: addr, RawData: d, DontCompress: true}) require.NoError(t, err) t.Run("valid but wrong storage id", func(t *testing.T) { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 8b29119c..ec302d14 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -1,20 +1,31 @@ package blobovniczatree import ( + "context" "errors" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) // Put saves object in the maximum weight blobobnicza. // // returns error if could not save object in any blobovnicza. -func (b *Blobovniczas) Put(prm common.PutPrm) (common.PutRes, error) { +func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + if b.readOnly { return common.PutRes{}, common.ErrReadOnly } diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index 738cd7ee..8c6766dc 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -75,12 +75,12 @@ func TestCompression(t *testing.T) { testPut := func(t *testing.T, b *BlobStor, i int) { var prm common.PutPrm prm.Object = smallObj[i] - _, err := b.Put(prm) + _, err := b.Put(context.Background(), prm) require.NoError(t, err) prm = common.PutPrm{} prm.Object = bigObj[i] - _, err = b.Put(prm) + _, err = b.Put(context.Background(), prm) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index b5d18624..801d32c1 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -23,7 +23,7 @@ type Storage interface { Get(context.Context, GetPrm) (GetRes, error) GetRange(context.Context, GetRangePrm) (GetRangeRes, error) Exists(context.Context, ExistsPrm) (ExistsRes, error) - Put(PutPrm) (PutRes, error) - Delete(DeletePrm) (DeleteRes, error) + Put(context.Context, PutPrm) (PutRes, error) + Delete(context.Context, DeletePrm) (DeleteRes, error) Iterate(IteratePrm) (IterateRes, error) } diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index 8c5a7aba..377214fb 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -1,19 +1,31 @@ package blobstor import ( + "context" + "encoding/hex" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) -func (b *BlobStor) Delete(prm common.DeletePrm) (common.DeleteRes, error) { +func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Delete", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.Delete(prm) + res, err := b.storage[i].Storage.Delete(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { if err == nil { logOp(b.log, deleteOp, prm.Address, b.storage[i].Storage.Type(), prm.StorageID) @@ -31,7 +43,7 @@ func (b *BlobStor) Delete(prm common.DeletePrm) (common.DeleteRes, error) { st = b.storage[0].Storage } - res, err := st.Delete(prm) + res, err := st.Delete(ctx, prm) if err == nil { logOp(b.log, deleteOp, prm.Address, st.Type(), prm.StorageID) } diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index 805d7829..f5c5fbbe 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -36,7 +36,7 @@ func TestExists(t *testing.T) { for i := range objects { var prm common.PutPrm prm.Object = objects[i] - _, err := b.Put(prm) + _, err := b.Put(context.Background(), prm) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 462fbd63..8eb0d5be 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -196,7 +196,13 @@ func (t *FSTree) treePath(addr oid.Address) string { } // Delete removes the object with the specified address from the storage. -func (t *FSTree) Delete(prm common.DeletePrm) (common.DeleteRes, error) { +func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Delete", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + )) + defer span.End() + if t.readOnly { return common.DeleteRes{}, common.ErrReadOnly } @@ -230,7 +236,14 @@ func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exist } // Put puts an object in the storage. -func (t *FSTree) Put(prm common.PutPrm) (common.PutRes, error) { +func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + if t.readOnly { return common.PutRes{}, common.ErrReadOnly } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go index b2663be2..e31f3280 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "math/rand" "testing" @@ -67,7 +68,7 @@ func prepare(t *testing.T, count int, s common.Storage, min, max uint64) []objec prm.Object = objects[i].obj prm.RawData = objects[i].raw - putRes, err := s.Put(prm) + putRes, err := s.Put(context.Background(), prm) require.NoError(t, err) objects[i].storageID = putRes.StorageID diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go index 350bea96..96d54dec 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go @@ -36,7 +36,7 @@ func TestControl(t *testing.T, cons Constructor, min, max uint64) { prm.Object = NewObject(min + uint64(rand.Intn(int(max-min+1)))) prm.Address = objectCore.AddressOf(prm.Object) - _, err := s.Put(prm) + _, err := s.Put(context.Background(), prm) require.ErrorIs(t, err, common.ErrReadOnly) }) t.Run("delete fails", func(t *testing.T) { @@ -44,7 +44,7 @@ func TestControl(t *testing.T, cons Constructor, min, max uint64) { prm.Address = objects[0].addr prm.StorageID = objects[0].storageID - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.ErrorIs(t, err, common.ErrReadOnly) }) } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index ad004531..7532a5b5 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -22,7 +22,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { var prm common.DeletePrm prm.Address = oidtest.Address() - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.Error(t, err, new(apistatus.ObjectNotFound)) }) @@ -31,7 +31,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { prm.Address = objects[0].addr prm.StorageID = objects[0].storageID - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.NoError(t, err) t.Run("exists fail", func(t *testing.T) { @@ -55,7 +55,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { var prm common.DeletePrm prm.Address = objects[1].addr - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.NoError(t, err) }) @@ -64,10 +64,10 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { prm.Address = objects[2].addr prm.StorageID = objects[2].storageID - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.NoError(t, err) - _, err = s.Delete(prm) + _, err = s.Delete(context.Background(), prm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index f98cca63..83ada960 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "errors" "testing" @@ -22,7 +23,7 @@ func TestIterate(t *testing.T, cons Constructor, min, max uint64) { var delPrm common.DeletePrm delPrm.Address = objects[2].addr delPrm.StorageID = objects[2].storageID - _, err := s.Delete(delPrm) + _, err := s.Delete(context.Background(), delPrm) require.NoError(t, err) objects = append(objects[:delID], objects[delID+1:]...) diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index b2a7ddfb..6488ff5f 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "encoding/binary" "os" "testing" @@ -63,7 +64,7 @@ func TestIterateObjects(t *testing.T) { } for _, v := range mObjs { - _, err := blobStor.Put(common.PutPrm{Address: v.addr, RawData: v.data}) + _, err := blobStor.Put(context.Background(), common.PutPrm{Address: v.addr, RawData: v.data}) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index 4068d742..e435cfef 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -91,7 +91,7 @@ func (s *memstoreImpl) Exists(_ context.Context, req common.ExistsPrm) (common.E return common.ExistsRes{Exists: exists}, nil } -func (s *memstoreImpl) Put(req common.PutPrm) (common.PutRes, error) { +func (s *memstoreImpl) Put(_ context.Context, req common.PutPrm) (common.PutRes, error) { if s.readOnly { return common.PutRes{}, common.ErrReadOnly } @@ -108,7 +108,7 @@ func (s *memstoreImpl) Put(req common.PutPrm) (common.PutRes, error) { return common.PutRes{StorageID: []byte(s.rootPath)}, nil } -func (s *memstoreImpl) Delete(req common.DeletePrm) (common.DeleteRes, error) { +func (s *memstoreImpl) Delete(_ context.Context, req common.DeletePrm) (common.DeleteRes, error) { if s.readOnly { return common.DeleteRes{}, common.ErrReadOnly } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index 6482b2cf..12527629 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -28,7 +28,7 @@ func TestSimpleLifecycle(t *testing.T) { require.NoError(t, err) { - _, err := s.Put(common.PutPrm{Address: addr, RawData: d, DontCompress: true}) + _, err := s.Put(context.Background(), common.PutPrm{Address: addr, RawData: d, DontCompress: true}) require.NoError(t, err) } @@ -57,7 +57,7 @@ func TestSimpleLifecycle(t *testing.T) { } { - _, err := s.Delete(common.DeletePrm{Address: addr}) + _, err := s.Delete(context.Background(), common.DeletePrm{Address: addr}) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index d2359335..c88dc85e 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -114,7 +114,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { if err != nil { return fmt.Errorf("marshal: %v", err) } - _, err = st.Put(common.PutPrm{ + _, err = st.Put(context.Background(), common.PutPrm{ Address: addr, RawData: raw, }) @@ -165,7 +165,7 @@ func BenchmarkSubstorageWritePerf(b *testing.B) { addr := testutil.AddressFromObject(b, obj) raw, err := obj.Marshal() require.NoError(b, err) - if _, err := st.Put(common.PutPrm{ + if _, err := st.Put(context.Background(), common.PutPrm{ Address: addr, RawData: raw, }); err != nil { @@ -202,7 +202,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { addr := testutil.AddressFromObject(b, obj) raw, err := obj.Marshal() require.NoError(b, err) - if _, err := st.Put(common.PutPrm{ + if _, err := st.Put(context.Background(), common.PutPrm{ Address: addr, RawData: raw, }); err != nil { diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index a4009ae4..2ae7f0fe 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -1,12 +1,16 @@ package blobstor import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrNoPlaceFound is returned when object can't be saved to any sub-storage component @@ -21,7 +25,14 @@ var ErrNoPlaceFound = logicerr.New("couldn't find a place to store an object") // // Returns any error encountered that // did not allow to completely save the object. -func (b *BlobStor) Put(prm common.PutPrm) (common.PutRes, error) { +func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() @@ -39,7 +50,7 @@ func (b *BlobStor) Put(prm common.PutPrm) (common.PutRes, error) { for i := range b.storage { if b.storage[i].Policy == nil || b.storage[i].Policy(prm.Object, prm.RawData) { - res, err := b.storage[i].Storage.Put(prm) + res, err := b.storage[i].Storage.Put(ctx, prm) if err == nil { logOp(b.log, putOp, prm.Address, b.storage[i].Storage.Type(), res.StorageID) } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 03f64f0f..24d742fd 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -176,27 +176,27 @@ func (s *TestStore) Exists(ctx context.Context, req common.ExistsPrm) (common.Ex } } -func (s *TestStore) Put(req common.PutPrm) (common.PutRes, error) { +func (s *TestStore) Put(ctx context.Context, req common.PutPrm) (common.PutRes, error) { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Put != nil: return s.overrides.Put(req) case s.st != nil: - return s.st.Put(req) + return s.st.Put(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Put(%+v)", req)) } } -func (s *TestStore) Delete(req common.DeletePrm) (common.DeleteRes, error) { +func (s *TestStore) Delete(ctx context.Context, req common.DeletePrm) (common.DeleteRes, error) { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Delete != nil: return s.overrides.Delete(req) case s.st != nil: - return s.st.Delete(req) + return s.st.Delete(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Delete(%+v)", req)) } diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 0c422ccc..9ad4fcf9 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -308,7 +308,7 @@ loop: e.removeShards(shardsToRemove...) for _, p := range shardsToReload { - err := p.sh.Reload(p.opts...) + err := p.sh.Reload(ctx, p.opts...) if err != nil { e.log.Error(logs.EngineCouldNotReloadAShard, zap.Stringer("shard id", p.sh.ID()), diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 91bec63a..046968db 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -204,7 +204,7 @@ func TestExecBlocks(t *testing.T) { addr := object.AddressOf(obj) - require.NoError(t, Put(e, obj)) + require.NoError(t, Put(context.Background(), e, obj)) // block executions errBlock := errors.New("block exec err") diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 1f3c142a..f9b9c9a8 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -4,11 +4,14 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -47,6 +50,13 @@ func (p *DeletePrm) WithForceRemoval() { // on operations with that object) if WithForceRemoval option has // been provided. func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Delete", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + attribute.Bool("force_removal", prm.forceRemoval), + )) + defer span.End() + err = e.execIfNotBlocked(func() error { res, err = e.delete(ctx, prm) return err @@ -135,7 +145,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo } e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { - res, err := sh.Select(selectPrm) + res, err := sh.Select(ctx, selectPrm) if err != nil { e.log.Warn(logs.EngineErrorDuringSearchingForObjectChildren, zap.Stringer("addr", addr), diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 259a40a7..53c62981 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -59,9 +59,9 @@ func TestDeleteBigObject(t *testing.T) { defer e.Close() for i := range children { - require.NoError(t, Put(e, children[i])) + require.NoError(t, Put(context.Background(), e, children[i])) } - require.NoError(t, Put(e, link)) + require.NoError(t, Put(context.Background(), e, link)) var splitErr *objectSDK.SplitInfoError diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index ddaf88d1..4d2ddc10 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -60,7 +60,7 @@ func benchmarkExists(b *testing.B, shardNum int) { addr := oidtest.Address() for i := 0; i < 100; i++ { obj := testutil.GenerateObjectWithCID(cidtest.ID()) - err := Put(e, obj) + err := Put(context.Background(), e, obj) if err != nil { b.Fatal(err) } @@ -69,7 +69,7 @@ func benchmarkExists(b *testing.B, shardNum int) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - ok, err := e.exists(addr) + ok, err := e.exists(context.Background(), addr) if err != nil || ok { b.Fatalf("%t %v", ok, err) } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 4ff019e4..017b635d 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -98,7 +98,7 @@ func TestErrorReporting(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err := te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + _, err := te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err) @@ -132,7 +132,7 @@ func TestErrorReporting(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err := te.ng.shards[te.shards[0].id.String()].Put(prm) + _, err := te.ng.shards[te.shards[0].id.String()].Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err) @@ -185,7 +185,7 @@ func TestBlobstorFailback(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err = te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + _, err = te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err) objs = append(objs, obj) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index c116aeff..291bc2b7 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -57,7 +57,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := e.shards[sh.String()].Put(putPrm) + _, err := e.shards[sh.String()].Put(context.Background(), putPrm) require.NoError(t, err) } @@ -67,7 +67,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng var putPrm PutPrm putPrm.WithObject(objects[len(objects)-1]) - _, err := e.Put(putPrm) + err := e.Put(context.Background(), putPrm) require.NoError(t, err) res, err := e.shards[ids[len(ids)-1].String()].List() diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index 3a8e09a6..6208461e 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -10,14 +10,14 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -func (e *StorageEngine) exists(addr oid.Address) (bool, error) { +func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, error) { var shPrm shard.ExistsPrm shPrm.SetAddress(addr) alreadyRemoved := false exists := false e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { - res, err := sh.Exists(context.TODO(), shPrm) + res, err := sh.Exists(ctx, shPrm) if err != nil { if shard.IsErrRemoved(err) { alreadyRemoved = true diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 7d17b50f..683b7bde 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -48,6 +48,12 @@ func (r GetRes) Object() *objectSDK.Object { // // Returns an error if executions are blocked (see BlockExecution). func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Get", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + err = e.execIfNotBlocked(func() error { res, err = e.get(ctx, prm) return err @@ -57,12 +63,6 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er } func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.get", - trace.WithAttributes( - attribute.String("address", prm.addr.EncodeToString()), - )) - defer span.End() - if e.metrics != nil { defer elapsed(e.metrics.AddGetDuration)() } diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index e5fd4b04..bf00c428 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -55,11 +55,11 @@ func TestHeadRaw(t *testing.T) { putPrmLink.SetObject(link) // put most left object in one shard - _, err := s1.Put(putPrmLeft) + _, err := s1.Put(context.Background(), putPrmLeft) require.NoError(t, err) // put link object in another shard - _, err = s2.Put(putPrmLink) + _, err = s2.Put(context.Background(), putPrmLink) require.NoError(t, err) // head with raw flag should return SplitInfoError diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 696e7874..b1204ed9 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -4,12 +4,15 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -62,6 +65,9 @@ var errInhumeFailure = errors.New("inhume operation failed") // // Returns an error if executions are blocked (see BlockExecution). func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Inhume") + defer span.End() + err = e.execIfNotBlocked(func() error { res, err = e.inhume(ctx, prm) return err @@ -82,7 +88,7 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e for i := range prm.addrs { if !prm.forceRemoval { - locked, err := e.IsLocked(prm.addrs[i]) + locked, err := e.IsLocked(ctx, prm.addrs[i]) if err != nil { e.log.Warn(logs.EngineRemovingAnObjectWithoutFullLockingCheck, zap.Error(err), @@ -181,13 +187,19 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh } // IsLocked checks whether an object is locked according to StorageEngine's state. -func (e *StorageEngine) IsLocked(addr oid.Address) (bool, error) { +func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.IsLocked", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + var locked bool var err error var outErr error e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { - locked, err = h.Shard.IsLocked(addr) + locked, err = h.Shard.IsLocked(ctx, addr) if err != nil { e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("addr", addr)) outErr = err @@ -206,7 +218,7 @@ func (e *StorageEngine) IsLocked(addr oid.Address) (bool, error) { func (e *StorageEngine) processExpiredTombstones(ctx context.Context, addrs []meta.TombstonedObject) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - sh.HandleExpiredTombstones(addrs) + sh.HandleExpiredTombstones(ctx, addrs) select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 4f8c96b9..924cf518 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -42,7 +42,7 @@ func TestStorageEngine_Inhume(t *testing.T) { e := testNewEngine(t).setShardsNum(t, 1).engine defer e.Close() - err := Put(e, parent) + err := Put(context.Background(), e, parent) require.NoError(t, err) var inhumePrm InhumePrm @@ -51,7 +51,7 @@ func TestStorageEngine_Inhume(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - addrs, err := Select(e, cnr, fs) + addrs, err := Select(context.Background(), e, cnr, fs) require.NoError(t, err) require.Empty(t, addrs) }) @@ -65,12 +65,12 @@ func TestStorageEngine_Inhume(t *testing.T) { var putChild shard.PutPrm putChild.SetObject(child) - _, err := s1.Put(putChild) + _, err := s1.Put(context.Background(), putChild) require.NoError(t, err) var putLink shard.PutPrm putLink.SetObject(link) - _, err = s2.Put(putLink) + _, err = s2.Put(context.Background(), putLink) require.NoError(t, err) var inhumePrm InhumePrm @@ -79,7 +79,7 @@ func TestStorageEngine_Inhume(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - addrs, err := Select(e, cnr, fs) + addrs, err := Select(context.Background(), e, cnr, fs) require.NoError(t, err) require.Empty(t, addrs) }) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 1261de9d..fde799d0 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "os" "sort" @@ -35,7 +36,7 @@ func TestListWithCursor(t *testing.T) { var prm PutPrm prm.WithObject(obj) - _, err := e.Put(prm) + err := e.Put(context.Background(), prm) require.NoError(t, err) expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) } diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 60a1d9c9..4562c1a5 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -4,12 +4,15 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var errLockFailed = errors.New("lock operation failed") @@ -20,19 +23,27 @@ var errLockFailed = errors.New("lock operation failed") // Allows locking regular objects only (otherwise returns apistatus.LockNonRegularObject). // // Locked list should be unique. Panics if it is empty. -func (e *StorageEngine) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { +func (e *StorageEngine) Lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked []oid.ID) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Lock", + trace.WithAttributes( + attribute.String("container_id", idCnr.EncodeToString()), + attribute.String("locker", locker.EncodeToString()), + attribute.Int("locked_count", len(locked)), + )) + defer span.End() + return e.execIfNotBlocked(func() error { - return e.lock(idCnr, locker, locked) + return e.lock(ctx, idCnr, locker, locked) }) } -func (e *StorageEngine) lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { +func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked []oid.ID) error { for i := range locked { - switch e.lockSingle(idCnr, locker, locked[i], true) { + switch e.lockSingle(ctx, idCnr, locker, locked[i], true) { case 1: return logicerr.Wrap(apistatus.LockNonRegularObject{}) case 0: - switch e.lockSingle(idCnr, locker, locked[i], false) { + switch e.lockSingle(ctx, idCnr, locker, locked[i], false) { case 1: return logicerr.Wrap(apistatus.LockNonRegularObject{}) case 0: @@ -48,7 +59,7 @@ func (e *StorageEngine) lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error // - 0: fail // - 1: locking irregular object // - 2: ok -func (e *StorageEngine) lockSingle(idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { +func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { // code is pretty similar to inhumeAddr, maybe unify? root := false var errIrregular apistatus.LockNonRegularObject @@ -70,7 +81,7 @@ func (e *StorageEngine) lockSingle(idCnr cid.ID, locker, locked oid.ID, checkExi var existsPrm shard.ExistsPrm existsPrm.SetAddress(addrLocked) - exRes, err := sh.Exists(context.TODO(), existsPrm) + exRes, err := sh.Exists(ctx, existsPrm) if err != nil { var siErr *objectSDK.SplitInfoError if !errors.As(err, &siErr) { @@ -90,7 +101,7 @@ func (e *StorageEngine) lockSingle(idCnr cid.ID, locker, locked oid.ID, checkExi } } - err := sh.Lock(idCnr, locker, []oid.ID{locked}) + err := sh.Lock(ctx, idCnr, locker, []oid.ID{locked}) if err != nil { e.reportShardError(sh, "could not lock object in shard", err) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index fd3b04ef..4c89b922 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -99,7 +99,7 @@ func TestLockUserScenario(t *testing.T) { id, _ := obj.ID() objAddr.SetObject(id) - err = Put(e, obj) + err = Put(context.Background(), e, obj) require.NoError(t, err) // 2. @@ -107,10 +107,10 @@ func TestLockUserScenario(t *testing.T) { locker.WriteMembers([]oid.ID{id}) object.WriteLock(lockerObj, locker) - err = Put(e, lockerObj) + err = Put(context.Background(), e, lockerObj) require.NoError(t, err) - err = e.Lock(cnr, lockerID, []oid.ID{id}) + err = e.Lock(context.Background(), cnr, lockerID, []oid.ID{id}) require.NoError(t, err) // 3. @@ -125,7 +125,7 @@ func TestLockUserScenario(t *testing.T) { tombObj.SetID(tombForLockID) tombObj.SetAttributes(a) - err = Put(e, tombObj) + err = Put(context.Background(), e, tombObj) require.NoError(t, err) inhumePrm.WithTarget(tombForLockAddr, lockerAddr) @@ -180,7 +180,7 @@ func TestLockExpiration(t *testing.T) { // 1. obj := testutil.GenerateObjectWithCID(cnr) - err = Put(e, obj) + err = Put(context.Background(), e, obj) require.NoError(t, err) // 2. @@ -192,13 +192,13 @@ func TestLockExpiration(t *testing.T) { lock.SetType(object.TypeLock) lock.SetAttributes(a) - err = Put(e, lock) + err = Put(context.Background(), e, lock) require.NoError(t, err) id, _ := obj.ID() idLock, _ := lock.ID() - err = e.Lock(cnr, idLock, []oid.ID{id}) + err = e.Lock(context.Background(), cnr, idLock, []oid.ID{id}) require.NoError(t, err) var inhumePrm InhumePrm @@ -255,20 +255,20 @@ func TestLockForceRemoval(t *testing.T) { // 1. obj := testutil.GenerateObjectWithCID(cnr) - err = Put(e, obj) + err = Put(context.Background(), e, obj) require.NoError(t, err) // 2. lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(object.TypeLock) - err = Put(e, lock) + err = Put(context.Background(), e, lock) require.NoError(t, err) id, _ := obj.ID() idLock, _ := lock.ID() - err = e.Lock(cnr, idLock, []oid.ID{id}) + err = e.Lock(context.Background(), cnr, idLock, []oid.ID{id}) require.NoError(t, err) // 3. diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index aea296cc..0543f9f1 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -12,6 +13,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -20,9 +23,6 @@ type PutPrm struct { obj *objectSDK.Object } -// PutRes groups the resulting values of Put operation. -type PutRes struct{} - var errPutShard = errors.New("could not put object to any shard") // WithObject is a Put option to set object to save. @@ -40,16 +40,22 @@ func (p *PutPrm) WithObject(obj *objectSDK.Object) { // Returns an error if executions are blocked (see BlockExecution). // // Returns an error of type apistatus.ObjectAlreadyRemoved if the object has been marked as removed. -func (e *StorageEngine) Put(prm PutPrm) (res PutRes, err error) { +func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Put", + trace.WithAttributes( + attribute.String("address", object.AddressOf(prm.obj).EncodeToString()), + )) + defer span.End() + err = e.execIfNotBlocked(func() error { - res, err = e.put(prm) + err = e.put(ctx, prm) return err }) return } -func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { +func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { if e.metrics != nil { defer elapsed(e.metrics.AddPutDuration)() } @@ -58,9 +64,9 @@ func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { // In #1146 this check was parallelized, however, it became // much slower on fast machines for 4 shards. - _, err := e.exists(addr) + _, err := e.exists(ctx, addr) if err != nil { - return PutRes{}, err + return err } finished := false @@ -74,7 +80,7 @@ func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { return false } - putDone, exists := e.putToShard(context.TODO(), sh, ind, pool, addr, prm.obj) + putDone, exists := e.putToShard(ctx, sh, ind, pool, addr, prm.obj) finished = putDone || exists return finished }) @@ -83,7 +89,7 @@ func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { err = errPutShard } - return PutRes{}, err + return err } // putToShard puts object to sh. @@ -117,7 +123,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, var toMoveItPrm shard.ToMoveItPrm toMoveItPrm.SetAddress(addr) - _, err = sh.ToMoveIt(toMoveItPrm) + _, err = sh.ToMoveIt(ctx, toMoveItPrm) if err != nil { e.log.Warn(logs.EngineCouldNotMarkObjectForShardRelocation, zap.Stringer("shard", sh.ID()), @@ -132,7 +138,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err = sh.Put(putPrm) + _, err = sh.Put(ctx, putPrm) if err != nil { if errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, blobstor.ErrNoPlaceFound) || errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { @@ -157,11 +163,9 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, } // Put writes provided object to local storage. -func Put(storage *StorageEngine, obj *objectSDK.Object) error { +func Put(ctx context.Context, storage *StorageEngine, obj *objectSDK.Object) error { var putPrm PutPrm putPrm.WithObject(obj) - _, err := storage.Put(putPrm) - - return err + return storage.Put(ctx, putPrm) } diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index d365fc7b..1ea56992 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -129,7 +129,7 @@ func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address var deletePrm shard.DeletePrm deletePrm.SetAddresses(addr) - _, err = shards[i].Delete(deletePrm) + _, err = shards[i].Delete(ctx, deletePrm) if err != nil { return err } diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go index 4415d01c..c53e03bb 100644 --- a/pkg/local_object_storage/engine/remove_copies_test.go +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -49,10 +49,10 @@ func TestRebalance(t *testing.T) { te.ng.mtx.RLock() // Every 3rd object (i%3 == 0) is put to both shards, others are distributed. if i%3 != 1 { - _, err1 = te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + _, err1 = te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) } if i%3 != 2 { - _, err2 = te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + _, err2 = te.ng.shards[te.shards[1].id.String()].Shard.Put(context.Background(), prm) } te.ng.mtx.RUnlock() @@ -109,8 +109,8 @@ func TestRebalanceSingleThread(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) - _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) + _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err1) require.NoError(t, err2) @@ -162,8 +162,8 @@ func TestRebalanceExitByContext(t *testing.T) { prm.SetObject(objects[i]) te.ng.mtx.RLock() - _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) - _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) + _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err1) diff --git a/pkg/local_object_storage/engine/restore.go b/pkg/local_object_storage/engine/restore.go index 84c750cd..7cc2eaf6 100644 --- a/pkg/local_object_storage/engine/restore.go +++ b/pkg/local_object_storage/engine/restore.go @@ -1,11 +1,24 @@ package engine -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) // RestoreShard restores objects from dump to the shard with provided identifier. // // Returns an error if shard is not read-only. -func (e *StorageEngine) RestoreShard(id *shard.ID, prm shard.RestorePrm) error { +func (e *StorageEngine) RestoreShard(ctx context.Context, id *shard.ID, prm shard.RestorePrm) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.RestoreShard", + trace.WithAttributes( + attribute.String("shard_id", id.String()), + )) + defer span.End() + e.mtx.RLock() defer e.mtx.RUnlock() @@ -14,6 +27,6 @@ func (e *StorageEngine) RestoreShard(id *shard.ID, prm shard.RestorePrm) error { return errShardNotFound } - _, err := sh.Restore(prm) + _, err := sh.Restore(ctx, prm) return err } diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 7b9b8be6..e1039ea2 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -1,10 +1,15 @@ package engine import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // SelectPrm groups the parameters of Select operation. @@ -38,16 +43,22 @@ func (r SelectRes) AddressList() []oid.Address { // Returns any error encountered that did not allow to completely select the objects. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) Select(prm SelectPrm) (res SelectRes, err error) { +func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Select", + trace.WithAttributes( + attribute.String("container_id", prm.cnr.EncodeToString()), + )) + defer span.End() + err = e.execIfNotBlocked(func() error { - res, err = e._select(prm) + res, err = e._select(ctx, prm) return err }) return } -func (e *StorageEngine) _select(prm SelectPrm) (SelectRes, error) { +func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { if e.metrics != nil { defer elapsed(e.metrics.AddSearchDuration)() } @@ -62,7 +73,7 @@ func (e *StorageEngine) _select(prm SelectPrm) (SelectRes, error) { shPrm.SetFilters(prm.filters) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.Select(shPrm) + res, err := sh.Select(ctx, shPrm) if err != nil { e.reportShardError(sh, "could not select objects from shard", err) return false @@ -133,12 +144,12 @@ func (e *StorageEngine) list(limit uint64) (SelectRes, error) { } // Select selects objects from local storage using provided filters. -func Select(storage *StorageEngine, cnr cid.ID, fs object.SearchFilters) ([]oid.Address, error) { +func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs object.SearchFilters) ([]oid.Address, error) { var selectPrm SelectPrm selectPrm.WithContainerID(cnr) selectPrm.WithFilters(fs) - res, err := storage.Select(selectPrm) + res, err := storage.Select(ctx, selectPrm) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 611c691f..77573c9e 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "strconv" "testing" @@ -31,7 +32,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { for i := 0; i < objCount; i++ { obj := testutil.GenerateObjectWithCID(cid) testutil.AddAttribute(obj, pilorama.AttributeFilename, strconv.Itoa(i)) - err := Put(te.ng, obj) + err := Put(context.Background(), te.ng, obj) if err != nil { b.Fatal(err) } @@ -51,7 +52,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { prm.WithFilters(fs) for i := 0; i < b.N; i++ { - res, err := te.ng.Select(prm) + res, err := te.ng.Select(context.Background(), prm) if err != nil { b.Fatal(err) } diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 26600a3e..4effb2b1 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -1,7 +1,12 @@ package engine import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // FlushWriteCachePrm groups the parameters of FlushWriteCache operation. @@ -26,7 +31,14 @@ func (p *FlushWriteCachePrm) SetIgnoreErrors(ignore bool) { type FlushWriteCacheRes struct{} // FlushWriteCache flushes write-cache on a single shard. -func (e *StorageEngine) FlushWriteCache(p FlushWriteCachePrm) (FlushWriteCacheRes, error) { +func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) (FlushWriteCacheRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.FlushWriteCache", + trace.WithAttributes( + attribute.String("shard)id", p.shardID.String()), + attribute.Bool("ignore_errors", p.ignoreErrors), + )) + defer span.End() + e.mtx.RLock() sh, ok := e.shards[p.shardID.String()] e.mtx.RUnlock() @@ -38,5 +50,5 @@ func (e *StorageEngine) FlushWriteCache(p FlushWriteCachePrm) (FlushWriteCacheRe var prm shard.FlushWriteCachePrm prm.SetIgnoreErrors(p.ignoreErrors) - return FlushWriteCacheRes{}, sh.FlushWriteCache(prm) + return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm) } diff --git a/pkg/local_object_storage/metabase/control_test.go b/pkg/local_object_storage/metabase/control_test.go index 17f3b389..b67e748b 100644 --- a/pkg/local_object_storage/metabase/control_test.go +++ b/pkg/local_object_storage/metabase/control_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -55,6 +56,6 @@ func metaExists(db *meta.DB, addr oid.Address) (bool, error) { var existsPrm meta.ExistsPrm existsPrm.SetAddress(addr) - res, err := db.Exists(existsPrm) + res, err := db.Exists(context.Background(), existsPrm) return res.Exists(), err } diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index d93bc436..17a593b6 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -38,7 +39,7 @@ func TestCounters(t *testing.T) { for i := 0; i < objCount; i++ { prm.SetObject(oo[i]) - _, err = db.Put(prm) + _, err = db.Put(context.Background(), prm) require.NoError(t, err) c, err = db.ObjectCounters() @@ -58,7 +59,7 @@ func TestCounters(t *testing.T) { for i := objCount - 1; i >= 0; i-- { prm.SetAddresses(objectcore.AddressOf(oo[i])) - res, err := db.Delete(prm) + res, err := db.Delete(context.Background(), prm) require.NoError(t, err) require.Equal(t, uint64(1), res.AvailableObjectsRemoved()) @@ -89,7 +90,7 @@ func TestCounters(t *testing.T) { prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) - res, err := db.Inhume(prm) + res, err := db.Inhume(context.Background(), prm) require.NoError(t, err) require.Equal(t, uint64(len(inhumedObjs)), res.AvailableInhumed()) @@ -159,7 +160,7 @@ func TestCounters(t *testing.T) { prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) c, err = db.ObjectCounters() @@ -223,7 +224,7 @@ func TestCounters_Expired(t *testing.T) { inhumePrm.SetGCMark() inhumePrm.SetAddresses(oo[0]) - inhumeRes, err := db.Inhume(inhumePrm) + inhumeRes, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Equal(t, uint64(1), inhumeRes.AvailableInhumed()) @@ -240,7 +241,7 @@ func TestCounters_Expired(t *testing.T) { var deletePrm meta.DeletePrm deletePrm.SetAddresses(oo[0]) - deleteRes, err := db.Delete(deletePrm) + deleteRes, err := db.Delete(context.Background(), deletePrm) require.NoError(t, err) require.Zero(t, deleteRes.AvailableObjectsRemoved()) @@ -257,7 +258,7 @@ func TestCounters_Expired(t *testing.T) { deletePrm.SetAddresses(oo[0]) - deleteRes, err = db.Delete(deletePrm) + deleteRes, err = db.Delete(context.Background(), deletePrm) require.NoError(t, err) require.Equal(t, uint64(1), deleteRes.AvailableObjectsRemoved()) @@ -284,7 +285,7 @@ func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*object.Ob oo = append(oo, o) prm.SetObject(o) - _, err = db.Put(prm) + _, err = db.Put(context.Background(), prm) require.NoError(t, err) c, err := db.ObjectCounters() diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 79f87037..990d3997 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -2,15 +2,19 @@ package meta import ( "bytes" + "context" "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // DeletePrm groups the parameters of Delete operation. @@ -65,7 +69,13 @@ type referenceNumber struct { type referenceCounter map[string]*referenceNumber // Delete removed object records from metabase indexes. -func (db *DB) Delete(prm DeletePrm) (DeleteRes, error) { +func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Delete", + trace.WithAttributes( + attribute.Int("addr_count", len(prm.addrs)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index ee161a88..d2a4bfa7 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "errors" "testing" @@ -139,6 +140,6 @@ func metaDelete(db *meta.DB, addrs ...oid.Address) error { var deletePrm meta.DeletePrm deletePrm.SetAddresses(addrs...) - _, err := db.Delete(deletePrm) + _, err := db.Delete(context.Background(), deletePrm) return err } diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 686b6588..cfd37b0d 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -1,15 +1,19 @@ package meta import ( + "context" "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ExistsPrm groups the parameters of Exists operation. @@ -39,7 +43,13 @@ func (p ExistsRes) Exists() bool { // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (db *DB) Exists(prm ExistsPrm) (res ExistsRes, err error) { +func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Exists", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index c0feda06..fff32d6a 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -1,14 +1,18 @@ package meta import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // GetPrm groups the parameters of Get operation. @@ -46,7 +50,14 @@ func (r GetRes) Header() *objectSDK.Object { // Returns an error of type apistatus.ObjectNotFound if object is missing in DB. // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (db *DB) Get(prm GetPrm) (res GetRes, err error) { +func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Get", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + attribute.Bool("raw", prm.raw), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index a242a099..4d2a7682 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -2,6 +2,7 @@ package meta_test import ( "bytes" + "context" "fmt" "os" "runtime" @@ -132,7 +133,7 @@ func TestDB_Get(t *testing.T) { var prm meta.InhumePrm prm.SetAddresses(obj) - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) _, err = metaGet(db, obj, false) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) @@ -216,7 +217,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { getPrm.SetAddress(addrs[counter%len(addrs)]) counter++ - _, err := db.Get(getPrm) + _, err := db.Get(context.Background(), getPrm) if err != nil { b.Fatal(err) } @@ -235,7 +236,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { var getPrm meta.GetPrm getPrm.SetAddress(addrs[i%len(addrs)]) - _, err := db.Get(getPrm) + _, err := db.Get(context.Background(), getPrm) if err != nil { b.Fatal(err) } @@ -248,6 +249,6 @@ func metaGet(db *meta.DB, addr oid.Address, raw bool) (*objectSDK.Object, error) prm.SetAddress(addr) prm.SetRaw(raw) - res, err := db.Get(prm) + res, err := db.Get(context.Background(), prm) return res.Header(), err } diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index b8b66554..8cd09e3f 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -68,7 +69,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { inhumePrm.SetAddresses(object.AddressOf(obj1)) inhumePrm.SetGCMark() - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) var counter int @@ -138,14 +139,14 @@ func TestDB_IterateDeletedObjects(t *testing.T) { inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) inhumePrm.SetTombstoneAddress(addrTombstone) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) inhumePrm.SetAddresses(object.AddressOf(obj3), object.AddressOf(obj4)) inhumePrm.SetGCMark() // inhume with GC mark - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) var ( @@ -225,7 +226,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { object.AddressOf(obj3), object.AddressOf(obj4)) inhumePrm.SetTombstoneAddress(addrTombstone) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) expectedGraveyard := []oid.Address{ @@ -320,7 +321,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { object.AddressOf(obj3), object.AddressOf(obj4)) inhumePrm.SetGCMark() - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) expectedGarbage := []oid.Address{ @@ -404,7 +405,7 @@ func TestDB_DropGraves(t *testing.T) { inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) inhumePrm.SetTombstoneAddress(addrTombstone) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) buriedTS := make([]meta.TombstonedObject, 0) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index b6e6cadf..a6887a33 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -2,9 +2,11 @@ package meta import ( "bytes" + "context" "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -118,7 +120,10 @@ var ErrLockObjectRemoval = logicerr.New("lock object removal") // // NOTE: Marks any object with GC mark (despite any prohibitions on operations // with that object) if WithForceGCMark option has been provided. -func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { +func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Inhume") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/inhume_test.go b/pkg/local_object_storage/metabase/inhume_test.go index b7ee5ef2..0f077422 100644 --- a/pkg/local_object_storage/metabase/inhume_test.go +++ b/pkg/local_object_storage/metabase/inhume_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -50,40 +51,40 @@ func TestInhumeTombOnTomb(t *testing.T) { inhumePrm.SetTombstoneAddress(addr2) // inhume addr1 via addr2 - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) existsPrm.SetAddress(addr1) // addr1 should become inhumed {addr1:addr2} - _, err = db.Exists(existsPrm) + _, err = db.Exists(context.Background(), existsPrm) require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) inhumePrm.SetAddresses(addr3) inhumePrm.SetTombstoneAddress(addr1) // try to inhume addr3 via addr1 - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) // record with {addr1:addr2} should be removed from graveyard // as a tomb-on-tomb; metabase should return ObjectNotFound // NOT ObjectAlreadyRemoved since that record has been removed // from graveyard but addr1 is still marked with GC - _, err = db.Exists(existsPrm) + _, err = db.Exists(context.Background(), existsPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) existsPrm.SetAddress(addr3) // addr3 should be inhumed {addr3: addr1} - _, err = db.Exists(existsPrm) + _, err = db.Exists(context.Background(), existsPrm) require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) inhumePrm.SetAddresses(addr1) inhumePrm.SetTombstoneAddress(oidtest.Address()) // try to inhume addr1 (which is already a tombstone in graveyard) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) existsPrm.SetAddress(addr1) @@ -91,7 +92,7 @@ func TestInhumeTombOnTomb(t *testing.T) { // record with addr1 key should not appear in graveyard // (tomb can not be inhumed) but should be kept as object // with GC mark - _, err = db.Exists(existsPrm) + _, err = db.Exists(context.Background(), existsPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) } @@ -100,13 +101,13 @@ func TestInhumeLocked(t *testing.T) { locked := oidtest.Address() - err := db.Lock(locked.Container(), oidtest.ID(), []oid.ID{locked.Object()}) + err := db.Lock(context.Background(), locked.Container(), oidtest.ID(), []oid.ID{locked.Object()}) require.NoError(t, err) var prm meta.InhumePrm prm.SetAddresses(locked) - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) var e apistatus.ObjectLocked require.ErrorAs(t, err, &e) @@ -117,6 +118,6 @@ func metaInhume(db *meta.DB, target, tomb oid.Address) error { inhumePrm.SetAddresses(target) inhumePrm.SetTombstoneAddress(tomb) - _, err := db.Inhume(inhumePrm) + _, err := db.Inhume(context.Background(), inhumePrm) return err } diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 6b3a3612..e7d6ad04 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "strconv" "testing" @@ -34,7 +35,7 @@ func TestDB_IterateExpired(t *testing.T) { expiredLocked := putWithExpiration(t, db, object.TypeRegular, epoch-1) - require.NoError(t, db.Lock(expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) + require.NoError(t, db.Lock(context.Background(), expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) err := db.IterateExpired(epoch, func(exp *meta.ExpiredObject) error { if addr, ok := mAlive[exp.Type()]; ok { @@ -81,13 +82,13 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { prm.SetAddresses(protected1, protected2, protectedLocked) prm.SetTombstoneAddress(ts) - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) prm.SetAddresses(garbage) prm.SetGCMark() - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) var handled []oid.Address @@ -107,7 +108,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { require.Contains(t, handled, protected2) require.Contains(t, handled, protectedLocked) - err = db.Lock(protectedLocked.Container(), oidtest.ID(), []oid.ID{protectedLocked.Object()}) + err = db.Lock(context.Background(), protectedLocked.Container(), oidtest.ID(), []oid.ID{protectedLocked.Object()}) require.NoError(t, err) handled = handled[:0] diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 2dcc85d4..5c3c9720 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -2,14 +2,18 @@ package meta import ( "bytes" + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var bucketNameLocked = []byte{lockedPrefix} @@ -30,7 +34,15 @@ func bucketNameLockers(idCnr cid.ID, key []byte) []byte { // Allows locking regular objects only (otherwise returns apistatus.LockNonRegularObject). // // Locked list should be unique. Panics if it is empty. -func (db *DB) Lock(cnr cid.ID, locker oid.ID, locked []oid.ID) error { +func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid.ID) error { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Lock", + trace.WithAttributes( + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("locker", locker.EncodeToString()), + attribute.Int("locked_count", len(locked)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -266,7 +278,13 @@ func (i IsLockedRes) Locked() bool { // object is considered as non-locked. // // Returns only non-logical errors related to underlying database. -func (db *DB) IsLocked(prm IsLockedPrm) (res IsLockedRes, err error) { +func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.IsLocked", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index efa9fba0..1d6ea6ff 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -20,8 +21,8 @@ func TestDB_Lock(t *testing.T) { db := newDB(t) t.Run("empty locked list", func(t *testing.T) { - require.Panics(t, func() { _ = db.Lock(cnr, oid.ID{}, nil) }) - require.Panics(t, func() { _ = db.Lock(cnr, oid.ID{}, []oid.ID{}) }) + require.Panics(t, func() { _ = db.Lock(context.Background(), cnr, oid.ID{}, nil) }) + require.Panics(t, func() { _ = db.Lock(context.Background(), cnr, oid.ID{}, []oid.ID{}) }) }) t.Run("(ir)regular", func(t *testing.T) { @@ -44,7 +45,7 @@ func TestDB_Lock(t *testing.T) { id, _ := obj.ID() // try to lock it - err = db.Lock(cnr, oidtest.ID(), []oid.ID{id}) + err = db.Lock(context.Background(), cnr, oidtest.ID(), []oid.ID{id}) if typ == object.TypeRegular { require.NoError(t, err, typ) } else { @@ -65,27 +66,27 @@ func TestDB_Lock(t *testing.T) { // check locking relation inhumePrm.SetAddresses(objAddr) - _, err := db.Inhume(inhumePrm) + _, err := db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) // try to remove lock object inhumePrm.SetAddresses(lockAddr) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.Error(t, err) // check that locking relation has not been // dropped inhumePrm.SetAddresses(objAddr) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) }) @@ -105,7 +106,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetForceGCMark() inhumePrm.SetLockObjectHandling() - res, err := db.Inhume(inhumePrm) + res, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Len(t, res.DeletedLockObjects(), 1) require.Equal(t, objectcore.AddressOf(lockObj), res.DeletedLockObjects()[0]) @@ -117,7 +118,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetGCMark() // now we can inhume the object - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) }) @@ -134,7 +135,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetAddresses(objectcore.AddressOf(lockObj)) inhumePrm.SetLockObjectHandling() - res, err := db.Inhume(inhumePrm) + res, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Len(t, res.DeletedLockObjects(), 1) require.Equal(t, objectcore.AddressOf(lockObj), res.DeletedLockObjects()[0]) @@ -151,7 +152,7 @@ func TestDB_Lock(t *testing.T) { for i := 0; i < objsNum; i++ { inhumePrm.SetAddresses(objectcore.AddressOf(objs[i])) - res, err = db.Inhume(inhumePrm) + res, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Len(t, res.DeletedLockObjects(), 0) } @@ -164,7 +165,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetForceGCMark() inhumePrm.SetAddresses(objectcore.AddressOf(lockObj)) - res, err := db.Inhume(inhumePrm) + res, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Len(t, res.DeletedLockObjects(), 0) }) @@ -184,7 +185,7 @@ func TestDB_Lock_Expired(t *testing.T) { require.ErrorIs(t, err, meta.ErrObjectIsExpired) // lock the obj - require.NoError(t, db.Lock(addr.Container(), oidtest.ID(), []oid.ID{addr.Object()})) + require.NoError(t, db.Lock(context.Background(), addr.Container(), oidtest.ID(), []oid.ID{addr.Object()})) // object is expired but locked, thus, must be available _, err = metaGet(db, addr, false) @@ -202,7 +203,7 @@ func TestDB_IsLocked(t *testing.T) { for _, obj := range objs { prm.SetAddress(objectcore.AddressOf(obj)) - res, err := db.IsLocked(prm) + res, err := db.IsLocked(context.Background(), prm) require.NoError(t, err) require.True(t, res.Locked()) @@ -212,7 +213,7 @@ func TestDB_IsLocked(t *testing.T) { prm.SetAddress(oidtest.Address()) - res, err := db.IsLocked(prm) + res, err := db.IsLocked(context.Background(), prm) require.NoError(t, err) require.False(t, res.Locked()) @@ -224,12 +225,12 @@ func TestDB_IsLocked(t *testing.T) { var putPrm meta.PutPrm putPrm.SetObject(obj) - _, err = db.Put(putPrm) + _, err = db.Put(context.Background(), putPrm) require.NoError(t, err) prm.SetAddress(objectcore.AddressOf(obj)) - res, err = db.IsLocked(prm) + res, err = db.IsLocked(context.Background(), prm) require.NoError(t, err) require.False(t, res.Locked()) @@ -260,7 +261,7 @@ func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Ob err := putBig(db, lockObj) require.NoError(t, err) - err = db.Lock(cnr, lockID, lockedObjIDs) + err = db.Lock(context.Background(), cnr, lockID, lockedObjIDs) require.NoError(t, err) return lockedObjs, lockObj diff --git a/pkg/local_object_storage/metabase/movable.go b/pkg/local_object_storage/metabase/movable.go index e6990dc5..412c4639 100644 --- a/pkg/local_object_storage/metabase/movable.go +++ b/pkg/local_object_storage/metabase/movable.go @@ -1,10 +1,14 @@ package meta import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ToMoveItPrm groups the parameters of ToMoveIt operation. @@ -48,7 +52,13 @@ func (p MovableRes) AddressList() []oid.Address { // ToMoveIt marks objects to move it into another shard. This useful for // faster HRW fetching. -func (db *DB) ToMoveIt(prm ToMoveItPrm) (res ToMoveItRes, err error) { +func (db *DB) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (res ToMoveItRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.ToMoveIt", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/movable_test.go b/pkg/local_object_storage/metabase/movable_test.go index 6918dec2..51e7e6d7 100644 --- a/pkg/local_object_storage/metabase/movable_test.go +++ b/pkg/local_object_storage/metabase/movable_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -61,7 +62,7 @@ func metaToMoveIt(db *meta.DB, addr oid.Address) error { var toMovePrm meta.ToMoveItPrm toMovePrm.SetAddress(addr) - _, err := db.ToMoveIt(toMovePrm) + _, err := db.ToMoveIt(context.Background(), toMovePrm) return err } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index b0fea653..2c78bda0 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -1,11 +1,13 @@ package meta import ( + "context" "encoding/binary" "errors" "fmt" gio "io" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" @@ -14,6 +16,8 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/io" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type ( @@ -52,7 +56,13 @@ var ( // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (db *DB) Put(prm PutPrm) (res PutRes, err error) { +func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Put", + trace.WithAttributes( + attribute.String("address", objectCore.AddressOf(prm.obj).EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index 837d931a..a3a071d1 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "runtime" "strconv" "testing" @@ -117,7 +118,7 @@ func metaPut(db *meta.DB, obj *objectSDK.Object, id []byte) error { putPrm.SetObject(obj) putPrm.SetStorageID(id) - _, err := db.Put(putPrm) + _, err := db.Put(context.Background(), putPrm) return err } diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 74c261d3..ecd83f86 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -1,17 +1,21 @@ package meta import ( + "context" "encoding/binary" "errors" "fmt" "strings" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -56,7 +60,13 @@ func (r SelectRes) AddressList() []oid.Address { } // Select returns list of addresses of objects that match search filters. -func (db *DB) Select(prm SelectPrm) (res SelectRes, err error) { +func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Select", + trace.WithAttributes( + attribute.String("container_id", prm.cnr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 5d4cc75e..38679752 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "encoding/hex" "strconv" "testing" @@ -829,7 +830,7 @@ func benchmarkSelect(b *testing.B, db *meta.DB, cid cidSDK.ID, fs objectSDK.Sear prm.SetFilters(fs) for i := 0; i < b.N; i++ { - res, err := db.Select(prm) + res, err := db.Select(context.Background(), prm) if err != nil { b.Fatal(err) } @@ -844,6 +845,6 @@ func metaSelect(db *meta.DB, cnr cidSDK.ID, fs objectSDK.SearchFilters) ([]oid.A prm.SetFilters(fs) prm.SetContainerID(cnr) - res, err := db.Select(prm) + res, err := db.Select(context.Background(), prm) return res.AddressList(), err } diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index ae309d4b..794879a3 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -1,11 +1,15 @@ package meta import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // StorageIDPrm groups the parameters of StorageID operation. @@ -30,7 +34,13 @@ func (r StorageIDRes) StorageID() []byte { // StorageID returns storage descriptor for objects from the blobstor. // It is put together with the object can makes get/delete operation faster. -func (db *DB) StorageID(prm StorageIDPrm) (res StorageIDRes, err error) { +func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.StorageID", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index f8185abe..5b27cdc8 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -63,6 +64,6 @@ func metaStorageID(db *meta.DB, addr oid.Address) ([]byte, error) { var sidPrm meta.StorageIDPrm sidPrm.SetAddress(addr) - r, err := db.StorageID(sidPrm) + r, err := db.StorageID(context.Background(), sidPrm) return r.StorageID(), err } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 3d0f7292..e74f235f 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -80,7 +81,10 @@ func (s *Shard) Open() error { type metabaseSynchronizer Shard func (x *metabaseSynchronizer) Init() error { - return (*Shard)(x).refillMetabase() + ctx, span := tracing.StartSpanFromContext(context.TODO(), "metabaseSynchronizer.Init") + defer span.End() + + return (*Shard)(x).refillMetabase(ctx) } // Init initializes all Shard's components. @@ -158,7 +162,7 @@ func (s *Shard) Init(ctx context.Context) error { return nil } -func (s *Shard) refillMetabase() error { +func (s *Shard) refillMetabase(ctx context.Context) error { err := s.metaBase.Reset() if err != nil { return fmt.Errorf("could not reset metabase: %w", err) @@ -177,9 +181,9 @@ func (s *Shard) refillMetabase() error { var err error switch obj.Type() { case objectSDK.TypeTombstone: - err = s.refillTombstoneObject(obj) + err = s.refillTombstoneObject(ctx, obj) case objectSDK.TypeLock: - err = s.refillLockObject(obj) + err = s.refillLockObject(ctx, obj) default: } if err != nil { @@ -190,7 +194,7 @@ func (s *Shard) refillMetabase() error { mPrm.SetObject(obj) mPrm.SetStorageID(descriptor) - _, err = s.metaBase.Put(mPrm) + _, err = s.metaBase.Put(ctx, mPrm) if err != nil && !meta.IsErrRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { return err } @@ -209,7 +213,7 @@ func (s *Shard) refillMetabase() error { return nil } -func (s *Shard) refillLockObject(obj *objectSDK.Object) error { +func (s *Shard) refillLockObject(ctx context.Context, obj *objectSDK.Object) error { var lock objectSDK.Lock if err := lock.Unmarshal(obj.Payload()); err != nil { return fmt.Errorf("could not unmarshal lock content: %w", err) @@ -220,14 +224,14 @@ func (s *Shard) refillLockObject(obj *objectSDK.Object) error { cnr, _ := obj.ContainerID() id, _ := obj.ID() - err := s.metaBase.Lock(cnr, id, locked) + err := s.metaBase.Lock(ctx, cnr, id, locked) if err != nil { return fmt.Errorf("could not lock objects: %w", err) } return nil } -func (s *Shard) refillTombstoneObject(obj *objectSDK.Object) error { +func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object) error { tombstone := objectSDK.NewTombstone() if err := tombstone.Unmarshal(obj.Payload()); err != nil { @@ -250,7 +254,7 @@ func (s *Shard) refillTombstoneObject(obj *objectSDK.Object) error { inhumePrm.SetTombstoneAddress(tombAddr) inhumePrm.SetAddresses(tombMembers...) - _, err := s.metaBase.Inhume(inhumePrm) + _, err := s.metaBase.Inhume(ctx, inhumePrm) if err != nil { return fmt.Errorf("could not inhume objects: %w", err) } @@ -290,7 +294,10 @@ func (s *Shard) Close() error { // Reload reloads configuration portions that are necessary. // If a config option is invalid, it logs an error and returns nil. // If there was a problem with applying new configuration, an error is returned. -func (s *Shard) Reload(opts ...Option) error { +func (s *Shard) Reload(ctx context.Context, opts ...Option) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Reload") + defer span.End() + // Do not use defaultCfg here missing options need not be reloaded. var c cfg for i := range opts { @@ -314,7 +321,7 @@ func (s *Shard) Reload(opts ...Option) error { // Here we refill metabase only if a new instance was opened. This is a feature, // we don't want to hang for some time just because we forgot to change // config after the node was updated. - err = s.refillMetabase() + err = s.refillMetabase(ctx) } else { err = s.metaBase.Init() } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 50ea20bb..170052d6 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -126,6 +126,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { } sh := New( + WithID(NewIDFromBytes([]byte{})), WithBlobStorOptions(blobOpts...), WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))), WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(epochState{}))) @@ -138,12 +139,12 @@ func TestRefillMetabaseCorrupted(t *testing.T) { var putPrm PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) require.NoError(t, sh.Close()) addr := object.AddressOf(obj) - _, err = fsTree.Put(common.PutPrm{Address: addr, RawData: []byte("not an object")}) + _, err = fsTree.Put(context.Background(), common.PutPrm{Address: addr, RawData: []byte("not an object")}) require.NoError(t, err) sh = New( @@ -245,13 +246,13 @@ func TestRefillMetabase(t *testing.T) { for _, v := range mObjs { putPrm.SetObject(v.obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) } putPrm.SetObject(tombObj) - _, err = sh.Put(putPrm) + _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) // LOCK object handling @@ -263,11 +264,11 @@ func TestRefillMetabase(t *testing.T) { objectSDK.WriteLock(lockObj, lock) putPrm.SetObject(lockObj) - _, err = sh.Put(putPrm) + _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) lockID, _ := lockObj.ID() - require.NoError(t, sh.Lock(cnrLocked, lockID, locked)) + require.NoError(t, sh.Lock(context.Background(), cnrLocked, lockID, locked)) var inhumePrm InhumePrm inhumePrm.SetTarget(object.AddressOf(tombObj), tombMembers...) @@ -368,7 +369,7 @@ func TestRefillMetabase(t *testing.T) { checkObj(object.AddressOf(tombObj), nil) checkTombMembers(false) - err = sh.refillMetabase() + err = sh.refillMetabase(context.Background()) require.NoError(t, err) c, err = sh.metaBase.ObjectCounters() diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index ed05f998..f086aa30 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -1,13 +1,17 @@ package shard import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -28,14 +32,21 @@ func (p *DeletePrm) SetAddresses(addr ...oid.Address) { // Delete removes data from the shard's writeCache, metaBase and // blobStor. -func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) { +func (s *Shard) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Delete", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Int("addr_count", len(prm.addr)), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() - return s.delete(prm) + return s.delete(ctx, prm) } -func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { +func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { if s.info.Mode.ReadOnly() { return DeleteRes{}, ErrReadOnlyMode } else if s.info.Mode.NoMetabase() { @@ -48,7 +59,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { for i := range prm.addr { if s.hasWriteCache() { - err := s.writeCache.Delete(prm.addr[i]) + err := s.writeCache.Delete(ctx, prm.addr[i]) if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.String("error", err.Error())) } @@ -57,7 +68,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { var sPrm meta.StorageIDPrm sPrm.SetAddress(prm.addr[i]) - res, err := s.metaBase.StorageID(sPrm) + res, err := s.metaBase.StorageID(ctx, sPrm) if err != nil { s.log.Debug(logs.ShardCantGetStorageIDFromMetabase, zap.Stringer("object", prm.addr[i]), @@ -74,7 +85,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { var delPrm meta.DeletePrm delPrm.SetAddresses(prm.addr...) - res, err := s.metaBase.Delete(delPrm) + res, err := s.metaBase.Delete(ctx, delPrm) if err != nil { return DeleteRes{}, err // stop on metabase error ? } @@ -99,7 +110,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { id := smalls[prm.addr[i]] delPrm.StorageID = id - _, err = s.blobStor.Delete(delPrm) + _, err = s.blobStor.Delete(ctx, delPrm) if err != nil { s.log.Debug(logs.ShardCantRemoveObjectFromBlobStor, zap.Stringer("object_address", prm.addr[i]), diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index c37dfa28..9646e9aa 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -43,13 +43,13 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { var delPrm shard.DeletePrm delPrm.SetAddresses(object.AddressOf(obj)) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) _, err = testGet(t, sh, getPrm, hasWriteCache) require.NoError(t, err) - _, err = sh.Delete(delPrm) + _, err = sh.Delete(context.TODO(), delPrm) require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) @@ -67,13 +67,13 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { var delPrm shard.DeletePrm delPrm.SetAddresses(object.AddressOf(obj)) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) require.NoError(t, err) - _, err = sh.Delete(delPrm) + _, err = sh.Delete(context.Background(), delPrm) require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) diff --git a/pkg/local_object_storage/shard/dump_test.go b/pkg/local_object_storage/shard/dump_test.go index 9d585cc0..92171720 100644 --- a/pkg/local_object_storage/shard/dump_test.go +++ b/pkg/local_object_storage/shard/dump_test.go @@ -104,7 +104,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var prm shard.PutPrm prm.SetObject(objects[i]) - _, err := sh.Put(prm) + _, err := sh.Put(context.Background(), prm) require.NoError(t, err) } @@ -129,13 +129,13 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { t.Run("empty dump", func(t *testing.T) { var restorePrm shard.RestorePrm restorePrm.WithPath(outEmpty) - res, err := sh.Restore(restorePrm) + res, err := sh.Restore(context.Background(), restorePrm) require.NoError(t, err) require.Equal(t, 0, res.Count()) }) t.Run("invalid path", func(t *testing.T) { - _, err := sh.Restore(*new(shard.RestorePrm)) + _, err := sh.Restore(context.Background(), *new(shard.RestorePrm)) require.ErrorIs(t, err, os.ErrNotExist) }) @@ -147,7 +147,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var restorePrm shard.RestorePrm restorePrm.WithPath(out) - _, err := sh.Restore(restorePrm) + _, err := sh.Restore(context.Background(), restorePrm) require.ErrorIs(t, err, shard.ErrInvalidMagic) }) @@ -162,7 +162,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var restorePrm shard.RestorePrm restorePrm.WithPath(out) - _, err := sh.Restore(restorePrm) + _, err := sh.Restore(context.Background(), restorePrm) require.ErrorIs(t, err, io.ErrUnexpectedEOF) }) t.Run("incomplete object data", func(t *testing.T) { @@ -173,7 +173,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var restorePrm shard.RestorePrm restorePrm.WithPath(out) - _, err := sh.Restore(restorePrm) + _, err := sh.Restore(context.Background(), restorePrm) require.ErrorIs(t, err, io.EOF) }) t.Run("invalid object", func(t *testing.T) { @@ -184,7 +184,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var restorePrm shard.RestorePrm restorePrm.WithPath(out) - _, err := sh.Restore(restorePrm) + _, err := sh.Restore(context.Background(), restorePrm) require.Error(t, err) t.Run("skip errors", func(t *testing.T) { @@ -195,7 +195,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { restorePrm.WithPath(out) restorePrm.WithIgnoreErrors(true) - res, err := sh.Restore(restorePrm) + res, err := sh.Restore(context.Background(), restorePrm) require.NoError(t, err) require.Equal(t, objCount, res.Count()) require.Equal(t, 2, res.FailCount()) @@ -208,7 +208,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { t.Run("must allow write", func(t *testing.T) { require.NoError(t, sh.SetMode(mode.ReadOnly)) - _, err := sh.Restore(prm) + _, err := sh.Restore(context.Background(), prm) require.ErrorIs(t, err, shard.ErrReadOnlyMode) }) @@ -234,7 +234,7 @@ func TestStream(t *testing.T) { var prm shard.PutPrm prm.SetObject(objects[i]) - _, err := sh1.Put(prm) + _, err := sh1.Put(context.Background(), prm) require.NoError(t, err) } @@ -269,7 +269,7 @@ func TestStream(t *testing.T) { } func checkRestore(t *testing.T, sh *shard.Shard, prm shard.RestorePrm, objects []*objectSDK.Object) { - res, err := sh.Restore(prm) + res, err := sh.Restore(context.Background(), prm) require.NoError(t, err) require.Equal(t, len(objects), res.Count()) @@ -333,7 +333,7 @@ func TestDumpIgnoreErrors(t *testing.T) { var prm shard.PutPrm prm.SetObject(objects[i]) - _, err := sh.Put(prm) + _, err := sh.Put(context.Background(), prm) require.NoError(t, err) } diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 76e4347d..66c61fcc 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -3,9 +3,12 @@ package shard import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ExistsPrm groups the parameters of Exists operation. @@ -36,6 +39,13 @@ func (p ExistsRes) Exists() bool { // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Exists", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + var exists bool var err error @@ -54,7 +64,7 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { existsPrm.SetAddress(prm.addr) var res meta.ExistsRes - res, err = s.metaBase.Exists(existsPrm) + res, err = s.metaBase.Exists(ctx, existsPrm) exists = res.Exists() } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 5ea9eced..86995cd0 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -234,7 +234,7 @@ func (s *Shard) removeGarbage() { deletePrm.SetAddresses(buf...) // delete accumulated objects - _, err = s.delete(deletePrm) + _, err = s.delete(context.TODO(), deletePrm) if err != nil { s.log.Warn(logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), @@ -320,7 +320,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) inhumePrm.SetGCMark() // inhume the collected objects - res, err := s.metaBase.Inhume(inhumePrm) + res, err := s.metaBase.Inhume(ctx, inhumePrm) if err != nil { s.log.Warn(logs.ShardCouldNotInhumeTheObjects, zap.String("error", err.Error()), @@ -485,7 +485,7 @@ func (s *Shard) selectExpired(ctx context.Context, epoch uint64, addresses []oid // and clears up corresponding graveyard records. // // Does not modify tss. -func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { +func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.TombstonedObject) { if s.GetMode().NoMetabase() { return } @@ -502,7 +502,7 @@ func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { pInhume.SetAddresses(tsAddrs...) // inhume tombstones - res, err := s.metaBase.Inhume(pInhume) + res, err := s.metaBase.Inhume(ctx, pInhume) if err != nil { s.log.Warn(logs.ShardCouldNotMarkTombstonesAsGarbage, zap.String("error", err.Error()), @@ -547,7 +547,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] pInhume.SetAddresses(lockers...) pInhume.SetForceGCMark() - res, err := s.metaBase.Inhume(pInhume) + res, err := s.metaBase.Inhume(ctx, pInhume) if err != nil { s.log.Warn(logs.ShardFailureToMarkLockersAsGarbage, zap.String("error", err.Error()), diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 8012e60f..b0126fcd 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -100,14 +100,14 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) - err = sh.Lock(cnr, lockID, []oid.ID{objID}) + err = sh.Lock(context.Background(), cnr, lockID, []oid.ID{objID}) require.NoError(t, err) putPrm.SetObject(lock) - _, err = sh.Put(putPrm) + _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) epoch.Value = 105 diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 8a0296ac..5268ac79 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -96,7 +96,7 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { } skipMeta := prm.skipMeta || s.info.Mode.NoMetabase() - obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, cb, wc) + obj, hasMeta, err := s.fetchObjectData(ctx, prm.addr, skipMeta, cb, wc) return GetRes{ obj: obj, @@ -109,7 +109,7 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { var emptyStorageID = make([]byte, 0) // fetchObjectData looks through writeCache and blobStor to find object. -func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, wc func(w writecache.Cache) (*objectSDK.Object, error)) (*objectSDK.Object, bool, error) { +func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta bool, cb storFetcher, wc func(w writecache.Cache) (*objectSDK.Object, error)) (*objectSDK.Object, bool, error) { var ( mErr error mRes meta.ExistsRes @@ -118,7 +118,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, if !skipMeta { var mPrm meta.ExistsPrm mPrm.SetAddress(addr) - mRes, mErr = s.metaBase.Exists(mPrm) + mRes, mErr = s.metaBase.Exists(ctx, mPrm) if mErr != nil && !s.info.Mode.NoMetabase() { return nil, false, mErr } @@ -154,7 +154,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, var mPrm meta.StorageIDPrm mPrm.SetAddress(addr) - mExRes, err := s.metaBase.StorageID(mPrm) + mExRes, err := s.metaBase.StorageID(ctx, mPrm) if err != nil { return nil, true, fmt.Errorf("can't fetch blobovnicza id from metabase: %w", err) } diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index f670b286..ea28c8e3 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -40,7 +40,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) getPrm.SetAddress(object.AddressOf(obj)) @@ -58,7 +58,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) getPrm.SetAddress(object.AddressOf(obj)) @@ -86,7 +86,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { putPrm.SetObject(child) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) getPrm.SetAddress(object.AddressOf(child)) diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index 8e8ff943..a15cdfdc 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -73,7 +73,7 @@ func (s *Shard) Head(ctx context.Context, prm HeadPrm) (HeadRes, error) { headParams.SetRaw(prm.raw) var res meta.GetRes - res, err = s.metaBase.Get(headParams) + res, err = s.metaBase.Get(ctx, headParams) obj = res.Header() } diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 449626e9..11e7a8b0 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -37,7 +37,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) headPrm.SetAddress(object.AddressOf(obj)) @@ -62,7 +62,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) { putPrm.SetObject(child) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) headPrm.SetAddress(object.AddressOf(parent)) diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 3457188b..12a2900a 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -5,9 +5,12 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -62,6 +65,12 @@ var ErrLockObjectRemoval = meta.ErrLockObjectRemoval // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Inhume", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + s.m.RLock() if s.info.Mode.ReadOnly() { @@ -74,7 +83,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { if s.hasWriteCache() { for i := range prm.target { - _ = s.writeCache.Delete(prm.target[i]) + _ = s.writeCache.Delete(ctx, prm.target[i]) } } @@ -92,7 +101,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { metaPrm.SetForceGCMark() } - res, err := s.metaBase.Inhume(metaPrm) + res, err := s.metaBase.Inhume(ctx, metaPrm) if err != nil { if errors.Is(err, meta.ErrLockObjectRemoval) { s.m.RUnlock() diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 41845c41..0b4e5170 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -42,7 +42,7 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { var getPrm shard.GetPrm getPrm.SetAddress(object.AddressOf(obj)) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) _, err = testGet(t, sh, getPrm, hasWriteCache) diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index bab1090e..aaa1112c 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -1,6 +1,7 @@ package shard import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -85,7 +86,7 @@ func (s *Shard) List() (res SelectRes, err error) { sPrm.SetContainerID(lst[i]) sPrm.SetFilters(filters) - sRes, err := s.metaBase.Select(sPrm) // consider making List in metabase + sRes, err := s.metaBase.Select(context.TODO(), sPrm) // consider making List in metabase if err != nil { s.log.Debug(logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 33c9e489..8fac41a0 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -1,6 +1,7 @@ package shard_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -52,7 +53,7 @@ func testShardList(t *testing.T, sh *shard.Shard) { putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) } } diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index d8113cf3..cfbd94c5 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -1,11 +1,15 @@ package shard import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Lock marks objects as locked with another object. All objects from the @@ -14,7 +18,16 @@ import ( // Allows locking regular objects only (otherwise returns apistatus.LockNonRegularObject). // // Locked list should be unique. Panics if it is empty. -func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { +func (s *Shard) Lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked []oid.ID) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Lock", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", idCnr.EncodeToString()), + attribute.String("locker", locker.EncodeToString()), + attribute.Int("locked_count", len(locked)), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -25,7 +38,7 @@ func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { return ErrDegradedMode } - err := s.metaBase.Lock(idCnr, locker, locked) + err := s.metaBase.Lock(ctx, idCnr, locker, locked) if err != nil { return fmt.Errorf("metabase lock: %w", err) } @@ -35,7 +48,14 @@ func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { // IsLocked checks object locking relation of the provided object. Not found object is // considered as not locked. Requires healthy metabase, returns ErrDegradedMode otherwise. -func (s *Shard) IsLocked(addr oid.Address) (bool, error) { +func (s *Shard) IsLocked(ctx context.Context, addr oid.Address) (bool, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.IsLocked", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + m := s.GetMode() if m.NoMetabase() { return false, ErrDegradedMode @@ -44,7 +64,7 @@ func (s *Shard) IsLocked(addr oid.Address) (bool, error) { var prm meta.IsLockedPrm prm.SetAddress(addr) - res, err := s.metaBase.IsLocked(prm) + res, err := s.metaBase.IsLocked(ctx, prm) if err != nil { return false, err } diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 2bee6629..c577ae18 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -76,16 +76,16 @@ func TestShard_Lock(t *testing.T) { var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) // lock the object - err = sh.Lock(cnr, lockID, []oid.ID{objID}) + err = sh.Lock(context.Background(), cnr, lockID, []oid.ID{objID}) require.NoError(t, err) putPrm.SetObject(lock) - _, err = sh.Put(putPrm) + _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) t.Run("inhuming locked objects", func(t *testing.T) { @@ -158,21 +158,21 @@ func TestShard_IsLocked(t *testing.T) { var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) // not locked object is not locked - locked, err := sh.IsLocked(objectcore.AddressOf(obj)) + locked, err := sh.IsLocked(context.Background(), objectcore.AddressOf(obj)) require.NoError(t, err) require.False(t, locked) // locked object is locked - require.NoError(t, sh.Lock(cnrID, lockID, []oid.ID{objID})) + require.NoError(t, sh.Lock(context.Background(), cnrID, lockID, []oid.ID{objID})) - locked, err = sh.IsLocked(objectcore.AddressOf(obj)) + locked, err = sh.IsLocked(context.Background(), objectcore.AddressOf(obj)) require.NoError(t, err) require.True(t, locked) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 18e97e25..1578c662 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -109,7 +109,7 @@ func TestCounters(t *testing.T) { for i := 0; i < objNumber; i++ { prm.SetObject(oo[i]) - _, err := sh.Put(prm) + _, err := sh.Put(context.Background(), prm) require.NoError(t, err) } @@ -168,7 +168,7 @@ func TestCounters(t *testing.T) { deletedNumber := int(phy / 4) prm.SetAddresses(addrFromObjs(oo[:deletedNumber])...) - _, err := sh.Delete(prm) + _, err := sh.Delete(context.Background(), prm) require.NoError(t, err) require.Equal(t, phy-uint64(deletedNumber), mm.objCounters[physical]) @@ -207,6 +207,7 @@ func shardWithMetrics(t *testing.T, path string) (*shard.Shard, *metricsStore) { } sh := shard.New( + shard.WithID(shard.NewIDFromBytes([]byte{})), shard.WithBlobStorOptions(blobOpts...), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(path, "pilorama"))), shard.WithMetaBaseOptions( diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index f3199ac0..11991062 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -1,9 +1,14 @@ package shard import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -23,7 +28,14 @@ func (p *ToMoveItPrm) SetAddress(addr oid.Address) { // ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to // another shard. -func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) { +func (s *Shard) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (ToMoveItRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ToMoveIt", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -37,7 +49,7 @@ func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) { var toMovePrm meta.ToMoveItPrm toMovePrm.SetAddress(prm.addr) - _, err := s.metaBase.ToMoveIt(toMovePrm) + _, err := s.metaBase.ToMoveIt(ctx, toMovePrm) if err != nil { s.log.Debug(logs.ShardCouldNotMarkObjectForShardRelocationInMetabase, zap.String("error", err.Error()), diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index a4cb2cb1..d7d4ae53 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -1,13 +1,17 @@ package shard import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -30,7 +34,14 @@ func (p *PutPrm) SetObject(obj *object.Object) { // did not allow to completely save the object. // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. -func (s *Shard) Put(prm PutPrm) (PutRes, error) { +func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Put", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", objectCore.AddressOf(prm.obj).EncodeToString()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -55,7 +66,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { // ahead of `Put` by storage engine tryCache := s.hasWriteCache() && !m.NoMetabase() if tryCache { - res, err = s.writeCache.Put(putPrm) + res, err = s.writeCache.Put(ctx, putPrm) } if err != nil || !tryCache { if err != nil { @@ -63,7 +74,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { zap.String("err", err.Error())) } - res, err = s.blobStor.Put(putPrm) + res, err = s.blobStor.Put(ctx, putPrm) if err != nil { return PutRes{}, fmt.Errorf("could not put object to BLOB storage: %w", err) } @@ -73,7 +84,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { var pPrm meta.PutPrm pPrm.SetObject(prm.obj) pPrm.SetStorageID(res.StorageID) - if _, err := s.metaBase.Put(pPrm); err != nil { + if _, err := s.metaBase.Put(ctx, pPrm); err != nil { // may we need to handle this case in a special way // since the object has been successfully written to BlobStor return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err) diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 4355c31a..06aea2f8 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -123,7 +123,7 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { } skipMeta := prm.skipMeta || s.info.Mode.NoMetabase() - obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, cb, wc) + obj, hasMeta, err := s.fetchObjectData(ctx, prm.addr, skipMeta, cb, wc) return RngRes{ obj: obj, diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 16418121..c95dbae9 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -99,7 +99,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) var rngPrm shard.RngPrm diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 1bfa33dd..9ad05f52 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -44,6 +44,7 @@ func TestShardReload(t *testing.T) { meta.WithEpochState(epochState{})} opts := []Option{ + WithID(NewIDFromBytes([]byte{})), WithLogger(l), WithBlobStorOptions(blobOpts...), WithMetaBaseOptions(metaOpts...), @@ -75,7 +76,7 @@ func TestShardReload(t *testing.T) { checkHasObjects(t, true) t.Run("same config, no-op", func(t *testing.T) { - require.NoError(t, sh.Reload(opts...)) + require.NoError(t, sh.Reload(context.Background(), opts...)) checkHasObjects(t, true) }) @@ -86,7 +87,7 @@ func TestShardReload(t *testing.T) { } newOpts := newShardOpts(filepath.Join(p, "meta1"), false) - require.NoError(t, sh.Reload(newOpts...)) + require.NoError(t, sh.Reload(context.Background(), newOpts...)) checkHasObjects(t, false) // new path, but no resync @@ -97,7 +98,7 @@ func TestShardReload(t *testing.T) { }) newOpts = newShardOpts(filepath.Join(p, "meta2"), true) - require.NoError(t, sh.Reload(newOpts...)) + require.NoError(t, sh.Reload(context.Background(), newOpts...)) checkHasObjects(t, true) // all objects are restored, including the new one @@ -106,7 +107,7 @@ func TestShardReload(t *testing.T) { require.NoError(t, os.WriteFile(badPath, []byte{1}, 0)) newOpts = newShardOpts(badPath, true) - require.Error(t, sh.Reload(newOpts...)) + require.Error(t, sh.Reload(context.Background(), newOpts...)) // Cleanup is done, no panic. obj := newObject() @@ -117,7 +118,7 @@ func TestShardReload(t *testing.T) { // Successive reload produces no undesired effects. require.NoError(t, os.RemoveAll(badPath)) - require.NoError(t, sh.Reload(newOpts...)) + require.NoError(t, sh.Reload(context.Background(), newOpts...)) obj = newObject() require.NoError(t, putObject(sh, obj)) @@ -132,7 +133,7 @@ func putObject(sh *Shard, obj *objectSDK.Object) error { var prm PutPrm prm.SetObject(obj) - _, err := sh.Put(prm) + _, err := sh.Put(context.Background(), prm) return err } diff --git a/pkg/local_object_storage/shard/restore.go b/pkg/local_object_storage/shard/restore.go index 73dc1d17..2cb64a51 100644 --- a/pkg/local_object_storage/shard/restore.go +++ b/pkg/local_object_storage/shard/restore.go @@ -2,13 +2,17 @@ package shard import ( "bytes" + "context" "encoding/binary" "errors" "io" "os" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrInvalidMagic is returned when dump format is invalid. @@ -57,8 +61,15 @@ func (r RestoreRes) FailCount() int { // Restore restores objects from the dump prepared by Dump. // // Returns any error encountered. -func (s *Shard) Restore(prm RestorePrm) (RestoreRes, error) { - // Disallow changing mode during restore. +func (s *Shard) Restore(ctx context.Context, prm RestorePrm) (RestoreRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Restore", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("path", prm.path), + attribute.Bool("ignore_errors", prm.ignoreErrors), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -122,7 +133,7 @@ func (s *Shard) Restore(prm RestorePrm) (RestoreRes, error) { } putPrm.SetObject(obj) - _, err = s.Put(putPrm) + _, err = s.Put(ctx, putPrm) if err != nil && !IsErrObjectExpired(err) && !IsErrRemoved(err) { return RestoreRes{}, err } diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 4bb467d4..7f776c18 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -1,12 +1,16 @@ package shard import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // SelectPrm groups the parameters of Select operation. @@ -39,7 +43,14 @@ func (r SelectRes) AddressList() []oid.Address { // // Returns any error encountered that // did not allow to completely select the objects. -func (s *Shard) Select(prm SelectPrm) (SelectRes, error) { +func (s *Shard) Select(ctx context.Context, prm SelectPrm) (SelectRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Select", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", prm.cnr.EncodeToString()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -51,7 +62,7 @@ func (s *Shard) Select(prm SelectPrm) (SelectRes, error) { selectPrm.SetFilters(prm.filters) selectPrm.SetContainerID(prm.cnr) - mRes, err := s.metaBase.Select(selectPrm) + mRes, err := s.metaBase.Select(ctx, selectPrm) if err != nil { return SelectRes{}, fmt.Errorf("could not select objects from metabase: %w", err) } diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 5fd13221..714811b7 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -43,7 +43,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { for i := range objects { putPrm.SetObject(objects[i]) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) } require.NoError(t, sh.Close()) diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 7282f121..245eb4c7 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -1,7 +1,12 @@ package shard import ( + "context" "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // FlushWriteCachePrm represents parameters of a `FlushWriteCache` operation. @@ -19,7 +24,14 @@ func (p *FlushWriteCachePrm) SetIgnoreErrors(ignore bool) { var errWriteCacheDisabled = errors.New("write-cache is disabled") // FlushWriteCache flushes all data from the write-cache. -func (s *Shard) FlushWriteCache(p FlushWriteCachePrm) error { +func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.FlushWriteCache", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Bool("ignore_errors", p.ignoreErrors), + )) + defer span.End() + if !s.hasWriteCache() { return errWriteCacheDisabled } @@ -35,5 +47,5 @@ func (s *Shard) FlushWriteCache(p FlushWriteCachePrm) error { return ErrDegradedMode } - return s.writeCache.Flush(p.ignoreErrors) + return s.writeCache.Flush(ctx, p.ignoreErrors) } diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index dcfea8dd..c1aab9e5 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -1,16 +1,27 @@ package writecache import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Delete removes object from write-cache. // // Returns an error of type apistatus.ObjectNotFound if object is missing in write-cache. -func (c *cache) Delete(addr oid.Address) error { +func (c *cache) Delete(ctx context.Context, addr oid.Address) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Delete", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -45,7 +56,7 @@ func (c *cache) Delete(addr oid.Address) error { return nil } - _, err := c.fsTree.Delete(common.DeletePrm{Address: addr}) + _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, storagelog.AddressField(saddr), diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 3ca3aa90..9d0ffc98 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -2,9 +2,11 @@ package writecache import ( "bytes" + "context" "errors" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -15,6 +17,8 @@ import ( "github.com/mr-tron/base58" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -37,7 +41,7 @@ func (c *cache) runFlushLoop() { } c.wg.Add(1) - go c.flushBigObjects() + go c.flushBigObjects(context.TODO()) c.wg.Add(1) go func() { @@ -141,7 +145,7 @@ func (c *cache) flushDB() { } } -func (c *cache) flushBigObjects() { +func (c *cache) flushBigObjects(ctx context.Context) { defer c.wg.Done() tick := time.NewTicker(defaultFlushInterval * 10) @@ -157,7 +161,7 @@ func (c *cache) flushBigObjects() { continue } - _ = c.flushFSTree(true) + _ = c.flushFSTree(ctx, true) c.modeMtx.RUnlock() case <-c.closeCh: @@ -176,7 +180,7 @@ func (c *cache) reportFlushError(msg string, addr string, err error) { } } -func (c *cache) flushFSTree(ignoreErrors bool) error { +func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { var prm common.IteratePrm prm.IgnoreErrors = ignoreErrors prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { @@ -205,7 +209,7 @@ func (c *cache) flushFSTree(ignoreErrors bool) error { return err } - err = c.flushObject(&obj, data) + err = c.flushObject(ctx, &obj, data) if err != nil { if ignoreErrors { return nil @@ -236,7 +240,7 @@ func (c *cache) flushWorker(_ int) { return } - err := c.flushObject(obj, nil) + err := c.flushObject(context.TODO(), obj, nil) if err == nil { c.flushed.Add(objectCore.AddressOf(obj).EncodeToString(), true) } @@ -244,14 +248,14 @@ func (c *cache) flushWorker(_ int) { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(obj *object.Object, data []byte) error { +func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte) error { addr := objectCore.AddressOf(obj) var prm common.PutPrm prm.Object = obj prm.RawData = data - res, err := c.blobstor.Put(prm) + res, err := c.blobstor.Put(ctx, prm) if err != nil { if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) && !errors.Is(err, blobstor.ErrNoPlaceFound) { @@ -276,15 +280,21 @@ func (c *cache) flushObject(obj *object.Object, data []byte) error { // Flush flushes all objects from the write-cache to the main storage. // Write-cache must be in readonly mode to ensure correctness of an operation and // to prevent interference with background flush workers. -func (c *cache) Flush(ignoreErrors bool) error { +func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush", + trace.WithAttributes( + attribute.Bool("ignore_errors", ignoreErrors), + )) + defer span.End() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() - return c.flush(ignoreErrors) + return c.flush(ctx, ignoreErrors) } -func (c *cache) flush(ignoreErrors bool) error { - if err := c.flushFSTree(ignoreErrors); err != nil { +func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { + if err := c.flushFSTree(ctx, ignoreErrors); err != nil { return err } @@ -316,7 +326,7 @@ func (c *cache) flush(ignoreErrors bool) error { return err } - if err := c.flushObject(&obj, data); err != nil { + if err := c.flushObject(ctx, &obj, data); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 9dc216fb..2cec0708 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -89,7 +89,7 @@ func TestFlush(t *testing.T) { var mPrm meta.StorageIDPrm mPrm.SetAddress(objects[i].addr) - mRes, err := mb.StorageID(mPrm) + mRes, err := mb.StorageID(context.Background(), mPrm) require.NoError(t, err) var prm common.GetPrm @@ -112,12 +112,12 @@ func TestFlush(t *testing.T) { wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true) wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false) - require.NoError(t, wc.Flush(false)) + require.NoError(t, wc.Flush(context.Background(), false)) for i := 0; i < 2; i++ { var mPrm meta.GetPrm mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(mPrm) + _, err := mb.Get(context.Background(), mPrm) require.Error(t, err) _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) @@ -147,7 +147,7 @@ func TestFlush(t *testing.T) { for i := 0; i < 2; i++ { var mPrm meta.GetPrm mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(mPrm) + _, err := mb.Get(context.Background(), mPrm) require.Error(t, err) _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) @@ -171,9 +171,9 @@ func TestFlush(t *testing.T) { require.NoError(t, mb.SetMode(mode.ReadWrite)) require.Equal(t, uint32(0), errCount.Load()) - require.Error(t, wc.Flush(false)) + require.Error(t, wc.Flush(context.Background(), false)) require.True(t, errCount.Load() > 0) - require.NoError(t, wc.Flush(true)) + require.NoError(t, wc.Flush(context.Background(), true)) check(t, mb, bs, objects) } @@ -202,7 +202,7 @@ func TestFlush(t *testing.T) { prm.Address = objectCore.AddressOf(obj) prm.RawData = data - _, err := c.fsTree.Put(prm) + _, err := c.fsTree.Put(context.Background(), prm) require.NoError(t, err) p := prm.Address.Object().EncodeToString() + "." + prm.Address.Container().EncodeToString() @@ -218,7 +218,7 @@ func TestFlush(t *testing.T) { var prm common.PutPrm prm.Address = oidtest.Address() prm.RawData = []byte{1, 2, 3} - _, err := c.fsTree.Put(prm) + _, err := c.fsTree.Put(context.Background(), prm) require.NoError(t, err) }) }) @@ -245,19 +245,19 @@ func TestFlush(t *testing.T) { for i := range objects { var prm meta.PutPrm prm.SetObject(objects[i].obj) - _, err := mb.Put(prm) + _, err := mb.Put(context.Background(), prm) require.NoError(t, err) } var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(objects[0].addr, objects[1].addr) inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err := mb.Inhume(inhumePrm) + _, err := mb.Inhume(context.Background(), inhumePrm) require.NoError(t, err) var deletePrm meta.DeletePrm deletePrm.SetAddresses(objects[2].addr, objects[3].addr) - _, err = mb.Delete(deletePrm) + _, err = mb.Delete(context.Background(), deletePrm) require.NoError(t, err) require.NoError(t, bs.SetMode(mode.ReadOnly)) @@ -294,7 +294,7 @@ func putObject(t *testing.T, c Cache, size int) objectPair { prm.Object = obj prm.RawData = data - _, err := c.Put(prm) + _, err := c.Put(context.Background(), prm) require.NoError(t, err) return objectPair{prm.Address, prm.Object} diff --git a/pkg/local_object_storage/writecache/init.go b/pkg/local_object_storage/writecache/init.go index 0ac8cea9..d92e9a2d 100644 --- a/pkg/local_object_storage/writecache/init.go +++ b/pkg/local_object_storage/writecache/init.go @@ -15,21 +15,21 @@ import ( "go.uber.org/zap" ) -func (c *cache) initFlushMarks() { +func (c *cache) initFlushMarks(ctx context.Context) { var localWG sync.WaitGroup localWG.Add(1) go func() { defer localWG.Done() - c.fsTreeFlushMarkUpdate() + c.fsTreeFlushMarkUpdate(ctx) }() localWG.Add(1) go func() { defer localWG.Done() - c.dbFlushMarkUpdate() + c.dbFlushMarkUpdate(ctx) }() c.initWG.Add(1) @@ -54,7 +54,7 @@ func (c *cache) initFlushMarks() { var errStopIter = errors.New("stop iteration") -func (c *cache) fsTreeFlushMarkUpdate() { +func (c *cache) fsTreeFlushMarkUpdate(ctx context.Context) { c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInFSTree) var prm common.IteratePrm @@ -67,14 +67,14 @@ func (c *cache) fsTreeFlushMarkUpdate() { default: } - flushed, needRemove := c.flushStatus(addr) + flushed, needRemove := c.flushStatus(ctx, addr) if flushed { c.store.flushed.Add(addr.EncodeToString(), true) if needRemove { var prm common.DeletePrm prm.Address = addr - _, err := c.fsTree.Delete(prm) + _, err := c.fsTree.Delete(ctx, prm) if err == nil { storagelog.Write(c.log, storagelog.AddressField(addr), @@ -90,7 +90,7 @@ func (c *cache) fsTreeFlushMarkUpdate() { c.log.Info(logs.WritecacheFinishedUpdatingFSTreeFlushMarks) } -func (c *cache) dbFlushMarkUpdate() { +func (c *cache) dbFlushMarkUpdate(ctx context.Context) { c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInDatabase) var m []string @@ -125,7 +125,7 @@ func (c *cache) dbFlushMarkUpdate() { continue } - flushed, needRemove := c.flushStatus(addr) + flushed, needRemove := c.flushStatus(ctx, addr) if flushed { c.store.flushed.Add(addr.EncodeToString(), true) if needRemove { @@ -165,11 +165,11 @@ func (c *cache) dbFlushMarkUpdate() { // flushStatus returns info about the object state in the main storage. // First return value is true iff object exists. // Second return value is true iff object can be safely removed. -func (c *cache) flushStatus(addr oid.Address) (bool, bool) { +func (c *cache) flushStatus(ctx context.Context, addr oid.Address) (bool, bool) { var existsPrm meta.ExistsPrm existsPrm.SetAddress(addr) - _, err := c.metabase.Exists(existsPrm) + _, err := c.metabase.Exists(ctx, existsPrm) if err != nil { needRemove := errors.Is(err, meta.ErrObjectIsExpired) || errors.As(err, new(apistatus.ObjectAlreadyRemoved)) return needRemove, needRemove @@ -178,7 +178,7 @@ func (c *cache) flushStatus(addr oid.Address) (bool, bool) { var prm meta.StorageIDPrm prm.SetAddress(addr) - mRes, _ := c.metabase.StorageID(prm) - res, err := c.blobstor.Exists(context.TODO(), common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) + mRes, _ := c.metabase.StorageID(ctx, prm) + res, err := c.blobstor.Exists(ctx, common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) return err == nil && res.Exists, false } diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 939dc5b0..20b0cce2 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -1,12 +1,16 @@ package writecache import ( + "context" "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrReadOnly is returned when Put/Write is performed in a read-only mode. @@ -19,19 +23,25 @@ var ErrNotInitialized = logicerr.New("write-cache is not initialized yet") // When shard is put in read-only mode all objects in memory are flushed to disk // and all background jobs are suspended. func (c *cache) SetMode(m mode.Mode) error { + ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.SetMode", + trace.WithAttributes( + attribute.String("mode", m.String()), + )) + defer span.End() + c.modeMtx.Lock() defer c.modeMtx.Unlock() - return c.setMode(m) + return c.setMode(ctx, m) } // setMode applies new mode. Must be called with cache.modeMtx lock taken. -func (c *cache) setMode(m mode.Mode) error { +func (c *cache) setMode(ctx context.Context, m mode.Mode) error { var err error turnOffMeta := m.NoMetabase() if turnOffMeta && !c.mode.NoMetabase() { - err = c.flush(true) + err = c.flush(ctx, true) if err != nil { return err } @@ -45,7 +55,7 @@ func (c *cache) setMode(m mode.Mode) error { defer func() { if err == nil && !turnOffMeta { - c.initFlushMarks() + c.initFlushMarks(ctx) } }() } diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index cca8986b..3434e935 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -19,14 +19,14 @@ type Option func(*options) // meta is an interface for a metabase. type metabase interface { - Exists(meta.ExistsPrm) (meta.ExistsRes, error) - StorageID(meta.StorageIDPrm) (meta.StorageIDRes, error) + Exists(context.Context, meta.ExistsPrm) (meta.ExistsRes, error) + StorageID(context.Context, meta.StorageIDPrm) (meta.StorageIDRes, error) UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) } // blob is an interface for the blobstor. type blob interface { - Put(common.PutPrm) (common.PutRes, error) + Put(context.Context, common.PutPrm) (common.PutRes, error) NeedsCompression(obj *objectSDK.Object) bool Exists(ctx context.Context, res common.ExistsPrm) (common.ExistsRes, error) } diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 7791e93d..e2535d9e 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -1,11 +1,15 @@ package writecache import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var ( @@ -21,7 +25,14 @@ var ( // Returns ErrNotInitialized if write-cache has not been initialized yet. // Returns ErrOutOfSpace if saving an object leads to WC's size overflow. // Returns ErrBigObject if an objects exceeds maximum object size. -func (c *cache) Put(prm common.PutPrm) (common.PutRes, error) { +func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -44,7 +55,7 @@ func (c *cache) Put(prm common.PutPrm) (common.PutRes, error) { if sz <= c.smallObjectSize { return common.PutRes{}, c.putSmall(oi) } - return common.PutRes{}, c.putBig(oi.addr, prm) + return common.PutRes{}, c.putBig(ctx, oi.addr, prm) } // putSmall persists small objects to the write-cache database and @@ -71,13 +82,13 @@ func (c *cache) putSmall(obj objectInfo) error { } // putBig writes object to FSTree and pushes it to the flush workers queue. -func (c *cache) putBig(addr string, prm common.PutPrm) error { +func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error { cacheSz := c.estimateCacheSize() if c.maxCacheSize < c.incSizeFS(cacheSz) { return ErrOutOfSpace } - _, err := c.fsTree.Put(prm) + _, err := c.fsTree.Put(ctx, prm) if err != nil { return err } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index ff7eb1d6..aeae752e 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -1,6 +1,7 @@ package writecache import ( + "context" "errors" "fmt" "os" @@ -146,7 +147,7 @@ func (c *cache) deleteFromDisk(keys []string) []string { continue } - _, err := c.fsTree.Delete(common.DeletePrm{Address: addr}) + _, err := c.fsTree.Delete(context.TODO(), common.DeletePrm{Address: addr}) if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) { c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 24070dbd..0fc2e601 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -5,6 +5,7 @@ import ( "os" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -32,13 +33,13 @@ type Cache interface { // // Returns apistatus.ObjectNotFound if object is missing in the Cache. // Returns ErrReadOnly if the Cache is currently in the read-only mode. - Delete(oid.Address) error + Delete(context.Context, oid.Address) error Iterate(IterationPrm) error - Put(common.PutPrm) (common.PutRes, error) + Put(context.Context, common.PutPrm) (common.PutRes, error) SetMode(mode.Mode) error SetLogger(*logger.Logger) DumpInfo() Info - Flush(bool) error + Flush(context.Context, bool) error Init() error Open(readOnly bool) error @@ -152,7 +153,10 @@ func (c *cache) Open(readOnly bool) error { // Init runs necessary services. func (c *cache) Init() error { - c.initFlushMarks() + ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.Init") + defer span.End() + + c.initFlushMarks(ctx) c.runFlushLoop() return nil } @@ -163,7 +167,7 @@ func (c *cache) Close() error { defer c.modeMtx.Unlock() // Finish all in-progress operations. - if err := c.setMode(mode.ReadOnly); err != nil { + if err := c.setMode(context.TODO(), mode.ReadOnly); err != nil { return err } diff --git a/pkg/services/control/server/flush_cache.go b/pkg/services/control/server/flush_cache.go index fdfd136a..9ead530d 100644 --- a/pkg/services/control/server/flush_cache.go +++ b/pkg/services/control/server/flush_cache.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/status" ) -func (s *Server) FlushCache(_ context.Context, req *control.FlushCacheRequest) (*control.FlushCacheResponse, error) { +func (s *Server) FlushCache(ctx context.Context, req *control.FlushCacheRequest) (*control.FlushCacheResponse, error) { err := s.isValidRequest(req) if err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -19,7 +19,7 @@ func (s *Server) FlushCache(_ context.Context, req *control.FlushCacheRequest) ( var prm engine.FlushWriteCachePrm prm.SetShardID(shardID) - _, err = s.s.FlushWriteCache(prm) + _, err = s.s.FlushWriteCache(ctx, prm) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/restore.go b/pkg/services/control/server/restore.go index 0e636795..dba186f5 100644 --- a/pkg/services/control/server/restore.go +++ b/pkg/services/control/server/restore.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/status" ) -func (s *Server) RestoreShard(_ context.Context, req *control.RestoreShardRequest) (*control.RestoreShardResponse, error) { +func (s *Server) RestoreShard(ctx context.Context, req *control.RestoreShardRequest) (*control.RestoreShardResponse, error) { err := s.isValidRequest(req) if err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -21,7 +21,7 @@ func (s *Server) RestoreShard(_ context.Context, req *control.RestoreShardReques prm.WithPath(req.GetBody().GetFilepath()) prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) - err = s.s.RestoreShard(shardID, prm) + err = s.s.RestoreShard(ctx, shardID, prm) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index f9870f7e..f6341f02 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -120,7 +120,7 @@ func (w *putSvcWrapper) put(ctx context.Context, exec *execCtx) (*oid.ID, error) WithCommonPrm(exec.commonParameters()). WithObject(exec.tombstoneObj.CutPayload()) - err = streamer.Init(initPrm) + err = streamer.Init(ctx, initPrm) if err != nil { return nil, err } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 10a6af27..6beb6747 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -8,6 +8,7 @@ import ( "fmt" "io" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -402,6 +403,9 @@ func (x PutObjectRes) ID() oid.ID { // // Returns any error which prevented the operation from completing correctly in error return. func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "client.PutObject") + defer span.End() + var prmCli client.PrmObjectPutInit prmCli.MarkLocal() diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 15296f83..4b205680 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -117,7 +117,7 @@ func (x errIncompletePut) Error() string { return commonMsg } -func (t *distributedTarget) WriteHeader(obj *objectSDK.Object) error { +func (t *distributedTarget) WriteHeader(_ context.Context, obj *objectSDK.Object) error { t.obj = obj return nil diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index 2e6a496f..7aef9f06 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -14,15 +14,15 @@ import ( type ObjectStorage interface { // Put must save passed object // and return any appeared error. - Put(*object.Object) error + Put(context.Context, *object.Object) error // Delete must delete passed objects // and return any appeared error. Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error // Lock must lock passed objects // and return any appeared error. - Lock(locker oid.Address, toLock []oid.ID) error + Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error // IsLocked must clarify object's lock status. - IsLocked(oid.Address) (bool, error) + IsLocked(context.Context, oid.Address) (bool, error) } type localTarget struct { @@ -47,7 +47,7 @@ func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers return nil, fmt.Errorf("could not delete objects from tombstone locally: %w", err) } case object.TypeLock: - err := t.storage.Lock(objectCore.AddressOf(t.obj), t.meta.Objects()) + err := t.storage.Lock(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) if err != nil { return nil, fmt.Errorf("could not lock object from lock objects locally: %w", err) } @@ -55,7 +55,7 @@ func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers // objects that do not change meta storage } - if err := t.storage.Put(t.obj); err != nil { + if err := t.storage.Put(ctx, t.obj); err != nil { //TODO return nil, fmt.Errorf("(%T) could not put object to local storage: %w", t, err) } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index e355990a..ea885366 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -32,13 +32,13 @@ var errNotInit = errors.New("stream not initialized") var errInitRecall = errors.New("init recall") -func (p *Streamer) Init(prm *PutInitPrm) error { +func (p *Streamer) Init(ctx context.Context, prm *PutInitPrm) error { // initialize destination target if err := p.initTarget(prm); err != nil { return fmt.Errorf("(%T) could not initialize object target: %w", p, err) } - if err := p.target.WriteHeader(prm.hdr); err != nil { + if err := p.target.WriteHeader(ctx, prm.hdr); err != nil { return fmt.Errorf("(%T) could not write header to target: %w", p, err) } return nil diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 3b8d7b88..f7a97a95 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -15,6 +16,8 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type streamer struct { @@ -34,6 +37,9 @@ type sizes struct { } func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "putv2.streamer.Send") + defer span.End() + switch v := req.GetBody().GetObjectPart().(type) { case *object.PutObjectPartInit: var initPrm *putsvc.PutInitPrm @@ -43,7 +49,7 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) return err } - if err = s.stream.Init(initPrm); err != nil { + if err = s.stream.Init(ctx, initPrm); err != nil { err = fmt.Errorf("(%T) could not init object put stream: %w", s, err) } @@ -105,6 +111,9 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) } func (s *streamer) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "putv2.streamer.CloseAndRecv") + defer span.End() + if s.saveChunks { // check payload size correctness if s.writtenPayload != s.payloadSz { @@ -121,6 +130,9 @@ func (s *streamer) CloseAndRecv(ctx context.Context) (*object.PutResponse, error } func (s *streamer) relayRequest(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) error { + ctx, span := tracing.StartSpanFromContext(ctx, "putv2.streamer.relayRequest") + defer span.End() + // open stream resp := new(object.PutResponse) @@ -129,6 +141,12 @@ func (s *streamer) relayRequest(ctx context.Context, info client.NodeInfo, c cli var firstErr error info.AddressGroup().IterateAddresses(func(addr network.Address) (stop bool) { + ctx, span := tracing.StartSpanFromContext(ctx, "putv2.streamer.iterateAddress", + trace.WithAttributes( + attribute.String("address", addr.String()), + )) + defer span.End() + var err error defer func() { diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index 70c6974d..8c40d067 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -41,7 +41,7 @@ var ( ErrWrongPayloadSize = errors.New("wrong payload size") ) -func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error { +func (t *validatingTarget) WriteHeader(ctx context.Context, obj *objectSDK.Object) error { t.payloadSz = obj.PayloadSize() chunkLn := uint64(len(obj.Payload())) @@ -73,11 +73,11 @@ func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error { t.checksum = cs.Value() } - if err := t.fmt.Validate(obj, t.unpreparedObject); err != nil { + if err := t.fmt.Validate(ctx, obj, t.unpreparedObject); err != nil { return fmt.Errorf("(%T) coult not validate object format: %w", t, err) } - err := t.nextTarget.WriteHeader(obj) + err := t.nextTarget.WriteHeader(ctx, obj) if err != nil { return err } diff --git a/pkg/services/object/search/local.go b/pkg/services/object/search/local.go index f768c886..1af69caf 100644 --- a/pkg/services/object/search/local.go +++ b/pkg/services/object/search/local.go @@ -1,12 +1,14 @@ package searchsvc import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.uber.org/zap" ) -func (exec *execCtx) executeLocal() { - ids, err := exec.svc.localStorage.search(exec) +func (exec *execCtx) executeLocal(ctx context.Context) { + ids, err := exec.svc.localStorage.search(ctx, exec) if err != nil { exec.status = statusUndefined diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index b192e1d0..7a7cbfc5 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -27,7 +27,7 @@ func (exec *execCtx) execute(ctx context.Context) { exec.log.Debug(logs.ServingRequest) // perform local operation - exec.executeLocal() + exec.executeLocal(ctx) exec.analyzeStatus(ctx, true) } diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index e9597095..75059103 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -107,7 +107,7 @@ func (c *testClientCache) get(info clientcore.NodeInfo) (searchClient, error) { return v, nil } -func (s *testStorage) search(exec *execCtx) ([]oid.ID, error) { +func (s *testStorage) search(_ context.Context, exec *execCtx) ([]oid.ID, error) { v, ok := s.items[exec.containerID().EncodeToString()] if !ok { return nil, nil diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index b858e221..708979d7 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -37,7 +37,7 @@ type cfg struct { log *logger.Logger localStorage interface { - search(*execCtx) ([]oid.ID, error) + search(context.Context, *execCtx) ([]oid.ID, error) } clientConstructor interface { diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 49f3e5ef..b5b351a3 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -117,12 +117,12 @@ func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info c return res.IDList(), nil } -func (e *storageEngineWrapper) search(exec *execCtx) ([]oid.ID, error) { +func (e *storageEngineWrapper) search(ctx context.Context, exec *execCtx) ([]oid.ID, error) { var selectPrm engine.SelectPrm selectPrm.WithFilters(exec.searchFilters()) selectPrm.WithContainerID(exec.containerID()) - r, err := e.storage.Select(selectPrm) + r, err := e.storage.Select(ctx, selectPrm) if err != nil { return nil, err } diff --git a/pkg/services/object_manager/transformer/fmt.go b/pkg/services/object_manager/transformer/fmt.go index 462cc747..fbe8af2f 100644 --- a/pkg/services/object_manager/transformer/fmt.go +++ b/pkg/services/object_manager/transformer/fmt.go @@ -48,7 +48,7 @@ func NewFormatTarget(p *FormatterParams) ObjectTarget { } } -func (f *formatter) WriteHeader(obj *object.Object) error { +func (f *formatter) WriteHeader(_ context.Context, obj *object.Object) error { f.obj = obj return nil @@ -97,7 +97,7 @@ func (f *formatter) Close(ctx context.Context) (*AccessIdentifiers, error) { return nil, fmt.Errorf("could not finalize object: %w", err) } - if err := f.prm.NextTarget.WriteHeader(f.obj); err != nil { + if err := f.prm.NextTarget.WriteHeader(ctx, f.obj); err != nil { return nil, fmt.Errorf("could not write header to next target: %w", err) } diff --git a/pkg/services/object_manager/transformer/transformer.go b/pkg/services/object_manager/transformer/transformer.go index 199f5d0c..c23b4dca 100644 --- a/pkg/services/object_manager/transformer/transformer.go +++ b/pkg/services/object_manager/transformer/transformer.go @@ -56,7 +56,7 @@ func NewPayloadSizeLimiter(maxSize uint64, withoutHomomorphicHash bool, targetIn } } -func (s *payloadSizeLimiter) WriteHeader(hdr *object.Object) error { +func (s *payloadSizeLimiter) WriteHeader(_ context.Context, hdr *object.Object) error { s.current = fromObject(hdr) s.initialize() @@ -190,7 +190,7 @@ func (s *payloadSizeLimiter) release(ctx context.Context, finalize bool) (*Acces writeHashes(s.currentHashers) // release current, get its id - if err := s.target.WriteHeader(s.current); err != nil { + if err := s.target.WriteHeader(ctx, s.current); err != nil { return nil, fmt.Errorf("could not write header: %w", err) } diff --git a/pkg/services/object_manager/transformer/types.go b/pkg/services/object_manager/transformer/types.go index 3e6e2fef..73cea521 100644 --- a/pkg/services/object_manager/transformer/types.go +++ b/pkg/services/object_manager/transformer/types.go @@ -28,7 +28,7 @@ type ObjectTarget interface { // that depends on the implementation. // // Must not be called after Close call. - WriteHeader(*object.Object) error + WriteHeader(context.Context, *object.Object) error // Write writes object payload chunk. //