From c1e4130020dfde27387ddf26bdf1cafd5444a00b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 27 Sep 2023 11:02:06 +0300 Subject: [PATCH] [#146] node: Add trace_id to logs Signed-off-by: Alexander Chuprov --- .../blobovnicza/delete.go | 2 ++ .../blobstor/blobovniczatree/delete.go | 2 ++ .../blobstor/blobovniczatree/exists.go | 4 ++- .../blobstor/blobovniczatree/get.go | 2 ++ .../blobstor/blobovniczatree/get_range.go | 3 +- .../blobstor/blobovniczatree/put.go | 10 ++++-- pkg/local_object_storage/blobstor/exists.go | 4 ++- pkg/local_object_storage/engine/delete.go | 7 ++-- pkg/local_object_storage/engine/evacuate.go | 19 +++++++---- pkg/local_object_storage/engine/get.go | 4 ++- pkg/local_object_storage/engine/inhume.go | 7 ++-- pkg/local_object_storage/engine/put.go | 5 ++- pkg/local_object_storage/engine/range.go | 4 ++- pkg/local_object_storage/engine/tree.go | 34 +++++++++++++------ pkg/local_object_storage/shard/delete.go | 7 ++-- pkg/local_object_storage/shard/get.go | 7 ++-- pkg/local_object_storage/shard/inhume.go | 2 ++ pkg/local_object_storage/shard/list.go | 4 ++- pkg/local_object_storage/shard/move.go | 2 ++ pkg/services/object/put/single.go | 5 +-- pkg/services/replicator/process.go | 5 ++- pkg/services/tree/redirect.go | 5 ++- pkg/services/tree/replicator.go | 7 ++-- pkg/tracing/trace.go | 19 +++++++++++ 24 files changed, 129 insertions(+), 41 deletions(-) create mode 100644 pkg/tracing/trace.go diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 5a5da235f..4bc13cc95 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -4,6 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -73,6 +74,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err b.log.Debug(logs.BlobovniczaObjectWasRemovedFromBucket, zap.String("binary size", stringifyByteSize(dataSize)), zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) b.itemDeleted(sizeUpperBound) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index ca1e46a15..28e3a8f36 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -69,6 +70,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index c94881168..e1a6f5ed5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "go.opentelemetry.io/otel/attribute" @@ -56,7 +57,8 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index f2399e414..49849e759 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -69,6 +70,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 56b41351b..91d1e3ce0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -71,7 +72,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), - ) + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } if outOfBounds { return true, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index aae280e0b..6f9c8c0de 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -81,14 +82,16 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { i.B.log.Debug(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return false, nil } if active == nil { - i.B.log.Debug(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath)) + i.B.log.Debug(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return false, nil } defer active.Close() @@ -102,7 +105,8 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) } else { i.B.log.Debug(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", active.Path()), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return false, nil diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 03dad392a..43feec7c9 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -74,7 +75,8 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi for _, err := range errors[:len(errors)-1] { b.log.Warn(logs.BlobstorErrorOccurredDuringObjectExistenceChecking, zap.Stringer("address", prm.Address), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return common.ExistsRes{}, errors[len(errors)-1] diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 8e3180560..68a7325c6 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -150,7 +151,8 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo if err != nil { e.log.Warn(logs.EngineErrorDuringSearchingForObjectChildren, zap.Stringer("addr", addr), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return false } @@ -161,7 +163,8 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo if err != nil { e.log.Debug(logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), - zap.String("err", err.Error())) + zap.String("err", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) continue } } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 98a3a202d..192070c1c 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -11,6 +11,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -200,11 +201,13 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p e.evacuateLimiter.Complete(err) }() - e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) + e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField) + e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -266,7 +269,8 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break } - e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField) + e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -342,7 +346,8 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to res.failed.Add(1) continue } - e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) + e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -363,7 +368,8 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to err = prm.handler(ctx, addr, getRes.Object()) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) + e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } res.evacuated.Add(1) @@ -392,7 +398,8 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), zap.Stringer("addr", addr), - evacuationOperationLogField) + evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return true, nil } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 5d888238e..f77c44226 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -105,7 +106,8 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), zap.String("error", it.MetaError.Error()), - zap.Stringer("address", prm.addr)) + zap.Stringer("address", prm.addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index e0f9d7ab3..293746f70 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -93,7 +94,8 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e if err != nil { e.log.Warn(logs.EngineRemovingAnObjectWithoutFullLockingCheck, zap.Error(err), - zap.Stringer("addr", prm.addrs[i])) + zap.Stringer("addr", prm.addrs[i]), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else if locked { return InhumeRes{}, new(apistatus.ObjectLocked) } @@ -201,7 +203,8 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { locked, err = h.Shard.IsLocked(ctx, addr) if err != nil { - e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("addr", addr)) + e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("addr", addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err return false } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 2f96b8296..79ee3a997 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -128,6 +129,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, e.log.Warn(logs.EngineCouldNotMarkObjectForShardRelocation, zap.Stringer("shard", sh.ID()), zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } } @@ -144,7 +146,8 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index fb0873ee2..e45845d6c 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -118,7 +119,8 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), zap.String("error", it.MetaError.Error()), - zap.Stringer("address", prm.addr)) + zap.Stringer("address", prm.addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 55b9f9c50..6c3594bab 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" @@ -38,7 +39,8 @@ func (e *StorageEngine) TreeMove(ctx context.Context, d pilorama.CIDDescriptor, if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeMove`", err, zap.Stringer("cid", d.CID), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return nil, err @@ -71,7 +73,8 @@ func (e *StorageEngine) TreeAddByPath(ctx context.Context, d pilorama.CIDDescrip if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeAddByPath`", err, zap.Stringer("cid", d.CID), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return nil, err } @@ -99,7 +102,8 @@ func (e *StorageEngine) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID str if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeApply`", err, zap.Stringer("cid", cnr), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return err } @@ -130,7 +134,8 @@ func (e *StorageEngine) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't perform `TreeGetByPath`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -162,7 +167,8 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't perform `TreeGetMeta`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -193,7 +199,8 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't perform `TreeGetChildren`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -224,7 +231,8 @@ func (e *StorageEngine) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't perform `TreeGetOpLog`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -253,7 +261,8 @@ func (e *StorageEngine) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID stri if !errors.Is(err, pilorama.ErrTreeNotFound) && !errors.Is(err, shard.ErrReadOnlyMode) { e.reportShardError(sh, "can't perform `TreeDrop`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -281,7 +290,8 @@ func (e *StorageEngine) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, } e.reportShardError(sh, "can't perform `TreeList`", err, - zap.Stringer("cid", cid)) + zap.Stringer("cid", cid), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) // returns as much info about // trees as possible @@ -347,7 +357,8 @@ func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK if err != nil && !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't update tree synchronization height", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return err } @@ -373,7 +384,8 @@ func (e *StorageEngine) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, t if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't read tree synchronization height", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 662c0835a..ea481300b 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -94,7 +95,8 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { if err != nil { s.log.Debug(logs.StorageIDRetrievalFailure, zap.Stringer("object", addr), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } storageID := res.StorageID() @@ -106,7 +108,8 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { if err != nil { s.log.Debug(logs.ObjectRemovalFailureBlobStor, zap.Stringer("object_address", addr), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 91565a346..2e7c84bcd 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -10,6 +10,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -144,12 +145,14 @@ func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta if client.IsErrObjectNotFound(err) { s.log.Debug(logs.ShardObjectIsMissingInWritecache, zap.Stringer("addr", addr), - zap.Bool("skip_meta", skipMeta)) + zap.Bool("skip_meta", skipMeta), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else { s.log.Error(logs.ShardFailedToFetchObjectFromWritecache, zap.Error(err), zap.Stringer("addr", addr), - zap.Bool("skip_meta", skipMeta)) + zap.Bool("skip_meta", skipMeta), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } if skipMeta || mErr != nil { diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 6a2f9311d..a5f8960c3 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -110,6 +111,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { s.log.Debug(logs.ShardCouldNotMarkObjectToDeleteInMetabase, zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) s.m.RUnlock() diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index bc13c622b..d6e4d7e50 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -99,7 +100,8 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { if err != nil { s.log.Debug(logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) continue } diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index 9832c9c84..539dbc97f 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -53,6 +54,7 @@ func (s *Shard) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (ToMoveItRes, err if err != nil { s.log.Debug(logs.ShardCouldNotMarkObjectForShardRelocationInMetabase, zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index adac194a0..3865aabb9 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -22,6 +22,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -242,8 +243,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, info.AddressGroup().IterateAddresses(func(addr network.Address) (stop bool) { ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest.IterateAddresses", trace.WithAttributes( - attribute.String("address", addr.String()), - )) + attribute.String("address", addr.String()))) defer span.End() var err error @@ -257,6 +257,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, zap.Stringer("address", addr), zap.Stringer("object_id", objID), zap.Stringer("container_id", cnrID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 16bcec9c5..3a46e5f04 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" @@ -44,7 +45,8 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) if err != nil { p.log.Error(logs.ReplicatorCouldNotGetObjectFromLocalStorage, zap.Stringer("object", task.Addr), - zap.Error(err)) + zap.Error(err), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return } @@ -63,6 +65,7 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) log := p.log.With( zap.String("node", netmap.StringifyPublicKey(task.Nodes[i])), zap.Stringer("object", task.Addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) callCtx, cancel := context.WithTimeout(ctx, p.putTimeout) diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 0afd3439a..ec41a60d4 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -6,6 +6,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" @@ -39,7 +40,9 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo return false } - s.log.Debug(logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) + s.log.Debug(logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + called = true stop = f(c) return true diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 0ca30273e..49a37b8be 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -107,12 +108,14 @@ func (s *Service) replicationWorker(ctx context.Context) { if lastErr != nil { if errors.Is(lastErr, errRecentlyFailed) { s.log.Debug(logs.TreeDoNotSendUpdateToTheNode, - zap.String("last_error", lastErr.Error())) + zap.String("last_error", lastErr.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else { s.log.Warn(logs.TreeFailedToSentUpdateToTheNode, zap.String("last_error", lastErr.Error()), zap.String("address", lastAddr), - zap.String("key", hex.EncodeToString(task.n.PublicKey()))) + zap.String("key", hex.EncodeToString(task.n.PublicKey())), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } s.metrics.AddReplicateTaskDuration(time.Since(start), false) } else { diff --git a/pkg/tracing/trace.go b/pkg/tracing/trace.go new file mode 100644 index 000000000..dc2f90259 --- /dev/null +++ b/pkg/tracing/trace.go @@ -0,0 +1,19 @@ +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/trace" +) + +var emptyTraceID = [16]byte{} + +// GetTraceID retrieves the trace ID from the provided context. +// It returns an empty string if no trace ID is found. +func GetTraceID(ctx context.Context) string { + span := trace.SpanFromContext(ctx) + if span == nil || span.SpanContext().TraceID() == emptyTraceID { + return "" + } + return span.SpanContext().TraceID().String() +}