realloc/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Releases Packages Activity

[#242] node: Add tracing spans

Browse source 
Add tracing spans for PUT requests.
Add tracing spans for DELETE requests.
Add tracing spans for SELECT requests.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-04-12 17:01:29 +03:00 • committed by fyrchik
parent 200fc8b882
commit d62c6e4ce6
122 changed files with 863 additions and 417 deletions
Download patch file Download diff file Expand all files Collapse all files

15
pkg/local_object_storage/writecache/delete.go
View file

@ -1,16 +1,27 @@
package writecache
import (
"context"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"go.etcd.io/bbolt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
// Delete removes object from write-cache.
//
// Returns an error of type apistatus.ObjectNotFound if object is missing in write-cache.
func (c *cache) Delete(addr oid.Address) error {
func (c *cache) Delete(ctx context.Context, addr oid.Address) error {
ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Delete",
trace.WithAttributes(
attribute.String("address", addr.EncodeToString()),
))
defer span.End()
c.modeMtx.RLock()
defer c.modeMtx.RUnlock()
if c.readOnly() {
@ -45,7 +56,7 @@ func (c *cache) Delete(addr oid.Address) error {
return nil
}
_, err := c.fsTree.Delete(common.DeletePrm{Address: addr})
_, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr})
if err == nil {
storagelog.Write(c.log,
storagelog.AddressField(saddr),

36
pkg/local_object_storage/writecache/flush.go
View file

@ -2,9 +2,11 @@ package writecache
import (
"bytes"
"context"
"errors"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
@ -15,6 +17,8 @@ import (
"github.com/mr-tron/base58"
"github.com/nspcc-dev/neo-go/pkg/util/slice"
"go.etcd.io/bbolt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
)
@ -37,7 +41,7 @@ func (c *cache) runFlushLoop() {
}
c.wg.Add(1)
go c.flushBigObjects()
go c.flushBigObjects(context.TODO())
c.wg.Add(1)
go func() {
@ -141,7 +145,7 @@ func (c *cache) flushDB() {
}
}
func (c *cache) flushBigObjects() {
func (c *cache) flushBigObjects(ctx context.Context) {
defer c.wg.Done()
tick := time.NewTicker(defaultFlushInterval * 10)
@ -157,7 +161,7 @@ func (c *cache) flushBigObjects() {
continue
}
_ = c.flushFSTree(true)
_ = c.flushFSTree(ctx, true)
c.modeMtx.RUnlock()
case <-c.closeCh:
@ -176,7 +180,7 @@ func (c *cache) reportFlushError(msg string, addr string, err error) {
}
}
func (c *cache) flushFSTree(ignoreErrors bool) error {
func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error {
var prm common.IteratePrm
prm.IgnoreErrors = ignoreErrors
prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error {
@ -205,7 +209,7 @@ func (c *cache) flushFSTree(ignoreErrors bool) error {
return err
}
err = c.flushObject(&obj, data)
err = c.flushObject(ctx, &obj, data)
if err != nil {
if ignoreErrors {
return nil
@ -236,7 +240,7 @@ func (c *cache) flushWorker(_ int) {
return
}
err := c.flushObject(obj, nil)
err := c.flushObject(context.TODO(), obj, nil)
if err == nil {
c.flushed.Add(objectCore.AddressOf(obj).EncodeToString(), true)
}
@ -244,14 +248,14 @@ func (c *cache) flushWorker(_ int) {
}
// flushObject is used to write object directly to the main storage.
func (c *cache) flushObject(obj *object.Object, data []byte) error {
func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte) error {
addr := objectCore.AddressOf(obj)
var prm common.PutPrm
prm.Object = obj
prm.RawData = data
res, err := c.blobstor.Put(prm)
res, err := c.blobstor.Put(ctx, prm)
if err != nil {
if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) &&
!errors.Is(err, blobstor.ErrNoPlaceFound) {
@ -276,15 +280,21 @@ func (c *cache) flushObject(obj *object.Object, data []byte) error {
// Flush flushes all objects from the write-cache to the main storage.
// Write-cache must be in readonly mode to ensure correctness of an operation and
// to prevent interference with background flush workers.
func (c *cache) Flush(ignoreErrors bool) error {
func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error {
ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush",
trace.WithAttributes(
attribute.Bool("ignore_errors", ignoreErrors),
))
defer span.End()
c.modeMtx.RLock()
defer c.modeMtx.RUnlock()
return c.flush(ignoreErrors)
return c.flush(ctx, ignoreErrors)
}
func (c *cache) flush(ignoreErrors bool) error {
if err := c.flushFSTree(ignoreErrors); err != nil {
func (c *cache) flush(ctx context.Context, ignoreErrors bool) error {
if err := c.flushFSTree(ctx, ignoreErrors); err != nil {
return err
}
@ -316,7 +326,7 @@ func (c *cache) flush(ignoreErrors bool) error {
return err
}
if err := c.flushObject(&obj, data); err != nil {
if err := c.flushObject(ctx, &obj, data); err != nil {
return err
}
}

24
pkg/local_object_storage/writecache/flush_test.go
View file

@ -89,7 +89,7 @@ func TestFlush(t *testing.T) {
var mPrm meta.StorageIDPrm
mPrm.SetAddress(objects[i].addr)
mRes, err := mb.StorageID(mPrm)
mRes, err := mb.StorageID(context.Background(), mPrm)
require.NoError(t, err)
var prm common.GetPrm
@ -112,12 +112,12 @@ func TestFlush(t *testing.T) {
wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true)
wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false)
require.NoError(t, wc.Flush(false))
require.NoError(t, wc.Flush(context.Background(), false))
for i := 0; i < 2; i++ {
var mPrm meta.GetPrm
mPrm.SetAddress(objects[i].addr)
_, err := mb.Get(mPrm)
_, err := mb.Get(context.Background(), mPrm)
require.Error(t, err)
_, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr})
@ -147,7 +147,7 @@ func TestFlush(t *testing.T) {
for i := 0; i < 2; i++ {
var mPrm meta.GetPrm
mPrm.SetAddress(objects[i].addr)
_, err := mb.Get(mPrm)
_, err := mb.Get(context.Background(), mPrm)
require.Error(t, err)
_, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr})
@ -171,9 +171,9 @@ func TestFlush(t *testing.T) {
require.NoError(t, mb.SetMode(mode.ReadWrite))
require.Equal(t, uint32(0), errCount.Load())
require.Error(t, wc.Flush(false))
require.Error(t, wc.Flush(context.Background(), false))
require.True(t, errCount.Load() > 0)
require.NoError(t, wc.Flush(true))
require.NoError(t, wc.Flush(context.Background(), true))
check(t, mb, bs, objects)
}
@ -202,7 +202,7 @@ func TestFlush(t *testing.T) {
prm.Address = objectCore.AddressOf(obj)
prm.RawData = data
_, err := c.fsTree.Put(prm)
_, err := c.fsTree.Put(context.Background(), prm)
require.NoError(t, err)
p := prm.Address.Object().EncodeToString() + "." + prm.Address.Container().EncodeToString()
@ -218,7 +218,7 @@ func TestFlush(t *testing.T) {
var prm common.PutPrm
prm.Address = oidtest.Address()
prm.RawData = []byte{1, 2, 3}
_, err := c.fsTree.Put(prm)
_, err := c.fsTree.Put(context.Background(), prm)
require.NoError(t, err)
})
})
@ -245,19 +245,19 @@ func TestFlush(t *testing.T) {
for i := range objects {
var prm meta.PutPrm
prm.SetObject(objects[i].obj)
_, err := mb.Put(prm)
_, err := mb.Put(context.Background(), prm)
require.NoError(t, err)
}
var inhumePrm meta.InhumePrm
inhumePrm.SetAddresses(objects[0].addr, objects[1].addr)
inhumePrm.SetTombstoneAddress(oidtest.Address())
_, err := mb.Inhume(inhumePrm)
_, err := mb.Inhume(context.Background(), inhumePrm)
require.NoError(t, err)
var deletePrm meta.DeletePrm
deletePrm.SetAddresses(objects[2].addr, objects[3].addr)
_, err = mb.Delete(deletePrm)
_, err = mb.Delete(context.Background(), deletePrm)
require.NoError(t, err)
require.NoError(t, bs.SetMode(mode.ReadOnly))
@ -294,7 +294,7 @@ func putObject(t *testing.T, c Cache, size int) objectPair {
prm.Object = obj
prm.RawData = data
_, err := c.Put(prm)
_, err := c.Put(context.Background(), prm)
require.NoError(t, err)
return objectPair{prm.Address, prm.Object}

24
pkg/local_object_storage/writecache/init.go
View file

@ -15,21 +15,21 @@ import (
"go.uber.org/zap"
)
func (c *cache) initFlushMarks() {
func (c *cache) initFlushMarks(ctx context.Context) {
var localWG sync.WaitGroup
localWG.Add(1)
go func() {
defer localWG.Done()
c.fsTreeFlushMarkUpdate()
c.fsTreeFlushMarkUpdate(ctx)
}()
localWG.Add(1)
go func() {
defer localWG.Done()
c.dbFlushMarkUpdate()
c.dbFlushMarkUpdate(ctx)
}()
c.initWG.Add(1)
@ -54,7 +54,7 @@ func (c *cache) initFlushMarks() {
var errStopIter = errors.New("stop iteration")
func (c *cache) fsTreeFlushMarkUpdate() {
func (c *cache) fsTreeFlushMarkUpdate(ctx context.Context) {
c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInFSTree)
var prm common.IteratePrm
@ -67,14 +67,14 @@ func (c *cache) fsTreeFlushMarkUpdate() {
default:
}
flushed, needRemove := c.flushStatus(addr)
flushed, needRemove := c.flushStatus(ctx, addr)
if flushed {
c.store.flushed.Add(addr.EncodeToString(), true)
if needRemove {
var prm common.DeletePrm
prm.Address = addr
_, err := c.fsTree.Delete(prm)
_, err := c.fsTree.Delete(ctx, prm)
if err == nil {
storagelog.Write(c.log,
storagelog.AddressField(addr),
@ -90,7 +90,7 @@ func (c *cache) fsTreeFlushMarkUpdate() {
c.log.Info(logs.WritecacheFinishedUpdatingFSTreeFlushMarks)
}
func (c *cache) dbFlushMarkUpdate() {
func (c *cache) dbFlushMarkUpdate(ctx context.Context) {
c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInDatabase)
var m []string
@ -125,7 +125,7 @@ func (c *cache) dbFlushMarkUpdate() {
continue
}
flushed, needRemove := c.flushStatus(addr)
flushed, needRemove := c.flushStatus(ctx, addr)
if flushed {
c.store.flushed.Add(addr.EncodeToString(), true)
if needRemove {
@ -165,11 +165,11 @@ func (c *cache) dbFlushMarkUpdate() {
// flushStatus returns info about the object state in the main storage.
// First return value is true iff object exists.
// Second return value is true iff object can be safely removed.
func (c *cache) flushStatus(addr oid.Address) (bool, bool) {
func (c *cache) flushStatus(ctx context.Context, addr oid.Address) (bool, bool) {
var existsPrm meta.ExistsPrm
existsPrm.SetAddress(addr)
_, err := c.metabase.Exists(existsPrm)
_, err := c.metabase.Exists(ctx, existsPrm)
if err != nil {
needRemove := errors.Is(err, meta.ErrObjectIsExpired) || errors.As(err, new(apistatus.ObjectAlreadyRemoved))
return needRemove, needRemove
@ -178,7 +178,7 @@ func (c *cache) flushStatus(addr oid.Address) (bool, bool) {
var prm meta.StorageIDPrm
prm.SetAddress(addr)
mRes, _ := c.metabase.StorageID(prm)
res, err := c.blobstor.Exists(context.TODO(), common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()})
mRes, _ := c.metabase.StorageID(ctx, prm)
res, err := c.blobstor.Exists(ctx, common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()})
return err == nil && res.Exists, false
}

18
pkg/local_object_storage/writecache/mode.go
View file

@ -1,12 +1,16 @@
package writecache
import (
"context"
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
// ErrReadOnly is returned when Put/Write is performed in a read-only mode.
@ -19,19 +23,25 @@ var ErrNotInitialized = logicerr.New("write-cache is not initialized yet")
// When shard is put in read-only mode all objects in memory are flushed to disk
// and all background jobs are suspended.
func (c *cache) SetMode(m mode.Mode) error {
ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.SetMode",
trace.WithAttributes(
attribute.String("mode", m.String()),
))
defer span.End()
c.modeMtx.Lock()
defer c.modeMtx.Unlock()
return c.setMode(m)
return c.setMode(ctx, m)
}
// setMode applies new mode. Must be called with cache.modeMtx lock taken.
func (c *cache) setMode(m mode.Mode) error {
func (c *cache) setMode(ctx context.Context, m mode.Mode) error {
var err error
turnOffMeta := m.NoMetabase()
if turnOffMeta && !c.mode.NoMetabase() {
err = c.flush(true)
err = c.flush(ctx, true)
if err != nil {
return err
}
@ -45,7 +55,7 @@ func (c *cache) setMode(m mode.Mode) error {
defer func() {
if err == nil && !turnOffMeta {
c.initFlushMarks()
c.initFlushMarks(ctx)
}
}()
}

6
pkg/local_object_storage/writecache/options.go
View file

@ -19,14 +19,14 @@ type Option func(*options)
// meta is an interface for a metabase.
type metabase interface {
Exists(meta.ExistsPrm) (meta.ExistsRes, error)
StorageID(meta.StorageIDPrm) (meta.StorageIDRes, error)
Exists(context.Context, meta.ExistsPrm) (meta.ExistsRes, error)
StorageID(context.Context, meta.StorageIDPrm) (meta.StorageIDRes, error)
UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error)
}
// blob is an interface for the blobstor.
type blob interface {
Put(common.PutPrm) (common.PutRes, error)
Put(context.Context, common.PutPrm) (common.PutRes, error)
NeedsCompression(obj *objectSDK.Object) bool
Exists(ctx context.Context, res common.ExistsPrm) (common.ExistsRes, error)
}

19
pkg/local_object_storage/writecache/put.go
View file

@ -1,11 +1,15 @@
package writecache
import (
"context"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log"
"go.etcd.io/bbolt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
var (
@ -21,7 +25,14 @@ var (
// Returns ErrNotInitialized if write-cache has not been initialized yet.
// Returns ErrOutOfSpace if saving an object leads to WC's size overflow.
// Returns ErrBigObject if an objects exceeds maximum object size.
func (c *cache) Put(prm common.PutPrm) (common.PutRes, error) {
func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Put",
trace.WithAttributes(
attribute.String("address", prm.Address.EncodeToString()),
attribute.Bool("dont_compress", prm.DontCompress),
))
defer span.End()
c.modeMtx.RLock()
defer c.modeMtx.RUnlock()
if c.readOnly() {
@ -44,7 +55,7 @@ func (c *cache) Put(prm common.PutPrm) (common.PutRes, error) {
if sz <= c.smallObjectSize {
return common.PutRes{}, c.putSmall(oi)
}
return common.PutRes{}, c.putBig(oi.addr, prm)
return common.PutRes{}, c.putBig(ctx, oi.addr, prm)
}
// putSmall persists small objects to the write-cache database and
@ -71,13 +82,13 @@ func (c *cache) putSmall(obj objectInfo) error {
}
// putBig writes object to FSTree and pushes it to the flush workers queue.
func (c *cache) putBig(addr string, prm common.PutPrm) error {
func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error {
cacheSz := c.estimateCacheSize()
if c.maxCacheSize < c.incSizeFS(cacheSz) {
return ErrOutOfSpace
}
_, err := c.fsTree.Put(prm)
_, err := c.fsTree.Put(ctx, prm)
if err != nil {
return err
}

3
pkg/local_object_storage/writecache/storage.go
View file

@ -1,6 +1,7 @@
package writecache
import (
"context"
"errors"
"fmt"
"os"
@ -146,7 +147,7 @@ func (c *cache) deleteFromDisk(keys []string) []string {
continue
}
_, err := c.fsTree.Delete(common.DeletePrm{Address: addr})
_, err := c.fsTree.Delete(context.TODO(), common.DeletePrm{Address: addr})
if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) {
c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err))

14
pkg/local_object_storage/writecache/writecache.go
View file

@ -5,6 +5,7 @@ import (
"os"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
@ -32,13 +33,13 @@ type Cache interface {
//
// Returns apistatus.ObjectNotFound if object is missing in the Cache.
// Returns ErrReadOnly if the Cache is currently in the read-only mode.
Delete(oid.Address) error
Delete(context.Context, oid.Address) error
Iterate(IterationPrm) error
Put(common.PutPrm) (common.PutRes, error)
Put(context.Context, common.PutPrm) (common.PutRes, error)
SetMode(mode.Mode) error
SetLogger(*logger.Logger)
DumpInfo() Info
Flush(bool) error
Flush(context.Context, bool) error
Init() error
Open(readOnly bool) error
@ -152,7 +153,10 @@ func (c *cache) Open(readOnly bool) error {
// Init runs necessary services.
func (c *cache) Init() error {
c.initFlushMarks()
ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.Init")
defer span.End()
c.initFlushMarks(ctx)
c.runFlushLoop()
return nil
}
@ -163,7 +167,7 @@ func (c *cache) Close() error {
defer c.modeMtx.Unlock()
// Finish all in-progress operations.
if err := c.setMode(mode.ReadOnly); err != nil {
if err := c.setMode(context.TODO(), mode.ReadOnly); err != nil {
return err
}