2020-11-17 12:26:03 +00:00
|
|
|
package engine
|
|
|
|
|
|
|
|
import (
|
2023-03-13 11:37:35 +00:00
|
|
|
"context"
|
2020-11-17 12:26:03 +00:00
|
|
|
"errors"
|
|
|
|
|
2023-04-12 14:35:10 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
|
2023-03-07 13:38:26 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
|
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
|
2023-09-27 08:02:06 +00:00
|
|
|
tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing"
|
2023-03-07 13:38:26 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
|
2023-05-31 09:24:04 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
|
2023-03-07 13:38:26 +00:00
|
|
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
|
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
2023-04-12 14:01:29 +00:00
|
|
|
"go.opentelemetry.io/otel/attribute"
|
|
|
|
"go.opentelemetry.io/otel/trace"
|
2020-11-17 12:26:03 +00:00
|
|
|
"go.uber.org/zap"
|
|
|
|
)
|
|
|
|
|
|
|
|
// PutPrm groups the parameters of Put operation.
|
|
|
|
type PutPrm struct {
|
2022-03-03 14:19:05 +00:00
|
|
|
obj *objectSDK.Object
|
2020-11-17 12:26:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var errPutShard = errors.New("could not put object to any shard")
|
|
|
|
|
|
|
|
// WithObject is a Put option to set object to save.
|
|
|
|
//
|
|
|
|
// Option is required.
|
2022-05-23 13:12:32 +00:00
|
|
|
func (p *PutPrm) WithObject(obj *objectSDK.Object) {
|
2022-09-14 14:36:37 +00:00
|
|
|
p.obj = obj
|
2020-11-17 12:26:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Put saves the object to local storage.
|
|
|
|
//
|
|
|
|
// Returns any error encountered that
|
|
|
|
// did not allow to completely save the object.
|
2021-11-09 15:46:12 +00:00
|
|
|
//
|
|
|
|
// Returns an error if executions are blocked (see BlockExecution).
|
2022-03-17 08:03:58 +00:00
|
|
|
//
|
2022-04-21 11:28:05 +00:00
|
|
|
// Returns an error of type apistatus.ObjectAlreadyRemoved if the object has been marked as removed.
|
2023-04-12 14:01:29 +00:00
|
|
|
func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) {
|
|
|
|
ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Put",
|
|
|
|
trace.WithAttributes(
|
|
|
|
attribute.String("address", object.AddressOf(prm.obj).EncodeToString()),
|
|
|
|
))
|
|
|
|
defer span.End()
|
|
|
|
|
2021-11-10 15:00:30 +00:00
|
|
|
err = e.execIfNotBlocked(func() error {
|
2023-04-12 14:01:29 +00:00
|
|
|
err = e.put(ctx, prm)
|
2021-11-09 15:46:12 +00:00
|
|
|
return err
|
|
|
|
})
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error {
|
2021-03-16 08:14:56 +00:00
|
|
|
if e.metrics != nil {
|
2023-06-13 16:48:15 +00:00
|
|
|
defer elapsed("Put", e.metrics.AddMethodDuration)()
|
2021-03-15 13:09:27 +00:00
|
|
|
}
|
|
|
|
|
2022-03-03 14:19:05 +00:00
|
|
|
addr := object.AddressOf(prm.obj)
|
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
_, err := e.exists(ctx, addr)
|
2020-12-01 11:06:14 +00:00
|
|
|
if err != nil {
|
2023-04-12 14:01:29 +00:00
|
|
|
return err
|
2020-12-01 11:06:14 +00:00
|
|
|
}
|
2020-11-17 12:26:03 +00:00
|
|
|
|
2020-12-01 10:18:25 +00:00
|
|
|
finished := false
|
2020-11-17 12:26:03 +00:00
|
|
|
|
2022-03-03 14:19:05 +00:00
|
|
|
e.iterateOverSortedShards(addr, func(ind int, sh hashedShard) (stop bool) {
|
2021-10-08 12:25:45 +00:00
|
|
|
e.mtx.RLock()
|
[#2188] engine: Fix panic during setting shard mode
Under load changing shard mode can lead to it being removed from the
list during some other PUT.
```
Dec 28 07:01:26 az neofs-node[364505]: panic: runtime error: invalid memory address or nil pointer dereference
Dec 28 07:01:26 az neofs-node[364505]: [signal SIGSEGV: segmentation violation code=0x1 addr=0x20 pc=0xc9fbb1]
Dec 28 07:01:26 az neofs-node[364505]: goroutine 11791912 [running]:
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).putToShard(0xc000435490, {0xc0003f7a28?, 0xc0001192c0?}, 0x2, {0x0, 0x>
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:91 +0x1b1
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).put.func1(0xc000435490?, {0xc0003f7a28?, 0xc0001192c0?})
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:71 +0x19c
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).iterateOverSortedShards(0x1?, {{0x62, 0x23, 0xfe, 0x60, 0x67, 0xd5, 0x>
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/shards.go:225 +0xc8
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).put(0xc000435490, {0x1?})
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:66 +0x2a9
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).Put.func1()
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:43 +0x2a
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).execIfNotBlocked(0x8?, 0x38?)
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/control.go:147 +0xcf
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).Put(0xc4df775a80?, {0x0?})
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:42 +0x65
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.Put(0xc06d928b80?, 0xc06b1b8dc8?)
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:158 +0x19
Dec 28 07:01:26 az neofs-node[364505]: main.engineWithoutNotifications.Put({0x20301b?}, 0x20301b?)
```
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2022-12-29 08:12:01 +00:00
|
|
|
pool, ok := e.shardPools[sh.ID().String()]
|
2021-10-08 12:25:45 +00:00
|
|
|
e.mtx.RUnlock()
|
[#2188] engine: Fix panic during setting shard mode
Under load changing shard mode can lead to it being removed from the
list during some other PUT.
```
Dec 28 07:01:26 az neofs-node[364505]: panic: runtime error: invalid memory address or nil pointer dereference
Dec 28 07:01:26 az neofs-node[364505]: [signal SIGSEGV: segmentation violation code=0x1 addr=0x20 pc=0xc9fbb1]
Dec 28 07:01:26 az neofs-node[364505]: goroutine 11791912 [running]:
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).putToShard(0xc000435490, {0xc0003f7a28?, 0xc0001192c0?}, 0x2, {0x0, 0x>
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:91 +0x1b1
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).put.func1(0xc000435490?, {0xc0003f7a28?, 0xc0001192c0?})
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:71 +0x19c
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).iterateOverSortedShards(0x1?, {{0x62, 0x23, 0xfe, 0x60, 0x67, 0xd5, 0x>
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/shards.go:225 +0xc8
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).put(0xc000435490, {0x1?})
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:66 +0x2a9
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).Put.func1()
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:43 +0x2a
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).execIfNotBlocked(0x8?, 0x38?)
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/control.go:147 +0xcf
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.(*StorageEngine).Put(0xc4df775a80?, {0x0?})
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:42 +0x65
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine.Put(0xc06d928b80?, 0xc06b1b8dc8?)
Dec 28 07:01:26 az neofs-node[364505]: github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine/put.go:158 +0x19
Dec 28 07:01:26 az neofs-node[364505]: main.engineWithoutNotifications.Put({0x20301b?}, 0x20301b?)
```
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2022-12-29 08:12:01 +00:00
|
|
|
if !ok {
|
|
|
|
// Shard was concurrently removed, skip.
|
|
|
|
return false
|
|
|
|
}
|
2021-10-08 12:25:45 +00:00
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
putDone, exists := e.putToShard(ctx, sh, ind, pool, addr, prm.obj)
|
2022-09-13 11:18:00 +00:00
|
|
|
finished = putDone || exists
|
2022-09-12 11:48:06 +00:00
|
|
|
return finished
|
|
|
|
})
|
2021-10-08 12:25:45 +00:00
|
|
|
|
2022-09-12 11:48:06 +00:00
|
|
|
if !finished {
|
|
|
|
err = errPutShard
|
|
|
|
}
|
2021-10-08 12:25:45 +00:00
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
return err
|
2022-09-12 11:48:06 +00:00
|
|
|
}
|
2022-07-27 18:38:28 +00:00
|
|
|
|
2022-09-13 11:18:00 +00:00
|
|
|
// putToShard puts object to sh.
|
|
|
|
// First return value is true iff put has been successfully done.
|
|
|
|
// Second return value is true iff object already exists.
|
2023-03-13 11:37:35 +00:00
|
|
|
func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, pool util.WorkerPool, addr oid.Address, obj *objectSDK.Object) (bool, bool) {
|
2022-09-13 11:18:00 +00:00
|
|
|
var putSuccess, alreadyExists bool
|
2020-12-01 10:18:25 +00:00
|
|
|
|
2022-09-12 11:48:06 +00:00
|
|
|
exitCh := make(chan struct{})
|
2020-11-17 12:26:03 +00:00
|
|
|
|
2022-09-12 11:48:06 +00:00
|
|
|
if err := pool.Submit(func() {
|
|
|
|
defer close(exitCh)
|
2020-11-17 12:26:03 +00:00
|
|
|
|
2022-09-12 11:48:06 +00:00
|
|
|
var existPrm shard.ExistsPrm
|
|
|
|
existPrm.SetAddress(addr)
|
2021-10-08 12:25:45 +00:00
|
|
|
|
2023-03-13 11:37:35 +00:00
|
|
|
exists, err := sh.Exists(ctx, existPrm)
|
2022-09-12 11:48:06 +00:00
|
|
|
if err != nil {
|
|
|
|
if shard.IsErrObjectExpired(err) {
|
|
|
|
// object is already found but
|
|
|
|
// expired => do nothing with it
|
2022-09-13 11:18:00 +00:00
|
|
|
alreadyExists = true
|
2022-09-12 11:48:06 +00:00
|
|
|
}
|
2020-11-17 12:26:03 +00:00
|
|
|
|
2022-09-12 11:48:06 +00:00
|
|
|
return // this is not ErrAlreadyRemoved error so we can go to the next shard
|
|
|
|
}
|
2020-11-17 12:26:03 +00:00
|
|
|
|
2022-09-13 11:18:00 +00:00
|
|
|
alreadyExists = exists.Exists()
|
|
|
|
if alreadyExists {
|
2022-09-12 11:48:06 +00:00
|
|
|
if ind != 0 {
|
|
|
|
var toMoveItPrm shard.ToMoveItPrm
|
|
|
|
toMoveItPrm.SetAddress(addr)
|
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
_, err = sh.ToMoveIt(ctx, toMoveItPrm)
|
2022-09-12 11:48:06 +00:00
|
|
|
if err != nil {
|
2023-04-12 14:35:10 +00:00
|
|
|
e.log.Warn(logs.EngineCouldNotMarkObjectForShardRelocation,
|
2022-09-12 11:48:06 +00:00
|
|
|
zap.Stringer("shard", sh.ID()),
|
|
|
|
zap.String("error", err.Error()),
|
2023-09-27 08:02:06 +00:00
|
|
|
zap.String("trace_id", tracingPkg.GetTraceID(ctx)),
|
2022-09-12 11:48:06 +00:00
|
|
|
)
|
|
|
|
}
|
2021-10-08 12:25:45 +00:00
|
|
|
}
|
|
|
|
|
2022-09-12 11:48:06 +00:00
|
|
|
return
|
2021-10-08 14:33:51 +00:00
|
|
|
}
|
2020-11-17 12:26:03 +00:00
|
|
|
|
2022-09-12 11:48:06 +00:00
|
|
|
var putPrm shard.PutPrm
|
|
|
|
putPrm.SetObject(obj)
|
2020-12-01 11:06:14 +00:00
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
_, err = sh.Put(ctx, putPrm)
|
2022-09-12 11:48:06 +00:00
|
|
|
if err != nil {
|
2022-09-28 08:10:46 +00:00
|
|
|
if errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, blobstor.ErrNoPlaceFound) ||
|
|
|
|
errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) {
|
2023-04-12 14:35:10 +00:00
|
|
|
e.log.Warn(logs.EngineCouldNotPutObjectToShard,
|
2022-09-28 08:10:46 +00:00
|
|
|
zap.Stringer("shard_id", sh.ID()),
|
2023-09-27 08:02:06 +00:00
|
|
|
zap.String("error", err.Error()),
|
|
|
|
zap.String("trace_id", tracingPkg.GetTraceID(ctx)))
|
2022-09-28 08:10:46 +00:00
|
|
|
return
|
|
|
|
}
|
2020-12-01 10:18:25 +00:00
|
|
|
|
2022-09-28 08:10:46 +00:00
|
|
|
e.reportShardError(sh, "could not put object to shard", err)
|
2022-09-12 11:48:06 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-09-13 11:18:00 +00:00
|
|
|
putSuccess = true
|
2022-09-12 11:48:06 +00:00
|
|
|
}); err != nil {
|
2023-07-11 13:13:03 +00:00
|
|
|
e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Error(err))
|
2022-09-12 11:48:06 +00:00
|
|
|
close(exitCh)
|
2020-11-17 12:26:03 +00:00
|
|
|
}
|
|
|
|
|
2022-09-12 11:48:06 +00:00
|
|
|
<-exitCh
|
|
|
|
|
2022-09-13 11:18:00 +00:00
|
|
|
return putSuccess, alreadyExists
|
2020-11-17 12:26:03 +00:00
|
|
|
}
|
2020-11-19 08:15:49 +00:00
|
|
|
|
|
|
|
// Put writes provided object to local storage.
|
2023-04-12 14:01:29 +00:00
|
|
|
func Put(ctx context.Context, storage *StorageEngine, obj *objectSDK.Object) error {
|
2022-05-23 13:12:32 +00:00
|
|
|
var putPrm PutPrm
|
|
|
|
putPrm.WithObject(obj)
|
|
|
|
|
2023-04-12 14:01:29 +00:00
|
|
|
return storage.Put(ctx, putPrm)
|
2020-11-19 08:15:49 +00:00
|
|
|
}
|