From 16f787698859cb3069400082f00cee9cc317597c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Mar 2024 11:36:58 +0300 Subject: [PATCH 1/5] [#1064] putsvc: Refactor distributed target Extract object builder. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/builder.go | 54 ++++++++++++++++++++++++++ pkg/services/object/put/distributed.go | 36 +---------------- pkg/services/object/put/streamer.go | 18 +++------ 3 files changed, 60 insertions(+), 48 deletions(-) create mode 100644 pkg/services/object/put/builder.go diff --git a/pkg/services/object/put/builder.go b/pkg/services/object/put/builder.go new file mode 100644 index 000000000..64baf4e05 --- /dev/null +++ b/pkg/services/object/put/builder.go @@ -0,0 +1,54 @@ +package putsvc + +import ( + "context" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" +) + +var _ transformer.ChunkedObjectWriter = (*inMemoryObjectBuilder)(nil) + +type inMemoryObjectBuilder struct { + objectWriter transformer.ObjectWriter + payload *payload + + obj *objectSDK.Object +} + +func newInMemoryObjectBuilder(objectWriter transformer.ObjectWriter) *inMemoryObjectBuilder { + return &inMemoryObjectBuilder{ + objectWriter: objectWriter, + payload: getPayload(), + } +} + +func (b *inMemoryObjectBuilder) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { + defer func() { + putPayload(b.payload) + b.payload = nil + }() + + b.obj.SetPayload(b.payload.Data) + + if err := b.objectWriter.WriteObject(ctx, b.obj); err != nil { + return nil, err + } + + id, _ := b.obj.ID() + return &transformer.AccessIdentifiers{ + SelfID: id, + }, nil +} + +func (b *inMemoryObjectBuilder) Write(_ context.Context, p []byte) (int, error) { + b.payload.Data = append(b.payload.Data, p...) + + return len(p), nil +} + +func (b *inMemoryObjectBuilder) WriteHeader(_ context.Context, obj *objectSDK.Object) error { + b.obj = obj + + return nil +} diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 509f4aee0..c71427b67 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) type preparedObjectTarget interface { @@ -15,16 +14,13 @@ type preparedObjectTarget interface { } type distributedTarget struct { - placementOpts []placement.Option - extraBroadcastEnabled bool + placementOpts []placement.Option obj *objectSDK.Object objMeta object.ContentMeta *cfg - payload *payload - nodeTargetInitializer func(nodeDesc) preparedObjectTarget relay func(context.Context, nodeDesc) error @@ -91,36 +87,6 @@ func (x errIncompletePut) Error() string { return commonMsg } -func (t *distributedTarget) WriteHeader(_ context.Context, obj *objectSDK.Object) error { - t.obj = obj - - return nil -} - -func (t *distributedTarget) Write(_ context.Context, p []byte) (n int, err error) { - t.payload.Data = append(t.payload.Data, p...) - - return len(p), nil -} - -func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { - defer func() { - putPayload(t.payload) - t.payload = nil - }() - - t.obj.SetPayload(t.payload.Data) - - if err := t.WriteObject(ctx, t.obj); err != nil { - return nil, err - } - - id, _ := t.obj.ID() - return &transformer.AccessIdentifiers{ - SelfID: id, - }, nil -} - // WriteObject implements the transformer.ObjectWriter interface. func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Object) error { t.obj = obj diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 90d580a1c..f32b2ab99 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -79,7 +78,7 @@ func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { // prepare untrusted-Put object target p.target = &validatingPreparedTarget{ - nextTarget: p.newCommonTarget(prm), + nextTarget: newInMemoryObjectBuilder(p.newObjectWriter(prm)), fmt: p.fmtValidator, maxPayloadSz: p.maxPayloadSz, @@ -133,7 +132,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ Key: sessionKey, - NextTargetInit: func() transformer.ObjectWriter { return p.newCommonTarget(prm) }, + NextTargetInit: func() transformer.ObjectWriter { return p.newObjectWriter(prm) }, NetworkState: p.networkState, MaxSize: p.maxPayloadSz, WithoutHomomorphicHash: containerSDK.IsHomomorphicHashingDisabled(prm.cnr), @@ -196,7 +195,7 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { return nil } -func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { +func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { @@ -213,16 +212,9 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { } } - // enable additional container broadcast on non-local operation - // if object has TOMBSTONE or LOCK type. - typ := prm.hdr.Type() - withBroadcast := !prm.common.LocalOnly() && (typ == objectSDK.TypeTombstone || typ == objectSDK.TypeLock) - return &distributedTarget{ - cfg: p.cfg, - placementOpts: prm.traverseOpts, - extraBroadcastEnabled: withBroadcast, - payload: getPayload(), + cfg: p.cfg, + placementOpts: prm.traverseOpts, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return localTarget{ -- 2.45.2 From 9b9a90ba0a1370372dfb25a750bd15d5eadca0a1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Mar 2024 13:46:19 +0300 Subject: [PATCH 2/5] [#1064] putsvc: Add EC put Signed-off-by: Dmitrii Stepanov --- go.mod | 1 + go.sum | Bin 41962 -> 42147 bytes internal/logs/logs.go | 2 + pkg/core/container/ec.go | 11 + pkg/core/object/ec.go | 13 + pkg/core/policy/ec.go | 20 ++ pkg/services/object/put/ec.go | 265 ++++++++++++++++++ pkg/services/object/put/single.go | 88 +++++- pkg/services/object/put/streamer.go | 48 +++- pkg/services/object/put/writer.go | 23 ++ .../object_manager/placement/traverser.go | 2 +- 11 files changed, 450 insertions(+), 23 deletions(-) create mode 100644 pkg/core/container/ec.go create mode 100644 pkg/core/object/ec.go create mode 100644 pkg/core/policy/ec.go create mode 100644 pkg/services/object/put/ec.go create mode 100644 pkg/services/object/put/writer.go diff --git a/go.mod b/go.mod index 3159b9c0a..baca99de2 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( github.com/ipfs/go-cid v0.4.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/reedsolomon v1.12.1 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect diff --git a/go.sum b/go.sum index e4bd28600d1f11e87b913a8409a3383830e4f25f..ab079ea9a0287adca2514d6a85e28d4fe0c7db97 100644 GIT binary patch delta 167 zcmaELoN4h%rVVq9CifX~Iu)g+rWEJr$XR0sf7L 0 +} + +// ECDataCount returns EC data count for EC placement policy. +func ECDataCount(policy netmapSDK.PlacementPolicy) int { + return int(policy.ReplicaDescriptor(0).GetECDataCount()) +} + +// ECParityCount returns EC parity count for EC placement policy. +func ECParityCount(policy netmapSDK.PlacementPolicy) int { + return int(policy.ReplicaDescriptor(0).GetECParityCount()) +} diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go new file mode 100644 index 000000000..74db3c31f --- /dev/null +++ b/pkg/services/object/put/ec.go @@ -0,0 +1,265 @@ +package putsvc + +import ( + "context" + "crypto/ecdsa" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" + svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +var _ transformer.ObjectWriter = (*ecWriter)(nil) + +var errUnsupportedECObject = errors.New("object is not supported for erasure coding") + +type ecWriter struct { + cfg *cfg + placementOpts []placement.Option + container containerSDK.Container + key *ecdsa.PrivateKey + commonPrm *svcutil.CommonPrm + relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error + + objMeta object.ContentMeta + objMetaValid bool +} + +func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + relayed, err := e.relayIfNotContainerNode(ctx) + if err != nil { + return err + } + if relayed { + return nil + } + + if !object.IsECSupported(obj) { + // must be resolved by caller + return errUnsupportedECObject + } + + if !e.objMetaValid { + if e.objMeta, err = e.cfg.fmtValidator.ValidateContent(obj); err != nil { + return fmt.Errorf("(%T) could not validate payload content: %w", e, err) + } + e.objMetaValid = true + } + + if obj.ECHeader() != nil { + return e.writeECPart(ctx, obj) + } + return e.writeRawObject(ctx, obj) +} + +func (e *ecWriter) relayIfNotContainerNode(ctx context.Context) (bool, error) { + if e.relay == nil { + return false, nil + } + currentNodeIsContainerNode, err := e.currentNodeIsContainerNode() + if err != nil { + return false, err + } + if currentNodeIsContainerNode { + // object can be splitted or saved local + return false, nil + } + if err := e.relayToContainerNode(ctx); err != nil { + return false, err + } + return true, nil +} + +func (e *ecWriter) currentNodeIsContainerNode() (bool, error) { + t, err := placement.NewTraverser(e.placementOpts...) + if err != nil { + return false, err + } + for { + nodes := t.Next() + if len(nodes) == 0 { + break + } + for _, node := range nodes { + if e.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + return true, nil + } + } + } + return false, nil +} + +func (e *ecWriter) relayToContainerNode(ctx context.Context) error { + t, err := placement.NewTraverser(e.placementOpts...) + if err != nil { + return err + } + var lastErr error + for { + nodes := t.Next() + if len(nodes) == 0 { + break + } + for _, node := range nodes { + var info client.NodeInfo + client.NodeInfoFromNetmapElement(&info, node) + + c, err := e.cfg.clientConstructor.Get(info) + if err != nil { + return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) + } + + completed := make(chan interface{}) + if poolErr := e.cfg.remotePool.Submit(func() { + defer close(completed) + err = e.relay(ctx, info, c) + }); poolErr != nil { + close(completed) + svcutil.LogWorkerPoolError(e.cfg.log, "PUT", poolErr) + return poolErr + } + <-completed + + if err == nil { + return nil + } + e.cfg.log.Logger.Warn(logs.ECFailedToSendToContainerNode, zap.Stringers("address_group", info.AddressGroup())) + lastErr = err + } + } + if lastErr == nil { + return nil + } + return errIncompletePut{ + singleErr: lastErr, + } +} + +func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error { + t, err := placement.NewTraverser(e.placementOpts...) + if err != nil { + return err + } + + eg, egCtx := errgroup.WithContext(ctx) + for { + nodes := t.Next() + if len(nodes) == 0 { + break + } + + eg.Go(func() error { + return e.writePart(egCtx, obj, int(obj.ECHeader().Index()), nodes) + }) + } + if err := eg.Wait(); err != nil { + return errIncompletePut{ + singleErr: err, + } + } + return nil +} + +func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) error { + // now only single EC policy is supported + c, err := erasurecode.NewConstructor(policy.ECDataCount(e.container.PlacementPolicy()), policy.ECParityCount(e.container.PlacementPolicy())) + if err != nil { + return err + } + parts, err := c.Split(obj, e.key) + if err != nil { + return err + } + t, err := placement.NewTraverser(e.placementOpts...) + if err != nil { + return err + } + + eg, egCtx := errgroup.WithContext(ctx) + for { + nodes := t.Next() + if len(nodes) == 0 { + break + } + + for idx := range parts { + idx := idx + eg.Go(func() error { + return e.writePart(egCtx, parts[idx], idx, nodes) + }) + } + } + if err := eg.Wait(); err != nil { + return errIncompletePut{ + singleErr: err, + } + } + return nil +} + +func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx int, nodes []placement.Node) error { + var err error + node := nodes[partIdx%len(nodes)] + if e.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + err = e.writePartLocal(ctx, obj) + } else { + err = e.writePartRemote(ctx, obj, node) + } + if err == nil { + return nil + } + e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("parent_object", object.AddressOf(obj)), zap.Error(err)) + return err +} + +func (e *ecWriter) writePartLocal(ctx context.Context, obj *objectSDK.Object) error { + var err error + localTarget := localTarget{ + storage: e.cfg.localStore, + } + completed := make(chan interface{}) + if poolErr := e.cfg.localPool.Submit(func() { + defer close(completed) + err = localTarget.WriteObject(ctx, obj, e.objMeta) + }); poolErr != nil { + close(completed) + return poolErr + } + <-completed + return err +} + +func (e *ecWriter) writePartRemote(ctx context.Context, obj *objectSDK.Object, node placement.Node) error { + var clientNodeInfo client.NodeInfo + client.NodeInfoFromNetmapElement(&clientNodeInfo, node) + + remoteTaget := remoteTarget{ + privateKey: e.key, + clientConstructor: e.cfg.clientConstructor, + commonPrm: e.commonPrm, + nodeInfo: clientNodeInfo, + } + + var err error + completed := make(chan interface{}) + if poolErr := e.cfg.remotePool.Submit(func() { + defer close(completed) + err = remoteTaget.WriteObject(ctx, obj, e.objMeta) + }); poolErr != nil { + close(completed) + return poolErr + } + <-completed + return err +} diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 09e1eb092..43b3b0ac1 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -25,6 +26,7 @@ import ( tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/tzhash/tz" "go.opentelemetry.io/otel/attribute" @@ -32,7 +34,10 @@ import ( "go.uber.org/zap" ) -var errInvalidPayloadChecksum = errors.New("incorrect payload checksum") +var ( + errInvalidPayloadChecksum = errors.New("incorrect payload checksum") + errInvalidECObject = errors.New("object must be splitted to EC parts") +) type putSingleRequestSigner struct { req *objectAPI.PutSingleRequest @@ -148,12 +153,20 @@ func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Ob func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { localOnly := req.GetMetaHeader().GetTTL() <= 1 - placementOptions, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) + placement, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) if err != nil { return err } - iter := s.cfg.newNodeIterator(placementOptions) + if placement.isEC { + return s.saveToECReplicas(ctx, placement, obj, req, meta) + } + + return s.saveToREPReplicas(ctx, placement, obj, localOnly, req, meta) +} + +func (s *Service) saveToREPReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, localOnly bool, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { + iter := s.cfg.newNodeIterator(placement.placementOptions) iter.extraBroadcastEnabled = needAdditionalBroadcast(obj, localOnly) signer := &putSingleRequestSigner{ @@ -167,38 +180,83 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o }) } -func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) ([]placement.Option, error) { - var result []placement.Option - if len(copiesNumber) > 0 { - result = append(result, placement.WithCopyNumbers(copiesNumber)) +func (s *Service) saveToECReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { + if obj.Type() == objectSDK.TypeRegular && obj.ECHeader() == nil { + return errInvalidECObject } + commonPrm, err := svcutil.CommonPrmFromV2(req) + if err != nil { + return err + } + key, err := s.cfg.keyStorage.GetKey(nil) + if err != nil { + return err + } + signer := &putSingleRequestSigner{ + req: req, + keyStorage: s.keyStorage, + signer: &sync.Once{}, + } + + w := ecWriter{ + cfg: s.cfg, + placementOpts: placement.placementOptions, + objMeta: meta, + objMetaValid: true, + commonPrm: commonPrm, + container: placement.container, + key: key, + relay: func(ctx context.Context, ni client.NodeInfo, mac client.MultiAddressClient) error { + return s.redirectPutSingleRequest(ctx, signer, obj, ni, mac) + }, + } + return w.WriteObject(ctx, obj) +} + +type putSinglePlacement struct { + placementOptions []placement.Option + isEC bool + container containerSDK.Container +} + +func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) (putSinglePlacement, error) { + var result putSinglePlacement + cnrID, ok := obj.ContainerID() if !ok { - return nil, errors.New("missing container ID") + return result, errors.New("missing container ID") } cnrInfo, err := s.cnrSrc.Get(cnrID) if err != nil { - return nil, fmt.Errorf("could not get container by ID: %w", err) + return result, fmt.Errorf("could not get container by ID: %w", err) } - result = append(result, placement.ForContainer(cnrInfo.Value)) + result.container = cnrInfo.Value + result.isEC = container.IsECContainer(cnrInfo.Value) && object.IsECSupported(obj) + if len(copiesNumber) > 0 && !result.isEC { + result.placementOptions = append(result.placementOptions, placement.WithCopyNumbers(copiesNumber)) + } + result.placementOptions = append(result.placementOptions, placement.ForContainer(cnrInfo.Value)) objID, ok := obj.ID() if !ok { - return nil, errors.New("missing object ID") + return result, errors.New("missing object ID") } - result = append(result, placement.ForObject(objID)) + if obj.ECHeader() != nil { + objID = obj.ECHeader().Parent() + } + result.placementOptions = append(result.placementOptions, placement.ForObject(objID)) latestNetmap, err := netmap.GetLatestNetworkMap(s.netMapSrc) if err != nil { - return nil, fmt.Errorf("could not get latest network map: %w", err) + return result, fmt.Errorf("could not get latest network map: %w", err) } builder := placement.NewNetworkMapBuilder(latestNetmap) if localOnly { - result = append(result, placement.SuccessAfter(1)) + result.placementOptions = append(result.placementOptions, placement.SuccessAfter(1)) builder = svcutil.NewLocalPlacement(builder, s.netmapKeys) } - result = append(result, placement.UseBuilder(builder)) + result.placementOptions = append(result.placementOptions, placement.UseBuilder(builder)) return result, nil } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index f32b2ab99..14dae38d5 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -7,7 +7,9 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -19,7 +21,7 @@ import ( type Streamer struct { *cfg - sessionKey *ecdsa.PrivateKey + privateKey *ecdsa.PrivateKey target transformer.ChunkedObjectWriter @@ -76,6 +78,12 @@ func (p *Streamer) initTarget(prm *PutInitPrm) error { func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { p.relay = prm.relay + nodeKey, err := p.cfg.keyStorage.GetKey(nil) + if err != nil { + return err + } + p.privateKey = nodeKey + // prepare untrusted-Put object target p.target = &validatingPreparedTarget{ nextTarget: newInMemoryObjectBuilder(p.newObjectWriter(prm)), @@ -102,7 +110,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { } } - sessionKey, err := p.keyStorage.GetKey(sessionInfo) + key, err := p.keyStorage.GetKey(sessionInfo) if err != nil { return fmt.Errorf("(%T) could not receive session key: %w", p, err) } @@ -116,7 +124,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { if sToken == nil { var ownerSession user.ID - user.IDFromKey(&ownerSession, sessionKey.PublicKey) + user.IDFromKey(&ownerSession, key.PublicKey) if !ownerObj.Equals(ownerSession) { return fmt.Errorf("(%T) session token is missing but object owner id is different from the default key", p) @@ -127,11 +135,11 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { } } - p.sessionKey = sessionKey + p.privateKey = key p.target = &validatingTarget{ fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ - Key: sessionKey, + Key: key, NextTargetInit: func() transformer.ObjectWriter { return p.newObjectWriter(prm) }, NetworkState: p.networkState, MaxSize: p.maxPayloadSz, @@ -171,7 +179,12 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { placement.ForContainer(prm.cnr), ) - if id, ok := prm.hdr.ID(); ok { + if ech := prm.hdr.ECHeader(); ech != nil { + prm.traverseOpts = append(prm.traverseOpts, + // set identifier of the processing object + placement.ForObject(ech.Parent()), + ) + } else if id, ok := prm.hdr.ID(); ok { prm.traverseOpts = append(prm.traverseOpts, // set identifier of the processing object placement.ForObject(id), @@ -196,6 +209,13 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { } func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { + if container.IsECContainer(prm.cnr) && object.IsECSupported(prm.hdr) { + return p.newECWriter(prm) + } + return p.newDefaultObjectWriter(prm) +} + +func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { @@ -223,7 +243,7 @@ func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { } rt := &remoteTarget{ - privateKey: p.sessionKey, + privateKey: p.privateKey, commonPrm: prm.common, clientConstructor: p.clientConstructor, } @@ -236,6 +256,20 @@ func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { } } +func (p *Streamer) newECWriter(prm *PutInitPrm) transformer.ObjectWriter { + return &objectWriterDispatcher{ + ecWriter: &ecWriter{ + cfg: p.cfg, + placementOpts: append(prm.traverseOpts, placement.WithCopyNumbers(nil)), // copies number ignored for EC + container: prm.cnr, + key: p.privateKey, + commonPrm: prm.common, + relay: p.relay, + }, + repWriter: p.newDefaultObjectWriter(prm), + } +} + func (p *Streamer) SendChunk(ctx context.Context, prm *PutChunkPrm) error { if p.target == nil { return errNotInit diff --git a/pkg/services/object/put/writer.go b/pkg/services/object/put/writer.go new file mode 100644 index 000000000..53eee6006 --- /dev/null +++ b/pkg/services/object/put/writer.go @@ -0,0 +1,23 @@ +package putsvc + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" +) + +var _ transformer.ObjectWriter = (*objectWriterDispatcher)(nil) + +type objectWriterDispatcher struct { + ecWriter transformer.ObjectWriter + repWriter transformer.ObjectWriter +} + +func (m *objectWriterDispatcher) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + if object.IsECSupported(obj) { + return m.ecWriter.WriteObject(ctx, obj) + } + return m.repWriter.WriteObject(ctx, obj) +} diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index a699b4454..306169571 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -137,7 +137,7 @@ func defaultCopiesVector(policy netmap.PlacementPolicy) []int { copyVector := make([]int, 0, replNum) for i := 0; i < replNum; i++ { - copyVector = append(copyVector, int(policy.ReplicaDescriptor(i).NumberOfObjects())) + copyVector = append(copyVector, int(policy.ReplicaDescriptor(i).NumberOfObjects()+policy.ReplicaDescriptor(i).GetECDataCount()+policy.ReplicaDescriptor(i).GetECParityCount())) } return copyVector -- 2.45.2 From 17a6eb26eaab5e158d7a2aa6bfe21945a8e1315e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Mar 2024 13:47:36 +0300 Subject: [PATCH 3/5] [#1064] dev: Add `IR + 4 storage nodes` configuration Signed-off-by: Dmitrii Stepanov --- Makefile | 12 +- dev/.vscode-example/launch.json | 197 +++++++++++++++++++-- dev/storage/{wallet.json => wallet01.json} | 0 dev/storage/wallet02.json | 30 ++++ dev/storage/wallet03.json | 30 ++++ dev/storage/wallet04.json | 30 ++++ 6 files changed, 278 insertions(+), 21 deletions(-) rename dev/storage/{wallet.json => wallet01.json} (100%) create mode 100644 dev/storage/wallet02.json create mode 100644 dev/storage/wallet03.json create mode 100644 dev/storage/wallet04.json diff --git a/Makefile b/Makefile index 20c84d041..fbf93facd 100755 --- a/Makefile +++ b/Makefile @@ -276,15 +276,19 @@ env-up: all echo "Frostfs contracts not found"; exit 1; \ fi ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph init --contracts ${FROSTFS_CONTRACTS_PATH} - ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet01.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet02.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet03.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet04.json --gas 10.0 @if [ ! -f "$(LOCODE_DB_PATH)" ]; then \ make locode-download; \ fi + mkdir -p ./$(TMP_DIR)/state + mkdir -p ./$(TMP_DIR)/storage # Shutdown dev environment env-down: docker compose -f dev/docker-compose.yml down docker volume rm -f frostfs-node_neo-go - rm -f ./.cache/.frostfs-ir-state - rm -f ./.cache/.frostfs-node-state - rm -rf ./.cache/storage + rm -rf ./$(TMP_DIR)/state + rm -rf ./$(TMP_DIR)/storage diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index 459350145..6aedde85e 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -27,12 +27,12 @@ "FROSTFS_IR_NETMAP_CLEANER_THRESHOLD":"3", "FROSTFS_IR_LOCODE_DB_PATH":"${workspaceFolder}/.cache/locode_db", "FROSTFS_IR_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8090", - "FROSTFS_IR_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/.frostfs-ir-state" + "FROSTFS_IR_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-ir-state" }, "postDebugTask": "env-down" }, { - "name": "Storage node", + "name": "Storage node 1", "type": "go", "request": "launch", "mode": "debug", @@ -42,7 +42,8 @@ "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", - "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet.json", + "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet01.json", "FROSTFS_NODE_WALLET_PASSWORD":"", "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8080", "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8080", @@ -50,31 +51,187 @@ "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", - "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/.frostfs-node-state", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s1-state", "FROSTFS_TREE_ENABLED":"true", "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", - "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/wc0", - "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/meta0", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s1/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/s1/meta0", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", - "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s1/blobovnicza0", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", - "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s1/fstree0", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", - "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/pilorama0", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s1/pilorama0", "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", - "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/wc1", - "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/meta1", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s1/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/s1/meta1", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", - "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s1/blobovnicza1", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", - "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s1/fstree1", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", - "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/pilorama1" + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s1/pilorama1", + "FROSTFS_PROMETHEUS_ENABLED":"true", + "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9090", + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + }, + "postDebugTask": "env-down" + }, + { + "name": "Storage node 2", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/frostfs-node", + "env": { + "FROSTFS_LOGGER_LEVEL":"debug", + "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", + "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", + "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", + "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet02.json", + "FROSTFS_NODE_WALLET_PASSWORD":"", + "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8082", + "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8082", + "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8083", + "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", + "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", + "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s2-state", + "FROSTFS_TREE_ENABLED":"true", + "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s2/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/s2/meta0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s2/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s2/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s2/pilorama0", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s2/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/s2/meta1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s2/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s2/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s2/pilorama1", + "FROSTFS_PROMETHEUS_ENABLED":"true", + "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9091", + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + }, + "postDebugTask": "env-down" + }, + { + "name": "Storage node 3", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/frostfs-node", + "env": { + "FROSTFS_LOGGER_LEVEL":"debug", + "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", + "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", + "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", + "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet03.json", + "FROSTFS_NODE_WALLET_PASSWORD":"", + "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8084", + "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8084", + "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8085", + "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", + "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", + "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s3-state", + "FROSTFS_TREE_ENABLED":"true", + "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s3/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/s3/meta0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s3/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s3/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s3/pilorama0", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s3/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/s3/meta1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s3/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s3/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s3/pilorama1", + "FROSTFS_PROMETHEUS_ENABLED":"true", + "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9092", + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + }, + "postDebugTask": "env-down" + }, + { + "name": "Storage node 4", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/frostfs-node", + "env": { + "FROSTFS_LOGGER_LEVEL":"debug", + "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", + "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", + "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", + "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet04.json", + "FROSTFS_NODE_WALLET_PASSWORD":"", + "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8086", + "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8086", + "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8087", + "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", + "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", + "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s4-state", + "FROSTFS_TREE_ENABLED":"true", + "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s4/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/s4/meta0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s4/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s4/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s4/pilorama0", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s4/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/s4/meta1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s4/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s4/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s4/pilorama1", + "FROSTFS_PROMETHEUS_ENABLED":"true", + "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9093", + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" }, "postDebugTask": "env-down" } @@ -82,9 +239,15 @@ "compounds": [ { "name": "IR+Storage node", - "configurations": ["IR", "Storage node"], + "configurations": ["IR", "Storage node 1"], "preLaunchTask": "env-up", "stopAll": true - } + }, + { + "name": "IR + 4 storage nodes", + "configurations": ["IR", "Storage node 1", "Storage node 2", "Storage node 3", "Storage node 4"], + "preLaunchTask": "env-up", + "stopAll": true + } ] -} +} \ No newline at end of file diff --git a/dev/storage/wallet.json b/dev/storage/wallet01.json similarity index 100% rename from dev/storage/wallet.json rename to dev/storage/wallet01.json diff --git a/dev/storage/wallet02.json b/dev/storage/wallet02.json new file mode 100644 index 000000000..9c073deef --- /dev/null +++ b/dev/storage/wallet02.json @@ -0,0 +1,30 @@ +{ + "version":"3.0", + "accounts":[ + { + "address":"NVXXy3hNTvwVEZa2dAibALyJB3Q86aiHvL", + "key":"6PYXd9hxMYfaCkgeZp3q1RoMB921RQFkRxYftcacTJ2S7MUwnivrxi6Yk5", + "label":"", + "contract":{ + "script":"DCED/2W2rnkTSk3OnQ0504Uem6tO6Xq/hugeHFu8UM0oJq5BVuezJw==", + "parameters":[ + { + "name":"parameter0", + "type":"Signature" + } + ], + "deployed":false + }, + "lock":false, + "isDefault":false + } + ], + "scrypt":{ + "n":16384, + "r":8, + "p":8 + }, + "extra":{ + "Tokens":null + } + } diff --git a/dev/storage/wallet03.json b/dev/storage/wallet03.json new file mode 100644 index 000000000..c054a3160 --- /dev/null +++ b/dev/storage/wallet03.json @@ -0,0 +1,30 @@ +{ + "version":"3.0", + "accounts":[ + { + "address":"NPTmih9X14Y7xLvmD6RVtDHdH1Y9qJwoTe", + "key":"6PYXNeQzge9fWztVnWYRbr5Mh9q1y4npKVARHYGb484Hct1iNd3vXGR1kk", + "label":"", + "contract":{ + "script":"DCECrJIM198LYbKJBy5rlG4tpOGjG5qxxiG7R14w+kqxAsNBVuezJw==", + "parameters":[ + { + "name":"parameter0", + "type":"Signature" + } + ], + "deployed":false + }, + "lock":false, + "isDefault":false + } + ], + "scrypt":{ + "n":16384, + "r":8, + "p":8 + }, + "extra":{ + "Tokens":null + } + } diff --git a/dev/storage/wallet04.json b/dev/storage/wallet04.json new file mode 100644 index 000000000..cb4676df6 --- /dev/null +++ b/dev/storage/wallet04.json @@ -0,0 +1,30 @@ +{ + "version":"3.0", + "accounts":[ + { + "address":"Ne2DAQbWvP1s7TbtFc7BStKMnjKJdBaVRm", + "key":"6PYWCsGWx8uSVYK94tvK7Ccit8x8Z3f3dHADTFTgLhT9NBXTBqBECL8AyC", + "label":"", + "contract":{ + "script":"DCEDjIYpWeVrQ+IPeRh8T+ngvHyMZsFgPmzw7H+Hq2sI3DVBVuezJw==", + "parameters":[ + { + "name":"parameter0", + "type":"Signature" + } + ], + "deployed":false + }, + "lock":false, + "isDefault":false + } + ], + "scrypt":{ + "n":16384, + "r":8, + "p":8 + }, + "extra":{ + "Tokens":null + } + } -- 2.45.2 From 23db20582cd766b0c2efe01a842062f55ed74cc3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Mar 2024 17:27:22 +0300 Subject: [PATCH 4/5] [#1064] cli: Add EC header output to object head Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/head.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index db466e588..f97ef952d 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -171,6 +171,15 @@ func printHeader(cmd *cobra.Command, obj *objectSDK.Object) error { cmd.Printf(" signature: %s\n", hex.EncodeToString(sigV2.GetSign())) } + if ecHeader := obj.ECHeader(); ecHeader != nil { + cmd.Print("EC header:\n") + + cmd.Printf(" parent object ID: %s\n", ecHeader.Parent().EncodeToString()) + cmd.Printf(" index: %d\n", ecHeader.Index()) + cmd.Printf(" total: %d\n", ecHeader.Total()) + cmd.Printf(" header length: %d\n", ecHeader.HeaderLength()) + } + return printSplitHeader(cmd, obj) } -- 2.45.2 From 04a23efef6149dbbbf17a511d295a45556ce2fdb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Mar 2024 17:29:30 +0300 Subject: [PATCH 5/5] [#1064] policer: Disable EC processing Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index d2297fed7..d68d99d18 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + policycore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -40,6 +41,10 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add } policy := cnr.Value.PlacementPolicy() + if policycore.IsECPlacement(policy) { + // EC not supported yet by policer + return nil + } nn, err := p.placementBuilder.BuildPlacement(idCnr, &idObj, policy) if err != nil { -- 2.45.2