From 7c66b4cbe29aae082b167086b650eb72db045346 Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Wed, 26 Mar 2025 16:51:42 +0300 Subject: [PATCH 1/3] [42] Client: add splitId patch Signed-off-by: Pavel Gross --- .../Mappers/Object/ObjectHeaderMapper.cs | 33 ++++++++++--------- .../Models/Object/FrostFsSplit.cs | 26 +++++++++++++++ .../Parameters/PrmObjectPatch.cs | 3 ++ .../Services/ObjectServiceProvider.cs | 5 +++ src/FrostFS.SDK.Protos/object/service.proto | 4 +++ 5 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs index 317b31c..2b4bd43 100644 --- a/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs +++ b/src/FrostFS.SDK.Client/Mappers/Object/ObjectHeaderMapper.cs @@ -24,25 +24,14 @@ public static class ObjectHeaderMapper _ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.") }; - FrostFsSplit? split = null; - - if (header.Split != null) - { - var children = header.Split.Children.Count != 0 ? new ReadOnlyCollection( - header.Split.Children.Select(x => x.ToModel()).ToList()) : null; - - split = new FrostFsSplit(new SplitId(header.Split.SplitId.ToUuid()), - header.Split.Previous?.ToModel(), - header.Split.Parent?.ToModel(), - header.Split.ParentHeader?.ToModel(), - null, - children); - } + FrostFsSplit? split = header!.Split != null + ? header.Split.ToModel() + : null; var model = new FrostFsObjectHeader( new FrostFsContainerId(Base58.Encode(header.ContainerId.Value.Span)), objTypeName, - header.Attributes.Select(attribute => attribute.ToModel()).ToArray(), + [.. header.Attributes.Select(attribute => attribute.ToModel())], split, header.OwnerId.ToModel(), header.Version.ToModel()) @@ -52,4 +41,18 @@ public static class ObjectHeaderMapper return model; } + + public static FrostFsSplit ToModel(this Header.Types.Split split) + { + var children = split!.Children.Count != 0 + ? new ReadOnlyCollection([.. split.Children.Select(x => x.ToModel())]) + : null; + + return new FrostFsSplit(new SplitId(split.SplitId.ToUuid()), + split.Previous?.ToModel(), + split.Parent?.ToModel(), + split.ParentHeader?.ToModel(), + null, + children); + } } diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs index bd850cb..5afd46b 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs @@ -1,4 +1,7 @@ using System.Collections.ObjectModel; +using System.Linq; +using FrostFS.Object; +using FrostFS.SDK.Client.Mappers.GRPC; namespace FrostFS.SDK; @@ -9,6 +12,8 @@ public class FrostFsSplit(SplitId splitId, FrostFsSignature? parentSignature = null, ReadOnlyCollection? children = null) { + private Header.Types.Split? _split; + public FrostFsSplit() : this(new SplitId()) { } @@ -24,4 +29,25 @@ public class FrostFsSplit(SplitId splitId, public FrostFsObjectHeader? ParentHeader { get; set; } = parentHeader; public ReadOnlyCollection? Children { get; } = children; + + public Header.Types.Split GetSplit() + { + if (_split == null) + { + _split = new Header.Types.Split + { + SplitId = SplitId?.GetSplitId(), + Parent = Parent?.ToMessage(), + ParentHeader = ParentHeader?.GetHeader(), + ParentSignature = ParentSignature?.ToMessage() + }; + + if (Children != null) + { + _split.Children.AddRange(Children.Select(x => x.ToMessage())); + } + } + + return _split; + } } diff --git a/src/FrostFS.SDK.Client/Parameters/PrmObjectPatch.cs b/src/FrostFS.SDK.Client/Parameters/PrmObjectPatch.cs index 7b26f5b..fc02c7c 100644 --- a/src/FrostFS.SDK.Client/Parameters/PrmObjectPatch.cs +++ b/src/FrostFS.SDK.Client/Parameters/PrmObjectPatch.cs @@ -10,6 +10,7 @@ public readonly struct PrmObjectPatch( FrostFsSessionToken? sessionToken = null, bool replaceAttributes = false, FrostFsAttributePair[]? newAttributes = null, + FrostFsSplit? newSplitHeader = null, string[]? xheaders = null) : ISessionToken, System.IEquatable { public FrostFsAddress Address { get; } = address; @@ -23,6 +24,8 @@ public readonly struct PrmObjectPatch( public FrostFsAttributePair[]? NewAttributes { get; } = newAttributes; + public FrostFsSplit? NewSplitHeader { get; } = newSplitHeader; + public bool ReplaceAttributes { get; } = replaceAttributes; public int MaxChunkLength { get; } = maxChunkLength; diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index 3ce543d..62dc2be 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -356,6 +356,11 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl } } + if (args.NewSplitHeader != null) + { + request.Body.NewSplitHeader = args.NewSplitHeader.GetSplit(); + } + isFirstChunk = false; } else diff --git a/src/FrostFS.SDK.Protos/object/service.proto b/src/FrostFS.SDK.Protos/object/service.proto index 2b8042b..d490dca 100644 --- a/src/FrostFS.SDK.Protos/object/service.proto +++ b/src/FrostFS.SDK.Protos/object/service.proto @@ -887,6 +887,10 @@ message PatchRequest { // key, then it just replaces it while merging the lists. bool replace_attributes = 3; + // New split header for the object. This defines how the object will relate + // to other objects in a split operation. + neo.fs.v2.object.Header.Split new_split_header = 5; + // The patch for the object's payload. message Patch { // The range of the source object for which the payload is replaced by the From e179c32a3c38eaa9ac3d3b32c2143b1a3c8a7aef Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Fri, 28 Mar 2025 09:24:44 +0300 Subject: [PATCH 2/3] Add attr-only patch without body modification --- .../Models/Object/FrostFsSplit.cs | 2 +- .../Services/ObjectServiceProvider.cs | 163 +++++++++--------- src/FrostFS.SDK.Client/Tools/ObjectTools.cs | 20 +++ 3 files changed, 104 insertions(+), 81 deletions(-) diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs index 5afd46b..dcd7ccf 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs @@ -22,7 +22,7 @@ public class FrostFsSplit(SplitId splitId, public FrostFsObjectId? Previous { get; } = previous; - public FrostFsObjectId? Parent { get; } = parent; + public FrostFsObjectId? Parent { get; internal set; } = parent; public FrostFsSignature? ParentSignature { get; } = parentSignature; diff --git a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs index 62dc2be..27d8dfb 100644 --- a/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.Client/Services/ObjectServiceProvider.cs @@ -1,13 +1,11 @@ using System; using System.Buffers; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Threading.Tasks; using FrostFS.Object; using FrostFS.Refs; -using FrostFS.SDK.Client; using FrostFS.SDK.Client.Interfaces; using FrostFS.SDK.Client.Mappers.GRPC; using FrostFS.Session; @@ -295,91 +293,96 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl internal async Task PatchObjectAsync(PrmObjectPatch args, CallContext ctx) { var chunkSize = args.MaxChunkLength; - Stream payload = args.Payload ?? throw new ArgumentNullException(nameof(args), "Stream parameter is null"); - + var call = client.Patch(null, ctx.GetDeadline(), ctx.CancellationToken); - byte[]? chunkBuffer = null; - try + var sessionToken = args.SessionToken ?? await GetDefaultSession(args, ctx).ConfigureAwait(false); + var address = new Address { - // common - chunkBuffer = ArrayPool.Shared.Rent(chunkSize); - - bool isFirstChunk = true; - ulong currentPos = args.Range.Offset; - - var address = new Address + ObjectId = args.Address.ObjectId, + ContainerId = args.Address.ContainerId + }; + var request = new PatchRequest + { + Body = new() { - ObjectId = args.Address.ObjectId, - ContainerId = args.Address.ContainerId - }; + Address = address, + } + }; + var protoToken = sessionToken.CreateObjectTokenContext( + address, + ObjectSessionContext.Types.Verb.Patch, + ClientContext.Key); - while (true) + request.AddMetaHeader(args.XHeaders, protoToken); + + if (args.NewAttributes is { Length: > 0 }) + { + request.Body.ReplaceAttributes = args.ReplaceAttributes; + foreach (var attr in args.NewAttributes) { - var bytesCount = await payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken).ConfigureAwait(false); - - if (bytesCount == 0) - { - break; - } - - var request = new PatchRequest() - { - Body = new() - { - Address = address, - Patch = new PatchRequest.Types.Body.Types.Patch - { - Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), - SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount } - } - } - }; - - if (isFirstChunk) - { - var sessionToken = args.SessionToken ?? await GetDefaultSession(args, ctx).ConfigureAwait(false); - - var protoToken = sessionToken.CreateObjectTokenContext( - address, - ObjectSessionContext.Types.Verb.Patch, - ClientContext.Key); - - request.AddMetaHeader(args.XHeaders, protoToken); - - if (args.NewAttributes != null && args.NewAttributes.Length > 0) - { - foreach (var attr in args.NewAttributes) - { - request.Body.NewAttributes.Add(attr.ToMessage()); - request.Body.ReplaceAttributes = args.ReplaceAttributes; - } - } - - if (args.NewSplitHeader != null) - { - request.Body.NewSplitHeader = args.NewSplitHeader.GetSplit(); - } - - isFirstChunk = false; - } - else - { - request.AddMetaHeader(args.XHeaders); - } - - request.Sign(ClientContext.Key); - - await call.RequestStream.WriteAsync(request).ConfigureAwait(false); - - currentPos += (ulong)bytesCount; + request.Body.NewAttributes.Add(attr.ToMessage()); } } - finally + + if (args.NewSplitHeader != null) { - if (chunkBuffer != null) + request.Body.NewSplitHeader = ObjectTools.CreateSplit(args.NewSplitHeader, ClientContext.Key, + ClientContext.Owner, ClientContext.Version); + if (request.Body.NewSplitHeader.Parent != null) { - ArrayPool.Shared.Return(chunkBuffer); + args.NewSplitHeader.Parent = request.Body.NewSplitHeader.Parent.ToModel(); + } + } + request.Sign(ClientContext.Key); + await call.RequestStream.WriteAsync(request).ConfigureAwait(false); + + if (args.Payload != null) + { + byte[]? chunkBuffer = null; + try + { + // common + chunkBuffer = ArrayPool.Shared.Rent(chunkSize); + ulong currentPos = args.Range.Offset; + + while (true) + { + var bytesCount = await args.Payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken) + .ConfigureAwait(false); + + if (bytesCount == 0) + { + break; + } + + request = new PatchRequest() + { + Body = new() + { + Address = address, + Patch = new PatchRequest.Types.Body.Types.Patch + { + Chunk = UnsafeByteOperations.UnsafeWrap(chunkBuffer.AsMemory(0, bytesCount)), + SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount } + } + } + }; + + request.AddMetaHeader(args.XHeaders); + request.Sign(ClientContext.Key); + + await call.RequestStream.WriteAsync(request).ConfigureAwait(false); + + currentPos += (ulong)bytesCount; + } + } + finally + { + if (chunkBuffer != null) + { + ArrayPool.Shared.Return(chunkBuffer); + } } } @@ -390,7 +393,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return response.Body.ObjectId.ToModel(); } - + internal async Task PutClientCutObjectAsync(PrmObjectClientCutPut args, CallContext ctx) { var stream = args.Payload!; @@ -572,7 +575,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl break; sentBytes += bytesCount; - + var chunkRequest = new PutRequest { Body = new PutRequest.Types.Body @@ -627,7 +630,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl var initRequest = new PutRequest { Body = new PutRequest.Types.Body - { + { Init = new PutRequest.Types.Body.Types.Init { Header = grpcHeader, diff --git a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs index ac5d019..701ba05 100644 --- a/src/FrostFS.SDK.Client/Tools/ObjectTools.cs +++ b/src/FrostFS.SDK.Client/Tools/ObjectTools.cs @@ -131,6 +131,26 @@ public static class ObjectTools grpcHeader.Split.Previous = split.Previous?.ToMessage(); } + internal static Header.Types.Split CreateSplit(FrostFsSplit split, ClientKey key, FrostFsOwner owner, FrostFsVersion version) + { + if (split.ParentHeader == null) + { + return split.GetSplit(); + } + split.ParentHeader.PayloadCheckSum ??= Array.Empty().Sha256(); + var grpcParentHeader = CreateHeader(split.ParentHeader, split.ParentHeader.PayloadCheckSum, owner, version); + + var res = split.GetSplit(); + res.ParentHeader = grpcParentHeader; + res.Parent ??= new ObjectID { Value = grpcParentHeader.Sha256() }; + res.ParentSignature ??= new Signature + { + Key = key.PublicKeyProto, + Sign = key.ECDsaKey.SignData(res.Parent.ToByteArray()), + }; + return res; + } + internal static Header CreateHeader( FrostFsObjectHeader header, ReadOnlyMemory payloadChecksum, From 586703636ae4ec8c377f1b01b42c0517c00ba32e Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Fri, 28 Mar 2025 14:02:07 +0300 Subject: [PATCH 3/3] Add previous field handling in FrostFsSplit grpc export --- src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs index dcd7ccf..a173b34 100644 --- a/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs +++ b/src/FrostFS.SDK.Client/Models/Object/FrostFsSplit.cs @@ -39,7 +39,8 @@ public class FrostFsSplit(SplitId splitId, SplitId = SplitId?.GetSplitId(), Parent = Parent?.ToMessage(), ParentHeader = ParentHeader?.GetHeader(), - ParentSignature = ParentSignature?.ToMessage() + ParentSignature = ParentSignature?.ToMessage(), + Previous = Previous?.ToMessage(), }; if (Children != null)