From bad4d5a1df82617037dac083f349caa43c5f9261 Mon Sep 17 00:00:00 2001 From: Pavel Gros Date: Tue, 5 Nov 2024 21:08:38 +0300 Subject: [PATCH] patch --- src/FrostFS.SDK.ClientV2/FrostFSClient.cs | 30 +++ .../Interfaces/IFrostFSClient.cs | 6 + .../Mappers/ContainerId.cs | 11 +- .../Models/Object/FrostFsAddress.cs | 48 ++++ .../Models/Object/FrostFsRange.cs | 27 ++ .../Models/Object/IObjectReader.cs | 2 +- .../Parameters/PrmObjectPatch.cs | 24 ++ .../Parameters/PrmRangeGet.cs | 20 ++ .../Parameters/PrmRangeHashGet.cs | 20 ++ .../{Poll => Pool}/ClientStatusMonitor.cs | 0 .../{Poll => Pool}/ClientWrapper.cs | 4 +- .../{Poll => Pool}/HealthyStatus.cs | 0 .../{Poll => Pool}/IClientStatus.cs | 0 .../{Poll => Pool}/InitParameters.cs | 0 .../{Poll => Pool}/InnerPool.cs | 0 .../{Poll => Pool}/MethodIndex.cs | 0 .../{Poll => Pool}/MethodStatus.cs | 0 .../{Poll => Pool}/NodeParam.cs | 0 .../{Poll => Pool}/NodeStatistic.cs | 0 .../{Poll => Pool}/NodesParam.cs | 0 .../{Poll => Pool}/Pool.cs | 57 +++- .../{Poll => Pool}/RebalanceParameters.cs | 0 .../{Poll => Pool}/RequestInfo.cs | 0 .../{Poll => Pool}/Sampler.cs | 0 .../{Poll => Pool}/SessionCache.cs | 0 .../{Poll => Pool}/Statistic.cs | 0 .../{Poll => Pool}/StatusSnapshot.cs | 0 .../{Poll => Pool}/WorkList.cs | 0 .../{Poll => Pool}/WrapperPrm.cs | 0 .../Services/ObjectServiceProvider.cs | 226 ++++++++++++++-- .../Tools/ObjectReader.cs | 2 +- .../Tools/ObjectStreamer.cs | 10 +- src/FrostFS.SDK.ClientV2/Tools/RangeReader.cs | 38 +++ .../accounting/service.proto | 18 +- src/FrostFS.SDK.ProtosV2/acl/types.proto | 28 +- src/FrostFS.SDK.ProtosV2/netmap/service.proto | 8 +- src/FrostFS.SDK.ProtosV2/netmap/types.proto | 74 +++-- .../object/Extension.Message.cs | 56 ++++ src/FrostFS.SDK.ProtosV2/object/service.proto | 127 ++++++++- src/FrostFS.SDK.ProtosV2/object/types.proto | 14 +- src/FrostFS.SDK.ProtosV2/refs/types.proto | 11 +- .../session/service.proto | 2 +- src/FrostFS.SDK.ProtosV2/session/types.proto | 11 +- src/FrostFS.SDK.ProtosV2/status/types.proto | 19 +- .../tombstone/types.proto | 6 +- .../Mocks/AsyncStreamRangeReaderMock.cs | 43 +++ src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs | 124 +++++++-- .../Mocks/PatchStreamWriter.cs | 36 +++ src/FrostFS.SDK.Tests/ObjectTest.cs | 121 +++++++++ src/FrostFS.SDK.Tests/SmokeClientTests.cs | 252 +++++++++++++++--- src/FrostFS.SDK.Tests/SmokeTestsBase.cs | 2 + 51 files changed, 1340 insertions(+), 137 deletions(-) create mode 100644 src/FrostFS.SDK.ClientV2/Models/Object/FrostFsAddress.cs create mode 100644 src/FrostFS.SDK.ClientV2/Models/Object/FrostFsRange.cs create mode 100644 src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPatch.cs create mode 100644 src/FrostFS.SDK.ClientV2/Parameters/PrmRangeGet.cs create mode 100644 src/FrostFS.SDK.ClientV2/Parameters/PrmRangeHashGet.cs rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/ClientStatusMonitor.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/ClientWrapper.cs (96%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/HealthyStatus.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/IClientStatus.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/InitParameters.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/InnerPool.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/MethodIndex.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/MethodStatus.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/NodeParam.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/NodeStatistic.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/NodesParam.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/Pool.cs (93%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/RebalanceParameters.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/RequestInfo.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/Sampler.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/SessionCache.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/Statistic.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/StatusSnapshot.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/WorkList.cs (100%) rename src/FrostFS.SDK.ClientV2/{Poll => Pool}/WrapperPrm.cs (100%) create mode 100644 src/FrostFS.SDK.ClientV2/Tools/RangeReader.cs create mode 100644 src/FrostFS.SDK.Tests/Mocks/AsyncStreamRangeReaderMock.cs create mode 100644 src/FrostFS.SDK.Tests/Mocks/PatchStreamWriter.cs diff --git a/src/FrostFS.SDK.ClientV2/FrostFSClient.cs b/src/FrostFS.SDK.ClientV2/FrostFSClient.cs index e13b6b6..830eb94 100644 --- a/src/FrostFS.SDK.ClientV2/FrostFSClient.cs +++ b/src/FrostFS.SDK.ClientV2/FrostFSClient.cs @@ -285,6 +285,25 @@ public class FrostFSClient : IFrostFSClient return service.GetObjectAsync(args); } + public Task GetRangeAsync(PrmRangeGet args) + { + if (args is null) + throw new ArgumentNullException(nameof(args)); + + var service = GetObjectService(args); + return service.GetRangeAsync(args); + } + + public Task>> GetRangeHashAsync(PrmRangeHashGet args) + { + if (args is null) + throw new ArgumentNullException(nameof(args)); + + var service = GetObjectService(args); + return service.GetRangeHashAsync(args); + } + + public Task PutObjectAsync(PrmObjectPut args) { if (args is null) @@ -303,6 +322,17 @@ public class FrostFSClient : IFrostFSClient return service.PutSingleObjectAsync(args); } + public Task PatchObjectAsync(PrmObjectPatch args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var service = GetObjectService(args); + return service.PatchObjectAsync(args); + } + public Task DeleteObjectAsync(PrmObjectDelete args) { if (args is null) diff --git a/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs b/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs index e81343b..48ab8e8 100644 --- a/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs +++ b/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs @@ -42,10 +42,16 @@ public interface IFrostFSClient : IDisposable Task GetObjectAsync(PrmObjectGet args); + Task GetRangeAsync(PrmRangeGet args); + + Task>> GetRangeHashAsync(PrmRangeHashGet args); + Task PutObjectAsync(PrmObjectPut args); Task PutSingleObjectAsync(PrmSingleObjectPut args); + Task PatchObjectAsync(PrmObjectPatch args); + Task DeleteObjectAsync(PrmObjectDelete args); IAsyncEnumerable SearchObjectsAsync(PrmObjectSearch args); diff --git a/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs b/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs index 7a5359d..c4c805c 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs @@ -33,7 +33,16 @@ public static class ContainerIdMapper Caches.Containers.Set(containerId, message, _oneHourExpiration); } - return message!; } + + public static FrostFsContainerId ToModel(this ContainerID message) + { + if (message is null) + { + throw new ArgumentNullException(nameof(message)); + } + + return new FrostFsContainerId(Base58.Encode(message.Value.ToByteArray())); + } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsAddress.cs b/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsAddress.cs new file mode 100644 index 0000000..46581d1 --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsAddress.cs @@ -0,0 +1,48 @@ +using FrostFS.Refs; +using FrostFS.SDK.ClientV2.Mappers.GRPC; + +namespace FrostFS.SDK; + +public class FrostFsAddress +{ + private FrostFsObjectId? frostFsObjectId; + private FrostFsContainerId? frostFsContainerId; + private ObjectID? objectId; + private ContainerID? containerId; + + public FrostFsAddress(FrostFsContainerId frostFsContainerId, FrostFsObjectId frostFsObjectId) + { + FrostFsObjectId = frostFsObjectId ?? throw new System.ArgumentNullException(nameof(frostFsObjectId)); + FrostFsContainerId = frostFsContainerId ?? throw new System.ArgumentNullException(nameof(frostFsContainerId)); + } + + internal FrostFsAddress(ObjectID objectId, ContainerID containerId) + { + ObjectId = objectId ?? throw new System.ArgumentNullException(nameof(objectId)); + ContainerId = containerId ?? throw new System.ArgumentNullException(nameof(containerId)); + } + + public FrostFsObjectId FrostFsObjectId + { + get => frostFsObjectId ??= objectId!.ToModel(); + set => frostFsObjectId = value; + } + + public FrostFsContainerId FrostFsContainerId + { + get => frostFsContainerId ??= containerId!.ToModel(); + set => frostFsContainerId = value; + } + + public ObjectID ObjectId + { + get => objectId ??= frostFsObjectId!.ToMessage(); + set => objectId = value; + } + + public ContainerID ContainerId + { + get => containerId ??= frostFsContainerId!.ToMessage(); + set => containerId = value; + } +} diff --git a/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsRange.cs b/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsRange.cs new file mode 100644 index 0000000..b50568f --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsRange.cs @@ -0,0 +1,27 @@ +namespace FrostFS.SDK; + +public readonly struct FrostFsRange(ulong offset, ulong length) : System.IEquatable +{ + public ulong Offset { get; } = offset; + + public ulong Length { get; } = length; + + public override readonly bool Equals(object obj) => this == (FrostFsRange)obj; + + public override readonly int GetHashCode() => $"{Offset}{Length}".GetHashCode(); + + public static bool operator ==(FrostFsRange left, FrostFsRange right) + { + return left.Equals(right); + } + + public static bool operator !=(FrostFsRange left, FrostFsRange right) + { + return !(left == right); + } + + public readonly bool Equals(FrostFsRange other) + { + return this == other; + } +} diff --git a/src/FrostFS.SDK.ClientV2/Models/Object/IObjectReader.cs b/src/FrostFS.SDK.ClientV2/Models/Object/IObjectReader.cs index 51211fd..8b08e6b 100644 --- a/src/FrostFS.SDK.ClientV2/Models/Object/IObjectReader.cs +++ b/src/FrostFS.SDK.ClientV2/Models/Object/IObjectReader.cs @@ -6,5 +6,5 @@ namespace FrostFS.SDK; public interface IObjectReader : IDisposable { - Task?> ReadChunk(CancellationToken cancellationToken = default); + ValueTask?> ReadChunk(CancellationToken cancellationToken = default); } diff --git a/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPatch.cs b/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPatch.cs new file mode 100644 index 0000000..de292ea --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPatch.cs @@ -0,0 +1,24 @@ +using System.IO; + +namespace FrostFS.SDK.ClientV2; + +public sealed class PrmObjectPatch(FrostFsAddress address, CallContext? ctx = null) : PrmBase(ctx), ISessionToken +{ + public FrostFsAddress Address { get; } = address; + + public FrostFsRange Range { get; set; } + + /// + /// A stream with source data + /// + public Stream? Payload { get; set; } + + public FrostFsAttributePair[]? NewAttributes { get; set; } + + public bool ReplaceAttributes { get; set; } + + public int MaxPayloadPatchChunkLength { get; set; } + + /// + public FrostFsSessionToken? SessionToken { get; set; } +} diff --git a/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeGet.cs b/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeGet.cs new file mode 100644 index 0000000..5bcad9f --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeGet.cs @@ -0,0 +1,20 @@ +namespace FrostFS.SDK.ClientV2; + +public sealed class PrmRangeGet( + FrostFsContainerId containerId, + FrostFsObjectId objectId, + FrostFsRange range, + bool raw = false, + CallContext? ctx = null) : PrmBase(ctx), ISessionToken +{ + public FrostFsContainerId ContainerId { get; } = containerId; + + public FrostFsObjectId ObjectId { get; } = objectId; + + public FrostFsRange Range { get; } = range; + + public bool Raw { get; } = raw; + + /// + public FrostFsSessionToken? SessionToken { get; set; } +} diff --git a/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeHashGet.cs b/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeHashGet.cs new file mode 100644 index 0000000..661ed64 --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeHashGet.cs @@ -0,0 +1,20 @@ +namespace FrostFS.SDK.ClientV2; + +public sealed class PrmRangeHashGet( + FrostFsContainerId containerId, + FrostFsObjectId objectId, + FrostFsRange[] ranges, + byte[] salt, + CallContext? ctx = null) : PrmBase(ctx), ISessionToken +{ + public FrostFsContainerId ContainerId { get; } = containerId; + + public FrostFsObjectId ObjectId { get; } = objectId; + + public FrostFsRange[] Ranges { get; } = ranges; + + public byte[] Salt { get; } = salt; + + /// + public FrostFsSessionToken? SessionToken { get; set; } +} diff --git a/src/FrostFS.SDK.ClientV2/Poll/ClientStatusMonitor.cs b/src/FrostFS.SDK.ClientV2/Pool/ClientStatusMonitor.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/ClientStatusMonitor.cs rename to src/FrostFS.SDK.ClientV2/Pool/ClientStatusMonitor.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/ClientWrapper.cs b/src/FrostFS.SDK.ClientV2/Pool/ClientWrapper.cs similarity index 96% rename from src/FrostFS.SDK.ClientV2/Poll/ClientWrapper.cs rename to src/FrostFS.SDK.ClientV2/Pool/ClientWrapper.cs index ddde002..206f69e 100644 --- a/src/FrostFS.SDK.ClientV2/Poll/ClientWrapper.cs +++ b/src/FrostFS.SDK.ClientV2/Pool/ClientWrapper.cs @@ -101,9 +101,9 @@ public class ClientWrapper : ClientStatusMonitor await ScheduleGracefulClose().ConfigureAwait(false); } -#pragma warning disable CA2000 // Dispose objects before losing scope: will be disposed manually + //#pragma warning disable CA2000 // Dispose objects before losing scope: will be disposed manually FrostFSClient client = new(WrapperPrm, sessionCache); -#pragma warning restore CA2000 + //#pragma warning restore CA2000 //TODO: set additioanl params var error = await client.Dial(ctx).ConfigureAwait(false); diff --git a/src/FrostFS.SDK.ClientV2/Poll/HealthyStatus.cs b/src/FrostFS.SDK.ClientV2/Pool/HealthyStatus.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/HealthyStatus.cs rename to src/FrostFS.SDK.ClientV2/Pool/HealthyStatus.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/IClientStatus.cs b/src/FrostFS.SDK.ClientV2/Pool/IClientStatus.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/IClientStatus.cs rename to src/FrostFS.SDK.ClientV2/Pool/IClientStatus.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/InitParameters.cs b/src/FrostFS.SDK.ClientV2/Pool/InitParameters.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/InitParameters.cs rename to src/FrostFS.SDK.ClientV2/Pool/InitParameters.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/InnerPool.cs b/src/FrostFS.SDK.ClientV2/Pool/InnerPool.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/InnerPool.cs rename to src/FrostFS.SDK.ClientV2/Pool/InnerPool.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/MethodIndex.cs b/src/FrostFS.SDK.ClientV2/Pool/MethodIndex.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/MethodIndex.cs rename to src/FrostFS.SDK.ClientV2/Pool/MethodIndex.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/MethodStatus.cs b/src/FrostFS.SDK.ClientV2/Pool/MethodStatus.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/MethodStatus.cs rename to src/FrostFS.SDK.ClientV2/Pool/MethodStatus.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/NodeParam.cs b/src/FrostFS.SDK.ClientV2/Pool/NodeParam.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/NodeParam.cs rename to src/FrostFS.SDK.ClientV2/Pool/NodeParam.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/NodeStatistic.cs b/src/FrostFS.SDK.ClientV2/Pool/NodeStatistic.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/NodeStatistic.cs rename to src/FrostFS.SDK.ClientV2/Pool/NodeStatistic.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/NodesParam.cs b/src/FrostFS.SDK.ClientV2/Pool/NodesParam.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/NodesParam.cs rename to src/FrostFS.SDK.ClientV2/Pool/NodesParam.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/Pool.cs b/src/FrostFS.SDK.ClientV2/Pool/Pool.cs similarity index 93% rename from src/FrostFS.SDK.ClientV2/Poll/Pool.cs rename to src/FrostFS.SDK.ClientV2/Pool/Pool.cs index 3e20634..9e34551 100644 --- a/src/FrostFS.SDK.ClientV2/Poll/Pool.cs +++ b/src/FrostFS.SDK.ClientV2/Pool/Pool.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -711,6 +710,62 @@ public partial class Pool : IFrostFSClient return await client.Client!.PutSingleObjectAsync(args).ConfigureAwait(false); } + public async Task PatchObjectAsync(PrmObjectPatch args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var client = Connection(); + + args.Context.PoolErrorHandler = client.HandleError; + + return await client.Client!.PatchObjectAsync(args).ConfigureAwait(false); + } + + public async Task GetRangeAsync(PrmRangeGet args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var client = Connection(); + + args.Context.PoolErrorHandler = client.HandleError; + + return await client.Client!.GetRangeAsync(args).ConfigureAwait(false); + } + + public async Task>> GetRangeHashAsync(PrmRangeHashGet args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var client = Connection(); + + args.Context.PoolErrorHandler = client.HandleError; + + return await client.Client!.GetRangeHashAsync(args).ConfigureAwait(false); + } + + public async Task PatchAsync(PrmObjectPatch args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var client = Connection(); + + args.Context.PoolErrorHandler = client.HandleError; + + return await client.Client!.PatchObjectAsync(args).ConfigureAwait(false); + } + public async Task DeleteObjectAsync(PrmObjectDelete args) { if (args is null) diff --git a/src/FrostFS.SDK.ClientV2/Poll/RebalanceParameters.cs b/src/FrostFS.SDK.ClientV2/Pool/RebalanceParameters.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/RebalanceParameters.cs rename to src/FrostFS.SDK.ClientV2/Pool/RebalanceParameters.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/RequestInfo.cs b/src/FrostFS.SDK.ClientV2/Pool/RequestInfo.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/RequestInfo.cs rename to src/FrostFS.SDK.ClientV2/Pool/RequestInfo.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/Sampler.cs b/src/FrostFS.SDK.ClientV2/Pool/Sampler.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/Sampler.cs rename to src/FrostFS.SDK.ClientV2/Pool/Sampler.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/SessionCache.cs b/src/FrostFS.SDK.ClientV2/Pool/SessionCache.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/SessionCache.cs rename to src/FrostFS.SDK.ClientV2/Pool/SessionCache.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/Statistic.cs b/src/FrostFS.SDK.ClientV2/Pool/Statistic.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/Statistic.cs rename to src/FrostFS.SDK.ClientV2/Pool/Statistic.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/StatusSnapshot.cs b/src/FrostFS.SDK.ClientV2/Pool/StatusSnapshot.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/StatusSnapshot.cs rename to src/FrostFS.SDK.ClientV2/Pool/StatusSnapshot.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/WorkList.cs b/src/FrostFS.SDK.ClientV2/Pool/WorkList.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/WorkList.cs rename to src/FrostFS.SDK.ClientV2/Pool/WorkList.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/WrapperPrm.cs b/src/FrostFS.SDK.ClientV2/Pool/WrapperPrm.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/WrapperPrm.cs rename to src/FrostFS.SDK.ClientV2/Pool/WrapperPrm.cs diff --git a/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs index bcea7c7..bb742a5 100644 --- a/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs @@ -1,6 +1,7 @@ using System; using System.Buffers; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; @@ -107,6 +108,101 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return await GetObject(request, ctx).ConfigureAwait(false); } + internal async Task GetRangeAsync(PrmRangeGet args) + { + var ctx = args.Context!; + + ctx.Key ??= ClientContext.Key?.ECDsaKey; + + if (ctx.Key == null) + throw new ArgumentNullException(nameof(args), "Key is null"); + + var request = new GetRangeRequest + { + Body = new GetRangeRequest.Types.Body + { + Address = new Address + { + ContainerId = args.ContainerId.ToMessage(), + ObjectId = args.ObjectId.ToMessage() + }, + Range = new Object.Range + { + Offset = args.Range.Offset, + Length = args.Range.Length + }, + Raw = args.Raw + } + }; + + var sessionToken = await GetOrCreateSession(args, ctx).ConfigureAwait(false); + + sessionToken.CreateObjectTokenContext( + request.Body.Address, + ObjectSessionContext.Types.Verb.Range, + ctx.Key); + + request.AddMetaHeader(args.XHeaders, sessionToken); + + request.Sign(ctx.Key); + + var call = client.GetRange(request, null, ctx.Deadline, ctx.CancellationToken); + return new RangeReader(call); + } + + internal async Task>> GetRangeHashAsync(PrmRangeHashGet args) + { + var ctx = args.Context!; + + ctx.Key ??= ClientContext.Key?.ECDsaKey; + + if (ctx.Key == null) + throw new ArgumentNullException(nameof(args), "Key is null"); + + var request = new GetRangeHashRequest + { + Body = new GetRangeHashRequest.Types.Body + { + Address = new Address + { + ContainerId = args.ContainerId.ToMessage(), + ObjectId = args.ObjectId.ToMessage() + }, + Type = ChecksumType.Sha256, + Salt = ByteString.CopyFrom(args.Salt) // TODO: create a type with calculated cashed ByteString inside + } + }; + + foreach (var range in args.Ranges) + { + request.Body.Ranges.Add(new Object.Range + { + Length = range.Length, + Offset = range.Offset + }); + } + + var sessionToken = await GetOrCreateSession(args, ctx).ConfigureAwait(false); + + sessionToken.CreateObjectTokenContext( + request.Body.Address, + ObjectSessionContext.Types.Verb.Rangehash, + ctx.Key); + + request.AddMetaHeader(args.XHeaders, sessionToken); + + request.Sign(ctx.Key); + + var response = await client.GetRangeHashAsync(request, null, ctx.Deadline, ctx.CancellationToken); + + Verifier.CheckResponse(response); + + var hashCollection = response.Body.HashList.ToArray().Select(h => h.Memory); + + return hashCollection; + } + + internal async Task DeleteObjectAsync(PrmObjectDelete args) { var ctx = args.Context!; @@ -191,7 +287,9 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl throw new ArgumentNullException(nameof(args), "Payload is null"); if (args.ClientCut) + { return await PutClientCutObject(args).ConfigureAwait(false); + } else { if (args.Header.PayloadLength > 0) @@ -199,7 +297,9 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl else if (args.Payload.CanSeek) args.FullLength = (ulong)args.Payload.Length; - return (await PutStreamObject(args).ConfigureAwait(false)).ObjectId; + var response = await PutStreamObject(args).ConfigureAwait(false); + + return response.ObjectId; } } @@ -235,6 +335,100 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return FrostFsObjectId.FromHash(grpcObject.ObjectId.Value.ToByteArray()); } + internal async Task PatchObjectAsync(PrmObjectPatch args) + { + var ctx = args.Context!; + if (ctx.Key == null) + throw new ArgumentNullException(nameof(args), "Key is null"); + + var chunkSize = args.MaxPayloadPatchChunkLength; + Stream payload = args.Payload ?? throw new ArgumentNullException(nameof(args), "Stream parameter is null"); + + var call = client.Patch(null, ctx.Deadline, ctx.CancellationToken); + + byte[]? chunkBuffer = null; + try + { + // common + chunkBuffer = ClientContext.GetArrayPool(Constants.ObjectChunkSize).Rent(chunkSize); + + var address = new Address + { + ObjectId = args.Address.ObjectId, + ContainerId = args.Address.ContainerId + }; + + var sessionToken = await GetOrCreateSession(args, ctx).ConfigureAwait(false); + + sessionToken.CreateObjectTokenContext( + address, + ObjectSessionContext.Types.Verb.Patch, + ctx.Key + ); + + var request = new PatchRequest() + { + Body = new() + { + Address = address, + ReplaceAttributes = args.ReplaceAttributes, + } + }; + + bool isFirstChunk = true; + ulong currentPos = args.Range.Offset; + + while (true) + { + var bytesCount = await payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken).ConfigureAwait(false); + + if (bytesCount == 0) + { + break; + } + + if (isFirstChunk && args.NewAttributes != null && args.NewAttributes.Length > 0) + { + foreach (var attr in args.NewAttributes) + { + request.Body.NewAttributes.Add(attr.ToMessage()); + } + } + + request.Body.Patch = new PatchRequest.Types.Body.Types.Patch + { + Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount), + SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount } + }; + + currentPos += (ulong)bytesCount; + + request.AddMetaHeader(args.XHeaders, sessionToken); + + request.Sign(ctx.Key); + + await call.RequestStream.WriteAsync(request).ConfigureAwait(false); + + isFirstChunk = false; + } + } + finally + { + await call.RequestStream.CompleteAsync().ConfigureAwait(false); + + if (chunkBuffer != null) + { + ArrayPool.Shared.Return(chunkBuffer); + } + } + + var response = await call.ResponseAsync.ConfigureAwait(false); + + Verifier.CheckResponse(response); + + return response.Body.ObjectId.ToModel(); + } + private async Task PutClientCutObject(PrmObjectPut args) { var ctx = args.Context!; @@ -406,7 +600,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl } } - private async Task GetUploadStream(PrmObjectPut args, CallContext ctx) + private async Task> GetUploadStream(PrmObjectPut args, CallContext ctx) { var header = args.Header!; @@ -451,6 +645,20 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return await PutObjectInit(initRequest, ctx).ConfigureAwait(false); } + private async Task> PutObjectInit(PutRequest initRequest, CallContext ctx) + { + if (initRequest is null) + { + throw new ArgumentNullException(nameof(initRequest)); + } + + var call = client.Put(null, ctx.Deadline, ctx.CancellationToken); + + await call.RequestStream.WriteAsync(initRequest).ConfigureAwait(false); + + return new ObjectStreamer(call); + } + private async Task GetObject(GetRequest request, CallContext ctx) { var reader = GetObjectInit(request, ctx); @@ -473,20 +681,6 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return new ObjectReader(call); } - private async Task PutObjectInit(PutRequest initRequest, CallContext ctx) - { - if (initRequest is null) - { - throw new ArgumentNullException(nameof(initRequest)); - } - - var call = client.Put(null, ctx.Deadline, ctx.CancellationToken); - - await call.RequestStream.WriteAsync(initRequest).ConfigureAwait(false); - - return new ObjectStreamer(call); - } - private async IAsyncEnumerable SearchObjects(SearchRequest request, CallContext ctx) { using var stream = GetSearchReader(request, ctx); diff --git a/src/FrostFS.SDK.ClientV2/Tools/ObjectReader.cs b/src/FrostFS.SDK.ClientV2/Tools/ObjectReader.cs index 7df2343..c6af0c8 100644 --- a/src/FrostFS.SDK.ClientV2/Tools/ObjectReader.cs +++ b/src/FrostFS.SDK.ClientV2/Tools/ObjectReader.cs @@ -32,7 +32,7 @@ public sealed class ObjectReader(AsyncServerStreamingCall call) : I }; } - public async Task?> ReadChunk(CancellationToken cancellationToken = default) + public async ValueTask?> ReadChunk(CancellationToken cancellationToken = default) { if (!await Call.ResponseStream.MoveNext(cancellationToken).ConfigureAwait(false)) return null; diff --git a/src/FrostFS.SDK.ClientV2/Tools/ObjectStreamer.cs b/src/FrostFS.SDK.ClientV2/Tools/ObjectStreamer.cs index acd0d28..a23c109 100644 --- a/src/FrostFS.SDK.ClientV2/Tools/ObjectStreamer.cs +++ b/src/FrostFS.SDK.ClientV2/Tools/ObjectStreamer.cs @@ -1,17 +1,15 @@ using System; using System.Threading.Tasks; -using FrostFS.Object; - using Grpc.Core; namespace FrostFS.SDK.ClientV2; -internal sealed class ObjectStreamer(AsyncClientStreamingCall call) : IDisposable +internal sealed class ObjectStreamer(AsyncClientStreamingCall call) : IDisposable { - public AsyncClientStreamingCall Call { get; private set; } = call; + public AsyncClientStreamingCall Call { get; private set; } = call; - public async Task Write(PutRequest request) + public async Task Write(TRequest request) { if (request is null) { @@ -21,7 +19,7 @@ internal sealed class ObjectStreamer(AsyncClientStreamingCall Close() + public async Task Close() { await Call.RequestStream.CompleteAsync().ConfigureAwait(false); diff --git a/src/FrostFS.SDK.ClientV2/Tools/RangeReader.cs b/src/FrostFS.SDK.ClientV2/Tools/RangeReader.cs new file mode 100644 index 0000000..d1128c8 --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Tools/RangeReader.cs @@ -0,0 +1,38 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +using FrostFS.Object; + +using Grpc.Core; + +namespace FrostFS.SDK.ClientV2; + +public sealed class RangeReader(AsyncServerStreamingCall call) : IObjectReader +{ + private bool disposed; + + public AsyncServerStreamingCall Call { get; private set; } = call; + + public async ValueTask?> ReadChunk(CancellationToken cancellationToken = default) + { + if (!await Call.ResponseStream.MoveNext(cancellationToken).ConfigureAwait(false)) + return null; + + var response = Call.ResponseStream.Current; + Verifier.CheckResponse(response); + + return response.Body.Chunk.Memory; + } + + public void Dispose() + { + if (!disposed) + { + Call?.Dispose(); + GC.SuppressFinalize(this); + + disposed = true; + } + } +} diff --git a/src/FrostFS.SDK.ProtosV2/accounting/service.proto b/src/FrostFS.SDK.ProtosV2/accounting/service.proto index 715ef63..6049e0f 100644 --- a/src/FrostFS.SDK.ProtosV2/accounting/service.proto +++ b/src/FrostFS.SDK.ProtosV2/accounting/service.proto @@ -9,13 +9,13 @@ import "accounting/types.proto"; import "refs/types.proto"; import "session/types.proto"; -// Accounting service provides methods for interaction with NeoFS sidechain via -// other NeoFS nodes to get information about the account balance. Deposit and -// Withdraw operations can't be implemented here, as they require Mainnet NeoFS -// smart contract invocation. Transfer operations between internal NeoFS -// accounts are possible if both use the same token type. +// Accounting service provides methods for interaction with FrostFS sidechain +// via other FrostFS nodes to get information about the account balance. Deposit +// and Withdraw operations can't be implemented here, as they require Mainnet +// FrostFS smart contract invocation. Transfer operations between internal +// FrostFS accounts are possible if both use the same token type. service AccountingService { - // Returns the amount of funds in GAS token for the requested NeoFS account. + // Returns the amount of funds in GAS token for the requested FrostFS account. // // Statuses: // - **OK** (0, SECTION_SUCCESS): @@ -27,9 +27,9 @@ service AccountingService { // BalanceRequest message message BalanceRequest { // To indicate the account for which the balance is requested, its identifier - // is used. It can be any existing account in NeoFS sidechain `Balance` smart - // contract. If omitted, client implementation MUST set it to the request's - // signer `OwnerID`. + // is used. It can be any existing account in FrostFS sidechain `Balance` + // smart contract. If omitted, client implementation MUST set it to the + // request's signer `OwnerID`. message Body { // Valid user identifier in `OwnerID` format for which the balance is // requested. Required field. diff --git a/src/FrostFS.SDK.ProtosV2/acl/types.proto b/src/FrostFS.SDK.ProtosV2/acl/types.proto index 186f08f..a1d9ae2 100644 --- a/src/FrostFS.SDK.ProtosV2/acl/types.proto +++ b/src/FrostFS.SDK.ProtosV2/acl/types.proto @@ -6,6 +6,7 @@ option go_package = "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl/grpc;ac option csharp_namespace = "FrostFS.Acl"; import "refs/types.proto"; +import "ape/types.proto"; // Target role of the access control rule in access control list. enum Role { @@ -88,14 +89,14 @@ enum HeaderType { // Filter object headers OBJECT = 2; - // Filter service headers. These are not processed by NeoFS nodes and + // Filter service headers. These are not processed by FrostFS nodes and // exist for service use only. SERVICE = 3; } // Describes a single eACL rule. message EACLRecord { - // NeoFS request Verb to match + // FrostFS request Verb to match Operation operation = 1 [ json_name = "operation" ]; // Rule execution result. Either allows or denies access if filters match. @@ -164,7 +165,7 @@ message EACLRecord { // Extended ACL rules table. A list of ACL rules defined additionally to Basic // ACL. Extended ACL rules can be attached to a container and can be updated // or may be defined in `BearerToken` structure. Please see the corresponding -// NeoFS Technical Specification section for detailed description. +// FrostFS Technical Specification section for detailed description. message EACLTable { // eACL format version. Effectively, the version of API library used to create // eACL Table. @@ -194,6 +195,9 @@ message BearerToken { // container. If it contains `container_id` field, bearer token is only // valid for this specific container. Otherwise, any container of the same // owner is allowed. + // + // Deprecated: eACL tables are no longer relevant - `APEOverrides` should be + // used instead. EACLTable eacl_table = 1 [ json_name = "eaclTable" ]; // `OwnerID` defines to whom the token was issued. It must match the request @@ -218,6 +222,24 @@ message BearerToken { // AllowImpersonate flag to consider token signer as request owner. // If this field is true extended ACL table in token body isn't processed. bool allow_impersonate = 4 [ json_name = "allowImpersonate" ]; + + // APEOverride is the list of APE chains defined for a target. + // These chains are meant to serve as overrides to the already defined (or + // even undefined) APE chains for the target (see contract `Policy`). + // + // The server-side processing of the bearer token with set APE overrides + // must verify if a client is permitted to override chains for the target, + // preventing unauthorized access through the APE mechanism. + message APEOverride { + // Target for which chains are applied. + frostfs.v2.ape.ChainTarget target = 1 [ json_name = "target" ]; + + // The list of APE chains. + repeated frostfs.v2.ape.Chain chains = 2 [ json_name = "chains" ]; + } + + // APE override for the target. + APEOverride ape_override = 5 [ json_name = "apeOverride" ]; } // Bearer Token body Body body = 1 [ json_name = "body" ]; diff --git a/src/FrostFS.SDK.ProtosV2/netmap/service.proto b/src/FrostFS.SDK.ProtosV2/netmap/service.proto index 8611d9a..11f8f96 100644 --- a/src/FrostFS.SDK.ProtosV2/netmap/service.proto +++ b/src/FrostFS.SDK.ProtosV2/netmap/service.proto @@ -12,7 +12,7 @@ import "session/types.proto"; // `NetmapService` provides methods to work with `Network Map` and the // information required to build it. The resulting `Network Map` is stored in // sidechain `Netmap` smart contract, while related information can be obtained -// from other NeoFS nodes. +// from other FrostFS nodes. service NetmapService { // Get NodeInfo structure from the particular node directly. // Node information can be taken from `Netmap` smart contract. In some cases, @@ -27,7 +27,7 @@ service NetmapService { // - Common failures (SECTION_FAILURE_COMMON). rpc LocalNodeInfo(LocalNodeInfoRequest) returns (LocalNodeInfoResponse); - // Read recent information about the NeoFS network. + // Read recent information about the FrostFS network. // // Statuses: // - **OK** (0, SECTION_SUCCESS): @@ -35,7 +35,7 @@ service NetmapService { // - Common failures (SECTION_FAILURE_COMMON). rpc NetworkInfo(NetworkInfoRequest) returns (NetworkInfoResponse); - // Returns network map snapshot of the current NeoFS epoch. + // Returns network map snapshot of the current FrostFS epoch. // // Statuses: // - **OK** (0, SECTION_SUCCESS): @@ -65,7 +65,7 @@ message LocalNodeInfoRequest { message LocalNodeInfoResponse { // Local Node Info, including API Version in use. message Body { - // Latest NeoFS API version in use + // Latest FrostFS API version in use neo.fs.v2.refs.Version version = 1; // NodeInfo structure with recent information from node itself diff --git a/src/FrostFS.SDK.ProtosV2/netmap/types.proto b/src/FrostFS.SDK.ProtosV2/netmap/types.proto index baaca04..5f0e93e 100644 --- a/src/FrostFS.SDK.ProtosV2/netmap/types.proto +++ b/src/FrostFS.SDK.ProtosV2/netmap/types.proto @@ -36,6 +36,9 @@ enum Operation { // Logical negation NOT = 9; + + // Matches pattern + LIKE = 10; } // Selector modifier shows how the node set will be formed. By default selector @@ -119,7 +122,7 @@ message PlacementPolicy { // bucket repeated Replica replicas = 1 [ json_name = "replicas" ]; - // Container backup factor controls how deep NeoFS will search for nodes + // Container backup factor controls how deep FrostFS will search for nodes // alternatives to include into container's nodes subset uint32 container_backup_factor = 2 [ json_name = "containerBackupFactor" ]; @@ -133,25 +136,25 @@ message PlacementPolicy { bool unique = 5 [ json_name = "unique" ]; } -// NeoFS node description +// FrostFS node description message NodeInfo { - // Public key of the NeoFS node in a binary format + // Public key of the FrostFS node in a binary format bytes public_key = 1 [ json_name = "publicKey" ]; // Ways to connect to a node repeated string addresses = 2 [ json_name = "addresses" ]; - // Administrator-defined Attributes of the NeoFS Storage Node. + // Administrator-defined Attributes of the FrostFS Storage Node. // // `Attribute` is a Key-Value metadata pair. Key name must be a valid UTF-8 // string. Value can't be empty. // // Attributes can be constructed into a chain of attributes: any attribute can // have a parent attribute and a child attribute (except the first and the - // last one). A string representation of the chain of attributes in NeoFS + // last one). A string representation of the chain of attributes in FrostFS // Storage Node configuration uses ":" and "/" symbols, e.g.: // - // `NEOFS_NODE_ATTRIBUTE_1=key1:val1/key2:val2` + // `FrostFS_NODE_ATTRIBUTE_1=key1:val1/key2:val2` // // Therefore the string attribute representation in the Node configuration // must use "\:", "\/" and "\\" escaped symbols if any of them appears in an @@ -198,8 +201,8 @@ message NodeInfo { // [ISO 3166-2](https://en.wikipedia.org/wiki/ISO_3166-2). Calculated // automatically from `UN-LOCODE` attribute. // * Continent \ - // Node's continent name according to the [Seven-Continent model] - // (https://en.wikipedia.org/wiki/Continent#Number). Calculated + // Node's continent name according to the [Seven-Continent + // model](https://en.wikipedia.org/wiki/Continent#Number). Calculated // automatically from `UN-LOCODE` attribute. // * ExternalAddr // Node's preferred way for communications with external clients. @@ -207,7 +210,7 @@ message NodeInfo { // Must contain a comma-separated list of multi-addresses. // // For detailed description of each well-known attribute please see the - // corresponding section in NeoFS Technical Specification. + // corresponding section in FrostFS Technical Specification. message Attribute { // Key of the node attribute string key = 1 [ json_name = "key" ]; @@ -219,13 +222,13 @@ message NodeInfo { // `Country`. repeated string parents = 3 [ json_name = "parents" ]; } - // Carries list of the NeoFS node attributes in a key-value form. Key name + // Carries list of the FrostFS node attributes in a key-value form. Key name // must be a node-unique valid UTF-8 string. Value can't be empty. NodeInfo // structures with duplicated attribute names or attributes with empty values // will be considered invalid. repeated Attribute attributes = 3 [ json_name = "attributes" ]; - // Represents the enumeration of various states of the NeoFS node. + // Represents the enumeration of various states of the FrostFS node. enum State { // Unknown state UNSPECIFIED = 0; @@ -240,7 +243,7 @@ message NodeInfo { MAINTENANCE = 3; } - // Carries state of the NeoFS node + // Carries state of the FrostFS node State state = 4 [ json_name = "state" ]; } @@ -253,7 +256,7 @@ message Netmap { repeated NodeInfo nodes = 2 [ json_name = "nodes" ]; } -// NeoFS network configuration +// FrostFS network configuration message NetworkConfig { // Single configuration parameter. Key MUST be network-unique. // @@ -272,7 +275,7 @@ message NetworkConfig { // Fee paid for container creation by the container owner. // Value: little-endian integer. Default: 0. // - **EpochDuration** \ - // NeoFS epoch duration measured in Sidechain blocks. + // FrostFS epoch duration measured in Sidechain blocks. // Value: little-endian integer. Default: 0. // - **HomomorphicHashingDisabled** \ // Flag of disabling the homomorphic hashing of objects' payload. @@ -284,8 +287,39 @@ message NetworkConfig { // Flag allowing setting the MAINTENANCE state to storage nodes. // Value: true if any byte != 0. Default: false. // - **MaxObjectSize** \ - // Maximum size of physically stored NeoFS object measured in bytes. + // Maximum size of physically stored FrostFS object measured in bytes. // Value: little-endian integer. Default: 0. + // + // This value refers to the maximum size of a **physically** stored object + // in FrostFS. However, from a user's perspective, the **logical** size of a + // stored object can be significantly larger. The relationship between the + // physical and logical object sizes is governed by the following formula + // + // ```math + // \mathrm{Stored\ Object\ Size} \le + // \frac{ + // \left(\mathrm{Max\ Object\ Size}\right)^2 + // }{ + // \mathrm{Object\ ID\ Size} + // } + // ``` + // + // This arises from the fact that a tombstone, also being an object, stores + // the IDs of inhumed objects and cannot be divided into smaller objects, + // thus having an upper limit for its size. + // + // For example, if: + // * Max Object Size Size = 64 MiB; + // * Object ID Size = 32 B; + // + // then: + // ```math + // \mathrm{Stored\ Object\ Size} \le + // \frac{\left(64\ \mathrm{MiB}\right)^2}{32\ \mathrm{B}} = + // \frac{2^{52}}{2^5}\ \mathrm{B} = + // 2^{47}\ \mathrm{B} = + // 128\ \mathrm{TiB} + // ``` // - **WithdrawFee** \ // Fee paid for withdrawal of funds paid by the account owner. // Value: little-endian integer. Default: 0. @@ -306,18 +340,18 @@ message NetworkConfig { repeated Parameter parameters = 1 [ json_name = "parameters" ]; } -// Information about NeoFS network +// Information about FrostFS network message NetworkInfo { - // Number of the current epoch in the NeoFS network + // Number of the current epoch in the FrostFS network uint64 current_epoch = 1 [ json_name = "currentEpoch" ]; - // Magic number of the sidechain of the NeoFS network + // Magic number of the sidechain of the FrostFS network uint64 magic_number = 2 [ json_name = "magicNumber" ]; - // MillisecondsPerBlock network parameter of the sidechain of the NeoFS + // MillisecondsPerBlock network parameter of the sidechain of the FrostFS // network int64 ms_per_block = 3 [ json_name = "msPerBlock" ]; - // NeoFS network configuration + // FrostFS network configuration NetworkConfig network_config = 4 [ json_name = "networkConfig" ]; } diff --git a/src/FrostFS.SDK.ProtosV2/object/Extension.Message.cs b/src/FrostFS.SDK.ProtosV2/object/Extension.Message.cs index 7c0c87d..6691400 100644 --- a/src/FrostFS.SDK.ProtosV2/object/Extension.Message.cs +++ b/src/FrostFS.SDK.ProtosV2/object/Extension.Message.cs @@ -458,4 +458,60 @@ namespace FrostFS.Object return Body; } } + + public partial class PatchRequest : IRequest + { + IMetaHeader IVerifiableMessage.GetMetaHeader() + { + return MetaHeader; + } + + IVerificationHeader IVerifiableMessage.GetVerificationHeader() + { + return VerifyHeader; + } + + void IVerifiableMessage.SetMetaHeader(IMetaHeader metaHeader) + { + MetaHeader = (RequestMetaHeader)metaHeader; + } + + void IVerifiableMessage.SetVerificationHeader(IVerificationHeader verificationHeader) + { + VerifyHeader = (RequestVerificationHeader)verificationHeader; + } + + public IMessage GetBody() + { + return Body; + } + } + + public partial class PatchResponse : IResponse + { + IMetaHeader IVerifiableMessage.GetMetaHeader() + { + return MetaHeader; + } + + IVerificationHeader IVerifiableMessage.GetVerificationHeader() + { + return VerifyHeader; + } + + void IVerifiableMessage.SetMetaHeader(IMetaHeader metaHeader) + { + MetaHeader = (ResponseMetaHeader)metaHeader; + } + + void IVerifiableMessage.SetVerificationHeader(IVerificationHeader verificationHeader) + { + VerifyHeader = (ResponseVerificationHeader)verificationHeader; + } + + public IMessage GetBody() + { + return Body; + } + } } \ No newline at end of file diff --git a/src/FrostFS.SDK.ProtosV2/object/service.proto b/src/FrostFS.SDK.ProtosV2/object/service.proto index 383e83b..2b8042b 100644 --- a/src/FrostFS.SDK.ProtosV2/object/service.proto +++ b/src/FrostFS.SDK.ProtosV2/object/service.proto @@ -151,7 +151,7 @@ service ObjectService { rpc Head(HeadRequest) returns (HeadResponse); // Search objects in container. Search query allows to match by Object - // Header's filed values. Please see the corresponding NeoFS Technical + // Header's filed values. Please see the corresponding FrostFS Technical // Specification section for more details. // // Extended headers can change `Search` behaviour: @@ -283,6 +283,55 @@ service ObjectService { // - **TOKEN_EXPIRED** (4097, SECTION_SESSION): \ // provided session token has expired. rpc PutSingle(PutSingleRequest) returns (PutSingleResponse); + + // Patch the object. Request uses gRPC stream. First message must set + // the address of the object that is going to get patched. If the object's + // attributes are patched, then these attrubutes must be set only within the + // first stream message. + // + // If the patch request is performed by NOT the object's owner but if the + // actor has the permission to perform the patch, then `OwnerID` of the object + // is changed. In this case the object's owner loses the object's ownership + // after the patch request is successfully done. + // + // As objects are content-addressable the patching causes new object ID + // generation for the patched object. This object id is set witihn + // `PatchResponse`. But the object id may remain unchanged in such cases: + // 1. The chunk of the applying patch contains the same value as the object's + // payload within the same range; + // 2. The patch that reverts the changes applied by preceding patch; + // 3. The application of the same patches for the object a few times. + // + // Extended headers can change `Patch` behaviour: + // * [ __SYSTEM__NETMAP_EPOCH \ + // (`__NEOFS__NETMAP_EPOCH` is deprecated) \ + // Will use the requsted version of Network Map for object placement + // calculation. + // + // Please refer to detailed `XHeader` description. + // + // Statuses: + // - **OK** (0, SECTION_SUCCESS): \ + // object has been successfully patched and saved in the container; + // - Common failures (SECTION_FAILURE_COMMON); + // - **ACCESS_DENIED** (2048, SECTION_OBJECT): \ + // write access to the container is denied; + // - **OBJECT_NOT_FOUND** (2049, SECTION_OBJECT): \ + // object not found in container; + // - **OBJECT_ALREADY_REMOVED** (2052, SECTION_OBJECT): \ + // the requested object has been marked as deleted. + // - **OUT_OF_RANGE** (2053, SECTION_OBJECT): \ + // the requested range is out of bounds; + // - **CONTAINER_NOT_FOUND** (3072, SECTION_CONTAINER): \ + // object storage container not found; + // - **CONTAINER_ACCESS_DENIED** (3074, SECTION_CONTAINER): \ + // access to container is denied; + // - **TOKEN_NOT_FOUND** (4096, SECTION_SESSION): \ + // (for trusted object preparation) session private key does not exist or + // has been deleted; + // - **TOKEN_EXPIRED** (4097, SECTION_SESSION): \ + // provided session token has expired. + rpc Patch(stream PatchRequest) returns (PatchResponse); } // GET object request @@ -583,6 +632,9 @@ message SearchRequest { // object_id of parent // * $Object:split.splitID \ // 16 byte UUIDv4 used to identify the split object hierarchy parts + // * $Object:ec.parent \ + // If the object is stored according to EC policy, then ec_parent + // attribute is set to return an id list of all related EC chunks. // // There are some well-known filter aliases to match objects by certain // properties: @@ -813,4 +865,75 @@ message PutSingleResponse { // authenticate the nodes of the message route and check the correctness of // transmission. neo.fs.v2.session.ResponseVerificationHeader verify_header = 3; -} \ No newline at end of file +} + +// Object PATCH request +message PatchRequest { + // PATCH request body + message Body { + // The address of the object that is requested to get patched. + neo.fs.v2.refs.Address address = 1; + + // New attributes for the object. See `replace_attributes` flag usage to + // define how new attributes should be set. + repeated neo.fs.v2.object.Header.Attribute new_attributes = 2; + + // If this flag is set, then the object's attributes will be entirely + // replaced by `new_attributes` list. The empty `new_attributes` list with + // `replace_attributes = true` just resets attributes list for the object. + // + // Default `false` value for this flag means the attributes will be just + // merged. If the incoming `new_attributes` list contains already existing + // key, then it just replaces it while merging the lists. + bool replace_attributes = 3; + + // The patch for the object's payload. + message Patch { + // The range of the source object for which the payload is replaced by the + // patch's chunk. If the range's `length = 0`, then the patch's chunk is + // just appended to the original payload starting from the `offest` + // without any replace. + Range source_range = 1; + + // The chunk that is being appended to or that replaces the original + // payload on the given range. + bytes chunk = 2; + } + + // The patch that is applied for the object. + Patch patch = 4; + } + + // Body for patch request message. + Body body = 1; + + // Carries request meta information. Header data is used only to regulate + // message transport and does not affect request execution. + neo.fs.v2.session.RequestMetaHeader meta_header = 2; + + // Carries request verification information. This header is used to + // authenticate the nodes of the message route and check the correctness of + // transmission. + neo.fs.v2.session.RequestVerificationHeader verify_header = 3; +} + +// Object PATCH response +message PatchResponse { + // PATCH response body + message Body { + // The object ID of the saved patched object. + neo.fs.v2.refs.ObjectID object_id = 1; + } + + // Body for patch response message. + Body body = 1; + + // Carries response meta information. Header data is used only to regulate + // message transport and does not affect request execution. + neo.fs.v2.session.ResponseMetaHeader meta_header = 2; + + // Carries response verification information. This header is used to + // authenticate the nodes of the message route and check the correctness of + // transmission. + neo.fs.v2.session.ResponseVerificationHeader verify_header = 3; +} diff --git a/src/FrostFS.SDK.ProtosV2/object/types.proto b/src/FrostFS.SDK.ProtosV2/object/types.proto index 6e62b86..b838c8e 100644 --- a/src/FrostFS.SDK.ProtosV2/object/types.proto +++ b/src/FrostFS.SDK.ProtosV2/object/types.proto @@ -155,7 +155,7 @@ message Header { // MIME Content Type of object's payload // // For detailed description of each well-known attribute please see the - // corresponding section in NeoFS Technical Specification. + // corresponding section in FrostFS Technical Specification. message Attribute { // string key to the object attribute string key = 1 [ json_name = "key" ]; @@ -208,6 +208,18 @@ message Header { uint32 header_length = 4 [ json_name = "headerLength" ]; // Chunk of a parent header. bytes header = 5 [ json_name = "header" ]; + // As the origin object is EC-splitted its identifier is known to all + // chunks as parent. But parent itself can be a part of Split (does not + // relate to EC-split). In this case parent_split_id should be set. + bytes parent_split_id = 6 [ json_name = "parentSplitID" ]; + // EC-parent's parent ID. parent_split_parent_id is set if EC-parent, + // itself, is a part of Split and if an object ID of its parent is + // presented. The field allows to determine how EC-chunk is placed in Split + // hierarchy. + neo.fs.v2.refs.ObjectID parent_split_parent_id = 7 + [ json_name = "parentSplitParentID" ]; + // EC parent's attributes. + repeated Attribute parent_attributes = 8 [ json_name = "parentAttributes" ]; } // Erasure code chunk information. EC ec = 12 [ json_name = "ec" ]; diff --git a/src/FrostFS.SDK.ProtosV2/refs/types.proto b/src/FrostFS.SDK.ProtosV2/refs/types.proto index 15d32c1..014c736 100644 --- a/src/FrostFS.SDK.ProtosV2/refs/types.proto +++ b/src/FrostFS.SDK.ProtosV2/refs/types.proto @@ -5,7 +5,7 @@ package neo.fs.v2.refs; option go_package = "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs/grpc;refs"; option csharp_namespace = "FrostFS.Refs"; -// Objects in NeoFS are addressed by their ContainerID and ObjectID. +// Objects in FrostFS are addressed by their ContainerID and ObjectID. // // String presentation of `Address` is a concatenation of string encoded // `ContainerID` and `ObjectID` delimited by '/' character. @@ -16,8 +16,9 @@ message Address { ObjectID object_id = 2 [ json_name = "objectID" ]; } -// NeoFS Object unique identifier. Objects are immutable and content-addressed. -// It means `ObjectID` will change if the `header` or the `payload` changes. +// FrostFS Object unique identifier. Objects are immutable and +// content-addressed. It means `ObjectID` will change if the `header` or the +// `payload` changes. // // `ObjectID` is a 32 byte long // [SHA256](https://csrc.nist.gov/publications/detail/fips/180/4/final) hash of @@ -37,7 +38,7 @@ message ObjectID { bytes value = 1 [ json_name = "value" ]; } -// NeoFS container identifier. Container structures are immutable and +// FrostFS container identifier. Container structures are immutable and // content-addressed. // // `ContainerID` is a 32 byte long @@ -90,7 +91,7 @@ message Version { uint32 minor = 2 [ json_name = "minor" ]; } -// Signature of something in NeoFS. +// Signature of something in FrostFS. message Signature { // Public key used for signing bytes key = 1 [ json_name = "key" ]; diff --git a/src/FrostFS.SDK.ProtosV2/session/service.proto b/src/FrostFS.SDK.ProtosV2/session/service.proto index 6f48e3a..6511f3b 100644 --- a/src/FrostFS.SDK.ProtosV2/session/service.proto +++ b/src/FrostFS.SDK.ProtosV2/session/service.proto @@ -11,7 +11,7 @@ import "session/types.proto"; // `SessionService` allows to establish a temporary trust relationship between // two peer nodes and generate a `SessionToken` as the proof of trust to be // attached in requests for further verification. Please see corresponding -// section of NeoFS Technical Specification for details. +// section of FrostFS Technical Specification for details. service SessionService { // Open a new session between two peers. // diff --git a/src/FrostFS.SDK.ProtosV2/session/types.proto b/src/FrostFS.SDK.ProtosV2/session/types.proto index d1a9ef1..bc0d7f1 100644 --- a/src/FrostFS.SDK.ProtosV2/session/types.proto +++ b/src/FrostFS.SDK.ProtosV2/session/types.proto @@ -36,6 +36,9 @@ message ObjectSessionContext { // Refers to object.GetRangeHash RPC call RANGEHASH = 7; + + // Refers to object.Patch RPC call + PATCH = 8; } // Type of request for which the token is issued Verb verb = 1 [ json_name = "verb" ]; @@ -47,7 +50,7 @@ message ObjectSessionContext { refs.ContainerID container = 1 [ json_name = "container" ]; // Indicates which objects the session is spread to. Objects are expected - // to be stored in the NeoFS container referenced by `container` field. + // to be stored in the FrostFS container referenced by `container` field. // Each element MUST have correct format. repeated refs.ObjectID objects = 2 [ json_name = "objects" ]; } @@ -85,7 +88,7 @@ message ContainerSessionContext { refs.ContainerID container_id = 3 [ json_name = "containerID" ]; } -// NeoFS Session Token. +// FrostFS Session Token. message SessionToken { // Session Token body message Body { @@ -123,7 +126,7 @@ message SessionToken { } // Session Token contains the proof of trust between peers to be attached in // requests for further verification. Please see corresponding section of - // NeoFS Technical Specification for details. + // FrostFS Technical Specification for details. Body body = 1 [ json_name = "body" ]; // Signature of `SessionToken` information @@ -183,7 +186,7 @@ message RequestMetaHeader { // `RequestMetaHeader` of the origin request RequestMetaHeader origin = 7 [ json_name = "origin" ]; - // NeoFS network magic. Must match the value for the network + // FrostFS network magic. Must match the value for the network // that the server belongs to. uint64 magic_number = 8 [ json_name = "magicNumber" ]; } diff --git a/src/FrostFS.SDK.ProtosV2/status/types.proto b/src/FrostFS.SDK.ProtosV2/status/types.proto index 8ab2f40..694f969 100644 --- a/src/FrostFS.SDK.ProtosV2/status/types.proto +++ b/src/FrostFS.SDK.ProtosV2/status/types.proto @@ -5,12 +5,12 @@ package neo.fs.v2.status; option go_package = "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/status/grpc;status"; option csharp_namespace = "FrostFS.Status"; -// Declares the general format of the status returns of the NeoFS RPC protocol. -// Status is present in all response messages. Each RPC of NeoFS protocol -// describes the possible outcomes and details of the operation. +// Declares the general format of the status returns of the FrostFS RPC +// protocol. Status is present in all response messages. Each RPC of FrostFS +// protocol describes the possible outcomes and details of the operation. // // Each status is assigned a one-to-one numeric code. Any unique result of an -// operation in NeoFS is unambiguously associated with the code value. +// operation in FrostFS is unambiguously associated with the code value. // // Numerical set of codes is split into 1024-element sections. An enumeration // is defined for each section. Values can be referred to in the following ways: @@ -78,7 +78,7 @@ enum Section { SECTION_APE_MANAGER = 5; } -// Section of NeoFS successful return codes. +// Section of FrostFS successful return codes. enum Success { // [**0**] Default success. Not detailed. // If the server cannot match successful outcome to the code, it should @@ -93,9 +93,9 @@ enum CommonFail { // use this code. INTERNAL = 0; - // [**1025**] Wrong magic of the NeoFS network. + // [**1025**] Wrong magic of the FrostFS network. // Details: - // - [**0**] Magic number of the served NeoFS network (big-endian 64-bit + // - [**0**] Magic number of the served FrostFS network (big-endian 64-bit // unsigned integer). WRONG_MAGIC_NUMBER = 1; @@ -104,6 +104,11 @@ enum CommonFail { // [**1027**] Node is under maintenance. NODE_UNDER_MAINTENANCE = 3; + + // [**1028**] Invalid argument error. If the server fails on validation of a + // request parameter as the client sent it incorrectly, then this code should + // be used. + INVALID_ARGUMENT = 4; } // Section of statuses for object-related operations. diff --git a/src/FrostFS.SDK.ProtosV2/tombstone/types.proto b/src/FrostFS.SDK.ProtosV2/tombstone/types.proto index 739bef4..8780317 100644 --- a/src/FrostFS.SDK.ProtosV2/tombstone/types.proto +++ b/src/FrostFS.SDK.ProtosV2/tombstone/types.proto @@ -8,10 +8,10 @@ option csharp_namespace = "FrostFS.Tombstone"; import "refs/types.proto"; // Tombstone keeps record of deleted objects for a few epochs until they are -// purged from the NeoFS network. +// purged from the FrostFS network. message Tombstone { - // Last NeoFS epoch number of the tombstone lifetime. It's set by the - // tombstone creator depending on the current NeoFS network settings. A + // Last FrostFS epoch number of the tombstone lifetime. It's set by the + // tombstone creator depending on the current FrostFS network settings. A // tombstone object must have the same expiration epoch value in // `__SYSTEM__EXPIRATION_EPOCH` (`__NEOFS__EXPIRATION_EPOCH` is deprecated) // attribute. Otherwise, the tombstone will be rejected by a storage node. diff --git a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamRangeReaderMock.cs b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamRangeReaderMock.cs new file mode 100644 index 0000000..e60b600 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamRangeReaderMock.cs @@ -0,0 +1,43 @@ +using System.Security.Cryptography; + +using FrostFS.Object; +using FrostFS.SDK.ClientV2; +using FrostFS.SDK.ClientV2.Mappers.GRPC; +using FrostFS.SDK.Cryptography; +using FrostFS.Session; + +using Google.Protobuf; + +using Grpc.Core; + +namespace FrostFS.SDK.Tests; + +public class AsyncStreamRangeReaderMock(string key, byte[] response) : ServiceBase(key), IAsyncStreamReader +{ + private readonly byte[] _response = response; + + public GetRangeResponse Current + { + get + { + var response = new GetRangeResponse + { + Body = new GetRangeResponse.Types.Body + { + Chunk = ByteString.CopyFrom(_response) + }, + MetaHeader = new ResponseMetaHeader() + }; + + response.VerifyHeader = GetResponseVerificationHeader(response); + + return response; + } + } + + public Task MoveNext(CancellationToken cancellationToken) + { + return Task.FromResult(true); + } +} + diff --git a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs index d0b900d..7c44fc8 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs @@ -5,6 +5,7 @@ using FrostFS.Object; using FrostFS.SDK.ClientV2; using FrostFS.SDK.ClientV2.Mappers.GRPC; using FrostFS.SDK.Cryptography; +using FrostFS.Session; using Google.Protobuf; @@ -16,6 +17,32 @@ namespace FrostFS.SDK.Tests; public class ObjectMocker(string key) : ObjectServiceBase(key) { + public FrostFsObjectId? ObjectId { get; set; } + + public FrostFsObjectHeader? ObjectHeader { get; set; } + + public Header? HeadResponse { get; set; } + + public Collection? ResultObjectIds { get; } = []; + + public ClientStreamWriter? ClientStreamWriter { get; } = new(); + + public PatchStreamWriter? PatchStreamWriter { get; } = new(); + + public Collection PutSingleRequests { get; } = []; + + public Collection DeleteRequests { get; } = []; + + public Collection HeadRequests { get; } = []; + + public byte[] RangeResponse { get; set; } = []; + + public GetRangeRequest? GetRangeRequest { get; set; } + + public GetRangeHashRequest? GetRangeHashRequest { get; set; } + + public Collection RangeHashResponses { get; } = []; + public override Mock GetMock() { var mock = new Mock(); @@ -189,23 +216,88 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) }); } + mock.Setup(x => x.GetRange( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((GetRangeRequest r, Metadata m, DateTime? dt, CancellationToken ct) => + { + Verifier.CheckRequest(r); + + GetRangeRequest = r; + + return new AsyncServerStreamingCall( + new AsyncStreamRangeReaderMock(StringKey, RangeResponse), + Task.FromResult(ResponseMetaData), + () => new Grpc.Core.Status(StatusCode.OK, string.Empty), + () => ResponseMetaData, + () => { }); + }); + + mock.Setup(x => x.GetRangeHashAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((GetRangeHashRequest r, Metadata m, DateTime? dt, CancellationToken ct) => + { + Verifier.CheckRequest(r); + + GetRangeHashRequest = r; + + var response = new GetRangeHashResponse + { + Body = new GetRangeHashResponse.Types.Body(), + MetaHeader = ResponseMetaHeader + }; + + if (RangeHashResponses != null) + { + foreach (var hash in RangeHashResponses) + { + response.Body.HashList.Add(hash); + } + } + + response.VerifyHeader = GetResponseVerificationHeader(response); + + return new AsyncUnaryCall( + Task.FromResult(response), + Task.FromResult(ResponseMetaData), + () => new Grpc.Core.Status(StatusCode.OK, string.Empty), + () => ResponseMetaData, + () => { }); + }); + + + mock.Setup(x => x.Patch( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((Metadata m, DateTime? dt, CancellationToken ct) => + { + var patchResponse = new PatchResponse + { + Body = new PatchResponse.Types.Body + { + ObjectId = new Refs.ObjectID { Value = ByteString.CopyFrom(SHA256.HashData([1,2,3])) }, + }, + MetaHeader = ResponseMetaHeader + }; + + patchResponse.VerifyHeader = GetResponseVerificationHeader(patchResponse); + + return new AsyncClientStreamingCall( + PatchStreamWriter!, + Task.FromResult(patchResponse), + Task.FromResult(ResponseMetaData), + () => new Grpc.Core.Status(StatusCode.OK, string.Empty), + () => ResponseMetaData, + () => { }); + }); + return mock; } - - public FrostFsObjectId? ObjectId { get; set; } - - public FrostFsObjectHeader? ObjectHeader { get; set; } - - public Header? HeadResponse { get; set; } - - public Collection? ResultObjectIds { get; } = []; - - public ClientStreamWriter? ClientStreamWriter { get; private set; } = new(); - - public Collection PutSingleRequests { get; private set; } = []; - - public Collection DeleteRequests { get; private set; } = []; - - public Collection HeadRequests { get; private set; } = []; } diff --git a/src/FrostFS.SDK.Tests/Mocks/PatchStreamWriter.cs b/src/FrostFS.SDK.Tests/Mocks/PatchStreamWriter.cs new file mode 100644 index 0000000..a2672c8 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Mocks/PatchStreamWriter.cs @@ -0,0 +1,36 @@ +using System.Collections.ObjectModel; + +using FrostFS.SDK.ProtosV2.Interfaces; + +using Grpc.Core; + +namespace FrostFS.SDK.Tests; + +public class PatchStreamWriter : IClientStreamWriter +{ + private WriteOptions? _options; + + public Collection Messages { get; } = []; + + public bool CompletedTask { get; private set; } + + public WriteOptions? WriteOptions + { + get => _options; + set => _options = value; + } + + public Task CompleteAsync() + { + CompletedTask = true; + return Task.CompletedTask; + } + + public Task WriteAsync(IRequest message) + { + Object.PatchRequest pr = new((Object.PatchRequest)message); + Messages.Add(pr); + return Task.CompletedTask; + } +} + diff --git a/src/FrostFS.SDK.Tests/ObjectTest.cs b/src/FrostFS.SDK.Tests/ObjectTest.cs index 85b42ca..435c042 100644 --- a/src/FrostFS.SDK.Tests/ObjectTest.cs +++ b/src/FrostFS.SDK.Tests/ObjectTest.cs @@ -10,6 +10,8 @@ using FrostFS.SDK.Cryptography; using Google.Protobuf; +using static FrostFS.Object.ECInfo.Types; + namespace FrostFS.SDK.Tests; [SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] @@ -223,4 +225,123 @@ public class ObjectTest : ObjectTestsBase Assert.Null(response.Split); } + + + [Fact] + public async void GetRangeTest() + { + Mocker.ResultObjectIds!.Add(SHA256.HashData([])); + + Random rnd = new(); + var bytes = new byte[1024]; + rnd.NextBytes(bytes); + + Mocker.RangeResponse = bytes; + + Mocker.ObjectId = new ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Encoding.UTF8.GetBytes("test"))) }.ToModel(); + + var param = new PrmRangeGet(ContainerId, Mocker.ObjectId, new FrostFsRange(100, (ulong)Mocker.RangeResponse.Length)); + + var result = await GetClient().GetRangeAsync(param); + + Assert.NotNull(Mocker.GetRangeRequest); + + Assert.Equal(param.Range.Offset, Mocker.GetRangeRequest.Body.Range.Offset); + Assert.Equal(param.Range.Length, Mocker.GetRangeRequest.Body.Range.Length); + + Assert.NotNull(result); + + var chunk = await result.ReadChunk(); + + var chunkBytes = chunk.Value.Span.ToArray(); + + Assert.Equal(chunkBytes.Length, Mocker.RangeResponse.Length); + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(Mocker.RangeResponse)); + } + + [Fact] + public async void GetRangeHashTest() + { + Mocker.ResultObjectIds!.Add(SHA256.HashData([])); + + Random rnd = new(); + var bytes = new byte[1024]; + rnd.NextBytes(bytes); + + var salt = new byte[32]; + rnd.NextBytes(salt); + + var hash = new byte[32]; + rnd.NextBytes(hash); + + Mocker.RangeResponse = bytes; + var len = (ulong)bytes.Length; + + Mocker.RangeHashResponses.Add(ByteString.CopyFrom(hash)); + + Mocker.ObjectId = new ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Encoding.UTF8.GetBytes("test"))) }.ToModel(); + + var param = new PrmRangeHashGet(ContainerId, Mocker.ObjectId, [new FrostFsRange(100, len)], salt); + + var result = await GetClient().GetRangeHashAsync(param); + + Assert.NotNull(Mocker.GetRangeHashRequest); + + Assert.Equal(param.Ranges[0].Offset, Mocker.GetRangeHashRequest.Body.Ranges[0].Offset); + Assert.Equal(param.Ranges[0].Length, Mocker.GetRangeHashRequest.Body.Ranges[0].Length); + + Assert.NotNull(result); + Assert.Single(result); + + Assert.Equal(SHA256.HashData(hash), SHA256.HashData(result.First().ToArray())); + } + + + [Fact] + public async void PatchTest() + { + Mocker.ObjectId = new ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Encoding.UTF8.GetBytes("test"))) }.ToModel(); + + var address = new FrostFsAddress(ContainerId, Mocker.ObjectId); + + Mocker.ResultObjectIds!.Add(SHA256.HashData([])); + + Random rnd = new(); + var patch = new byte[32]; + rnd.NextBytes(patch); + + var range = new FrostFsRange(8, (ulong)patch.Length); + + var param = new PrmObjectPatch(address) + { + Payload = new MemoryStream(patch), + MaxPayloadPatchChunkLength = 32, + Range = range + }; + + var result = await GetClient().PatchObjectAsync(param); + + Assert.NotNull(result); + + Assert.NotNull(result.Value); + + Assert.NotNull(Mocker.PatchStreamWriter); + Assert.Single(Mocker.PatchStreamWriter.Messages); + + var sentMessages = Mocker.PatchStreamWriter!.Messages; + + var body = sentMessages.First().GetBody() as Object.PatchRequest.Types.Body; + + Assert.NotNull(body); + + Assert.True(Mocker.PatchStreamWriter.CompletedTask); + + Assert.Equal(address.ContainerId, body.Address.ContainerId); + Assert.Equal(address.ObjectId, body.Address.ObjectId); + + Assert.Equal(32, body.Patch.Chunk.Length); + + Assert.Equal(SHA256.HashData(patch), SHA256.HashData(body.Patch.Chunk.ToArray())); + } } diff --git a/src/FrostFS.SDK.Tests/SmokeClientTests.cs b/src/FrostFS.SDK.Tests/SmokeClientTests.cs index 426aa9b..56acfc3 100644 --- a/src/FrostFS.SDK.Tests/SmokeClientTests.cs +++ b/src/FrostFS.SDK.Tests/SmokeClientTests.cs @@ -247,37 +247,108 @@ public class SmokeClientTests : SmokeTestsBase [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB [InlineData(6 * 1024 * 1024 + 100)] public async void SimpleScenarioTest(int objectSize) + { + using var client = FrostFSClient.GetSingleOwnerInstance(GetSingleOwnerOptions(this.keyString, this.url)); + + await Cleanup(client); + + bool callbackInvoked = false; + var ctx = new CallContext + { + // Timeout = TimeSpan.FromSeconds(20), + Callback = new((CallStatistics cs) => + { + callbackInvoked = true; + Assert.True(cs.ElapsedMicroSeconds > 0); + }) + }; + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")]), ctx); + + var createdContainer = await client.CreateContainerAsync(createContainerParam); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer, ctx)); + Assert.NotNull(container); + Assert.True(callbackInvoked); + + var bytes = GetRandomBytes(objectSize); + + var param = new PrmObjectPut(new CallContext + { + Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0)) + }) + { + Header = new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + Payload = new MemoryStream(bytes), + ClientCut = false + }; + + var objectId = await client.PutObjectAsync(param); + + var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); + + bool hasObject = false; + await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(createdContainer) { Filters = [filter] })) + { + hasObject = true; + + var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId)); + Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); + Assert.NotNull(objHeader.Attributes); + Assert.Single(objHeader.Attributes); + Assert.Equal("fileName", objHeader.Attributes.First().Key); + Assert.Equal("test", objHeader.Attributes.First().Value); + } + + Assert.True(hasObject); + + var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId)); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync()) + { + Assert.Fail("Containers exist"); + } + } + + [Fact] + public async void PatchTest() { using var client = FrostFSClient.GetSingleOwnerInstance(GetSingleOwnerOptions(this.keyString, this.url)); await Cleanup(client); - bool callbackInvoked = false; - var ctx = new CallContext - { - // Timeout = TimeSpan.FromSeconds(20), - Callback = new((CallStatistics cs) => - { - callbackInvoked = true; - Assert.True(cs.ElapsedMicroSeconds > 0); - }) - }; - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")]), ctx); + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")])); var createdContainer = await client.CreateContainerAsync(createContainerParam); - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer, ctx)); + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer)); Assert.NotNull(container); - Assert.True(callbackInvoked); - var bytes = GetRandomBytes(objectSize); - - var param = new PrmObjectPut(new CallContext + var bytes = new byte[1024]; + for (int i = 0; i < 1024; i++) { - Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0)) - }) + bytes[i] = (byte)31; + } + + var param = new PrmObjectPut { Header = new FrostFsObjectHeader( containerId: createdContainer, @@ -289,24 +360,24 @@ public class SmokeClientTests : SmokeTestsBase var objectId = await client.PutObjectAsync(param); - var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"); - - bool hasObject = false; - await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(createdContainer) { Filters = [filter] })) + var patch = new byte[16]; + for (int i = 0; i < 16; i++) { - hasObject = true; - - var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId)); - Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength); - Assert.NotNull(objHeader.Attributes); - Assert.Single(objHeader.Attributes); - Assert.Equal("fileName", objHeader.Attributes.First().Key); - Assert.Equal("test", objHeader.Attributes.First().Value); + patch[i] = (byte)32; } - Assert.True(hasObject); + var range = new FrostFsRange(8, (ulong)patch.Length); - var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId)); + var patchParams = new PrmObjectPatch(new FrostFsAddress(createdContainer, objectId)) + { + Payload = new MemoryStream(patch), + MaxPayloadPatchChunkLength = 32, + Range = range + }; + + var newIbjId = await client.PatchObjectAsync(patchParams); + + var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, newIbjId)); var downloadedBytes = new byte[@object.Header.PayloadLength]; MemoryStream ms = new(downloadedBytes); @@ -317,8 +388,121 @@ public class SmokeClientTests : SmokeTestsBase ms.Write(chunk.Value.Span); } - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); + for(int i = 0; i < (int)range.Offset; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); + var rangeEnd = range.Offset + range.Length; + + for (int i = (int)range.Offset; i < (int)rangeEnd; i++) + Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); + + for (int i = (int)rangeEnd; i < bytes.Length; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync()) + { + Assert.Fail("Containers exist"); + } + } + + [Fact] + public async void RangeTest() + { + using var client = FrostFSClient.GetSingleOwnerInstance(GetSingleOwnerOptions(this.keyString, this.url)); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")])); + + var createdContainer = await client.CreateContainerAsync(createContainerParam); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer)); + Assert.NotNull(container); + + var bytes = new byte[256]; + for (int i = 0; i < 256; i++) + { + bytes[i] = (byte)i; + } + + var param = new PrmObjectPut + { + Header = new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular), + Payload = new MemoryStream(bytes), + ClientCut = false + }; + + var objectId = await client.PutObjectAsync(param); + + var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange(100, 64)); + + var rangeReader = await client.GetRangeAsync(rangeParam); + + var downloadedBytes = new byte[rangeParam.Range.Length]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await rangeReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice(100, 64)), SHA256.HashData(downloadedBytes)); + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync()) + { + Assert.Fail("Containers exist"); + } + } + + [Fact] + public async void RangeHashTest() + { + using var client = FrostFSClient.GetSingleOwnerInstance(GetSingleOwnerOptions(this.keyString, this.url)); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")])); + + var createdContainer = await client.CreateContainerAsync(createContainerParam); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer)); + Assert.NotNull(container); + + var bytes = new byte[256]; + for (int i = 0; i < 256; i++) + { + bytes[i] = (byte)i; + } + + var param = new PrmObjectPut + { + Header = new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular), + Payload = new MemoryStream(bytes), + ClientCut = false + }; + + var objectId = await client.PutObjectAsync(param); + + var rangeParam = new PrmRangeHashGet(createdContainer, objectId, [ new FrostFsRange(100, 64)], bytes); + + var hashes = await client.GetRangeHashAsync(rangeParam); + + foreach (var hash in hashes) + { + var x = hash.Slice(0, 32).ToArray(); + } + await Cleanup(client); await foreach (var _ in client.ListContainersAsync()) diff --git a/src/FrostFS.SDK.Tests/SmokeTestsBase.cs b/src/FrostFS.SDK.Tests/SmokeTestsBase.cs index 6b42144..e0d53d7 100644 --- a/src/FrostFS.SDK.Tests/SmokeTestsBase.cs +++ b/src/FrostFS.SDK.Tests/SmokeTestsBase.cs @@ -7,6 +7,8 @@ namespace FrostFS.SDK.SmokeTests; public abstract class SmokeTestsBase { + // internal readonly string keyString = "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK"; + internal readonly string keyString = "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK"; internal readonly string url = "http://172.23.32.4:8080";