diff --git a/src/FrostFS.SDK.ClientV2/FrostFSClient.cs b/src/FrostFS.SDK.ClientV2/FrostFSClient.cs index 1a34a53..830eb94 100644 --- a/src/FrostFS.SDK.ClientV2/FrostFSClient.cs +++ b/src/FrostFS.SDK.ClientV2/FrostFSClient.cs @@ -294,6 +294,16 @@ public class FrostFSClient : IFrostFSClient return service.GetRangeAsync(args); } + public Task>> GetRangeHashAsync(PrmRangeHashGet args) + { + if (args is null) + throw new ArgumentNullException(nameof(args)); + + var service = GetObjectService(args); + return service.GetRangeHashAsync(args); + } + + public Task PutObjectAsync(PrmObjectPut args) { if (args is null) @@ -311,8 +321,8 @@ public class FrostFSClient : IFrostFSClient var service = GetObjectService(args); return service.PutSingleObjectAsync(args); } - - public Task PatchObjectAsync(PrmObjectPatch args) + + public Task PatchObjectAsync(PrmObjectPatch args) { if (args is null) { diff --git a/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs b/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs index ac862b8..48ab8e8 100644 --- a/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs +++ b/src/FrostFS.SDK.ClientV2/Interfaces/IFrostFSClient.cs @@ -44,11 +44,13 @@ public interface IFrostFSClient : IDisposable Task GetRangeAsync(PrmRangeGet args); + Task>> GetRangeHashAsync(PrmRangeHashGet args); + Task PutObjectAsync(PrmObjectPut args); Task PutSingleObjectAsync(PrmSingleObjectPut args); - Task PatchObjectAsync(PrmObjectPatch args); + Task PatchObjectAsync(PrmObjectPatch args); Task DeleteObjectAsync(PrmObjectDelete args); diff --git a/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs b/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs index bc691cb..c4c805c 100644 --- a/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs +++ b/src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs @@ -1,5 +1,4 @@ using System; -using System.Security.Cryptography; using FrostFS.Refs; using FrostFS.SDK.Cryptography; diff --git a/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsAddress.cs b/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsAddress.cs index 598ecda..46581d1 100644 --- a/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsAddress.cs +++ b/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsAddress.cs @@ -10,7 +10,7 @@ public class FrostFsAddress private ObjectID? objectId; private ContainerID? containerId; - public FrostFsAddress(FrostFsObjectId frostFsObjectId, FrostFsContainerId frostFsContainerId) + public FrostFsAddress(FrostFsContainerId frostFsContainerId, FrostFsObjectId frostFsObjectId) { FrostFsObjectId = frostFsObjectId ?? throw new System.ArgumentNullException(nameof(frostFsObjectId)); FrostFsContainerId = frostFsContainerId ?? throw new System.ArgumentNullException(nameof(frostFsContainerId)); @@ -24,25 +24,25 @@ public class FrostFsAddress public FrostFsObjectId FrostFsObjectId { - get => frostFsObjectId ??= objectId!.ToModel(); - set => frostFsObjectId = value; + get => frostFsObjectId ??= objectId!.ToModel(); + set => frostFsObjectId = value; } - public FrostFsContainerId FrostFsContainerId - { + public FrostFsContainerId FrostFsContainerId + { get => frostFsContainerId ??= containerId!.ToModel(); - set => frostFsContainerId = value; + set => frostFsContainerId = value; } - - public ObjectID ObjectId + + public ObjectID ObjectId { get => objectId ??= frostFsObjectId!.ToMessage(); - set => objectId = value; + set => objectId = value; } public ContainerID ContainerId { get => containerId ??= frostFsContainerId!.ToMessage(); - set => containerId = value; + set => containerId = value; } } diff --git a/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsRange.cs b/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsRange.cs index 22f5d40..b50568f 100644 --- a/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsRange.cs +++ b/src/FrostFS.SDK.ClientV2/Models/Object/FrostFsRange.cs @@ -1,10 +1,10 @@ namespace FrostFS.SDK; -public struct FrostFsRange : System.IEquatable +public readonly struct FrostFsRange(ulong offset, ulong length) : System.IEquatable { - public ulong Offset {get; set;} + public ulong Offset { get; } = offset; - public ulong Length { get; set; } + public ulong Length { get; } = length; public override readonly bool Equals(object obj) => this == (FrostFsRange)obj; diff --git a/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPatch.cs b/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPatch.cs index 180c634..de292ea 100644 --- a/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPatch.cs +++ b/src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPatch.cs @@ -1,5 +1,4 @@ using System.IO; -using System.Security.Cryptography; namespace FrostFS.SDK.ClientV2; diff --git a/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeGet.cs b/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeGet.cs index 5f03770..5bcad9f 100644 --- a/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeGet.cs +++ b/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeGet.cs @@ -1,9 +1,10 @@ namespace FrostFS.SDK.ClientV2; public sealed class PrmRangeGet( - FrostFsContainerId containerId, + FrostFsContainerId containerId, FrostFsObjectId objectId, FrostFsRange range, + bool raw = false, CallContext? ctx = null) : PrmBase(ctx), ISessionToken { public FrostFsContainerId ContainerId { get; } = containerId; @@ -12,6 +13,8 @@ public sealed class PrmRangeGet( public FrostFsRange Range { get; } = range; + public bool Raw { get; } = raw; + /// public FrostFsSessionToken? SessionToken { get; set; } } diff --git a/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeHashGet.cs b/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeHashGet.cs new file mode 100644 index 0000000..661ed64 --- /dev/null +++ b/src/FrostFS.SDK.ClientV2/Parameters/PrmRangeHashGet.cs @@ -0,0 +1,20 @@ +namespace FrostFS.SDK.ClientV2; + +public sealed class PrmRangeHashGet( + FrostFsContainerId containerId, + FrostFsObjectId objectId, + FrostFsRange[] ranges, + byte[] salt, + CallContext? ctx = null) : PrmBase(ctx), ISessionToken +{ + public FrostFsContainerId ContainerId { get; } = containerId; + + public FrostFsObjectId ObjectId { get; } = objectId; + + public FrostFsRange[] Ranges { get; } = ranges; + + public byte[] Salt { get; } = salt; + + /// + public FrostFsSessionToken? SessionToken { get; set; } +} diff --git a/src/FrostFS.SDK.ClientV2/Poll/ClientStatusMonitor.cs b/src/FrostFS.SDK.ClientV2/Pool/ClientStatusMonitor.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/ClientStatusMonitor.cs rename to src/FrostFS.SDK.ClientV2/Pool/ClientStatusMonitor.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/ClientWrapper.cs b/src/FrostFS.SDK.ClientV2/Pool/ClientWrapper.cs similarity index 99% rename from src/FrostFS.SDK.ClientV2/Poll/ClientWrapper.cs rename to src/FrostFS.SDK.ClientV2/Pool/ClientWrapper.cs index 777c311..206f69e 100644 --- a/src/FrostFS.SDK.ClientV2/Poll/ClientWrapper.cs +++ b/src/FrostFS.SDK.ClientV2/Pool/ClientWrapper.cs @@ -104,7 +104,7 @@ public class ClientWrapper : ClientStatusMonitor //#pragma warning disable CA2000 // Dispose objects before losing scope: will be disposed manually FrostFSClient client = new(WrapperPrm, sessionCache); //#pragma warning restore CA2000 - + //TODO: set additioanl params var error = await client.Dial(ctx).ConfigureAwait(false); if (!string.IsNullOrEmpty(error)) diff --git a/src/FrostFS.SDK.ClientV2/Poll/HealthyStatus.cs b/src/FrostFS.SDK.ClientV2/Pool/HealthyStatus.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/HealthyStatus.cs rename to src/FrostFS.SDK.ClientV2/Pool/HealthyStatus.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/IClientStatus.cs b/src/FrostFS.SDK.ClientV2/Pool/IClientStatus.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/IClientStatus.cs rename to src/FrostFS.SDK.ClientV2/Pool/IClientStatus.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/InitParameters.cs b/src/FrostFS.SDK.ClientV2/Pool/InitParameters.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/InitParameters.cs rename to src/FrostFS.SDK.ClientV2/Pool/InitParameters.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/InnerPool.cs b/src/FrostFS.SDK.ClientV2/Pool/InnerPool.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/InnerPool.cs rename to src/FrostFS.SDK.ClientV2/Pool/InnerPool.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/MethodIndex.cs b/src/FrostFS.SDK.ClientV2/Pool/MethodIndex.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/MethodIndex.cs rename to src/FrostFS.SDK.ClientV2/Pool/MethodIndex.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/MethodStatus.cs b/src/FrostFS.SDK.ClientV2/Pool/MethodStatus.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/MethodStatus.cs rename to src/FrostFS.SDK.ClientV2/Pool/MethodStatus.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/NodeParam.cs b/src/FrostFS.SDK.ClientV2/Pool/NodeParam.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/NodeParam.cs rename to src/FrostFS.SDK.ClientV2/Pool/NodeParam.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/NodeStatistic.cs b/src/FrostFS.SDK.ClientV2/Pool/NodeStatistic.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/NodeStatistic.cs rename to src/FrostFS.SDK.ClientV2/Pool/NodeStatistic.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/NodesParam.cs b/src/FrostFS.SDK.ClientV2/Pool/NodesParam.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/NodesParam.cs rename to src/FrostFS.SDK.ClientV2/Pool/NodesParam.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/Pool.cs b/src/FrostFS.SDK.ClientV2/Pool/Pool.cs similarity index 94% rename from src/FrostFS.SDK.ClientV2/Poll/Pool.cs rename to src/FrostFS.SDK.ClientV2/Pool/Pool.cs index 8bb44f2..9e34551 100644 --- a/src/FrostFS.SDK.ClientV2/Poll/Pool.cs +++ b/src/FrostFS.SDK.ClientV2/Pool/Pool.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -711,7 +710,7 @@ public partial class Pool : IFrostFSClient return await client.Client!.PutSingleObjectAsync(args).ConfigureAwait(false); } - public async Task PatchAsync(PrmObjectPatch args) + public async Task PatchObjectAsync(PrmObjectPatch args) { if (args is null) { @@ -722,7 +721,49 @@ public partial class Pool : IFrostFSClient args.Context.PoolErrorHandler = client.HandleError; - await client.Client!.PatchObjectAsync(args).ConfigureAwait(false); + return await client.Client!.PatchObjectAsync(args).ConfigureAwait(false); + } + + public async Task GetRangeAsync(PrmRangeGet args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var client = Connection(); + + args.Context.PoolErrorHandler = client.HandleError; + + return await client.Client!.GetRangeAsync(args).ConfigureAwait(false); + } + + public async Task>> GetRangeHashAsync(PrmRangeHashGet args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var client = Connection(); + + args.Context.PoolErrorHandler = client.HandleError; + + return await client.Client!.GetRangeHashAsync(args).ConfigureAwait(false); + } + + public async Task PatchAsync(PrmObjectPatch args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var client = Connection(); + + args.Context.PoolErrorHandler = client.HandleError; + + return await client.Client!.PatchObjectAsync(args).ConfigureAwait(false); } public async Task DeleteObjectAsync(PrmObjectDelete args) @@ -787,14 +828,4 @@ public partial class Pool : IFrostFSClient { throw new NotImplementedException(); } - - public Task PatchObjectAsync(PrmObjectPatch args) - { - throw new NotImplementedException(); - } - - public Task GetRangeAsync(PrmRangeGet args) - { - throw new NotImplementedException(); - } } diff --git a/src/FrostFS.SDK.ClientV2/Poll/RebalanceParameters.cs b/src/FrostFS.SDK.ClientV2/Pool/RebalanceParameters.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/RebalanceParameters.cs rename to src/FrostFS.SDK.ClientV2/Pool/RebalanceParameters.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/RequestInfo.cs b/src/FrostFS.SDK.ClientV2/Pool/RequestInfo.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/RequestInfo.cs rename to src/FrostFS.SDK.ClientV2/Pool/RequestInfo.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/Sampler.cs b/src/FrostFS.SDK.ClientV2/Pool/Sampler.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/Sampler.cs rename to src/FrostFS.SDK.ClientV2/Pool/Sampler.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/SessionCache.cs b/src/FrostFS.SDK.ClientV2/Pool/SessionCache.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/SessionCache.cs rename to src/FrostFS.SDK.ClientV2/Pool/SessionCache.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/Statistic.cs b/src/FrostFS.SDK.ClientV2/Pool/Statistic.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/Statistic.cs rename to src/FrostFS.SDK.ClientV2/Pool/Statistic.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/StatusSnapshot.cs b/src/FrostFS.SDK.ClientV2/Pool/StatusSnapshot.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/StatusSnapshot.cs rename to src/FrostFS.SDK.ClientV2/Pool/StatusSnapshot.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/WorkList.cs b/src/FrostFS.SDK.ClientV2/Pool/WorkList.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/WorkList.cs rename to src/FrostFS.SDK.ClientV2/Pool/WorkList.cs diff --git a/src/FrostFS.SDK.ClientV2/Poll/WrapperPrm.cs b/src/FrostFS.SDK.ClientV2/Pool/WrapperPrm.cs similarity index 100% rename from src/FrostFS.SDK.ClientV2/Poll/WrapperPrm.cs rename to src/FrostFS.SDK.ClientV2/Pool/WrapperPrm.cs diff --git a/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs b/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs index 9af8fb7..bb742a5 100644 --- a/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs +++ b/src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs @@ -130,7 +130,8 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl { Offset = args.Range.Offset, Length = args.Range.Length - } + }, + Raw = args.Raw } }; @@ -149,6 +150,59 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return new RangeReader(call); } + internal async Task>> GetRangeHashAsync(PrmRangeHashGet args) + { + var ctx = args.Context!; + + ctx.Key ??= ClientContext.Key?.ECDsaKey; + + if (ctx.Key == null) + throw new ArgumentNullException(nameof(args), "Key is null"); + + var request = new GetRangeHashRequest + { + Body = new GetRangeHashRequest.Types.Body + { + Address = new Address + { + ContainerId = args.ContainerId.ToMessage(), + ObjectId = args.ObjectId.ToMessage() + }, + Type = ChecksumType.Sha256, + Salt = ByteString.CopyFrom(args.Salt) // TODO: create a type with calculated cashed ByteString inside + } + }; + + foreach (var range in args.Ranges) + { + request.Body.Ranges.Add(new Object.Range + { + Length = range.Length, + Offset = range.Offset + }); + } + + var sessionToken = await GetOrCreateSession(args, ctx).ConfigureAwait(false); + + sessionToken.CreateObjectTokenContext( + request.Body.Address, + ObjectSessionContext.Types.Verb.Rangehash, + ctx.Key); + + request.AddMetaHeader(args.XHeaders, sessionToken); + + request.Sign(ctx.Key); + + var response = await client.GetRangeHashAsync(request, null, ctx.Deadline, ctx.CancellationToken); + + Verifier.CheckResponse(response); + + var hashCollection = response.Body.HashList.ToArray().Select(h => h.Memory); + + return hashCollection; + } + + internal async Task DeleteObjectAsync(PrmObjectDelete args) { var ctx = args.Context!; @@ -281,19 +335,19 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl return FrostFsObjectId.FromHash(grpcObject.ObjectId.Value.ToByteArray()); } - internal async Task PatchObjectAsync(PrmObjectPatch args) + internal async Task PatchObjectAsync(PrmObjectPatch args) { var ctx = args.Context!; if (ctx.Key == null) throw new ArgumentNullException(nameof(args), "Key is null"); - + var chunkSize = args.MaxPayloadPatchChunkLength; Stream payload = args.Payload ?? throw new ArgumentNullException(nameof(args), "Stream parameter is null"); var call = client.Patch(null, ctx.Deadline, ctx.CancellationToken); byte[]? chunkBuffer = null; - try + try { // common chunkBuffer = ClientContext.GetArrayPool(Constants.ObjectChunkSize).Rent(chunkSize); @@ -312,7 +366,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl ctx.Key ); - var request = new PatchRequest + var request = new PatchRequest() { Body = new() { @@ -322,8 +376,9 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl }; bool isFirstChunk = true; + ulong currentPos = args.Range.Offset; - while (true) + while (true) { var bytesCount = await payload.ReadAsync(chunkBuffer, 0, chunkSize, ctx.CancellationToken).ConfigureAwait(false); @@ -332,7 +387,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl break; } - if (isFirstChunk) + if (isFirstChunk && args.NewAttributes != null && args.NewAttributes.Length > 0) { foreach (var attr in args.NewAttributes) { @@ -340,14 +395,18 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl } } - request.Sign(ctx.Key); - request.Body.Patch = new PatchRequest.Types.Body.Types.Patch { Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount), - SourceRange = new Object.Range { Offset = 0, Length = (ulong)bytesCount } + SourceRange = new Object.Range { Offset = currentPos, Length = (ulong)bytesCount } }; + currentPos += (ulong)bytesCount; + + request.AddMetaHeader(args.XHeaders, sessionToken); + + request.Sign(ctx.Key); + await call.RequestStream.WriteAsync(request).ConfigureAwait(false); isFirstChunk = false; @@ -362,6 +421,12 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl ArrayPool.Shared.Return(chunkBuffer); } } + + var response = await call.ResponseAsync.ConfigureAwait(false); + + Verifier.CheckResponse(response); + + return response.Body.ObjectId.ToModel(); } private async Task PutClientCutObject(PrmObjectPut args) @@ -534,7 +599,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl } } } - + private async Task> GetUploadStream(PrmObjectPut args, CallContext ctx) { var header = args.Header!; @@ -553,7 +618,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl } var oid = new ObjectID { Value = grpcHeader.Sha256() }; - + var initRequest = new PutRequest { Body = new PutRequest.Types.Body diff --git a/src/FrostFS.SDK.ClientV2/Tools/ObjectStreamer.cs b/src/FrostFS.SDK.ClientV2/Tools/ObjectStreamer.cs index 3389b8e..a23c109 100644 --- a/src/FrostFS.SDK.ClientV2/Tools/ObjectStreamer.cs +++ b/src/FrostFS.SDK.ClientV2/Tools/ObjectStreamer.cs @@ -1,8 +1,6 @@ using System; using System.Threading.Tasks; -using FrostFS.Object; - using Grpc.Core; namespace FrostFS.SDK.ClientV2; diff --git a/src/FrostFS.SDK.Tests/Mocks/AsyncStreamRangeReaderMock.cs b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamRangeReaderMock.cs new file mode 100644 index 0000000..e60b600 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Mocks/AsyncStreamRangeReaderMock.cs @@ -0,0 +1,43 @@ +using System.Security.Cryptography; + +using FrostFS.Object; +using FrostFS.SDK.ClientV2; +using FrostFS.SDK.ClientV2.Mappers.GRPC; +using FrostFS.SDK.Cryptography; +using FrostFS.Session; + +using Google.Protobuf; + +using Grpc.Core; + +namespace FrostFS.SDK.Tests; + +public class AsyncStreamRangeReaderMock(string key, byte[] response) : ServiceBase(key), IAsyncStreamReader +{ + private readonly byte[] _response = response; + + public GetRangeResponse Current + { + get + { + var response = new GetRangeResponse + { + Body = new GetRangeResponse.Types.Body + { + Chunk = ByteString.CopyFrom(_response) + }, + MetaHeader = new ResponseMetaHeader() + }; + + response.VerifyHeader = GetResponseVerificationHeader(response); + + return response; + } + } + + public Task MoveNext(CancellationToken cancellationToken) + { + return Task.FromResult(true); + } +} + diff --git a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs index d0b900d..7c44fc8 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs @@ -5,6 +5,7 @@ using FrostFS.Object; using FrostFS.SDK.ClientV2; using FrostFS.SDK.ClientV2.Mappers.GRPC; using FrostFS.SDK.Cryptography; +using FrostFS.Session; using Google.Protobuf; @@ -16,6 +17,32 @@ namespace FrostFS.SDK.Tests; public class ObjectMocker(string key) : ObjectServiceBase(key) { + public FrostFsObjectId? ObjectId { get; set; } + + public FrostFsObjectHeader? ObjectHeader { get; set; } + + public Header? HeadResponse { get; set; } + + public Collection? ResultObjectIds { get; } = []; + + public ClientStreamWriter? ClientStreamWriter { get; } = new(); + + public PatchStreamWriter? PatchStreamWriter { get; } = new(); + + public Collection PutSingleRequests { get; } = []; + + public Collection DeleteRequests { get; } = []; + + public Collection HeadRequests { get; } = []; + + public byte[] RangeResponse { get; set; } = []; + + public GetRangeRequest? GetRangeRequest { get; set; } + + public GetRangeHashRequest? GetRangeHashRequest { get; set; } + + public Collection RangeHashResponses { get; } = []; + public override Mock GetMock() { var mock = new Mock(); @@ -189,23 +216,88 @@ public class ObjectMocker(string key) : ObjectServiceBase(key) }); } + mock.Setup(x => x.GetRange( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((GetRangeRequest r, Metadata m, DateTime? dt, CancellationToken ct) => + { + Verifier.CheckRequest(r); + + GetRangeRequest = r; + + return new AsyncServerStreamingCall( + new AsyncStreamRangeReaderMock(StringKey, RangeResponse), + Task.FromResult(ResponseMetaData), + () => new Grpc.Core.Status(StatusCode.OK, string.Empty), + () => ResponseMetaData, + () => { }); + }); + + mock.Setup(x => x.GetRangeHashAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((GetRangeHashRequest r, Metadata m, DateTime? dt, CancellationToken ct) => + { + Verifier.CheckRequest(r); + + GetRangeHashRequest = r; + + var response = new GetRangeHashResponse + { + Body = new GetRangeHashResponse.Types.Body(), + MetaHeader = ResponseMetaHeader + }; + + if (RangeHashResponses != null) + { + foreach (var hash in RangeHashResponses) + { + response.Body.HashList.Add(hash); + } + } + + response.VerifyHeader = GetResponseVerificationHeader(response); + + return new AsyncUnaryCall( + Task.FromResult(response), + Task.FromResult(ResponseMetaData), + () => new Grpc.Core.Status(StatusCode.OK, string.Empty), + () => ResponseMetaData, + () => { }); + }); + + + mock.Setup(x => x.Patch( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((Metadata m, DateTime? dt, CancellationToken ct) => + { + var patchResponse = new PatchResponse + { + Body = new PatchResponse.Types.Body + { + ObjectId = new Refs.ObjectID { Value = ByteString.CopyFrom(SHA256.HashData([1,2,3])) }, + }, + MetaHeader = ResponseMetaHeader + }; + + patchResponse.VerifyHeader = GetResponseVerificationHeader(patchResponse); + + return new AsyncClientStreamingCall( + PatchStreamWriter!, + Task.FromResult(patchResponse), + Task.FromResult(ResponseMetaData), + () => new Grpc.Core.Status(StatusCode.OK, string.Empty), + () => ResponseMetaData, + () => { }); + }); + return mock; } - - public FrostFsObjectId? ObjectId { get; set; } - - public FrostFsObjectHeader? ObjectHeader { get; set; } - - public Header? HeadResponse { get; set; } - - public Collection? ResultObjectIds { get; } = []; - - public ClientStreamWriter? ClientStreamWriter { get; private set; } = new(); - - public Collection PutSingleRequests { get; private set; } = []; - - public Collection DeleteRequests { get; private set; } = []; - - public Collection HeadRequests { get; private set; } = []; } diff --git a/src/FrostFS.SDK.Tests/Mocks/PatchStreamWriter.cs b/src/FrostFS.SDK.Tests/Mocks/PatchStreamWriter.cs new file mode 100644 index 0000000..a2672c8 --- /dev/null +++ b/src/FrostFS.SDK.Tests/Mocks/PatchStreamWriter.cs @@ -0,0 +1,36 @@ +using System.Collections.ObjectModel; + +using FrostFS.SDK.ProtosV2.Interfaces; + +using Grpc.Core; + +namespace FrostFS.SDK.Tests; + +public class PatchStreamWriter : IClientStreamWriter +{ + private WriteOptions? _options; + + public Collection Messages { get; } = []; + + public bool CompletedTask { get; private set; } + + public WriteOptions? WriteOptions + { + get => _options; + set => _options = value; + } + + public Task CompleteAsync() + { + CompletedTask = true; + return Task.CompletedTask; + } + + public Task WriteAsync(IRequest message) + { + Object.PatchRequest pr = new((Object.PatchRequest)message); + Messages.Add(pr); + return Task.CompletedTask; + } +} + diff --git a/src/FrostFS.SDK.Tests/ObjectTest.cs b/src/FrostFS.SDK.Tests/ObjectTest.cs index 85b42ca..435c042 100644 --- a/src/FrostFS.SDK.Tests/ObjectTest.cs +++ b/src/FrostFS.SDK.Tests/ObjectTest.cs @@ -10,6 +10,8 @@ using FrostFS.SDK.Cryptography; using Google.Protobuf; +using static FrostFS.Object.ECInfo.Types; + namespace FrostFS.SDK.Tests; [SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] @@ -223,4 +225,123 @@ public class ObjectTest : ObjectTestsBase Assert.Null(response.Split); } + + + [Fact] + public async void GetRangeTest() + { + Mocker.ResultObjectIds!.Add(SHA256.HashData([])); + + Random rnd = new(); + var bytes = new byte[1024]; + rnd.NextBytes(bytes); + + Mocker.RangeResponse = bytes; + + Mocker.ObjectId = new ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Encoding.UTF8.GetBytes("test"))) }.ToModel(); + + var param = new PrmRangeGet(ContainerId, Mocker.ObjectId, new FrostFsRange(100, (ulong)Mocker.RangeResponse.Length)); + + var result = await GetClient().GetRangeAsync(param); + + Assert.NotNull(Mocker.GetRangeRequest); + + Assert.Equal(param.Range.Offset, Mocker.GetRangeRequest.Body.Range.Offset); + Assert.Equal(param.Range.Length, Mocker.GetRangeRequest.Body.Range.Length); + + Assert.NotNull(result); + + var chunk = await result.ReadChunk(); + + var chunkBytes = chunk.Value.Span.ToArray(); + + Assert.Equal(chunkBytes.Length, Mocker.RangeResponse.Length); + + Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(Mocker.RangeResponse)); + } + + [Fact] + public async void GetRangeHashTest() + { + Mocker.ResultObjectIds!.Add(SHA256.HashData([])); + + Random rnd = new(); + var bytes = new byte[1024]; + rnd.NextBytes(bytes); + + var salt = new byte[32]; + rnd.NextBytes(salt); + + var hash = new byte[32]; + rnd.NextBytes(hash); + + Mocker.RangeResponse = bytes; + var len = (ulong)bytes.Length; + + Mocker.RangeHashResponses.Add(ByteString.CopyFrom(hash)); + + Mocker.ObjectId = new ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Encoding.UTF8.GetBytes("test"))) }.ToModel(); + + var param = new PrmRangeHashGet(ContainerId, Mocker.ObjectId, [new FrostFsRange(100, len)], salt); + + var result = await GetClient().GetRangeHashAsync(param); + + Assert.NotNull(Mocker.GetRangeHashRequest); + + Assert.Equal(param.Ranges[0].Offset, Mocker.GetRangeHashRequest.Body.Ranges[0].Offset); + Assert.Equal(param.Ranges[0].Length, Mocker.GetRangeHashRequest.Body.Ranges[0].Length); + + Assert.NotNull(result); + Assert.Single(result); + + Assert.Equal(SHA256.HashData(hash), SHA256.HashData(result.First().ToArray())); + } + + + [Fact] + public async void PatchTest() + { + Mocker.ObjectId = new ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Encoding.UTF8.GetBytes("test"))) }.ToModel(); + + var address = new FrostFsAddress(ContainerId, Mocker.ObjectId); + + Mocker.ResultObjectIds!.Add(SHA256.HashData([])); + + Random rnd = new(); + var patch = new byte[32]; + rnd.NextBytes(patch); + + var range = new FrostFsRange(8, (ulong)patch.Length); + + var param = new PrmObjectPatch(address) + { + Payload = new MemoryStream(patch), + MaxPayloadPatchChunkLength = 32, + Range = range + }; + + var result = await GetClient().PatchObjectAsync(param); + + Assert.NotNull(result); + + Assert.NotNull(result.Value); + + Assert.NotNull(Mocker.PatchStreamWriter); + Assert.Single(Mocker.PatchStreamWriter.Messages); + + var sentMessages = Mocker.PatchStreamWriter!.Messages; + + var body = sentMessages.First().GetBody() as Object.PatchRequest.Types.Body; + + Assert.NotNull(body); + + Assert.True(Mocker.PatchStreamWriter.CompletedTask); + + Assert.Equal(address.ContainerId, body.Address.ContainerId); + Assert.Equal(address.ObjectId, body.Address.ObjectId); + + Assert.Equal(32, body.Patch.Chunk.Length); + + Assert.Equal(SHA256.HashData(patch), SHA256.HashData(body.Patch.Chunk.ToArray())); + } } diff --git a/src/FrostFS.SDK.Tests/SmokeClientTests.cs b/src/FrostFS.SDK.Tests/SmokeClientTests.cs index 470e0e4..56acfc3 100644 --- a/src/FrostFS.SDK.Tests/SmokeClientTests.cs +++ b/src/FrostFS.SDK.Tests/SmokeClientTests.cs @@ -1,7 +1,6 @@ using System.Diagnostics.CodeAnalysis; using System.Security.Cryptography; -using FrostFS.Refs; using FrostFS.SDK.ClientV2; using FrostFS.SDK.ClientV2.Interfaces; using FrostFS.SDK.Cryptography; @@ -249,8 +248,6 @@ public class SmokeClientTests : SmokeTestsBase [InlineData(6 * 1024 * 1024 + 100)] public async void SimpleScenarioTest(int objectSize) { - try - { using var client = FrostFSClient.GetSingleOwnerInstance(GetSingleOwnerOptions(this.keyString, this.url)); await Cleanup(client); @@ -327,89 +324,86 @@ public class SmokeClientTests : SmokeTestsBase await foreach (var _ in client.ListContainersAsync()) { Assert.Fail("Containers exist"); - } - } - catch (Exception ex) - { - - } + } } [Fact] public async void PatchTest() { - try + using var client = FrostFSClient.GetSingleOwnerInstance(GetSingleOwnerOptions(this.keyString, this.url)); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")])); + + var createdContainer = await client.CreateContainerAsync(createContainerParam); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer)); + Assert.NotNull(container); + + var bytes = new byte[1024]; + for (int i = 0; i < 1024; i++) { - using var client = FrostFSClient.GetSingleOwnerInstance(GetSingleOwnerOptions(this.keyString, this.url)); - - await Cleanup(client); - - var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")])); - - var createdContainer = await client.CreateContainerAsync(createContainerParam); - - var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer)); - Assert.NotNull(container); - - var bytes = new byte[4 * 1024]; - for (int i = 0; i < 4 * 1024; i++) - { - bytes[i] = (byte)31; - } - - var param = new PrmObjectPut - { - Header = new FrostFsObjectHeader( - containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), - Payload = new MemoryStream(bytes), - ClientCut = false - }; - - var objectId = await client.PutObjectAsync(param); - - var patch = new byte[1024]; - for (int i = 0; i < 1024; i++) - { - bytes[i] = (byte)32; - } - - var patchParams = new PrmObjectPatch(new FrostFsAddress(objectId, createdContainer)) - { - Payload = new MemoryStream(patch), - MaxPayloadPatchChunkLength = 1024, - NewAttributes = [new FrostFsAttributePair("testKey", "testValue")], - Range = new FrostFsRange() { Length = 1024, Offset = 10 }, - ReplaceAttributes = false - }; - - await client.PatchObjectAsync(patchParams); - - var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId)); - - var downloadedBytes = new byte[@object.Header.PayloadLength]; - MemoryStream ms = new(downloadedBytes); - - ReadOnlyMemory? chunk = null; - while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) - { - ms.Write(chunk.Value.Span); - } - - Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes)); - - await Cleanup(client); - - await foreach (var _ in client.ListContainersAsync()) - { - Assert.Fail("Containers exist"); - } + bytes[i] = (byte)31; } - catch (Exception ex) - { + var param = new PrmObjectPut + { + Header = new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular, + [new FrostFsAttributePair("fileName", "test")]), + Payload = new MemoryStream(bytes), + ClientCut = false + }; + + var objectId = await client.PutObjectAsync(param); + + var patch = new byte[16]; + for (int i = 0; i < 16; i++) + { + patch[i] = (byte)32; + } + + var range = new FrostFsRange(8, (ulong)patch.Length); + + var patchParams = new PrmObjectPatch(new FrostFsAddress(createdContainer, objectId)) + { + Payload = new MemoryStream(patch), + MaxPayloadPatchChunkLength = 32, + Range = range + }; + + var newIbjId = await client.PatchObjectAsync(patchParams); + + var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, newIbjId)); + + var downloadedBytes = new byte[@object.Header.PayloadLength]; + MemoryStream ms = new(downloadedBytes); + + ReadOnlyMemory? chunk = null; + while ((chunk = await @object.ObjectReader!.ReadChunk()) != null) + { + ms.Write(chunk.Value.Span); + } + + for(int i = 0; i < (int)range.Offset; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); + + var rangeEnd = range.Offset + range.Length; + + for (int i = (int)range.Offset; i < (int)rangeEnd; i++) + Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]); + + for (int i = (int)rangeEnd; i < bytes.Length; i++) + Assert.Equal(downloadedBytes[i], bytes[i]); + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync()) + { + Assert.Fail("Containers exist"); } } @@ -438,15 +432,14 @@ public class SmokeClientTests : SmokeTestsBase { Header = new FrostFsObjectHeader( containerId: createdContainer, - type: FrostFsObjectType.Regular, - [new FrostFsAttributePair("fileName", "test")]), + type: FrostFsObjectType.Regular), Payload = new MemoryStream(bytes), ClientCut = false }; var objectId = await client.PutObjectAsync(param); - var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange { Offset = 100, Length = 64 }); + var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange(100, 64)); var rangeReader = await client.GetRangeAsync(rangeParam); @@ -469,6 +462,55 @@ public class SmokeClientTests : SmokeTestsBase } } + [Fact] + public async void RangeHashTest() + { + using var client = FrostFSClient.GetSingleOwnerInstance(GetSingleOwnerOptions(this.keyString, this.url)); + + await Cleanup(client); + + var createContainerParam = new PrmContainerCreate( + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")])); + + var createdContainer = await client.CreateContainerAsync(createContainerParam); + + var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer)); + Assert.NotNull(container); + + var bytes = new byte[256]; + for (int i = 0; i < 256; i++) + { + bytes[i] = (byte)i; + } + + var param = new PrmObjectPut + { + Header = new FrostFsObjectHeader( + containerId: createdContainer, + type: FrostFsObjectType.Regular), + Payload = new MemoryStream(bytes), + ClientCut = false + }; + + var objectId = await client.PutObjectAsync(param); + + var rangeParam = new PrmRangeHashGet(createdContainer, objectId, [ new FrostFsRange(100, 64)], bytes); + + var hashes = await client.GetRangeHashAsync(rangeParam); + + foreach (var hash in hashes) + { + var x = hash.Slice(0, 32).ToArray(); + } + + await Cleanup(client); + + await foreach (var _ in client.ListContainersAsync()) + { + Assert.Fail("Containers exist"); + } + } + [Theory] [InlineData(1)] [InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB