[#17] Client: Add extra parameter
All checks were successful
DCO / DCO (pull_request) Successful in 47s

API methods' parameters types with optional session, polling settings, xHeaders etc. and corresponding handlers have been added

Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
Pavel Gross 2024-07-18 15:08:53 +03:00 committed by p.gross
parent 00a1e9412f
commit 7b9c19f37c
42 changed files with 1054 additions and 386 deletions

View file

@ -8,10 +8,10 @@ using FrostFS.SDK.ModelsV2.Netmap;
using FrostFS.SDK.ModelsV2.Enums;
using Google.Protobuf;
using FrostFS.SDK.ClientV2.Interfaces;
using FrostFS.SDK.ClientV2.Parameters;
namespace FrostFS.SDK.Tests;
public abstract class ContainerTestsBase
{
protected readonly string key = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq";
@ -38,12 +38,12 @@ public abstract class ContainerTestsBase
protected IFrostFSClient GetClient()
{
return ClientV2.Client.GetTestInstance(
Settings,
null,
new NetworkMocker(this.key).GetMock().Object,
new SessionMocker(this.key).GetMock().Object,
Mocker.GetMock().Object,
new ObjectMocker(this.key).GetMock().Object);
Settings,
null,
new NetworkMocker(this.key).GetMock().Object,
new SessionMocker(this.key).GetMock().Object,
Mocker.GetMock().Object,
new ObjectMocker(this.key).GetMock().Object);
}
}
@ -52,7 +52,9 @@ public class ContainerTest : ContainerTestsBase
[Fact]
public async void CreateContainerTest()
{
var result = await GetClient().CreateContainerAsync(new ModelsV2.Container(BasicAcl.PublicRW, Mocker.PlacementPolicy));
var param = new PrmCreateContainer(new ModelsV2.Container(BasicAcl.PublicRW, Mocker.PlacementPolicy));
var result = await GetClient().CreateContainerAsync(param);
Assert.NotNull(result);
Assert.NotNull(result.Value);
@ -66,7 +68,7 @@ public class ContainerTest : ContainerTestsBase
Mocker.Acl = BasicAcl.PublicRO;
var result = await GetClient().GetContainerAsync(cid);
var result = await GetClient().GetContainerAsync(new PrmGetContainer(cid));
Assert.NotNull(result);
Assert.Equal(Mocker.Acl, result.BasicAcl);
@ -99,9 +101,10 @@ public class ContainerTest : ContainerTestsBase
[Fact]
public async void DeleteContainerAsyncTest()
{
Mocker.ReturnContainerRemoved = true;
var cid = new ContainerId(Base58.Encode(Mocker.ContainerGuid.ToBytes()));
await GetClient().DeleteContainerAsync(cid);
await GetClient().DeleteContainerAsync(new PrmDeleteContainer(cid));
Assert.Single(Mocker.Requests);

View file

@ -77,6 +77,21 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
getResponse.VerifyHeader = GetResponseVerificationHeader(getResponse);
var getNoContainerResponse = new GetResponse
{
Body = new (),
MetaHeader = new ResponseMetaHeader
{
Status = new Status.Status
{
Code = 3072,
Message = "container not found"
}
}
};
getNoContainerResponse.VerifyHeader = GetResponseVerificationHeader(getNoContainerResponse);
mock.Setup(x => x.GetAsync(
It.IsAny<GetRequest>(),
It.IsAny<Metadata>(),
@ -86,12 +101,22 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
{
Verifier.CheckRequest(r);
if (ReturnContainerRemoved)
{
return new AsyncUnaryCall<GetResponse>(
Task.FromResult(getNoContainerResponse),
Task.FromResult(ResponseMetaData),
() => new Grpc.Core.Status(StatusCode.NotFound, string.Empty),
() => ResponseMetaData,
() => { });
}
return new AsyncUnaryCall<GetResponse>(
Task.FromResult(getResponse),
Task.FromResult(ResponseMetaData),
() => new Grpc.Core.Status(StatusCode.OK, string.Empty),
() => ResponseMetaData,
() => { });
() => { });
});
var listResponse = new ListResponse
@ -154,6 +179,8 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
return mock;
}
public bool ReturnContainerRemoved { get; set; }
public List<byte[]> ContainerIds { get; set; } = [];
public List<RequestData<DeleteRequest>> Requests { get; set; } = [];

View file

@ -2,6 +2,7 @@ using FrostFS.Refs;
using FrostFS.SDK.ClientV2;
using FrostFS.SDK.ClientV2.Interfaces;
using FrostFS.SDK.ClientV2.Mappers.GRPC;
using FrostFS.SDK.ClientV2.Parameters;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ModelsV2.Enums;
@ -76,7 +77,7 @@ public class ObjectTest : ObjectTestsBase
Timeout = TimeSpan.FromSeconds(2)
};
var result = await client.GetObjectAsync(ContainerId, objectId, context);
var result = await client.GetObjectAsync(new PrmGetObject(ContainerId, objectId) { Context = context });
Assert.NotNull(result);
@ -97,7 +98,7 @@ public class ObjectTest : ObjectTestsBase
var bytes = new byte[1024];
rnd.NextBytes(bytes);
var param = new PutObjectParameters
var param = new PrmPutObject
{
Header = Mocker.ObjectHeader,
Payload = new MemoryStream(bytes),
@ -132,7 +133,7 @@ public class ObjectTest : ObjectTestsBase
byte[] bytes = File.ReadAllBytes(@".\..\..\..\TestData\cat.jpg");
var fileLength = bytes.Length;
var param = new PutObjectParameters
var param = new PrmPutObject
{
Header = Mocker.ObjectHeader,
Payload = new MemoryStream(bytes),
@ -196,7 +197,7 @@ public class ObjectTest : ObjectTestsBase
{
Mocker.ObjectId = new ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Encoding.UTF8.GetBytes("test"))) }.ToModel();
await GetClient().DeleteObjectAsync(ContainerId, Mocker.ObjectId);
await GetClient().DeleteObjectAsync(new PrmDeleteObject(ContainerId, Mocker.ObjectId));
var request = Mocker.DeleteRequests.FirstOrDefault();
Assert.NotNull(request);
@ -209,7 +210,7 @@ public class ObjectTest : ObjectTestsBase
{
Mocker.ObjectId = new ObjectID { Value = ByteString.CopyFrom(SHA256.HashData(Encoding.UTF8.GetBytes("test"))) }.ToModel();
var response = await GetClient().GetObjectHeadAsync(ContainerId, Mocker.ObjectId);
var response = await GetClient().GetObjectHeadAsync(new PrmGetObjectHead(ContainerId, Mocker.ObjectId));
var request = Mocker.HeadRequests.FirstOrDefault();
Assert.NotNull(request);

View file

@ -4,24 +4,30 @@ using FrostFS.SDK.ClientV2.Interfaces;
using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ModelsV2.Enums;
using FrostFS.SDK.ModelsV2.Netmap;
using FrostFS.SDK.Cryptography;
using Grpc.Core;
using Microsoft.Extensions.Options;
using Grpc.Core.Interceptors;
using System.Diagnostics;
using static FrostFS.Session.SessionToken.Types.Body;
using FrostFS.SDK.ClientV2.Parameters;
namespace FrostFS.SDK.SmokeTests;
public class SmokeTests
{
private static PrmWait lightWait = new (100, 1);
private readonly string key = "KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq";
private readonly string url = "http://172.29.238.97:8080";
[Fact]
public async void NetworkMapTest()
{
using var fsClient = Client.GetInstance(GetOptions(this.key, this.url));
using var client = Client.GetInstance(GetOptions(this.key, this.url));
var result = await fsClient.GetNetmapSnapshotAsync();
var result = await client.GetNetmapSnapshotAsync();
Assert.True(result.Epoch > 0);
Assert.Single(result.NodeInfoCollection);
@ -38,9 +44,9 @@ public class SmokeTests
[Fact]
public async void NodeInfoTest()
{
using var fsClient = Client.GetInstance(GetOptions(this.key, this.url));
using var client = Client.GetInstance(GetOptions(this.key, this.url));
var result = await fsClient.GetNodeInfoAsync();
var result = await client.GetNodeInfoAsync();
Assert.Equal(2, result.Version.Major);
Assert.Equal(13, result.Version.Minor);
@ -51,80 +57,74 @@ public class SmokeTests
}
[Fact]
public async void NodeInfo_Statictics_Test()
public async void NodeInfo_Statistics_Test()
{
var ctx = new Context
{
Callback = (cs) => Console.WriteLine($"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds")
};
using var fsClient = Client.GetInstance(GetOptions(this.key, this.url));
using var client = Client.GetInstance(GetOptions(this.key, this.url));
var result = await fsClient.GetNodeInfoAsync();
var result = await client.GetNodeInfoAsync();
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioTest(int objectSize)
[Fact]
public async void GetSessionTest()
{
using var fsClient = Client.GetInstance(GetOptions(this.key, this.url));
using var client = Client.GetInstance(GetOptions(this.key, this.url));
await Cleanup(fsClient);
var token = await client.CreateSessionAsync(new PrmCreateSession(100));
var containerId = await fsClient.CreateContainerAsync(
new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1))),
new Context
{
Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
}
);
var session = new Session.SessionToken().Deserialize(token.Token);
var ecdsaKey = this.key.LoadWif();
var owner = OwnerId.FromKey(ecdsaKey);
var context = new Context
{
Timeout = TimeSpan.FromSeconds(10),
Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
};
var ownerHash = Base58.Decode(owner.Value);
Assert.NotNull(session);
Assert.Null(session.Body.Container);
Assert.Null(session.Body.Object);
Assert.Equal(16, session.Body.Id.Length);
Assert.Equal(100ul, session.Body.Lifetime.Exp);
Assert.Equal(ownerHash, session.Body.OwnerId.Value);
Assert.Equal(33, session.Body.SessionKey.Length);
Assert.Equal(ContextOneofCase.None, session.Body.ContextCase);
}
var container = await GetContainer(fsClient, containerId, context);
[Fact]
public async void CreateObjectWithSessionToken()
{
using var client = Client.GetInstance(GetOptions(this.key, this.url));
await Cleanup(client);
Assert.NotNull(container);
var token = await client.CreateSessionAsync(new PrmCreateSession(int.MaxValue));
var bytes = GetRandomBytes(objectSize);
var createContainerParam = new PrmCreateContainer(
new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1))));
var param = new PutObjectParameters
createContainerParam.XHeaders.Add("key1", "value1");
var containerId = await client.CreateContainerAsync(createContainerParam);
var bytes = GetRandomBytes(1024);
var param = new PrmPutObject
{
Header = new ObjectHeader(
containerId: containerId,
type: ObjectType.Regular,
new ObjectAttribute("fileName", "test")),
Payload = new MemoryStream(bytes),
ClientCut = false
ClientCut = false,
SessionToken = token
};
var objectId = await client.PutObjectAsync(param);
var objectId = await fsClient.PutObjectAsync(param, new Context
{
Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
});
var filter = new ObjectFilter(ObjectMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in fsClient.SearchObjectsAsync(containerId, [filter]))
{
hasObject = true;
var objHeader = await fsClient.GetObjectHeadAsync(containerId, objectId);
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await fsClient.GetObjectAsync(containerId, objectId!);
var @object = await client.GetObjectAsync(new PrmGetObject(containerId, objectId));
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
@ -137,11 +137,177 @@ public class SmokeTests
Assert.Equal(MD5.HashData(bytes), MD5.HashData(downloadedBytes));
await Cleanup(fsClient);
await Cleanup(client);
}
await Task.Delay(2000);
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioTest(int objectSize)
{
using var client = Client.GetInstance(GetOptions(this.key, this.url));
await Cleanup(client);
await foreach (var _ in fsClient.ListContainersAsync())
bool callbackInvoked = false;
var ctx = new Context
{
Timeout = TimeSpan.FromSeconds(20),
Callback = new((CallStatistics cs) =>
{
callbackInvoked = true;
Assert.True(cs.ElapsedMicroSeconds > 0);
})
};
var createContainerParam = new PrmCreateContainer(
new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1))))
{
Context = ctx
};
var containerId = await client.CreateContainerAsync(createContainerParam);
var container = await client.GetContainerAsync(new PrmGetContainer(containerId,ctx));
Assert.NotNull(container);
Assert.True(callbackInvoked);
var bytes = GetRandomBytes(objectSize);
var param = new PrmPutObject
{
Header = new ObjectHeader(
containerId: containerId,
type: ObjectType.Regular,
new ObjectAttribute("fileName", "test")),
Payload = new MemoryStream(bytes),
ClientCut = false,
Context = new Context
{
Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
}
};
var objectId = await client.PutObjectAsync(param);
var filter = new ObjectFilter(ObjectMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in client.SearchObjectsAsync(new PrmSearchObject(containerId) { Filters = [filter] }))
{
hasObject = true;
var objHeader = await client.GetObjectHeadAsync(new PrmGetObjectHead(containerId, objectId));
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await client.GetObjectAsync(new PrmGetObject(containerId, objectId));
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
byte[]? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk);
}
Assert.Equal(MD5.HashData(bytes), MD5.HashData(downloadedBytes));
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync())
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioWithSessionTest(int objectSize)
{
using var client = Client.GetInstance(GetOptions(this.key, this.url));
var token = await client.CreateSessionAsync(new PrmCreateSession(int.MaxValue));
await Cleanup(client);
var ctx = new Context
{
Timeout = TimeSpan.FromSeconds(20),
Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
};
var createContainerParam = new PrmCreateContainer(
new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1))))
{
Context = ctx
};
var containerId = await client.CreateContainerAsync(createContainerParam);
var container = await client.GetContainerAsync(new PrmGetContainer(containerId,ctx));
Assert.NotNull(container);
var bytes = GetRandomBytes(objectSize);
var param = new PrmPutObject
{
Header = new ObjectHeader(
containerId: containerId,
type: ObjectType.Regular,
new ObjectAttribute("fileName", "test")),
Payload = new MemoryStream(bytes),
ClientCut = false,
Context = new Context
{
Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
},
SessionToken = token
};
var objectId = await client.PutObjectAsync(param);
var filter = new ObjectFilter(ObjectMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in client.SearchObjectsAsync(new PrmSearchObject(containerId) { Filters = [filter], SessionToken = token }))
{
hasObject = true;
var objHeader = await client.GetObjectHeadAsync(new PrmGetObjectHead(containerId, objectId) { SessionToken = token });
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await client.GetObjectAsync(new PrmGetObject(containerId, objectId) { SessionToken = token });
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
byte[]? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk);
}
Assert.Equal(MD5.HashData(bytes), MD5.HashData(downloadedBytes));
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync())
{
Assert.Fail("Containers exist");
}
@ -155,13 +321,16 @@ public class SmokeTests
[InlineData(2 * 64 * 1024 * 1024 + 256)]
public async void ClientCutScenarioTest(int objectSize)
{
using var fsClient = Client.GetInstance(GetOptions(this.key, this.url));
using var client = Client.GetInstance(GetOptions(this.key, this.url));
await Cleanup(fsClient);
await Cleanup(client);
var cnt = new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1)));
var createContainerParam = new PrmCreateContainer(new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1))))
{
WaitParams = lightWait
};
var containerId = await fsClient.CreateContainerAsync(cnt);
var containerId = await client.CreateContainerAsync(createContainerParam);
var context = new Context
{
@ -169,13 +338,13 @@ public class SmokeTests
Interceptors = new([new MetricsInterceptor()])
};
var container = await GetContainer(fsClient, containerId, context);
var container = await client.GetContainerAsync(new PrmGetContainer(containerId, context));
Assert.NotNull(container);
byte[] bytes = GetRandomBytes(objectSize);
var param = new PutObjectParameters
var param = new PrmPutObject
{
Header = new ObjectHeader(
containerId: containerId,
@ -185,16 +354,16 @@ public class SmokeTests
ClientCut = true
};
var objectId = await fsClient.PutObjectAsync(param);
var objectId = await client.PutObjectAsync(param);
var filter = new ObjectFilter(ObjectMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in fsClient.SearchObjectsAsync(containerId, [filter]))
await foreach (var objId in client.SearchObjectsAsync(new PrmSearchObject(containerId, filter)))
{
hasObject = true;
var objHeader = await fsClient.GetObjectHeadAsync(containerId, objectId);
var objHeader = await client.GetObjectHeadAsync(new PrmGetObjectHead(containerId, objectId));
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
@ -203,7 +372,7 @@ public class SmokeTests
Assert.True(hasObject);
var @object = await fsClient.GetObjectAsync(containerId, objectId!);
var @object = await client.GetObjectAsync(new PrmGetObject(containerId, objectId));
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
@ -216,14 +385,23 @@ public class SmokeTests
Assert.Equal(MD5.HashData(bytes), MD5.HashData(downloadedBytes));
await Cleanup(fsClient);
await Cleanup(client);
await Task.Delay(2000);
var deadline = DateTime.UtcNow.Add(TimeSpan.FromSeconds(5));
await foreach (var _ in fsClient.ListContainersAsync())
IAsyncEnumerator<ContainerId>? enumerator = null;
do
{
Assert.Fail("Containers exist");
if (deadline <= DateTime.UtcNow)
{
Assert.Fail("Containers exist");
break;
}
enumerator = client.ListContainersAsync().GetAsyncEnumerator();
await Task.Delay(500);
}
while (await enumerator!.MoveNextAsync());
}
private static byte[] GetRandomBytes(int size)
@ -243,36 +421,16 @@ public class SmokeTests
});
}
static async Task Cleanup(IFrostFSClient fsClient)
static async Task Cleanup(IFrostFSClient client)
{
await foreach (var cid in fsClient.ListContainersAsync())
await foreach (var cid in client.ListContainersAsync())
{
await fsClient.DeleteContainerAsync(cid);
}
}
static async Task<ModelsV2.Container> GetContainer(IFrostFSClient fsClient, ContainerId id, Context ctx)
{
while (true)
{
try
{
await Task.Delay(100);
return await fsClient.GetContainerAsync(id, ctx);
}
catch (ApplicationException)
{
if (DateTime.UtcNow >= ctx.Deadline)
throw new TimeoutException();
}
catch (RpcException)
{
throw;
}
await client.DeleteContainerAsync(new PrmDeleteContainer(cid) { WaitParams = lightWait });
}
}
}
public class MetricsInterceptor() : Interceptor
{
public override AsyncUnaryCall<TResponse> AsyncUnaryCall<TRequest, TResponse>(