frostfs-sdk-csharp/src/FrostFS.SDK.Tests/SmokeClientTests.cs
Pavel Gross 749000a090
All checks were successful
DCO / DCO (pull_request) Successful in 33s
lint-build / dotnet8.0 (pull_request) Successful in 1m4s
lint-build / dotnet8.0 (push) Successful in 49s
[#28] Client: Apply code optimizations
Signed-off-by: Pavel Gross <p.gross@yadro.com>
2024-11-18 17:00:19 +03:00

720 lines
24 KiB
C#

using System.Diagnostics.CodeAnalysis;
using System.Security.Cryptography;
using FrostFS.SDK.Client;
using FrostFS.SDK.Client.Interfaces;
using FrostFS.SDK.Cryptography;
using Microsoft.Extensions.Options;
namespace FrostFS.SDK.SmokeTests;
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
public class SmokeClientTests : SmokeTestsBase
{
private static readonly PrmWait lightWait = new(100, 1);
[Fact]
public async void AccountTest()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
var result = await client.GetBalanceAsync();
Assert.NotNull(result);
Assert.True(result.Value == 0);
}
[Fact]
public async void NetworkMapTest()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
var result = await client.GetNetmapSnapshotAsync();
Assert.True(result.Epoch > 0);
Assert.Single(result.NodeInfoCollection);
var item = result.NodeInfoCollection[0];
Assert.Equal(2, item.Version.Major);
Assert.Equal(13, item.Version.Minor);
Assert.Equal(NodeState.Online, item.State);
Assert.True(item.PublicKey.Length > 0);
Assert.Single(item.Addresses);
Assert.Equal(9, item.Attributes.Count);
}
[Fact]
public async void NodeInfoTest()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
var result = await client.GetNodeInfoAsync();
Assert.Equal(2, result.Version.Major);
Assert.Equal(13, result.Version.Minor);
Assert.Equal(NodeState.Online, result.State);
Assert.Equal(33, result.PublicKey.Length);
Assert.Single(result.Addresses);
Assert.Equal(9, result.Attributes.Count);
}
[Fact]
public async void NodeInfoStatisticsTest()
{
var options = GetClientOptions(this.keyString, this.url);
var callbackContent = string.Empty;
options.Value.Callback = (cs) => callbackContent = $"{cs.MethodName} took {cs.ElapsedMicroSeconds} microseconds";
using var client = FrostFSClient.GetSingleOwnerInstance(options);
var result = await client.GetNodeInfoAsync();
Assert.NotEmpty(callbackContent);
}
[Fact]
public async void GetSessionTest()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
PrmSessionCreate? prm = new(100);
var token = await client.CreateSessionAsync(prm);
var ownerHash = Base58.Decode(OwnerId!.Value);
Assert.NotNull(token);
Assert.NotEqual(Guid.Empty, token.Id);
Assert.Equal(33, token.SessionKey.Length);
}
[Fact]
public async void CreateObjectWithSessionToken()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
await Cleanup(client);
var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue));
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))));
createContainerParam.XHeaders.Add("key1", "value1");
var containerId = await client.CreateContainerAsync(createContainerParam);
var bytes = GetRandomBytes(1024);
var param = new PrmObjectPut
{
Header = new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
Payload = new MemoryStream(bytes),
ClientCut = false,
SessionToken = token
};
var objectId = await client.PutObjectAsync(param).ConfigureAwait(true);
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId))
.ConfigureAwait(true);
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk().ConfigureAwait(true)) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(client).ConfigureAwait(true);
}
[Fact]
public async void FilterTest()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))))
{
WaitParams = lightWait
};
var containerId = await client.CreateContainerAsync(createContainerParam);
var bytes = new byte[] { 1, 2, 3 };
var ParentHeader = new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular)
{
PayloadLength = 3
};
var param = new PrmObjectPut
{
Header = new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")],
new FrostFsSplit()),
Payload = new MemoryStream(bytes),
ClientCut = false
};
var objectId = await client.PutObjectAsync(param);
var head = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId));
var ecdsaKey = this.keyString.LoadWif();
var networkInfo = await client.GetNetmapSnapshotAsync();
await CheckFilter(client, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId));
await CheckFilter(client, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey)));
await CheckFilter(client, containerId, new FilterBySplitId(FrostFsMatchType.Equals, param.Header.Split!.SplitId));
await CheckFilter(client, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"));
await CheckFilter(client, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId));
await CheckFilter(client, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
await CheckFilter(client, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
await CheckFilter(client, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, 3));
var checkSum = CheckSum.CreateCheckSum(bytes);
await CheckFilter(client, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
await CheckFilter(client, containerId, new FilterByPhysicallyStored());
}
private static async Task CheckFilter(IFrostFSClient client, FrostFsContainerId containerId, IObjectFilter filter)
{
var resultObjectsCount = 0;
PrmObjectSearch searchParam = new(containerId) { Filters = [filter] };
await foreach (var objId in client.SearchObjectsAsync(searchParam))
{
resultObjectsCount++;
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId));
}
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioTest(int objectSize)
{
bool callbackInvoked = false;
var options = GetClientOptions(this.keyString, this.url);
options.Value.Callback = new((CallStatistics cs) =>
{
callbackInvoked = true;
Assert.True(cs.ElapsedMicroSeconds > 0);
});
using var client = FrostFSClient.GetSingleOwnerInstance(options);
await Cleanup(client);
var ctx = new CallContext { Timeout = TimeSpan.FromSeconds(20) };
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")]), ctx);
var createdContainer = await client.CreateContainerAsync(createContainerParam);
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer));
Assert.NotNull(container);
Assert.True(callbackInvoked);
var bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut
{
Header = new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
Payload = new MemoryStream(bytes),
ClientCut = false
};
var objectId = await client.PutObjectAsync(param);
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(createdContainer) { Filters = [filter] }))
{
hasObject = true;
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(createdContainer, objectId));
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, objectId));
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync())
{
Assert.Fail("Containers exist");
}
}
[Fact]
public async void PatchTest()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")]));
var createdContainer = await client.CreateContainerAsync(createContainerParam);
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer));
Assert.NotNull(container);
var bytes = new byte[1024];
for (int i = 0; i < 1024; i++)
{
bytes[i] = (byte)31;
}
var param = new PrmObjectPut
{
Header = new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
Payload = new MemoryStream(bytes),
ClientCut = false
};
var objectId = await client.PutObjectAsync(param);
var patch = new byte[16];
for (int i = 0; i < 16; i++)
{
patch[i] = (byte)32;
}
var range = new FrostFsRange(8, (ulong)patch.Length);
var patchParams = new PrmObjectPatch(new FrostFsAddress(createdContainer, objectId))
{
Payload = new MemoryStream(patch),
MaxPayloadPatchChunkLength = 32,
Range = range
};
var newIbjId = await client.PatchObjectAsync(patchParams);
var @object = await client.GetObjectAsync(new PrmObjectGet(createdContainer, newIbjId));
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
for (int i = 0; i < (int)range.Offset; i++)
Assert.Equal(downloadedBytes[i], bytes[i]);
var rangeEnd = range.Offset + range.Length;
for (int i = (int)range.Offset; i < (int)rangeEnd; i++)
Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]);
for (int i = (int)rangeEnd; i < bytes.Length; i++)
Assert.Equal(downloadedBytes[i], bytes[i]);
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync())
{
Assert.Fail("Containers exist");
}
}
[Fact]
public async void RangeTest()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")]));
var createdContainer = await client.CreateContainerAsync(createContainerParam);
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer));
Assert.NotNull(container);
var bytes = new byte[256];
for (int i = 0; i < 256; i++)
{
bytes[i] = (byte)i;
}
var param = new PrmObjectPut
{
Header = new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular),
Payload = new MemoryStream(bytes),
ClientCut = false
};
var objectId = await client.PutObjectAsync(param);
var rangeParam = new PrmRangeGet(createdContainer, objectId, new FrostFsRange(100, 64));
var rangeReader = await client.GetRangeAsync(rangeParam);
var downloadedBytes = new byte[rangeParam.Range.Length];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await rangeReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice(100, 64)), SHA256.HashData(downloadedBytes));
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync())
{
Assert.Fail("Containers exist");
}
}
[Fact]
public async void RangeHashTest()
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), [new("testKey", "testValue")]));
var createdContainer = await client.CreateContainerAsync(createContainerParam);
var container = await client.GetContainerAsync(new PrmContainerGet(createdContainer));
Assert.NotNull(container);
var bytes = new byte[256];
for (int i = 0; i < 256; i++)
{
bytes[i] = (byte)i;
}
var param = new PrmObjectPut
{
Header = new FrostFsObjectHeader(
containerId: createdContainer,
type: FrostFsObjectType.Regular),
Payload = new MemoryStream(bytes),
ClientCut = false
};
var objectId = await client.PutObjectAsync(param);
var rangeParam = new PrmRangeHashGet(createdContainer, objectId, [new FrostFsRange(100, 64)], bytes);
var hashes = await client.GetRangeHashAsync(rangeParam);
foreach (var hash in hashes)
{
var x = hash[..32].ToArray();
}
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync())
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(3 * 1024 * 1024)] // exactly one chunk size - 3MB
[InlineData(6 * 1024 * 1024 + 100)]
public async void SimpleScenarioWithSessionTest(int objectSize)
{
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
//Callback = new((CallStatistics cs) => Assert.True(cs.ElapsedMicroSeconds > 0))
var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue));
await Cleanup(client);
var ctx = new CallContext
{
Timeout = TimeSpan.FromSeconds(20),
};
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), ctx);
var container = await client.CreateContainerAsync(createContainerParam);
var containerInfo = await client.GetContainerAsync(new PrmContainerGet(container, ctx));
Assert.NotNull(containerInfo);
var bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut
{
Header = new FrostFsObjectHeader(
containerId: container,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
Payload = new MemoryStream(bytes),
ClientCut = false,
SessionToken = token
};
var objectId = await client.PutObjectAsync(param);
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(container) { Filters = [filter], SessionToken = token }))
{
hasObject = true;
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(container, objectId) { SessionToken = token });
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
}
Assert.True(hasObject);
var @object = await client.GetObjectAsync(new PrmObjectGet(container, objectId) { SessionToken = token });
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await Cleanup(client);
await foreach (var _ in client.ListContainersAsync())
{
Assert.Fail("Containers exist");
}
}
[Theory]
[InlineData(1)]
[InlineData(64 * 1024 * 1024)] // exactly 1 block size - 64MB
[InlineData(64 * 1024 * 1024 - 1)]
[InlineData(64 * 1024 * 1024 + 1)]
[InlineData(2 * 64 * 1024 * 1024 + 256)]
[InlineData(200)]
public async void ClientCutScenarioTest(int objectSize)
{
var options = GetClientOptions(this.keyString, this.url);
options.Value.Interceptors.Add(new CallbackInterceptor());
using var client = FrostFSClient.GetSingleOwnerInstance(GetClientOptions(this.keyString, this.url));
await Cleanup(client);
var createContainerParam = new PrmContainerCreate(new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))))
{
WaitParams = lightWait
};
var containerId = await client.CreateContainerAsync(createContainerParam);
var ctx = new CallContext { Timeout = TimeSpan.FromSeconds(10) };
var container = await client.GetContainerAsync(new PrmContainerGet(containerId, ctx));
Assert.NotNull(container);
byte[] bytes = GetRandomBytes(objectSize);
var param = new PrmObjectPut
{
Header = new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]),
Payload = new MemoryStream(bytes),
ClientCut = true
};
var objectId = await client.PutObjectAsync(param);
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
bool hasObject = false;
await foreach (var objId in client.SearchObjectsAsync(new PrmObjectSearch(containerId, null, filter)))
{
hasObject = true;
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId));
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.NotNull(objHeader.Attributes);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes[0].Key);
Assert.Equal("test", objHeader.Attributes[0].Value);
}
Assert.True(hasObject);
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, objectId));
var downloadedBytes = new byte[@object.Header.PayloadLength];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk = null;
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
{
ms.Write(chunk.Value.Span);
}
Assert.Equal(SHA256.HashData(bytes), SHA256.HashData(downloadedBytes));
await CheckFilter(client, containerId, new FilterByRootObject());
await Cleanup(client);
var deadline = DateTime.UtcNow.Add(TimeSpan.FromSeconds(5));
IAsyncEnumerator<FrostFsContainerId>? enumerator = null;
do
{
if (deadline <= DateTime.UtcNow)
{
Assert.Fail("Containers exist");
break;
}
enumerator = client.ListContainersAsync().GetAsyncEnumerator();
await Task.Delay(500);
}
while (await enumerator!.MoveNextAsync());
}
[Fact]
public async void NodeInfoCallbackAndInterceptorTest()
{
bool callbackInvoked = false;
bool intercepterInvoked = false;
var options = GetClientOptions(this.keyString, this.url);
options.Value.Callback = (CallStatistics cs) =>
{
callbackInvoked = true;
Assert.True(cs.ElapsedMicroSeconds > 0);
};
options.Value.Interceptors.Add(new CallbackInterceptor(s => intercepterInvoked = true));
using var client = FrostFSClient.GetSingleOwnerInstance(options);
var result = await client.GetNodeInfoAsync(new PrmNodeInfo());
Assert.True(callbackInvoked);
Assert.True(intercepterInvoked);
Assert.Equal(2, result.Version.Major);
Assert.Equal(13, result.Version.Minor);
Assert.Equal(NodeState.Online, result.State);
Assert.Equal(33, result.PublicKey.Length);
Assert.Single(result.Addresses);
Assert.Equal(9, result.Attributes.Count);
}
private static byte[] GetRandomBytes(int size)
{
Random rnd = new();
var bytes = new byte[size];
rnd.NextBytes(bytes);
return bytes;
}
private static IOptions<ClientSettings> GetClientOptions(string key, string url)
{
return Options.Create(new ClientSettings
{
Key = key,
Host = url
});
}
static async Task Cleanup(IFrostFSClient client)
{
await foreach (var cid in client.ListContainersAsync())
{
await client.DeleteContainerAsync(new PrmContainerDelete(cid) { WaitParams = lightWait });
}
}
}