342 lines
13 KiB
C#
342 lines
13 KiB
C#
using System.Diagnostics.CodeAnalysis;
|
|
using System.Security.Cryptography;
|
|
using FrostFS.SDK.Client;
|
|
using FrostFS.SDK.Client.Interfaces;
|
|
using FrostFS.SDK.Cryptography;
|
|
using Xunit.Abstractions;
|
|
|
|
namespace FrostFS.SDK.Tests.Smoke;
|
|
|
|
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
|
|
[SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "No secure purpose")]
|
|
public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
|
|
{
|
|
private readonly ITestOutputHelper _testOutputHelper = testOutputHelper;
|
|
|
|
const string clientCut = "clientCut";
|
|
const string serverCut = "serverCut";
|
|
const string singleObject = "singleObject";
|
|
|
|
[Theory]
|
|
[InlineData(true, 1, 1)]
|
|
[InlineData(false, 1, 1)]
|
|
[InlineData(true, 1, 3)]
|
|
[InlineData(false, 1, 3)]
|
|
[InlineData(true, 2, 3)]
|
|
[InlineData(false, 2, 3)]
|
|
[InlineData(true, 2, 1)]
|
|
[InlineData(false, 2, 1)]
|
|
public async void FullScenario(bool unique, uint backupFactor, int replicas)
|
|
{
|
|
var client = FrostFSClient.GetInstance(ClientOptions, GrpcChannel);
|
|
_testOutputHelper.WriteLine("client created");
|
|
|
|
await Cleanup(client);
|
|
_testOutputHelper.WriteLine("existing containers removed");
|
|
|
|
FrostFsContainerId containerId = await CreateContainer(client,
|
|
ctx: default,
|
|
token: null,
|
|
unique: unique,
|
|
backupFactor: backupFactor,
|
|
selectors: [],
|
|
filter: [],
|
|
containerAttributes: [],
|
|
new FrostFsReplica(replicas));
|
|
|
|
Assert.NotNull(containerId);
|
|
_testOutputHelper.WriteLine("container created");
|
|
|
|
await AddObjectRules(client, containerId);
|
|
_testOutputHelper.WriteLine("rules added");
|
|
|
|
await RunSuite(client, containerId);
|
|
}
|
|
|
|
private async Task RunSuite(IFrostFSClient client, FrostFsContainerId containerId)
|
|
{
|
|
int[] objectSizes = [1, 257, 6 * 1024, 20 * 1024];
|
|
|
|
string[] objectTypes = [clientCut, serverCut, singleObject];
|
|
|
|
foreach (var objectSize in objectSizes)
|
|
{
|
|
_testOutputHelper.WriteLine($"test set for object size {objectSize}");
|
|
|
|
var bytes = GetRandomBytes(objectSize);
|
|
var hash = SHA256.HashData(bytes);
|
|
|
|
FrostFsObjectId objectId;
|
|
foreach (var type in objectTypes)
|
|
{
|
|
switch (type)
|
|
{
|
|
case serverCut:
|
|
objectId = await CreateObjectServerCut(client, containerId, bytes);
|
|
break;
|
|
case clientCut:
|
|
objectId = await CreateObjectClientCut(client, containerId, bytes);
|
|
break;
|
|
case singleObject:
|
|
if (objectSize > 1 * 1024 * 1024)
|
|
continue;
|
|
objectId = await PutSingleObject(client, containerId, bytes);
|
|
|
|
break;
|
|
default:
|
|
throw new ArgumentException("unexpected object type");
|
|
}
|
|
|
|
Assert.NotNull(objectId);
|
|
|
|
_testOutputHelper.WriteLine($"\tobject created");
|
|
|
|
await ValidateContent(client, containerId, hash, objectId);
|
|
_testOutputHelper.WriteLine($"\tcontent validated");
|
|
|
|
await ValidateFilters(client, containerId, objectId, null, (ulong)bytes.Length);
|
|
_testOutputHelper.WriteLine($"\tfilters validated");
|
|
|
|
// if (type != clientCut)
|
|
// {
|
|
// await ValidatePatch(client, containerId, bytes, objectId);
|
|
// _testOutputHelper.WriteLine($"\tpatch validated");
|
|
// }
|
|
|
|
await ValidateRange(client, containerId, bytes, objectId);
|
|
_testOutputHelper.WriteLine($"\trange validated");
|
|
|
|
await ValidateRangeHash(client, containerId, bytes, objectId);
|
|
_testOutputHelper.WriteLine($"\trange hash validated");
|
|
|
|
await RemoveObject(client, containerId, objectId);
|
|
_testOutputHelper.WriteLine($"\tobject removed");
|
|
}
|
|
}
|
|
}
|
|
|
|
private static async Task ValidateRangeHash(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId)
|
|
{
|
|
if (bytes.Length < 200)
|
|
return;
|
|
|
|
var rangeParam = new PrmRangeHashGet(containerId, objectId, [new FrostFsRange(100, 64)], bytes);
|
|
|
|
var hashes = await client.GetRangeHashAsync(rangeParam, default);
|
|
|
|
var objectRange = bytes.AsMemory().Slice(100, 64).ToArray();
|
|
var expectedHash = SHA256.HashData(objectRange);
|
|
|
|
foreach (var h in hashes)
|
|
{
|
|
var x = h[..32].ToArray();
|
|
Assert.NotNull(x);
|
|
Assert.True(x.Length > 0);
|
|
|
|
Assert.True(expectedHash.SequenceEqual(h.ToArray()));
|
|
}
|
|
}
|
|
|
|
private async Task ValidateRange(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId)
|
|
{
|
|
if (bytes.Length < 200)
|
|
return;
|
|
|
|
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange(0, 50));
|
|
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange(50, 100));
|
|
|
|
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange((ulong)bytes.Length-100, 100));
|
|
|
|
if (bytes.Length >= 6200)
|
|
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange(6000, 100));
|
|
}
|
|
|
|
private async Task CheckRange(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId, FrostFsRange range)
|
|
{
|
|
var rangeParam = new PrmRangeGet(containerId, objectId, range);
|
|
|
|
var rangeReader = await client.GetRangeAsync(rangeParam, default);
|
|
|
|
var rangeBytes = new byte[rangeParam.Range.Length];
|
|
MemoryStream ms = new(rangeBytes);
|
|
|
|
ReadOnlyMemory<byte>? chunk;
|
|
while ((chunk = await rangeReader!.ReadChunk()) != null)
|
|
{
|
|
ms.Write(chunk.Value.Span);
|
|
}
|
|
|
|
Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice((int)range.Offset, (int)range.Length)), SHA256.HashData(rangeBytes));
|
|
|
|
_testOutputHelper.WriteLine($"\t\trange {range.Offset};{range.Length} validated");
|
|
}
|
|
|
|
private static async Task ValidatePatch(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId)
|
|
{
|
|
if (bytes.Length < 1024 + 64)
|
|
return;
|
|
|
|
var patch = new byte[1024];
|
|
for (int i = 0; i < patch.Length; i++)
|
|
{
|
|
patch[i] = 32;
|
|
}
|
|
|
|
var range = new FrostFsRange(64, (ulong)patch.Length);
|
|
|
|
var patchParams = new PrmObjectPatch(
|
|
new FrostFsAddress(containerId, objectId),
|
|
payload: new MemoryStream(patch),
|
|
maxChunkLength: 1024,
|
|
range: range);
|
|
|
|
var newIbjId = await client.PatchObjectAsync(patchParams, default);
|
|
|
|
var @object = await client.GetObjectAsync(new PrmObjectGet(containerId, newIbjId), default);
|
|
|
|
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
MemoryStream ms = new(downloadedBytes);
|
|
|
|
ReadOnlyMemory<byte>? chunk;
|
|
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
{
|
|
ms.Write(chunk.Value.Span);
|
|
}
|
|
|
|
for (int i = 0; i < (int)range.Offset; i++)
|
|
Assert.Equal(downloadedBytes[i], bytes[i]);
|
|
|
|
var rangeEnd = range.Offset + range.Length;
|
|
|
|
for (int i = (int)range.Offset; i < (int)rangeEnd; i++)
|
|
Assert.Equal(downloadedBytes[i], patch[i - (int)range.Offset]);
|
|
|
|
for (int i = (int)rangeEnd; i < bytes.Length; i++)
|
|
Assert.Equal(downloadedBytes[i], bytes[i]);
|
|
}
|
|
|
|
|
|
private async Task ValidateFilters(IFrostFSClient client, FrostFsContainerId containerId, FrostFsObjectId objectId, SplitId? splitId, ulong length)
|
|
{
|
|
var ecdsaKey = keyString.LoadWif();
|
|
|
|
var networkInfo = await client.GetNetmapSnapshotAsync(default);
|
|
|
|
await CheckFilter(client, containerId, new FilterByContainerId(FrostFsMatchType.Equals, containerId));
|
|
|
|
await CheckFilter(client, containerId, new FilterByOwnerId(FrostFsMatchType.Equals, FrostFsOwner.FromKey(ecdsaKey)));
|
|
|
|
if (splitId != null)
|
|
{
|
|
await CheckFilter(client, containerId, new FilterBySplitId(FrostFsMatchType.Equals, splitId));
|
|
}
|
|
|
|
await CheckFilter(client, containerId, new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test"));
|
|
|
|
await CheckFilter(client, containerId, new FilterByObjectId(FrostFsMatchType.Equals, objectId));
|
|
|
|
await CheckFilter(client, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
|
|
|
|
await CheckFilter(client, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
|
|
|
|
await CheckFilter(client, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, length));
|
|
|
|
// var checkSum = CheckSum.CreateCheckSum(hash);
|
|
// await CheckFilter(client, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
|
|
|
|
await CheckFilter(client, containerId, new FilterByPhysicallyStored());
|
|
}
|
|
|
|
private static async Task RemoveObject(IFrostFSClient client, FrostFsContainerId containerId, FrostFsObjectId objectId)
|
|
{
|
|
await client.DeleteObjectAsync(new PrmObjectDelete(containerId, objectId), default);
|
|
|
|
try
|
|
{
|
|
_ = await client.GetObjectAsync(
|
|
new PrmObjectGet(containerId, objectId),
|
|
default);
|
|
|
|
Assert.Fail("Exception is expected here");
|
|
}
|
|
catch (FrostFsResponseException ex)
|
|
{
|
|
Assert.Equal("object already removed", ex.Status!.Message);
|
|
}
|
|
}
|
|
|
|
private static async Task ValidateContent(IFrostFSClient client, FrostFsContainerId containerId, byte[] hash, FrostFsObjectId objectId)
|
|
{
|
|
var @object = await client.GetObjectAsync(
|
|
new PrmObjectGet(containerId, objectId),
|
|
default);
|
|
|
|
var downloadedBytes = new byte[@object.Header.PayloadLength];
|
|
MemoryStream ms = new(downloadedBytes);
|
|
|
|
ReadOnlyMemory<byte>? chunk = null;
|
|
while ((chunk = await @object.ObjectReader!.ReadChunk()) != null)
|
|
{
|
|
ms.Write(chunk.Value.Span);
|
|
}
|
|
|
|
Assert.Equal(hash, SHA256.HashData(downloadedBytes));
|
|
}
|
|
|
|
private static async Task<FrostFsObjectId> CreateObjectServerCut(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes)
|
|
{
|
|
var header = new FrostFsObjectHeader(
|
|
containerId: containerId,
|
|
type: FrostFsObjectType.Regular,
|
|
[new FrostFsAttributePair("fileName", "test")]);
|
|
|
|
var param = new PrmObjectPut(header);
|
|
|
|
var objectWriter = await client.PutObjectAsync(param, default).ConfigureAwait(true);
|
|
|
|
await objectWriter.WriteAsync(bytes);
|
|
return await objectWriter.CompleteAsync();
|
|
}
|
|
|
|
private static async Task<FrostFsObjectId> CreateObjectClientCut(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes)
|
|
{
|
|
var header = new FrostFsObjectHeader(
|
|
containerId: containerId,
|
|
type: FrostFsObjectType.Regular,
|
|
[new FrostFsAttributePair("fileName", "test")]);
|
|
|
|
var param = new PrmObjectClientCutPut(header, payload: new MemoryStream(bytes));
|
|
|
|
return await client.PutClientCutObjectAsync(param, default).ConfigureAwait(true);
|
|
}
|
|
|
|
private static async Task<FrostFsObjectId> PutSingleObject(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes)
|
|
{
|
|
var header = new FrostFsObjectHeader(
|
|
containerId: containerId,
|
|
type: FrostFsObjectType.Regular,
|
|
[new FrostFsAttributePair("fileName", "test")]);
|
|
|
|
|
|
var obj = new FrostFsObject(header) { SingleObjectPayload = bytes };
|
|
|
|
var param = new PrmSingleObjectPut(obj);
|
|
|
|
return await client.PutSingleObjectAsync(param, default).ConfigureAwait(true);
|
|
}
|
|
|
|
private static async Task CheckFilter(IFrostFSClient client, FrostFsContainerId containerId, IObjectFilter filter)
|
|
{
|
|
var resultObjectsCount = 0;
|
|
|
|
PrmObjectSearch searchParam = new(containerId, null, [], filter);
|
|
|
|
await foreach (var objId in client.SearchObjectsAsync(searchParam, default))
|
|
{
|
|
resultObjectsCount++;
|
|
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId), default);
|
|
}
|
|
|
|
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
|
|
}
|
|
}
|