TrueCloudLab/frostfs-sdk-csharp
20
1
Fork 5
Code Issues 3 Pull requests Projects Releases Packages 3 Activity Actions

[#35] Client: rollback to PutSingleObject for client cut upload
All checks were successful
DCO / DCO (pull_request) Successful in 27s
Details
lint-build / dotnet8.0 (pull_request) Successful in 41s
Details
lint-build / dotnet8.0 (push) Successful in 40s
Details

Browse source 
Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
Pavel Gross 2025-03-03 16:18:44 +03:00
parent 8835b23ed3
commit 6988fcedae
3 changed files with 187 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

106
src/FrostFS.SDK.Tests/Smoke/Client/ObjectTests/ObjectTests.cs
View file

@ -41,7 +41,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
backupFactor: backupFactor,
selectors: [],
filter: [],
containerAttributes: [],
containerAttributes: [new FrostFsAttributePair("contAttrKey", "contAttrValue")],
new FrostFsReplica(replicas));
Assert.NotNull(containerId);
@ -49,7 +49,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
await AddObjectRules(client, containerId);
_testOutputHelper.WriteLine("rules added");
await RunSuite(client, containerId);
}
@ -73,15 +73,18 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
{
case serverCut:
objectId = await CreateObjectServerCut(client, containerId, bytes);
_testOutputHelper.WriteLine($"\tserver side cut");
break;
case clientCut:
objectId = await CreateObjectClientCut(client, containerId, bytes);
_testOutputHelper.WriteLine($"\tclient side cut");
break;
case singleObject:
case singleObject:
if (objectSize > 1 * 1024 * 1024)
continue;
objectId = await PutSingleObject(client, containerId, bytes);
_testOutputHelper.WriteLine($"\tput single object");
break;
default:
throw new ArgumentException("unexpected object type");
@ -91,17 +94,38 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
_testOutputHelper.WriteLine($"\tobject created");
var ecdsaKey = ClientOptions.Value.Key.LoadWif();
var owner = FrostFsOwner.FromKey(ecdsaKey);
FrostFsHeaderResult expected = new()
{
HeaderInfo = new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
attributes: [new FrostFsAttributePair("fileName", "test")],
split: null,
owner: owner,
version: new FrostFsVersion(2, 13))
{
PayloadLength = (ulong)objectSize,
PayloadCheckSum = hash
}
};
await ValidateHeader(client, containerId, objectId, expected);
_testOutputHelper.WriteLine($"\theader validated");
await ValidateContent(client, containerId, hash, objectId);
_testOutputHelper.WriteLine($"\tcontent validated");
await ValidateFilters(client, containerId, objectId, null, (ulong)bytes.Length);
_testOutputHelper.WriteLine($"\tfilters validated");
// if (type != clientCut)
// {
// await ValidatePatch(client, containerId, bytes, objectId);
// _testOutputHelper.WriteLine($"\tpatch validated");
// }
if (type != clientCut)
{
await ValidatePatch(client, containerId, bytes, objectId);
_testOutputHelper.WriteLine($"\tpatch validated");
}
await ValidateRange(client, containerId, bytes, objectId);
_testOutputHelper.WriteLine($"\trange validated");
@ -126,27 +150,27 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
var objectRange = bytes.AsMemory().Slice(100, 64).ToArray();
var expectedHash = SHA256.HashData(objectRange);
foreach (var h in hashes)
{
var x = h[..32].ToArray();
Assert.NotNull(x);
Assert.True(x.Length > 0);
// Assert.True(expectedHash.SequenceEqual(h.ToArray()));
// Assert.True(expectedHash.SequenceEqual(h.ToArray()));
}
}
private async Task ValidateRange(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId)
{
if (bytes.Length < 200)
if (bytes.Length < 100)
return;
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange(0, 50));
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange(50, 100));
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange(50, 50));
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange((ulong)bytes.Length - 100, 100));
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange((ulong)bytes.Length-100, 100));
if (bytes.Length >= 6200)
await CheckRange(client, containerId, bytes, objectId, new FrostFsRange(6000, 100));
}
@ -161,11 +185,15 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
MemoryStream ms = new(rangeBytes);
ReadOnlyMemory<byte>? chunk;
int readBytes = 0;
while ((chunk = await rangeReader!.ReadChunk()) != null)
{
readBytes += chunk.Value.Length;
ms.Write(chunk.Value.Span);
}
Assert.Equal(range.Length, (ulong)readBytes);
Assert.Equal(SHA256.HashData(bytes.AsSpan().Slice((int)range.Offset, (int)range.Length)), SHA256.HashData(rangeBytes));
_testOutputHelper.WriteLine($"\t\trange {range.Offset};{range.Length} validated");
@ -173,7 +201,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
private static async Task ValidatePatch(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes, FrostFsObjectId objectId)
{
if (bytes.Length < 1024 + 64)
if (bytes.Length < 1024 + 64 || bytes.Length > 5900)
return;
var patch = new byte[1024];
@ -214,7 +242,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
for (int i = (int)rangeEnd; i < bytes.Length; i++)
Assert.Equal(downloadedBytes[i], bytes[i]);
}
private async Task ValidateFilters(IFrostFSClient client, FrostFsContainerId containerId, FrostFsObjectId objectId, SplitId? splitId, ulong length)
{
@ -237,13 +265,8 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
await CheckFilter(client, containerId, new FilterByVersion(FrostFsMatchType.Equals, networkInfo.NodeInfoCollection[0].Version));
await CheckFilter(client, containerId, new FilterByEpoch(FrostFsMatchType.Equals, networkInfo.Epoch));
await CheckFilter(client, containerId, new FilterByPayloadLength(FrostFsMatchType.Equals, length));
// var checkSum = CheckSum.CreateCheckSum(hash);
// await CheckFilter(client, containerId, new FilterByPayloadHash(FrostFsMatchType.Equals, checkSum));
await CheckFilter(client, containerId, new FilterByPhysicallyStored());
}
@ -265,7 +288,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
}
}
private static async Task ValidateContent(IFrostFSClient client, FrostFsContainerId containerId, byte[] hash, FrostFsObjectId objectId)
private static async Task ValidateContent(IFrostFSClient client, FrostFsContainerId containerId, byte[] hash, FrostFsObjectId objectId)
{
var @object = await client.GetObjectAsync(
new PrmObjectGet(containerId, objectId),
@ -283,6 +306,39 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
Assert.Equal(hash, SHA256.HashData(downloadedBytes));
}
private static async Task ValidateHeader(
IFrostFSClient client,
FrostFsContainerId containerId,
FrostFsObjectId objectId,
FrostFsHeaderResult expected)
{
var res = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId, default), default);
var objHeader = res.HeaderInfo;
Assert.NotNull(objHeader);
Assert.Equal(containerId.GetValue(), objHeader.ContainerId.GetValue());
Assert.Equal(expected.HeaderInfo!.OwnerId!.Value, objHeader.OwnerId!.Value);
Assert.Equal(expected.HeaderInfo.Version!.Major, objHeader.Version!.Major);
Assert.Equal(expected.HeaderInfo.Version!.Minor, objHeader.Version!.Minor);
Assert.Equal(expected.HeaderInfo.PayloadLength, objHeader.PayloadLength);
Assert.Equal(expected.HeaderInfo.ObjectType, objHeader.ObjectType);
if (expected.HeaderInfo.Attributes != null)
{
Assert.NotNull(objHeader.Attributes);
Assert.Equal(expected.HeaderInfo.Attributes.Count, objHeader.Attributes.Count);
Assert.True(expected.HeaderInfo.Attributes.SequenceEqual(objHeader.Attributes));
}
Assert.Null(objHeader.Split);
}
private static async Task<FrostFsObjectId> CreateObjectServerCut(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes)
{
var header = new FrostFsObjectHeader(
@ -310,7 +366,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
return await client.PutClientCutObjectAsync(param, default).ConfigureAwait(true);
}
private static async Task<FrostFsObjectId> PutSingleObject(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes)
private static async Task<FrostFsObjectId> PutSingleObject(IFrostFSClient client, FrostFsContainerId containerId, byte[] bytes)
{
var header = new FrostFsObjectHeader(
containerId: containerId,
@ -324,7 +380,7 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
return await client.PutSingleObjectAsync(param, default).ConfigureAwait(true);
}
private static async Task CheckFilter(IFrostFSClient client, FrostFsContainerId containerId, IObjectFilter filter)
{
var resultObjectsCount = 0;