[#34] Client: Add rules deserialization
All checks were successful
DCO / DCO (pull_request) Successful in 23s
lint-build / dotnet8.0 (pull_request) Successful in 44s
lint-build / dotnet8.0 (push) Successful in 43s

Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
Pavel Gross 2025-02-27 17:35:19 +03:00
parent bd8eb7cc60
commit 8835b23ed3
18 changed files with 426 additions and 52 deletions

View file

@ -6,7 +6,7 @@ public struct Actions(bool inverted, string[] names) : System.IEquatable<Actions
public string[] Names { get; set; } = names;
public override bool Equals(object obj)
public override readonly bool Equals(object obj)
{
if (obj == null || obj is not Actions)
return false;

View file

@ -8,7 +8,7 @@ public class FrostFsRule
public Actions Actions { get; set; }
// List of the resources the operation is applied to.
public Resource Resources { get; set; }
public Resources Resources { get; set; }
// True iff individual conditions must be combined with the logical OR.
// By default AND is used, so _each_ condition must pass.

View file

@ -1,6 +1,6 @@
namespace FrostFS.SDK.Client;
public struct Resource(bool inverted, string[] names) : System.IEquatable<Resource>
public struct Resources(bool inverted, string[] names) : System.IEquatable<Resources>
{
public bool Inverted { get; set; } = inverted;
@ -8,10 +8,10 @@ public struct Resource(bool inverted, string[] names) : System.IEquatable<Resour
public override bool Equals(object obj)
{
if (obj == null || obj is not Resource)
if (obj == null || obj is not Resources)
return false;
return Equals((Resource)obj);
return Equals((Resources)obj);
}
public override readonly int GetHashCode()
@ -19,17 +19,17 @@ public struct Resource(bool inverted, string[] names) : System.IEquatable<Resour
return Inverted.GetHashCode() ^ string.Join(string.Empty, Names).GetHashCode();
}
public static bool operator ==(Resource left, Resource right)
public static bool operator ==(Resources left, Resources right)
{
return left.Equals(right);
}
public static bool operator !=(Resource left, Resource right)
public static bool operator !=(Resources left, Resources right)
{
return !(left == right);
}
public readonly bool Equals(Resource other)
public readonly bool Equals(Resources other)
{
return this.GetHashCode().Equals(other.GetHashCode());
}

View file

@ -1,18 +1,18 @@
using System;
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FrostFS.SDK.Tests")]
namespace FrostFS.SDK.Client;
internal static class RuleSerializer
{
const byte Version = 0; // increase if breaking change
const int ByteSize = 1;
const int UInt8Size = ByteSize;
const int BoolSize = ByteSize;
const long NullSlice = -1;
const int NullSliceSize = 1;
const byte ByteTrue = 1;
const byte ByteFalse = 0;
@ -44,6 +44,42 @@ internal static class RuleSerializer
return buf;
}
internal static FrostFsChain Deserialize(byte[] data)
{
if (data is null)
{
throw new ArgumentNullException(nameof(data));
}
FrostFsChain chain = new();
var (offset, version) = UInt8Unmarshal(data, 0);
if (version != Version)
{
throw new FrostFsException($"unsupported marshaller version {version}");
}
(offset, byte chainVersion) = UInt8Unmarshal(data, offset);
if (chainVersion != ChainMarshalVersion)
{
throw new FrostFsException($"unsupported chain version {chainVersion}");
}
(offset, chain.ID) = SliceUnmarshal(data, offset, UInt8Unmarshal);
(offset, chain.Rules) = SliceUnmarshal(data, offset, UnmarshalRule);
(offset, var matchTypeV) = UInt8Unmarshal(data, offset);
chain.MatchType = (RuleMatchType)matchTypeV;
VerifyUnmarshal(data, offset);
return chain;
}
private static int Int64Size(long value)
{
// https://cs.opensource.google/go/go/+/master:src/encoding/binary/varint.go;l=92;drc=dac9b9ddbd5160c5f4552410f5f8281bd5eed38c
@ -83,8 +119,8 @@ internal static class RuleSerializer
}
private static int StringSize(string? s)
{
var len = s !=null ? s.Length : 0;
{
var len = s != null ? s.Length : 0;
return Int64Size(len) + len;
}
@ -94,7 +130,7 @@ internal static class RuleSerializer
+ SliceSize(action.Names, StringSize);
}
private static int ResourcesSize(Resource resource)
private static int ResourcesSize(Resources resource)
{
return BoolSize // Inverted
+ SliceSize(resource.Names, StringSize);
@ -108,7 +144,7 @@ internal static class RuleSerializer
+ StringSize(condition.Value);
}
public static int RuleSize(FrostFsRule rule)
private static int RuleSize(FrostFsRule rule)
{
if (rule is null)
{
@ -122,7 +158,7 @@ internal static class RuleSerializer
+ SliceSize(rule.Conditions!, ConditionSize);
}
public static int UInt8Marshal(byte[] buf, int offset, byte value)
private static int UInt8Marshal(byte[] buf, int offset, byte value)
{
if (buf.Length - offset < 1)
{
@ -134,7 +170,7 @@ internal static class RuleSerializer
return offset + 1;
}
public static int ByteMarshal(byte[] buf, int offset, byte value)
private static int ByteMarshal(byte[] buf, int offset, byte value)
{
return UInt8Marshal(buf, offset, value);
}
@ -167,7 +203,7 @@ internal static class RuleSerializer
return offset + 1;
}
public static int Int64Marshal(byte[] buf, int offset, long v)
private static int Int64Marshal(byte[] buf, int offset, long v)
{
if (buf.Length - offset < Int64Size(v))
{
@ -177,7 +213,7 @@ internal static class RuleSerializer
return PutVarint(buf, offset, v);
}
public static int SliceMarshal<T>(byte[] buf, int offset, T[] slice, Func<byte[], int, T, int> marshalT)
private static int SliceMarshal<T>(byte[] buf, int offset, T[] slice, Func<byte[], int, T, int> marshalT)
{
if (slice == null)
{
@ -269,7 +305,7 @@ internal static class RuleSerializer
return SliceMarshal(buf, offset, rule.Conditions!, MarshalCondition);
}
private static int MarshalResources(byte[] buf, int offset, Resource resources)
private static int MarshalResources(byte[] buf, int offset, Resources resources)
{
offset = BoolMarshal(buf, offset, resources.Inverted);
@ -283,4 +319,187 @@ internal static class RuleSerializer
throw new FrostFsException("actual data size differs from expected");
}
}
private static (int, bool) BoolUnmarshal(byte[] buf, int offset)
{
(offset, byte val) = UInt8Unmarshal(buf, offset);
return (offset, val == ByteTrue);
}
private static (int, string) StringUnmarshal(byte[] buf, int offset)
{
(offset, long size) = Int64Unmarshal(buf, offset);
if (size == 0)
{
return (offset, string.Empty);
}
if (size > MaxSliceLen)
{
throw new FrostFsException($"string is too long: '{size}'");
}
if (size < 0)
{
throw new FrostFsException($"invalid string size: '{size}'");
}
if (buf.Length - offset < size)
{
throw new FrostFsException($"not enough bytes left to string value");
}
return (offset + (int)size, System.Text.Encoding.UTF8.GetString(buf, offset, (int)size));
}
private static (int, Actions) UnmarshalActions(byte[] buf, int offset)
{
Actions action = new();
(offset, action.Inverted) = BoolUnmarshal(buf, offset);
(offset, action.Names) = SliceUnmarshal(buf, offset, StringUnmarshal);
return (offset, action);
}
private static (int, Resources) UnmarshalResources(byte[] buf, int offset)
{
Resources res = new();
(offset, res.Inverted) = BoolUnmarshal(buf, offset);
(offset, res.Names) = SliceUnmarshal(buf, offset, StringUnmarshal);
return (offset, res);
}
private static (int, Condition) UnmarshalCondition(byte[] buf, int offset)
{
Condition cond = new();
(offset, var op) = UInt8Unmarshal(buf, offset);
cond.Op = (ConditionType)op;
(offset, var kind) = UInt8Unmarshal(buf, offset);
cond.Kind = (ConditionKindType)kind;
(offset, cond.Key) = StringUnmarshal(buf, offset);
(offset, cond.Value) = StringUnmarshal(buf, offset);
return (offset, cond);
}
private static (int, FrostFsRule) UnmarshalRule(byte[] buf, int offset)
{
FrostFsRule rule = new();
(offset, byte statusV) = UInt8Unmarshal(buf, offset);
rule.Status = (RuleStatus)statusV;
(offset, rule.Actions) = UnmarshalActions(buf, offset);
(offset, rule.Resources) = UnmarshalResources(buf, offset);
(offset, rule.Any) = BoolUnmarshal(buf, offset);
(offset, rule.Conditions) = SliceUnmarshal(buf, offset, UnmarshalCondition);
return (offset, rule);
}
private static (int, byte) UInt8Unmarshal(byte[] buf, int offset)
{
if (buf.Length - offset < 1)
{
throw new FrostFsException($"not enough bytes left to read a value of type 'byte' from offset {offset}");
}
return (offset + 1, buf[offset]);
}
private static (int, long) Int64Unmarshal(byte[] buf, int offset)
{
if (buf.Length - offset < sizeof(long))
{
throw new FrostFsException($"not enough bytes left to read a value of type 'long' from offset {offset}");
}
return Varint(buf, offset);
}
private static (int, T[]) SliceUnmarshal<T>(byte[] buf, int offset, Func<byte[], int, (int, T)> unmarshalT)
{
var (newOffset, size) = Varint(buf, offset);
if (size == NullSlice)
{
return (newOffset, []);
}
if (size > MaxSliceLen)
{
throw new FrostFsException($"slice size is too big: '{size}'");
}
if (size < 0)
{
throw new FrostFsException($"invalid slice size: '{size}'");
}
var result = new T[size];
for (int i = 0; i < result.Length; i++)
{
(newOffset, result[i]) = unmarshalT(buf, newOffset);
}
return (newOffset, result);
}
private static void VerifyUnmarshal(byte[] buf, int offset)
{
if (buf.Length != offset)
{
throw new FrostFsException("unmarshalled bytes left");
}
}
private static int MaxVarIntLen64 = 10;
public static (int, long) Varint(byte[] buf, int offset)
{
var (ux, n) = Uvarint(buf, offset); // ok to continue in presence of error
long x = (long)ux >> 1;
if ((ux & 1) != 0)
{
x = ~x;
}
return (n, x);
}
public static (ulong, int) Uvarint(byte[] buf, int offset)
{
ulong x = 0;
int s = 0;
for (int i = offset; i < buf.Length; i++)
{
byte b = buf[i];
if (i == MaxVarIntLen64)
{
return (0, -(i + 1)); // overflow
}
if (b < 0x80)
{
if (i == MaxVarIntLen64 - 1 && b > 1)
{
return (0, -(i + 1)); // overflow
}
return (x | ((ulong)b << s), i + 1);
}
x |= (ulong)(b & 0x7f) << s;
s += 7;
}
return (0, 0);
}
}

View file

@ -2,8 +2,6 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Frostfs.V2.Ape;
using FrostFS.SDK.Client.Interfaces;
using FrostFS.SDK.Client.Services;
using FrostFS.SDK.Cryptography;
@ -195,7 +193,7 @@ public class FrostFSClient : IFrostFSClient
return GetApeManagerService().RemoveChainAsync(args, ctx);
}
public Task<Chain[]> ListChainAsync(PrmApeChainList args, CallContext ctx)
public Task<FrostFsChain[]> ListChainAsync(PrmApeChainList args, CallContext ctx)
{
return GetApeManagerService().ListChainAsync(args, ctx);
}

View file

@ -2,8 +2,6 @@ using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Frostfs.V2.Ape;
namespace FrostFS.SDK.Client.Interfaces;
public interface IFrostFSClient
@ -25,7 +23,7 @@ public interface IFrostFSClient
Task RemoveChainAsync(PrmApeChainRemove args, CallContext ctx);
Task<Chain[]> ListChainAsync(PrmApeChainList args, CallContext ctx);
Task<FrostFsChain[]> ListChainAsync(PrmApeChainList args, CallContext ctx);
#endregion
#region Container

View file

@ -1,5 +1,4 @@
using System;
using System.Diagnostics;
using FrostFS.Refs;
using FrostFS.SDK.Client;

View file

@ -1,6 +1,4 @@
using FrostFS.SDK.Client;
namespace FrostFS.SDK.Client;
namespace FrostFS.SDK.Client;
public readonly struct PrmApeChainAdd(FrostFsChainTarget target, FrostFsChain chain, string[]? xheaders = null) : System.IEquatable<PrmApeChainAdd>
{

View file

@ -1,6 +1,4 @@
using FrostFS.SDK.Client;
namespace FrostFS.SDK.Client;
namespace FrostFS.SDK.Client;
public readonly struct PrmApeChainList(FrostFsChainTarget target, string[]? xheaders = null) : System.IEquatable<PrmApeChainList>
{

View file

@ -1,7 +1,4 @@
using System;
using FrostFS.SDK.Client;
namespace FrostFS.SDK.Client;
namespace FrostFS.SDK.Client;
public readonly struct PrmApeChainRemove(
FrostFsChainTarget target,

View file

@ -5,8 +5,6 @@ using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using Frostfs.V2.Ape;
using FrostFS.Refs;
using FrostFS.SDK.Client.Interfaces;
using FrostFS.SDK.Client.Mappers.GRPC;
@ -550,7 +548,7 @@ public partial class Pool : IFrostFSClient
await client.Client!.RemoveChainAsync(args, ctx).ConfigureAwait(false);
}
public async Task<Chain[]> ListChainAsync(PrmApeChainList args, CallContext ctx)
public async Task<FrostFsChain[]> ListChainAsync(PrmApeChainList args, CallContext ctx)
{
var client = Connection();
return await client.Client!.ListChainAsync(args, ctx).ConfigureAwait(false);

View file

@ -1,7 +1,6 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Frostfs.V2.Ape;
using Frostfs.V2.Apemanager;
using Google.Protobuf;
@ -61,7 +60,7 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
Verifier.CheckResponse(response);
}
internal async Task<Chain[]> ListChainAsync(PrmApeChainList args, CallContext ctx)
internal async Task<FrostFsChain[]> ListChainAsync(PrmApeChainList args, CallContext ctx)
{
ListChainsRequest request = new()
{
@ -78,6 +77,6 @@ internal sealed class ApeManagerServiceProvider : ContextAccessor
Verifier.CheckResponse(response);
return [.. response.Body.Chains];
return [.. response.Body.Chains.Select(c => RuleSerializer.Deserialize([.. c.Raw]))];
}
}

View file

@ -1,8 +1,10 @@
using System.Diagnostics.CodeAnalysis;
using FrostFS.SDK.Client;
using FrostFS.SDK.SmokeTests;
namespace FrostFS.SDK.Tests.Smoke;
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
public class InterceptorTests() : SmokeTestsBase
{
[Fact]

View file

@ -124,11 +124,16 @@ public class ObjectTests(ITestOutputHelper testOutputHelper) : SmokeTestsBase
var hashes = await client.GetRangeHashAsync(rangeParam, default);
var objectRange = bytes.AsMemory().Slice(100, 64).ToArray();
var expectedHash = SHA256.HashData(objectRange);
foreach (var h in hashes)
{
var x = h[..32].ToArray();
Assert.NotNull(x);
Assert.True(x.Length > 0);
// Assert.True(expectedHash.SequenceEqual(h.ToArray()));
}
}

View file

@ -6,8 +6,6 @@ using FrostFS.SDK.Client.Interfaces;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.SmokeTests;
using Microsoft.Extensions.Options;
namespace FrostFS.SDK.Tests.Smoke;
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]

View file

@ -1,7 +1,6 @@
using System.Collections.ObjectModel;
using System.Diagnostics.CodeAnalysis;
using System.Security.Cryptography;
using System.Security.Principal;
using System.Text;
using FrostFS.SDK.Client;
using FrostFS.SDK.Client.Interfaces;
@ -10,7 +9,6 @@ using FrostFS.SDK.Cryptography;
using Grpc.Core;
using Microsoft.Extensions.Options;
using Xunit.Abstractions;
namespace FrostFS.SDK.Tests.Smoke;
@ -121,7 +119,7 @@ public abstract class SmokeTestsBase
{
Status = RuleStatus.Allow,
Actions = new Actions(inverted: false, names: ["*"]),
Resources = new Resource (inverted: false, names: [$"native:object/*"]),
Resources = new Resources (inverted: false, names: [$"native:object/*"]),
Any = false,
Conditions = []
}

View file

@ -0,0 +1,167 @@
using System.Text;
using FrostFS.SDK.Client;
namespace FrostFS.SDK.Tests.Unit;
public class ApeTests : ContainerTestsBase
{
[Fact]
public void ApeRule1Test()
{
var chain = new FrostFsChain
{
ID = Encoding.ASCII.GetBytes("chain-id-test"),
Rules = [
new FrostFsRule
{
Status = RuleStatus.Allow,
Actions = new Actions(inverted: false, names: ["*"]),
Resources = new Resources (inverted: false, names: [$"native:object/*"]),
Any = false,
Conditions = []
}
],
MatchType = RuleMatchType.DenyPriority
};
var serialized = RuleSerializer.Serialize(chain);
var restoredChain = RuleSerializer.Deserialize(serialized);
Assert.True(chain.ID.SequenceEqual(restoredChain.ID));
Assert.Equal(chain.MatchType, restoredChain.MatchType);
CompareRules(chain.Rules, restoredChain.Rules);
}
[Fact]
public void ApeRule2Test()
{
var chain = new FrostFsChain
{
ID = Encoding.ASCII.GetBytes("dumptext"),
Rules = [
new FrostFsRule
{
Status = RuleStatus.AccessDenied,
Actions = new Actions(inverted: true, names: ["put,get"]),
Resources = new Resources (inverted: true, names: [$"native:object/*,blablabla"]),
Any = true,
Conditions = [
new () {
Key = "key",
Value = "value",
Kind = ConditionKindType.Resource,
Op = ConditionType.CondStringEquals
},
new () {
Key = "key1",
Value = "value1",
Kind = ConditionKindType.Request,
Op = ConditionType.CondNumericGreaterThan
}
]
}
],
MatchType = RuleMatchType.FirstMatch
};
var serialized = RuleSerializer.Serialize(chain);
var restoredChain = RuleSerializer.Deserialize(serialized);
Assert.True(chain.ID.SequenceEqual(restoredChain.ID));
Assert.Equal(chain.MatchType, restoredChain.MatchType);
CompareRules(chain.Rules, restoredChain.Rules);
}
[Fact]
public void NegativeDeserialize1Test()
{
try
{
_ = RuleSerializer.Deserialize(null);
Assert.Fail("Error is expected");
}
catch (ArgumentNullException)
{
}
}
[Fact]
public void NegativeDeserialize2Test()
{
try
{
_ = RuleSerializer.Deserialize([]);
Assert.Fail("Error is expected");
}
catch (FrostFsException)
{
}
}
[Fact]
public void NegativeDeserialize3Test()
{
try
{
_ = RuleSerializer.Deserialize([1, 2, 3]);
Assert.Fail("Error is expected");
}
catch (FrostFsException)
{
}
}
[Fact]
public void NegativeDeserialize4Test()
{
try
{
//"\x00\x00:aws:iam::namespace:group/so\x82\x82\x82\x82\x82\x82u\x82"
_ = RuleSerializer.Deserialize([0x00, 0x00, 0x3A, 0x77, 0x73, 0x3A, 0x69, 0x61, 0x6D, 0x3A, 0x3A, 0x6E, 0x61, 0x6D, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x3A, 0x67, 0x72, 0x6F, 0x75, 0x70, 0x2F, 0x73, 0x6F, 0x82, 0x82, 0x82, 0x82, 0x82, 0x82, 0x75, 0x82]);
Assert.Fail("Error is expected");
}
catch (FrostFsException)
{
}
}
private static void CompareRules(FrostFsRule[] rules1, FrostFsRule[] rules2)
{
Assert.NotNull(rules1);
Assert.NotNull(rules2);
Assert.Equal(rules1.Length, rules2.Length);
for (int ri = 0; ri < rules1.Length; ri++)
{
var rule1 = rules1[ri];
var rule2 = rules2[ri];
Assert.Equal(rule1.Status, rule2.Status);
Assert.Equal(rule1.Any, rule2.Any);
Assert.Equal(rule1.Actions, rule2.Actions);
Assert.Equal(rule1.Resources, rule2.Resources);
bool cond1Empty = rule1.Conditions == null || rule1.Conditions.Length == 0;
bool cond2Empty = rule2.Conditions == null || rule2.Conditions.Length == 0;
if (cond1Empty && cond2Empty)
{
return;
}
Assert.Equal(cond1Empty, cond2Empty);
Assert.Equal(rule1.Conditions!.Length, rule2.Conditions!.Length);
for (int i = 0; i < rule1.Conditions.Length; i++)
{
Assert.Equal(rule1.Conditions[i], rule2.Conditions[i]);
}
}
}
}

View file

@ -111,12 +111,12 @@ public class ObjectTest : ObjectTestsBase
Assert.NotNull(header1.Split.SplitId);
Assert.Null(header1.Split.Previous);
Assert.Equal(bytes[..blockSize], payload1);
Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(0, blockSize).ToArray()), SHA256.HashData(payload1));
Assert.True(header1.Attributes.Count == 0);
Assert.Equal(header1.Split.SplitId, header2.Split.SplitId);
Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value);
Assert.Equal(bytes[blockSize..(blockSize * 2)], payload2);
Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(blockSize, blockSize).ToArray()), SHA256.HashData(payload2));
Assert.True(header2.Attributes.Count == 0);
// last part
@ -124,7 +124,7 @@ public class ObjectTest : ObjectTestsBase
Assert.NotNull(header3.Split.ParentHeader);
Assert.NotNull(header3.Split.ParentSignature);
Assert.Equal(header2.Split.SplitId, header3.Split.SplitId);
Assert.Equal(bytes[(fileLength / blockSize * blockSize)..fileLength], payload3);
Assert.Equal(SHA256.HashData(bytes.AsMemory().Slice(fileLength - fileLength % blockSize, fileLength % blockSize).ToArray()), SHA256.HashData(payload3));
Assert.True(header3.Attributes.Count == 0);
//link object