From 43e300c7730f7545f87f168a5f82cc15f8f96218 Mon Sep 17 00:00:00 2001 From: Pavel Gross Date: Mon, 13 Jan 2025 10:34:44 +0300 Subject: [PATCH] [#29] Client: Add PlacementVector unit tests Signed-off-by: Pavel Gross --- .../Mappers/Netmap/PlacementPolicy.cs | 3 + .../Mappers/Netmap/Replica.cs | 66 ++- .../Models/Netmap/FrostFsNetmapSnapshot.cs | 41 +- .../Models/Netmap/FrostFsPlacementPolicy.cs | 16 +- .../Models/Netmap/FrostFsSelector.cs | 4 +- .../Models/Netmap/Placement/Context.cs | 73 ++-- .../Models/Netmap/Placement/HasherList.cs | 8 + .../Models/Netmap/Placement/MeanAgg.cs | 2 +- .../Models/Netmap/Placement/Tools.cs | 24 +- .../ContainerServiceBase.cs | 2 +- .../Multithread/MultithreadPoolSmokeTests.cs | 10 +- .../MultithreadSmokeClientTests.cs | 16 +- src/FrostFS.SDK.Tests/Smoke/PoolSmokeTests.cs | 10 +- .../Smoke/SmokeClientTests.cs | 16 +- .../TestData/PlacementTests/cbf_default.json | 100 +++++ .../TestData/PlacementTests/cbf_minimal.json | 101 +++++ .../PlacementTests/cbf_requirements.json | 156 +++++++ .../PlacementTests/filter_complex.json | 345 +++++++++++++++ .../filter_invalid_integer.json | 81 ++++ .../PlacementTests/filter_simple.json | 397 ++++++++++++++++++ .../TestData/PlacementTests/hrw_sort.json | 225 ++++++++++ .../TestData/PlacementTests/issue213.json | 107 +++++ .../TestData/PlacementTests/many_selects.json | 279 ++++++++++++ .../TestData/PlacementTests/multiple_rep.json | 93 ++++ .../multiple_rep_asymmetric.json | 328 +++++++++++++++ .../TestData/PlacementTests/non_strict.json | 97 +++++ .../TestData/PlacementTests/rep_only.json | 113 +++++ .../PlacementTests/select_no_attribute.json | 116 +++++ .../PlacementTests/selector_invalid.json | 87 ++++ .../Unit/ContainerTestsBase.cs | 2 +- src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs | 2 +- .../Unit/PlacementVectorTests.cs | 366 ++++++++++------ .../Unit/SessionTestsBase.cs | 2 +- 33 files changed, 3054 insertions(+), 234 deletions(-) create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_default.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_minimal.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_requirements.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_complex.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_invalid_integer.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_simple.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/hrw_sort.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/issue213.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/many_selects.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/multiple_rep.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/multiple_rep_asymmetric.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/non_strict.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/rep_only.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/select_no_attribute.json create mode 100644 src/FrostFS.SDK.Tests/TestData/PlacementTests/selector_invalid.json diff --git a/src/FrostFS.SDK.Client/Mappers/Netmap/PlacementPolicy.cs b/src/FrostFS.SDK.Client/Mappers/Netmap/PlacementPolicy.cs index 5f1600c..30410aa 100644 --- a/src/FrostFS.SDK.Client/Mappers/Netmap/PlacementPolicy.cs +++ b/src/FrostFS.SDK.Client/Mappers/Netmap/PlacementPolicy.cs @@ -16,6 +16,9 @@ public static class PlacementPolicyMapper return new FrostFsPlacementPolicy( placementPolicy.Unique, + placementPolicy.ContainerBackupFactor, + new System.Collections.ObjectModel.Collection(placementPolicy.Selectors.Select(selector => selector.ToModel()).ToList()), + new System.Collections.ObjectModel.Collection(placementPolicy.Filters.Select(filter => filter.ToModel()).ToList()), placementPolicy.Replicas.Select(replica => replica.ToModel()).ToArray() ); } diff --git a/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs b/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs index f0ace94..bc415e1 100644 --- a/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs +++ b/src/FrostFS.SDK.Client/Mappers/Netmap/Replica.cs @@ -1,10 +1,11 @@ using System; +using System.Linq; using FrostFS.Netmap; namespace FrostFS.SDK.Client; -public static class ReplicaMapper +public static class PolicyMapper { public static Replica ToMessage(this FrostFsReplica replica) { @@ -24,4 +25,67 @@ public static class ReplicaMapper return new FrostFsReplica((int)replica.Count, replica.Selector); } + + public static Selector ToMessage(this FrostFsSelector selector) + { + if (selector is null) + { + throw new ArgumentNullException(nameof(selector)); + } + + return new Selector + { + Name = selector.Name, + Count = selector.Count, + Clause = (Clause)selector.Clause, + Attribute = selector.Attribute, + Filter = selector.Filter + }; + } + + public static FrostFsSelector ToModel(this Selector selector) + { + if (selector is null) + { + throw new ArgumentNullException(nameof(selector)); + } + + return new FrostFsSelector(selector.Name) + { + Count = selector.Count, + Clause = (int)selector.Clause, + Attribute = selector.Attribute, + Filter = selector.Filter + }; + } + + public static Filter ToMessage(this FrostFsFilter filter) + { + if (filter is null) + { + throw new ArgumentNullException(nameof(filter)); + } + + var message = new Filter + { + Name = filter.Name, + Key = filter.Key, + Op = (Operation)filter.Operation, + Value = filter.Value, + }; + + message.Filters.AddRange(filter.Filters.Select(f => f.ToMessage())); + + return message; + } + + public static FrostFsFilter ToModel(this Filter filter) + { + if (filter is null) + { + throw new ArgumentNullException(nameof(filter)); + } + + return new FrostFsFilter(filter.Name, filter.Key, (int)filter.Op, filter.Value, filter.Filters.Select(f => f.ToModel()).ToArray()); + } } \ No newline at end of file diff --git a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsNetmapSnapshot.cs b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsNetmapSnapshot.cs index 686c966..1d90b04 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsNetmapSnapshot.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsNetmapSnapshot.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Linq; using FrostFS.SDK.Client; @@ -58,9 +57,9 @@ public class FrostFsNetmapSnapshot(ulong epoch, IReadOnlyList n result[i][j] = vectors[i][j]; } - Tools.AppendWeightsTo(result[i], wf, spanWeigths); + Tools.AppendWeightsTo(result[i], wf, ref spanWeigths); - Tools.SortHasherSliceByWeightValue(result[i].ToList(), spanWeigths, hash); + result[i] = Tools.SortHasherSliceByWeightValue(result[i].ToList(), spanWeigths, hash).ToArray(); } return result; @@ -74,12 +73,7 @@ public class FrostFsNetmapSnapshot(ulong epoch, IReadOnlyList n // of the last select application. List> SelectFilterNodes(SelectFilterExpr expr) { - var policy = new FrostFsPlacementPolicy(false); - - foreach (var f in expr.Filters) - policy.Filters.Add(f); - - policy.Selectors.Add(expr.Selector); + var policy = new FrostFsPlacementPolicy(false, expr.Cbf, [expr.Selector], expr.Filters); var ctx = new Context(this) { @@ -166,7 +160,7 @@ public class FrostFsNetmapSnapshot(ulong epoch, IReadOnlyList n { var c = new Context(this) { - Cbf = p.BackupFactor + Cbf = p.BackupFactor == 0 ? 3 : p.BackupFactor }; if (pivot != null && pivot.Length > 0) @@ -198,20 +192,14 @@ public class FrostFsNetmapSnapshot(ulong epoch, IReadOnlyList n if (string.IsNullOrEmpty(sName) && !(p.Replicas.Length == 1 && p.Selectors.Count == 1)) { - var s = new FrostFsSelector(Context.mainFilterName) + var s = new FrostFsSelector(string.Empty) { - Count = p.Replicas[i].CountNodes() + Count = p.Replicas[i].CountNodes(), + Filter = Context.mainFilterName }; var nodes = c.GetSelection(s); - - var arg = new List>(nodes.Count); - for (int j = 0; j < nodes.Count; j++) - { - arg[i] = nodes[j]; - } - - result[i].AddRange(FlattenNodes(arg)); + result[i].AddRange(FlattenNodes(nodes)); if (unique) { @@ -243,23 +231,16 @@ public class FrostFsNetmapSnapshot(ulong epoch, IReadOnlyList n else { var nodes = c.Selections[sName]; - - var arg = new List>(nodes.Count); - for (int j = 0; j < nodes.Count; j++) - { - arg[i] = nodes[j]; - } - - result[i].AddRange(FlattenNodes(arg)); + result[i].AddRange(FlattenNodes(nodes)); } } var collection = new FrostFsNodeInfo[result.Count][]; - for(int i =0; i < result.Count; i++) + for (int i = 0; i < result.Count; i++) { collection[i] = [.. result[i]]; } - + return collection; } } diff --git a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs index 19fe755..6ca8ab9 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsPlacementPolicy.cs @@ -7,20 +7,24 @@ using FrostFS.SDK.Client; namespace FrostFS.SDK; -public struct FrostFsPlacementPolicy(bool unique, params FrostFsReplica[] replicas) +public struct FrostFsPlacementPolicy(bool unique, + uint backupFactor, + Collection selectors, + Collection filters, + params FrostFsReplica[] replicas) : IEquatable { private PlacementPolicy policy; - public FrostFsReplica[] Replicas { get; private set; } = replicas; + public FrostFsReplica[] Replicas { get; } = replicas; - public Collection Selectors { get; } = []; + public Collection Selectors { get; } = selectors; - public Collection Filters { get; } = []; + public Collection Filters { get; } = filters; - public bool Unique { get; private set; } = unique; + public bool Unique { get; } = unique; - public uint BackupFactor { get; set; } + public uint BackupFactor { get; } = backupFactor; public override readonly bool Equals(object obj) { diff --git a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsSelector.cs b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsSelector.cs index 74380f8..3369d6c 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/FrostFsSelector.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/FrostFsSelector.cs @@ -2,9 +2,9 @@ public class FrostFsSelector(string name) { - public string Name { get; } = name; + public string Name { get; set; } = name; public uint Count { get; set; } - public uint Clause { get; set; } + public int Clause { get; set; } public string? Attribute { get; set; } public string? Filter { get; set; } } diff --git a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs index fc6e80b..ed4ad82 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs @@ -104,20 +104,23 @@ internal struct Context return; } - if (filter.Operation == (int)Operation.EQ || - filter.Operation == (int)Operation.NE || - filter.Operation == (int)Operation.LIKE || - filter.Operation == (int)Operation.GT || - filter.Operation == (int)Operation.GE || - filter.Operation == (int)Operation.LT || - filter.Operation == (int)Operation.LE) + switch (filter.Operation) { - var n = uint.Parse(filter.Value, CultureInfo.InvariantCulture); - NumCache[filter.Value] = n; - } - else - { - throw new FrostFsException($"{errInvalidFilterOp}: {filter.Operation}"); + case (int)Operation.EQ: + case (int)Operation.NE: + case (int)Operation.LIKE: + break; + case (int)Operation.GT: + case (int)Operation.GE: + case (int)Operation.LT: + case (int)Operation.LE: + { + var n = uint.Parse(filter.Value, CultureInfo.InvariantCulture); + NumCache[filter.Value] = n; + break; + } + default: + throw new FrostFsException($"{errInvalidFilterOp}: {filter.Operation}"); } } @@ -153,7 +156,7 @@ internal struct Context // for the given selector. static (int bucketCount, int nodesInBucket) CalcNodesCount(FrostFsSelector selector) { - return selector.Clause == (uint)FrostFsClause.Same + return selector.Clause == (int)FrostFsClause.Same ? (1, (int)selector.Count) : ((int)selector.Count, 1); } @@ -211,11 +214,17 @@ internal struct Context { double[] ws = []; - foreach (var res in result) + var sortedNodes = new NodeAttrPair[result.Count]; + + for (int i = 0; i < result.Count; i++) { - Tools.AppendWeightsTo(res.nodes, weightFunc, ws); - Tools.SortHasherSliceByWeightValue(res.nodes.ToList(), ws, HrwSeedHash); + var res = result[i]; + Tools.AppendWeightsTo(res.nodes, weightFunc, ref ws); + sortedNodes[i].nodes = Tools.SortHasherSliceByWeightValue(res.nodes.ToList(), ws, HrwSeedHash).ToArray(); + sortedNodes[i].attr = result[i].attr; } + + return sortedNodes; } return [.. result]; } @@ -273,7 +282,7 @@ internal struct Context if (res.Count < bucketCount) { // Fallback to using minimum allowed backup factor (1). - res = fallback; + res.AddRange(fallback); if (Strict && res.Count < bucketCount) { @@ -293,7 +302,12 @@ internal struct Context } var hashers = res.Select(r => new HasherList(r)).ToList(); - Tools.SortHasherSliceByWeightValue(hashers, weights, HrwSeedHash); + hashers = Tools.SortHasherSliceByWeightValue(hashers, weights, HrwSeedHash); + + for (int i = 0; i < res.Count; i++) + { + res[i] = hashers[i].Nodes; + } } if (res.Count < bucketCount) @@ -331,7 +345,7 @@ internal struct Context switch (f.Operation) { case (int)Operation.EQ: - return nodeInfo.Attributes[f.Key] == f.Value; + return nodeInfo.Attributes.TryGetValue(f.Key, out var val) && val == f.Value; case (int)Operation.LIKE: { var hasPrefix = f.Value.StartsWith(likeWildcard, StringComparison.Ordinal); @@ -357,12 +371,21 @@ internal struct Context return nodeInfo.Attributes[f.Key] != f.Value; default: { - var attr = f.Key switch + ulong attr; + switch (f.Key) { - FrostFsNodeInfo.AttrPrice => nodeInfo.Price, - FrostFsNodeInfo.AttrCapacity => nodeInfo.GetCapacity(), - _ => uint.Parse(nodeInfo.Attributes[f.Key], CultureInfo.InvariantCulture), - }; + case FrostFsNodeInfo.AttrPrice: + attr = nodeInfo.Price; + break; + + case FrostFsNodeInfo.AttrCapacity: + attr = nodeInfo.GetCapacity(); + break; + default: + if (!ulong.TryParse(nodeInfo.Attributes[f.Key], NumberStyles.Integer, CultureInfo.InvariantCulture, out attr)) + return false; + break; + } switch (f.Operation) { diff --git a/src/FrostFS.SDK.Client/Models/Netmap/Placement/HasherList.cs b/src/FrostFS.SDK.Client/Models/Netmap/Placement/HasherList.cs index 117980c..17fabfe 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/Placement/HasherList.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/Placement/HasherList.cs @@ -11,6 +11,14 @@ internal sealed class HasherList : IHasher _nodes = nodes; } + internal List Nodes + { + get + { + return _nodes; + } + } + public ulong Hash() { return _nodes.Count > 0 ? _nodes[0].Hash() : 0; diff --git a/src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanAgg.cs b/src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanAgg.cs index 0febba1..98187b3 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanAgg.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanAgg.cs @@ -8,7 +8,7 @@ internal struct MeanAgg internal void Add(double n) { int c = count + 1; - mean *= (double)count / c + n / c; + mean = mean * count / c + n / c; count++; } diff --git a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Tools.cs b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Tools.cs index b6de113..e87994a 100644 --- a/src/FrostFS.SDK.Client/Models/Netmap/Placement/Tools.cs +++ b/src/FrostFS.SDK.Client/Models/Netmap/Placement/Tools.cs @@ -36,7 +36,7 @@ public static class Tools return x / (1 + x); } - internal static void AppendWeightsTo(FrostFsNodeInfo[] nodes, Func wf, double[] weights) + internal static void AppendWeightsTo(FrostFsNodeInfo[] nodes, Func wf, ref double[] weights) { if (weights.Length < nodes.Length) { @@ -49,11 +49,11 @@ public static class Tools } } - internal static void SortHasherSliceByWeightValue(List nodes, Span weights, ulong hash) where T : IHasher + internal static List SortHasherSliceByWeightValue(List nodes, Span weights, ulong hash) where T : IHasher { if (nodes.Count == 0) { - return; + return nodes; } var allEquals = true; @@ -70,7 +70,7 @@ public static class Tools } } - var dist = new ulong[nodes.Count]; + var dist = new double[nodes.Count]; if (allEquals) { @@ -80,20 +80,19 @@ public static class Tools dist[i] = Distance(x, hash); } - SortHasherByDistance(nodes, dist, true); - return; + return SortHasherByDistance(nodes, dist, true); } for (int i = 0; i < dist.Length; i++) { var d = Distance(nodes[i].Hash(), hash); - dist[i] = ulong.MaxValue - (ulong)(d * weights[i]); + dist[i] = (ulong.MaxValue - d) * weights[i]; } - SortHasherByDistance(nodes, dist, false); + return SortHasherByDistance(nodes, dist, false); } - internal static void SortHasherByDistance(List nodes, N[] dist, bool asc) + internal static List SortHasherByDistance(List nodes, N[] dist, bool asc) { IndexedValue[] indexes = new IndexedValue[nodes.Count]; for (int i = 0; i < dist.Length; i++) @@ -103,16 +102,15 @@ public static class Tools if (asc) { - nodes = new List(indexes + return new List(indexes .OrderBy(x => x.dist) .Select(x => x.nodeInfo).ToArray()); } else { - nodes = new List(indexes + return new List(indexes .OrderByDescending(x => x.dist) - .Select(x => x.nodeInfo) - .ToArray()); + .Select(x => x.nodeInfo)); } } diff --git a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs index 8d4230c..ab52c19 100644 --- a/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs +++ b/src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs @@ -23,7 +23,7 @@ public abstract class ServiceBase(string key) public FrostFsPlacementPolicy PlacementPolicy { get; set; } = DefaultPlacementPolicy; public static FrostFsVersion DefaultVersion { get; } = new(2, 13); - public static FrostFsPlacementPolicy DefaultPlacementPolicy { get; } = new FrostFsPlacementPolicy(true, new FrostFsReplica(1)); + public static FrostFsPlacementPolicy DefaultPlacementPolicy { get; } = new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1)); #pragma warning disable CA2227 // this is specific object, should be treated as is public Metadata? Metadata { get; set; } diff --git a/src/FrostFS.SDK.Tests/Multithread/MultithreadPoolSmokeTests.cs b/src/FrostFS.SDK.Tests/Multithread/MultithreadPoolSmokeTests.cs index 0cd3d6b..4cb7e4d 100644 --- a/src/FrostFS.SDK.Tests/Multithread/MultithreadPoolSmokeTests.cs +++ b/src/FrostFS.SDK.Tests/Multithread/MultithreadPoolSmokeTests.cs @@ -166,7 +166,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["key1", "value1"]); @@ -216,7 +216,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase await Cleanup(pool); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), lightWait); var containerId = await pool.CreateContainerAsync(createContainerParam, default); @@ -311,7 +311,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase await Cleanup(pool); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -396,7 +396,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase var ctx = new CallContext(TimeSpan.FromSeconds(20)); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams); var container = await pool.CreateContainerAsync(createContainerParam, ctx); @@ -479,7 +479,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase await Cleanup(pool); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), lightWait); var containerId = await pool.CreateContainerAsync(createContainerParam, default); diff --git a/src/FrostFS.SDK.Tests/Multithread/MultithreadSmokeClientTests.cs b/src/FrostFS.SDK.Tests/Multithread/MultithreadSmokeClientTests.cs index e72e657..b239e58 100644 --- a/src/FrostFS.SDK.Tests/Multithread/MultithreadSmokeClientTests.cs +++ b/src/FrostFS.SDK.Tests/Multithread/MultithreadSmokeClientTests.cs @@ -99,7 +99,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["key1", "value1"]); @@ -144,7 +144,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), lightWait); var containerId = await client.CreateContainerAsync(createContainerParam, default); @@ -237,7 +237,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase var ctx = new CallContext(TimeSpan.FromSeconds(20)); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -308,7 +308,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -388,7 +388,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -442,7 +442,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -499,7 +499,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase var ctx = new CallContext(TimeSpan.FromSeconds(20)); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams); var container = await client.CreateContainerAsync(createContainerParam, ctx); @@ -580,7 +580,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), lightWait); var containerId = await client.CreateContainerAsync(createContainerParam, default); diff --git a/src/FrostFS.SDK.Tests/Smoke/PoolSmokeTests.cs b/src/FrostFS.SDK.Tests/Smoke/PoolSmokeTests.cs index ce626d5..94b2c8b 100644 --- a/src/FrostFS.SDK.Tests/Smoke/PoolSmokeTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/PoolSmokeTests.cs @@ -164,7 +164,7 @@ public class PoolSmokeTests : SmokeTestsBase var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["key1", "value1"]); @@ -214,7 +214,7 @@ public class PoolSmokeTests : SmokeTestsBase await Cleanup(pool); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), lightWait); var containerId = await pool.CreateContainerAsync(createContainerParam, default); @@ -311,7 +311,7 @@ public class PoolSmokeTests : SmokeTestsBase var ctx = new CallContext(TimeSpan.Zero); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -396,7 +396,7 @@ public class PoolSmokeTests : SmokeTestsBase var ctx = new CallContext(TimeSpan.FromSeconds(20)); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams); var container = await pool.CreateContainerAsync(createContainerParam, ctx); @@ -480,7 +480,7 @@ public class PoolSmokeTests : SmokeTestsBase await Cleanup(pool); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), lightWait); var containerId = await pool.CreateContainerAsync(createContainerParam, default); diff --git a/src/FrostFS.SDK.Tests/Smoke/SmokeClientTests.cs b/src/FrostFS.SDK.Tests/Smoke/SmokeClientTests.cs index db43eef..d774b47 100644 --- a/src/FrostFS.SDK.Tests/Smoke/SmokeClientTests.cs +++ b/src/FrostFS.SDK.Tests/Smoke/SmokeClientTests.cs @@ -81,7 +81,7 @@ public class SmokeClientTests : SmokeTestsBase var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["key1", "value1"]); @@ -126,7 +126,7 @@ public class SmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), lightWait); var containerId = await client.CreateContainerAsync(createContainerParam, default); @@ -219,7 +219,7 @@ public class SmokeClientTests : SmokeTestsBase var ctx = new CallContext(TimeSpan.FromSeconds(20)); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -300,7 +300,7 @@ public class SmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -380,7 +380,7 @@ public class SmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -436,7 +436,7 @@ public class SmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams, xheaders: ["testKey", "testValue"]); @@ -493,7 +493,7 @@ public class SmokeClientTests : SmokeTestsBase var ctx = new CallContext(TimeSpan.FromSeconds(20)); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), PrmWait.DefaultParams); var container = await client.CreateContainerAsync(createContainerParam, ctx); @@ -573,7 +573,7 @@ public class SmokeClientTests : SmokeTestsBase await Cleanup(client); var createContainerParam = new PrmContainerCreate( - new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))), + new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))), lightWait); var containerId = await client.CreateContainerAsync(createContainerParam, default); diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_default.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_default.json new file mode 100644 index 0000000..25675f2 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_default.json @@ -0,0 +1,100 @@ +{ + "name": "default CBF is 3", + "nodes": [ + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "DE" + }, + { + "key": "City", + "value": "Berlin" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "FR" + }, + { + "key": "City", + "value": "Paris" + } + ] + } + ], + "tests": [ + { + "name": "set default CBF", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "EU" + } + ], + "containerBackupFactor": 0, + "selectors": [ + { + "name": "EU", + "count": 1, + "clause": "SAME", + "attribute": "Location", + "filter": "*" + } + ], + "filters": [] + }, + "result": [ + [ + 0, + 1, + 2 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_minimal.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_minimal.json new file mode 100644 index 0000000..7553dd6 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_minimal.json @@ -0,0 +1,101 @@ +{ + "name": "Real node count multiplier is in range [1, specified CBF]", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "Country", + "value": "DE" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "Country", + "value": "DE" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "Country", + "value": "DE" + } + ] + } + ], + "tests": [ + { + "name": "select 2, CBF is 2", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "X" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "X", + "count": 2, + "clause": "SAME", + "attribute": "Country", + "filter": "*" + } + ], + "filters": [] + }, + "result": [ + [ + 0, + 1, + 2 + ] + ] + }, + { + "name": "select 3, CBF is 2", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "X" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "X", + "count": 3, + "clause": "SAME", + "attribute": "Country", + "filter": "*" + } + ], + "filters": [] + }, + "result": [ + [ + 0, + 1, + 2 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_requirements.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_requirements.json new file mode 100644 index 0000000..279d919 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_requirements.json @@ -0,0 +1,156 @@ +{ + "name": "CBF requirements", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "Attr", + "value": "Same" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "Attr", + "value": "Same" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "Attr", + "value": "Same" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "4" + }, + { + "key": "Attr", + "value": "Same" + } + ] + } + ], + "tests": [ + { + "name": "default CBF, no selector", + "policy": { + "replicas": [ + { + "count": 2 + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [] + }, + "result": [ + [ + 0, + 2, + 1, + 3 + ] + ] + }, + { + "name": "explicit CBF, no selector", + "policy": { + "replicas": [ + { + "count": 2 + } + ], + "containerBackupFactor": 3, + "selectors": [], + "filters": [] + }, + "result": [ + [ + 0, + 2, + 1, + 3 + ] + ] + }, + { + "name": "select distinct, weak CBF", + "policy": { + "replicas": [ + { + "count": 2, + "selector": "X" + } + ], + "containerBackupFactor": 3, + "selectors": [ + { + "name": "X", + "count": 2, + "clause": "DISTINCT", + "filter": "*" + } + ], + "filters": [] + }, + "result": [ + [ + 0, + 2, + 1, + 3 + ] + ] + }, + { + "name": "select same, weak CBF", + "policy": { + "replicas": [ + { + "count": 2, + "selector": "X" + } + ], + "containerBackupFactor": 3, + "selectors": [ + { + "name": "X", + "count": 2, + "clause": "SAME", + "attribute": "Attr", + "filter": "*" + } + ], + "filters": [] + }, + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_complex.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_complex.json new file mode 100644 index 0000000..c2c19c0 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_complex.json @@ -0,0 +1,345 @@ +{ + "name": "compound filter", + "nodes": [ + { + "attributes": [ + { + "key": "Storage", + "value": "SSD" + }, + { + "key": "Rating", + "value": "10" + }, + { + "key": "IntField", + "value": "100" + }, + { + "key": "Param", + "value": "Value1" + } + ] + } + ], + "tests": [ + { + "name": "good", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "StorageSSD", + "key": "Storage", + "op": "EQ", + "value": "SSD", + "filters": [] + }, + { + "name": "GoodRating", + "key": "Rating", + "op": "GE", + "value": "4", + "filters": [] + }, + { + "name": "Main", + "op": "AND", + "filters": [ + { + "name": "StorageSSD", + "op": "Unspecified", + "filters": [] + }, + { + "name": "", + "key": "IntField", + "op": "LT", + "value": "123", + "filters": [] + }, + { + "name": "GoodRating", + "op": "Unspecified", + "filters": [] + }, + { + "op": "OR", + "filters": [ + { + "key": "Param", + "op": "EQ", + "value": "Value1", + "filters": [] + }, + { + "key": "Param", + "op": "EQ", + "value": "Value2", + "filters": [] + } + ] + } + ] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + { + "name": "bad storage type", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "StorageSSD", + "key": "Storage", + "op": "EQ", + "value": "HDD", + "filters": [] + }, + { + "name": "GoodRating", + "key": "Rating", + "op": "GE", + "value": "4", + "filters": [] + }, + { + "name": "Main", + "op": "AND", + "filters": [ + { + "name": "StorageSSD", + "op": "Unspecified", + "filters": [] + }, + { + "name": "", + "key": "IntField", + "op": "LT", + "value": "123", + "filters": [] + }, + { + "name": "GoodRating", + "op": "Unspecified", + "filters": [] + }, + { + "name": "", + "op": "OR", + "filters": [ + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value1", + "filters": [] + }, + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value2", + "filters": [] + } + ] + } + ] + } + ] + } + }, + { + "name": "bad rating", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "StorageSSD", + "key": "Storage", + "op": "EQ", + "value": "SSD", + "filters": [] + }, + { + "name": "GoodRating", + "key": "Rating", + "op": "GE", + "value": "15", + "filters": [] + }, + { + "name": "Main", + "op": "AND", + "filters": [ + { + "name": "StorageSSD", + "op": "Unspecified", + "filters": [] + }, + { + "name": "", + "key": "IntField", + "op": "LT", + "value": "123", + "filters": [] + }, + { + "name": "GoodRating", + "op": "Unspecified", + "filters": [] + }, + { + "name": "", + "op": "OR", + "filters": [ + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value1", + "filters": [] + }, + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value2", + "filters": [] + } + ] + } + ] + } + ] + } + }, + { + "name": "bad param", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "StorageSSD", + "key": "Storage", + "op": "EQ", + "value": "SSD", + "filters": [] + }, + { + "name": "GoodRating", + "key": "Rating", + "op": "GE", + "value": "4", + "filters": [] + }, + { + "name": "Main", + "op": "AND", + "filters": [ + { + "name": "StorageSSD", + "op": "Unspecified", + "filters": [] + }, + { + "name": "", + "key": "IntField", + "op": "LT", + "value": "123", + "filters": [] + }, + { + "name": "GoodRating", + "op": "Unspecified", + "filters": [] + }, + { + "name": "", + "op": "OR", + "filters": [ + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value0", + "filters": [] + }, + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value2", + "filters": [] + } + ] + } + ] + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_invalid_integer.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_invalid_integer.json new file mode 100644 index 0000000..16cdb9e --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_invalid_integer.json @@ -0,0 +1,81 @@ +{ + "name": "invalid integer field", + "nodes": [ + { + "attributes": [ + { + "key": "IntegerField", + "value": "true" + } + ] + }, + { + "attributes": [ + { + "key": "IntegerField", + "value": "str" + } + ] + } + ], + "tests": [ + { + "name": "empty string is not casted to 0", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "IntegerField", + "op": "LE", + "value": "8", + "filters": [] + } + ] + } + }, + { + "name": "non-empty string is not casted to a number", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "IntegerField", + "op": "GE", + "value": "0", + "filters": [] + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_simple.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_simple.json new file mode 100644 index 0000000..43142d4 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_simple.json @@ -0,0 +1,397 @@ +{ + "name": "single-op filters", + "nodes": [ + { + "attributes": [ + { + "key": "Rating", + "value": "4" + }, + { + "key": "Country", + "value": "Germany" + } + ] + } + ], + "tests": [ + { + "name": "GE true", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "GE", + "value": "4", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + { + "name": "GE false", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "GE", + "value": "5", + "filters": [] + } + ] + } + }, + { + "name": "GT true", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "GT", + "value": "3", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + { + "name": "GT false", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "GT", + "value": "4", + "filters": [] + } + ] + } + }, + { + "name": "LE true", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "LE", + "value": "4", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + { + "name": "LE false", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "LE", + "value": "3", + "filters": [] + } + ] + } + }, + { + "name": "LT true", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "LT", + "value": "5", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + { + "name": "LT false", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "LT", + "value": "4", + "filters": [] + } + ] + } + }, + { + "name": "EQ true", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Country", + "op": "EQ", + "value": "Germany", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + { + "name": "EQ false", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Country", + "op": "EQ", + "value": "China", + "filters": [] + } + ] + } + }, + { + "name": "NE true", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Country", + "op": "NE", + "value": "France", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + { + "name": "NE false", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Country", + "op": "NE", + "value": "Germany", + "filters": [] + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/hrw_sort.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/hrw_sort.json new file mode 100644 index 0000000..a6dc75c --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/hrw_sort.json @@ -0,0 +1,225 @@ +{ + "name": "HRW ordering", + "nodes": [ + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + }, + { + "key": "Price", + "value": "2" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + }, + { + "key": "Price", + "value": "4" + }, + { + "key": "Capacity", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "France" + }, + { + "key": "Price", + "value": "3" + }, + { + "key": "Capacity", + "value": "10" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Price", + "value": "2" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Price", + "value": "1" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "France" + }, + { + "key": "Price", + "value": "100" + }, + { + "key": "Capacity", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "France" + }, + { + "key": "Price", + "value": "7" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Price", + "value": "2" + }, + { + "key": "Capacity", + "value": "1" + } + ] + } + ], + "tests": [ + { + "name":"select 3 nodes in 3 distinct countries, same placement", + "policy": { + "containerBackupFactor": 1, + "filters": [], + "replicas": [ + { + "count": 1, + "selector": "Main" + } + ], + "selectors": [ + { + "attribute": "Country", + "clause": "DISTINCT", + "count": 3, + "filter": "*", + "name": "Main" + } + ] + }, + "pivot": "Y29udGFpbmVySUQ=", + "result": [[ + 5, + 0, + 7 + ]], + "placement": { + "pivot": "b2JqZWN0SUQ=", + "result": [[ + 5, + 0, + 7 + ]] + } + }, + { + "name":"select 6 nodes in 3 distinct countries, different placement", + "policy": { + "containerBackupFactor": 2, + "filters": [ + ], + "replicas": [ + { + "count": 1, + "selector": "Main" + } + ], + "selectors": [ + { + "attribute": "Country", + "clause": "DISTINCT", + "count": 3, + "filter": "*", + "name": "Main" + } + ] + }, + "pivot": "Y29udGFpbmVySUQ=", + "result": [[ + 5, + 4, + 0, + 1, + 7, + 2]], + + "placement": { + "pivot": "b2JqZWN0SUQ=", + "result": [[ + 5, + 4, + 0, + 7, + 2, + 1]] + } + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/issue213.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/issue213.json new file mode 100644 index 0000000..4430297 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/issue213.json @@ -0,0 +1,107 @@ +{ + "name": "unnamed selector (nspcc-dev/neofs-api-go#213)", + "nodes": [ + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "Russia" + }, + { + "key": "City", + "value": "Moscow" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "Russia" + }, + { + "key": "City", + "value": "Saint-Petersburg" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "Sweden" + }, + { + "key": "City", + "value": "Stockholm" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "Finalnd" + }, + { + "key": "City", + "value": "Helsinki" + } + ] + } + ], + "tests": [ + { + "name": "test", + "policy": { + "replicas": [ + { + "count": 4 + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "", + "count": 4, + "clause": "DISTINCT", + "filter": "LOC_EU" + } + ], + "filters": [ + { + "name": "LOC_EU", + "key": "Location", + "op": "EQ", + "value": "Europe", + "filters": [] + } + ] + }, + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/many_selects.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/many_selects.json new file mode 100644 index 0000000..e76a441 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/many_selects.json @@ -0,0 +1,279 @@ +{ + "name": "single-op filters", + "nodes": [ + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Rating", + "value": "1" + }, + { + "key": "City", + "value": "SPB" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + }, + { + "key": "Rating", + "value": "5" + }, + { + "key": "City", + "value": "Berlin" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Rating", + "value": "6" + }, + { + "key": "City", + "value": "Moscow" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "France" + }, + { + "key": "Rating", + "value": "4" + }, + { + "key": "City", + "value": "Paris" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "France" + }, + { + "key": "Rating", + "value": "1" + }, + { + "key": "City", + "value": "Lyon" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Rating", + "value": "5" + }, + { + "key": "City", + "value": "SPB" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Rating", + "value": "7" + }, + { + "key": "City", + "value": "Moscow" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + }, + { + "key": "Rating", + "value": "3" + }, + { + "key": "City", + "value": "Darmstadt" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + }, + { + "key": "Rating", + "value": "7" + }, + { + "key": "City", + "value": "Frankfurt" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Rating", + "value": "9" + }, + { + "key": "City", + "value": "SPB" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Rating", + "value": "9" + }, + { + "key": "City", + "value": "SPB" + } + ] + } + ], + "tests": [ + { + "name": "Select", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "SameRU" + }, + { + "count": 1, + "selector": "DistinctRU" + }, + { + "count": 1, + "selector": "Good" + }, + { + "count": 1, + "selector": "Main" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "SameRU", + "count": 2, + "clause": "SAME", + "attribute": "City", + "filter": "FromRU" + }, + { + "name": "DistinctRU", + "count": 2, + "clause": "DISTINCT", + "attribute": "City", + "filter": "FromRU" + }, + { + "name": "Good", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "Good" + }, + { + "name": "Main", + "count": 3, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "*" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia" + }, + { + "name": "Good", + "key": "Rating", + "op": "GE", + "value": "4" + } + ] + }, + "result": [ + [ + 0, + 5, + 9, + 10 + ], + [ + 2, + 6, + 0, + 5 + ], + [ + 1, + 8, + 2, + 5 + ], + [ + 3, + 4, + 1, + 7, + 0, + 2 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/multiple_rep.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/multiple_rep.json new file mode 100644 index 0000000..e01ad62 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/multiple_rep.json @@ -0,0 +1,93 @@ +{ + "name": "multiple replicas (#215)", + "nodes": [ + { + "attributes": [ + { + "key": "City", + "value": "Saint-Petersburg" + } + ] + }, + { + "attributes": [ + { + "key": "City", + "value": "Moscow" + } + ] + }, + { + "attributes": [ + { + "key": "City", + "value": "Berlin" + } + ] + }, + { + "attributes": [ + { + "key": "City", + "value": "Paris" + } + ] + } + ], + "tests": [ + { + "name": "test", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "LOC_SPB_PLACE" + }, + { + "count": 1, + "selector": "LOC_MSK_PLACE" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "LOC_SPB_PLACE", + "count": 1, + "clause": "UNSPECIFIED", + "filter": "LOC_SPB" + }, + { + "name": "LOC_MSK_PLACE", + "count": 1, + "clause": "UNSPECIFIED", + "filter": "LOC_MSK" + } + ], + "filters": [ + { + "name": "LOC_SPB", + "key": "City", + "op": "EQ", + "value": "Saint-Petersburg", + "filters": [] + }, + { + "name": "LOC_MSK", + "key": "City", + "op": "EQ", + "value": "Moscow", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ], + [ + 1 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/multiple_rep_asymmetric.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/multiple_rep_asymmetric.json new file mode 100644 index 0000000..55390b2 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/multiple_rep_asymmetric.json @@ -0,0 +1,328 @@ +{ + "name": "multiple REP, asymmetric", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "0" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "4" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "5" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "6" + }, + { + "key": "Continent", + "value": "NA" + }, + { + "key": "City", + "value": "NewYork" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "7" + }, + { + "key": "Continent", + "value": "AF" + }, + { + "key": "City", + "value": "Cairo" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "8" + }, + { + "key": "Continent", + "value": "AF" + }, + { + "key": "City", + "value": "Cairo" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "9" + }, + { + "key": "Continent", + "value": "SA" + }, + { + "key": "City", + "value": "Lima" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "10" + }, + { + "key": "Continent", + "value": "AF" + }, + { + "key": "City", + "value": "Cairo" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "11" + }, + { + "key": "Continent", + "value": "NA" + }, + { + "key": "City", + "value": "NewYork" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "12" + }, + { + "key": "Continent", + "value": "NA" + }, + { + "key": "City", + "value": "LosAngeles" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "13" + }, + { + "key": "Continent", + "value": "SA" + }, + { + "key": "City", + "value": "Lima" + } + ] + } + ], + "tests": [ + { + "name": "test", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "SPB" + }, + { + "count": 2, + "selector": "Americas" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "SPB", + "count": 1, + "clause": "SAME", + "attribute": "City", + "filter": "SPBSSD" + }, + { + "name": "Americas", + "count": 2, + "clause": "DISTINCT", + "attribute": "City", + "filter": "Americas" + } + ], + "filters": [ + { + "name": "SPBSSD", + "op": "AND", + "filters": [ + { + "name": "", + "key": "Country", + "op": "EQ", + "value": "RU", + "filters": [] + }, + { + "name": "", + "key": "City", + "op": "EQ", + "value": "St.Petersburg", + "filters": [] + }, + { + "name": "", + "key": "SSD", + "op": "EQ", + "value": "1", + "filters": [] + } + ] + }, + { + "name": "Americas", + "op": "OR", + "filters": [ + { + "name": "", + "key": "Continent", + "op": "EQ", + "value": "NA", + "filters": [] + }, + { + "name": "", + "key": "Continent", + "op": "EQ", + "value": "SA", + "filters": [] + } + ] + } + ] + }, + "result": [ + [ + 1, + 4 + ], + [ + 8, + 12, + 5, + 10 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/non_strict.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/non_strict.json new file mode 100644 index 0000000..d8e010e --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/non_strict.json @@ -0,0 +1,97 @@ +{ + "name": "non-strict selections", + "comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.", + "nodes": [ + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + } + ] + }, + { + "attributes": [] + } + ], + "tests": [ + { + "name": "not enough nodes (backup factor)", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromRU" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + { + "name": "not enough nodes (buckets)", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromRU" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [] + } + ] + }, + "result": [ + [ + 0 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/rep_only.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/rep_only.json new file mode 100644 index 0000000..7af5eb0 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/rep_only.json @@ -0,0 +1,113 @@ +{ + "name": "REP X", + "nodes": [ + { + "publicKey": "", + "addresses": [], + "attributes": [ + { + "key": "City", + "value": "Saint-Petersburg" + } + ], + "state": "Unspecified" + }, + { + "publicKey": "", + "addresses": [], + "attributes": [ + { + "key": "City", + "value": "Moscow" + } + ], + "state": "Unspecified" + }, + { + "publicKey": "", + "addresses": [], + "attributes": [ + { + "key": "City", + "value": "Berlin" + } + ], + "state": "Unspecified" + }, + { + "publicKey": "", + "addresses": [], + "attributes": [ + { + "key": "City", + "value": "Paris" + } + ], + "state": "Unspecified" + } + ], + "tests": [ + { + "name": "REP 1", + "policy": { + "replicas": [ + { + "count": 1 + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [] + }, + "result": [ + [ + 0, + 1, + 2 + ] + ] + }, + { + "name": "REP 3", + "policy": { + "replicas": [ + { + "count": 3 + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [] + }, + "result": [ + [ + 0, + 3, + 1, + 2 + ] + ] + }, + { + "name": "REP 5", + "policy": { + "replicas": [ + { + "count": 5 + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [] + }, + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/select_no_attribute.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/select_no_attribute.json new file mode 100644 index 0000000..6a49d68 --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/select_no_attribute.json @@ -0,0 +1,116 @@ +{ + "name": "select with unspecified attribute", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "0" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "4" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + }, + { + "key": "SSD", + "value": "1" + } + ] + } + ], + "tests": [ + { + "name": "test", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "X" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "X", + "count": 4, + "clause": "DISTINCT", + "filter": "*" + } + ], + "filters": [] + }, + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/TestData/PlacementTests/selector_invalid.json b/src/FrostFS.SDK.Tests/TestData/PlacementTests/selector_invalid.json new file mode 100644 index 0000000..08e1a8d --- /dev/null +++ b/src/FrostFS.SDK.Tests/TestData/PlacementTests/selector_invalid.json @@ -0,0 +1,87 @@ +{ + "name": "invalid selections", + "nodes": [ + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + } + ] + }, + { + "attributes": [] + } + ], + "tests": [ + { + "name": "missing filter", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "MyStore", + "count": 1, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromNL" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [] + } + ] + }, + "error": "filter not found" + }, + { + "name": "not enough nodes (filter results in empty set)", + "policy": { + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromMoon" + } + ], + "filters": [ + { + "name": "FromMoon", + "key": "Country", + "op": "EQ", + "value": "Moon", + "filters": [] + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/FrostFS.SDK.Tests/Unit/ContainerTestsBase.cs b/src/FrostFS.SDK.Tests/Unit/ContainerTestsBase.cs index b6cd61c..60579f6 100644 --- a/src/FrostFS.SDK.Tests/Unit/ContainerTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Unit/ContainerTestsBase.cs @@ -21,7 +21,7 @@ public abstract class ContainerTestsBase Mocker = new ContainerMocker(key) { - PlacementPolicy = new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), + PlacementPolicy = new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1)), Version = new FrostFsVersion(2, 13), ContainerGuid = Guid.NewGuid() }; diff --git a/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs b/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs index cff9064..18e3981 100644 --- a/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Unit/ObjectTestsBase.cs @@ -30,7 +30,7 @@ public abstract class ObjectTestsBase Mocker = new ObjectMocker(key) { - PlacementPolicy = new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), + PlacementPolicy = new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1)), Version = new FrostFsVersion(2, 13), ContainerGuid = Guid.NewGuid() }; diff --git a/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs b/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs index e578f90..98d3ee9 100644 --- a/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs +++ b/src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs @@ -1,161 +1,275 @@ +using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; using FrostFS.SDK.Client.Models.Netmap.Placement; +using Xunit.Abstractions; + namespace FrostFS.SDK.Tests.Unit; [SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")] -public class PlacementVectorTests +public class PlacementVectorTests(ITestOutputHelper testOutputHelper) { - Dictionary[] attribs; - - public PlacementVectorTests() + private static readonly JsonSerializerOptions serializeOptions = new() { - var attribs1 = new Dictionary - { - {"Country", "Germany" }, - {"Price", "2" }, - {"Capacity", "10000"} - }; + PropertyNameCaseInsensitive = true + }; - var attribs2 = new Dictionary - { - {"Country", "Germany" }, - {"Price", "4" }, - {"Capacity", "1"} - }; - - var attribs3 = new Dictionary - { - {"Country", "France" }, - {"Price", "3" }, - {"Capacity", "10"} - }; - - var attribs4 = new Dictionary - { - {"Country", "Russia" }, - {"Price", "2" }, - {"Capacity", "10000"} - }; - - var attribs5 = new Dictionary - { - {"Country", "Russia" }, - {"Price", "1" }, - {"Capacity", "10000"} - }; - - var attribs6 = new Dictionary - { - {"Country", "Russia" }, - {"Capacity", "10000"} - }; - var attribs7 = new Dictionary - { - {"Country", "France" }, - {"Price", "100" }, - {"Capacity", "10000"} - }; - var attribs8 = new Dictionary - { - {"Country", "France" }, - {"Price", "7" }, - {"Capacity", "10000"} - }; - var attribs9 = new Dictionary - { - {"Country", "Russia" }, - {"Price", "2" }, - {"Capacity", "1"} - }; - - attribs = [attribs1, attribs2, attribs3, attribs4, attribs5, attribs6, attribs7, attribs8, attribs9]; - } + private readonly ITestOutputHelper _testOutputHelper = testOutputHelper; [Fact] - public void PlacementVectorTest() + public void PlacementTest() { + var path = ".\\..\\..\\..\\TestData\\PlacementTests"; + Assert.True(Directory.Exists(path)); + + var files = Directory.GetFiles(path); + FrostFsVersion v = new(2, 13); var addresses = new string[] { "localhost", "server1" }; - var key1 = new byte[] { 1 }; - var key2 = new byte[] { 2 }; - var key3 = new byte[] { 3 }; - var nodes = new List + foreach (var file in files.Where(f => f.EndsWith(".json", StringComparison.OrdinalIgnoreCase))) { - new(v, NodeState.Online, addresses.AsReadOnly(), attribs[5].AsReadOnly(), key1), - new(v, NodeState.Online, addresses.AsReadOnly(), attribs[0].AsReadOnly(), key2), - new(v, NodeState.Online, addresses.AsReadOnly(), attribs[8].AsReadOnly(), key3) - }; + //if (!file.EndsWith("selector_invalid.json")) + // continue; - var netmap = new FrostFsNetmapSnapshot(100, nodes.AsReadOnly()); + var fileName = file[(file.LastIndexOf("..\\") + 3)..]; + _testOutputHelper.WriteLine($"Open file {fileName}"); - var arg = new FrostFsNodeInfo[1][]; - var pivot = "objectID"u8.ToArray(); + var str = File.ReadAllText(file); + Assert.False(string.IsNullOrEmpty(str)); - arg[0] = [.. nodes]; - var result = netmap.PlacementVectors(arg, pivot); + var testCase = JsonSerializer.Deserialize(str, serializeOptions); - Assert.Single(result); - Assert.Equal(3, result[0].Length); - Assert.Equal(key1, result[0][0].PublicKey); - Assert.Equal(key2, result[0][1].PublicKey); - Assert.Equal(key3, result[0][2].PublicKey); - } + Assert.NotNull(testCase); + Assert.NotNull(testCase.Nodes); + Assert.True(testCase.Nodes.Length > 0); - [Fact] - public void TestPlacementPolicyUnique() - { - FrostFsVersion version = new(2, 13); - var p = new FrostFsPlacementPolicy(true, [new FrostFsReplica(1, "S"), new FrostFsReplica(1, "S")]) - { - BackupFactor = 2 - }; - p.Selectors.Add(new FrostFsSelector("S") - { - Attribute = "City", - Count = 1, - Filter = "*", - Clause = (int)FrostFsClause.Same - }); + _testOutputHelper.WriteLine($"Test case: \"{testCase.Name}\""); - List nodes = []; + var nodes = testCase.Nodes + .Select(n => new FrostFsNodeInfo(v, + n.State, + addresses.AsReadOnly(), + n.Attributes?.ToDictionary(x => x.Key, x => x.Value) ?? [], + n.PublicKeyBytes + ) + ) + .ToArray() + .AsReadOnly(); - var cities = new string[] { "Moscow", "Berlin", "Shenzhen" }; - for (int i = 0; i < 3; i++) - { - for (int j = 0; j < 3; j++) + var netmap = new FrostFsNetmapSnapshot(100, nodes); + + Assert.NotNull(testCase.Tests); + + foreach (var test in testCase.Tests) { - var attr = new Dictionary { { "City", cities[i] } }; - var key = new byte[] { (byte)(i * 4 + j) }; - var node = new FrostFsNodeInfo(version, NodeState.Online, [], attr, key); + _testOutputHelper.WriteLine($"Start test \"{test.Name}\""); - nodes.Add(node); - } - } + var policy = new FrostFsPlacementPolicy( + test.Policy!.Unique, + test.Policy.ContainerBackupFactor, + new Collection(test.Policy.Selectors?.Select(s => s.Selector).ToList() ?? []), + new Collection(test.Policy.Filters?.Select(f => f.Filter).ToList() ?? []), + test.Policy.Replicas?.Select(r => new FrostFsReplica(r.Count, r.Selector)).ToArray() ?? [] + ); - var netMap = new FrostFsNetmapSnapshot(100, nodes.AsReadOnly()); - - var v = netMap.ContainerNodes(p, null); - - Assert.Equal(2, v.Length); - Assert.Equal(2, v[0].Length); - Assert.Equal(2, v[1].Length); - - - for (int i = 0; i < v.Length; i++) - { - foreach (var ni in v[i]) - { - for (int j = 0; j < i; j++) + try { - foreach (var nj in v[j]) + var result = netmap.ContainerNodes(policy, test.PivotBytes); + + if (test.Result == null) { - Assert.NotEqual(ni.Hash, nj.Hash); + if (!string.IsNullOrEmpty(test.Error)) + { + Assert.Fail("Error is expected but has not been thrown"); + } + else + { + Assert.NotNull(test.Policy?.Replicas); + Assert.Equal(result.Length, test.Policy.Replicas.Length); + + for (int i = 0; i < result.Length; i++) + { + Assert.Empty(result[i]); + } + } + } + else + { + Assert.Equal(test.Result.Length, result.Length); + + for (var i = 0; i < test.Result.Length; i++) + { + Assert.Equal(test.Result[i].Length, result[i].Length); + for (var j = 0; j < test.Result[i].Length; j++) + { + CompareNodes(nodes[test.Result[i][j]].Attributes, result[i][j]); + } + } + + if (test.Placement?.Result != null && test.Placement.PivotBytes != null) + { + var placementResult = netmap.PlacementVectors(result, test.Placement.PivotBytes); + + Assert.Equal(test.Placement.Result.Length, placementResult.Length); + + for (int i = 0; i < placementResult.Length; i++) + { + Assert.Equal(test.Placement.Result[i].Length, placementResult[i].Length); + for (int j = 0; j < placementResult[i].Length; j++) + { + CompareNodes(nodes[test.Placement.Result[i][j]].Attributes, placementResult[i][j]); + } + } + } } } + catch (Exception ex) + { + if (!string.IsNullOrEmpty(test.Error)) + { + Assert.Contains(test.Error, ex.Message, StringComparison.InvariantCulture); + } + else + { + throw; + } + } + + _testOutputHelper.WriteLine($"Done"); } } } + + + private static void CompareNodes(IReadOnlyDictionary attrs, FrostFsNodeInfo nodeInfo) + { + Assert.Equal(attrs.Count, nodeInfo.Attributes.Count); + Assert.True(attrs.OrderBy(k => k.Key).SequenceEqual(nodeInfo.Attributes.OrderBy(x => x.Key))); + } +} + +public class TestCase +{ + public string? Name { get; set; } + + public Node[]? Nodes { get; set; } + + public TestData[]? Tests { get; set; } +} + + + +public class Node +{ + [JsonPropertyName("attributes")] + public KeyValuePair[]? Attributes { get; set; } + + public string? PublicKey { get; set; } + + internal byte[]? PublicKeyBytes => string.IsNullOrEmpty(PublicKey) ? [] : Convert.FromBase64String(PublicKey); + + public string[]? Addresses { get; set; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public NodeState State { get; set; } = NodeState.Online; +} + +public class TestData +{ + public string? Name { get; set; } + + public PolicyDto? Policy { get; set; } + + public string? Pivot { get; set; } + + public int[][]? Result { get; set; } + + public string? Error { get; set; } + + internal byte[]? PivotBytes => Pivot != null ? Convert.FromBase64String(Pivot) : null; + + public ResultData? Placement { get; set; } +} + +public class PolicyDto +{ + public bool Unique { get; set; } + + public uint ContainerBackupFactor { get; set; } + + public FilterDto[]? Filters { get; set; } + + public ReplicaDto[]? Replicas { get; set; } + + public SelectorDto[]? Selectors { get; set; } +} + +public class SelectorDto() +{ + public uint Count { get; set; } + + public string? Name { get; set; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public ClauseValues Clause { get; set; } + + public string? Attribute { get; set; } + + public string? Filter { get; set; } + + public FrostFsSelector Selector => new(Name ?? string.Empty) + { + Count = Count, + Clause = (int)Clause, + Filter = Filter, + Attribute = Attribute + }; +} + +public class FilterDto +{ + public string? Name { get; set; } + + public string? Key { get; set; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public Operation Op { get; set; } + + public string? Value { get; set; } + + public FilterDto[]? Filters { get; set; } + + public FrostFsFilter Filter => new( + Name ?? string.Empty, + Key ?? string.Empty, + (int)Op, + Value ?? string.Empty, + Filters != null ? Filters.Select(f => f.Filter).ToArray() : []); +} + +public class ReplicaDto +{ + public int Count { get; set; } + + public string? Selector { get; set; } +} + +public class ResultData +{ + public string? Pivot { get; set; } + + public int[][]? Result { get; set; } + + internal byte[]? PivotBytes => Pivot != null ? Convert.FromBase64String(Pivot) : null; +} + +public enum ClauseValues +{ + UNSPECIFIED = 0, + SAME, + DISTINCT } diff --git a/src/FrostFS.SDK.Tests/Unit/SessionTestsBase.cs b/src/FrostFS.SDK.Tests/Unit/SessionTestsBase.cs index 3706a91..eef8a45 100644 --- a/src/FrostFS.SDK.Tests/Unit/SessionTestsBase.cs +++ b/src/FrostFS.SDK.Tests/Unit/SessionTestsBase.cs @@ -30,7 +30,7 @@ public abstract class SessionTestsBase Mocker = new SessionMocker(key) { - PlacementPolicy = new FrostFsPlacementPolicy(true, new FrostFsReplica(1)), + PlacementPolicy = new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1)), Version = new FrostFsVersion(2, 13) }; }