Compare commits
5 commits
feature/ad
...
master
Author | SHA1 | Date | |
---|---|---|---|
43e300c773 | |||
568bdc67e8 | |||
8637515869 | |||
db9b93b2e6 | |||
543247e4d9 |
53 changed files with 4306 additions and 77 deletions
|
@ -16,6 +16,9 @@ public static class PlacementPolicyMapper
|
|||
|
||||
return new FrostFsPlacementPolicy(
|
||||
placementPolicy.Unique,
|
||||
placementPolicy.ContainerBackupFactor,
|
||||
new System.Collections.ObjectModel.Collection<FrostFsSelector>(placementPolicy.Selectors.Select(selector => selector.ToModel()).ToList()),
|
||||
new System.Collections.ObjectModel.Collection<FrostFsFilter>(placementPolicy.Filters.Select(filter => filter.ToModel()).ToList()),
|
||||
placementPolicy.Replicas.Select(replica => replica.ToModel()).ToArray()
|
||||
);
|
||||
}
|
||||
|
|
|
@ -1,10 +1,11 @@
|
|||
using System;
|
||||
using System.Linq;
|
||||
|
||||
using FrostFS.Netmap;
|
||||
|
||||
namespace FrostFS.SDK.Client;
|
||||
|
||||
public static class ReplicaMapper
|
||||
public static class PolicyMapper
|
||||
{
|
||||
public static Replica ToMessage(this FrostFsReplica replica)
|
||||
{
|
||||
|
@ -24,4 +25,67 @@ public static class ReplicaMapper
|
|||
|
||||
return new FrostFsReplica((int)replica.Count, replica.Selector);
|
||||
}
|
||||
|
||||
public static Selector ToMessage(this FrostFsSelector selector)
|
||||
{
|
||||
if (selector is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(selector));
|
||||
}
|
||||
|
||||
return new Selector
|
||||
{
|
||||
Name = selector.Name,
|
||||
Count = selector.Count,
|
||||
Clause = (Clause)selector.Clause,
|
||||
Attribute = selector.Attribute,
|
||||
Filter = selector.Filter
|
||||
};
|
||||
}
|
||||
|
||||
public static FrostFsSelector ToModel(this Selector selector)
|
||||
{
|
||||
if (selector is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(selector));
|
||||
}
|
||||
|
||||
return new FrostFsSelector(selector.Name)
|
||||
{
|
||||
Count = selector.Count,
|
||||
Clause = (int)selector.Clause,
|
||||
Attribute = selector.Attribute,
|
||||
Filter = selector.Filter
|
||||
};
|
||||
}
|
||||
|
||||
public static Filter ToMessage(this FrostFsFilter filter)
|
||||
{
|
||||
if (filter is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(filter));
|
||||
}
|
||||
|
||||
var message = new Filter
|
||||
{
|
||||
Name = filter.Name,
|
||||
Key = filter.Key,
|
||||
Op = (Operation)filter.Operation,
|
||||
Value = filter.Value,
|
||||
};
|
||||
|
||||
message.Filters.AddRange(filter.Filters.Select(f => f.ToMessage()));
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
public static FrostFsFilter ToModel(this Filter filter)
|
||||
{
|
||||
if (filter is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(filter));
|
||||
}
|
||||
|
||||
return new FrostFsFilter(filter.Name, filter.Key, (int)filter.Op, filter.Value, filter.Filters.Select(f => f.ToModel()).ToArray());
|
||||
}
|
||||
}
|
10
src/FrostFS.SDK.Client/Models/Netmap/FrostFsFilter.cs
Normal file
10
src/FrostFS.SDK.Client/Models/Netmap/FrostFsFilter.cs
Normal file
|
@ -0,0 +1,10 @@
|
|||
namespace FrostFS.SDK;
|
||||
|
||||
public class FrostFsFilter(string name, string key, int operation, string value, FrostFsFilter[] filters) : IFrostFsFilter
|
||||
{
|
||||
public string Name { get; } = name;
|
||||
public string Key { get; } = key;
|
||||
public int Operation { get; } = operation;
|
||||
public string Value { get; } = value;
|
||||
public FrostFsFilter[] Filters { get; } = filters;
|
||||
}
|
|
@ -1,4 +1,10 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
using FrostFS.SDK.Client;
|
||||
using FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
using FrostFS.SDK.Cryptography;
|
||||
|
||||
namespace FrostFS.SDK;
|
||||
|
||||
|
@ -7,4 +13,234 @@ public class FrostFsNetmapSnapshot(ulong epoch, IReadOnlyList<FrostFsNodeInfo> n
|
|||
public ulong Epoch { get; private set; } = epoch;
|
||||
|
||||
public IReadOnlyList<FrostFsNodeInfo> NodeInfoCollection { get; private set; } = nodeInfoCollection;
|
||||
}
|
||||
|
||||
internal static INormalizer NewReverseMinNorm(double minV)
|
||||
{
|
||||
return new ReverseMinNorm { min = minV };
|
||||
}
|
||||
|
||||
// newSigmoidNorm returns a normalizer which
|
||||
// normalize values in range of 0.0 to 1.0 to a scaled sigmoid.
|
||||
internal static INormalizer NewSigmoidNorm(double scale)
|
||||
{
|
||||
return new SigmoidNorm(scale);
|
||||
}
|
||||
|
||||
// PlacementVectors sorts container nodes returned by ContainerNodes method
|
||||
// and returns placement vectors for the entity identified by the given pivot.
|
||||
// For example, in order to build node list to store the object, binary-encoded
|
||||
// object identifier can be used as pivot. Result is deterministic for
|
||||
// the fixed NetMap and parameters.
|
||||
public FrostFsNodeInfo[][] PlacementVectors(FrostFsNodeInfo[][] vectors, byte[] pivot)
|
||||
{
|
||||
if (vectors is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(vectors));
|
||||
}
|
||||
|
||||
using var murmur3 = new Murmur3(0);
|
||||
var hash = murmur3.GetCheckSum64(pivot);
|
||||
|
||||
var wf = Tools.DefaultWeightFunc(NodeInfoCollection.ToArray());
|
||||
|
||||
var result = new FrostFsNodeInfo[vectors.Length][];
|
||||
var maxSize = vectors.Max(x => x.Length);
|
||||
|
||||
var spanWeigths = new double[maxSize];
|
||||
|
||||
for (int i = 0; i < vectors.Length; i++)
|
||||
{
|
||||
result[i] = new FrostFsNodeInfo[vectors[i].Length];
|
||||
|
||||
for (int j = 0; j < vectors[i].Length; j++)
|
||||
{
|
||||
result[i][j] = vectors[i][j];
|
||||
}
|
||||
|
||||
Tools.AppendWeightsTo(result[i], wf, ref spanWeigths);
|
||||
|
||||
result[i] = Tools.SortHasherSliceByWeightValue(result[i].ToList<FrostFsNodeInfo>(), spanWeigths, hash).ToArray();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// SelectFilterNodes returns a two-dimensional list of nodes as a result of applying the
|
||||
// given SelectFilterExpr to the NetMap.
|
||||
// If the SelectFilterExpr contains only filters, the result contains a single row with the
|
||||
// result of the last filter application.
|
||||
// If the SelectFilterExpr contains only selectors, the result contains the selection rows
|
||||
// of the last select application.
|
||||
List<List<FrostFsNodeInfo>> SelectFilterNodes(SelectFilterExpr expr)
|
||||
{
|
||||
var policy = new FrostFsPlacementPolicy(false, expr.Cbf, [expr.Selector], expr.Filters);
|
||||
|
||||
var ctx = new Context(this)
|
||||
{
|
||||
Cbf = expr.Cbf
|
||||
};
|
||||
|
||||
ctx.ProcessFilters(policy);
|
||||
ctx.ProcessSelectors(policy);
|
||||
|
||||
var ret = new List<List<FrostFsNodeInfo>>();
|
||||
|
||||
if (expr.Selector == null)
|
||||
{
|
||||
var lastFilter = expr.Filters[^1];
|
||||
|
||||
var subCollestion = new List<FrostFsNodeInfo>();
|
||||
ret.Add(subCollestion);
|
||||
|
||||
foreach (var nodeInfo in NodeInfoCollection)
|
||||
{
|
||||
if (ctx.Match(ctx.ProcessedFilters[lastFilter.Name], nodeInfo))
|
||||
{
|
||||
subCollestion.Add(nodeInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (expr.Selector.Name != null)
|
||||
{
|
||||
var sel = ctx.GetSelection(ctx.ProcessedSelectors[expr.Selector.Name]);
|
||||
|
||||
foreach (var ns in sel)
|
||||
{
|
||||
var subCollestion = new List<FrostFsNodeInfo>();
|
||||
ret.Add(subCollestion);
|
||||
foreach (var n in ns)
|
||||
{
|
||||
subCollestion.Add(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
internal static Func<FrostFsNodeInfo, double> NewWeightFunc(INormalizer capNorm, INormalizer priceNorm)
|
||||
{
|
||||
return new Func<FrostFsNodeInfo, double>((FrostFsNodeInfo nodeInfo) =>
|
||||
{
|
||||
return capNorm.Normalize(nodeInfo.GetCapacity()) * priceNorm.Normalize(nodeInfo.Price);
|
||||
});
|
||||
}
|
||||
|
||||
private static FrostFsNodeInfo[] FlattenNodes(List<List<FrostFsNodeInfo>> nodes)
|
||||
{
|
||||
int sz = 0;
|
||||
foreach (var ns in nodes)
|
||||
{
|
||||
sz += ns.Count;
|
||||
}
|
||||
|
||||
var result = new FrostFsNodeInfo[sz];
|
||||
|
||||
int i = 0;
|
||||
foreach (var ns in nodes)
|
||||
{
|
||||
foreach (var n in ns)
|
||||
{
|
||||
result[i++] = n;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ContainerNodes returns two-dimensional list of nodes as a result of applying
|
||||
// given PlacementPolicy to the NetMap. Each line of the list corresponds to a
|
||||
// replica descriptor. Line order corresponds to order of ReplicaDescriptor list
|
||||
// in the policy. Nodes are pre-filtered according to the Filter list from
|
||||
// the policy, and then selected by Selector list. Result is deterministic for
|
||||
// the fixed NetMap and parameters.
|
||||
//
|
||||
// Result can be used in PlacementVectors.
|
||||
public FrostFsNodeInfo[][] ContainerNodes(FrostFsPlacementPolicy p, byte[]? pivot)
|
||||
{
|
||||
var c = new Context(this)
|
||||
{
|
||||
Cbf = p.BackupFactor == 0 ? 3 : p.BackupFactor
|
||||
};
|
||||
|
||||
if (pivot != null && pivot.Length > 0)
|
||||
{
|
||||
c.HrwSeed = pivot;
|
||||
|
||||
using var murmur = new Murmur3(0);
|
||||
c.HrwSeedHash = murmur.GetCheckSum64(pivot);
|
||||
}
|
||||
|
||||
c.ProcessFilters(p);
|
||||
c.ProcessSelectors(p);
|
||||
|
||||
var unique = p.IsUnique();
|
||||
|
||||
var result = new List<List<FrostFsNodeInfo>>(p.Replicas.Length);
|
||||
for (int i = 0; i < p.Replicas.Length; i++)
|
||||
{
|
||||
result.Add([]);
|
||||
}
|
||||
|
||||
// Note that the cached selectors are not used when the policy contains the UNIQUE flag.
|
||||
// This is necessary because each selection vector affects potentially the subsequent vectors
|
||||
// and thus we call getSelection in such case, in order to take into account nodes previously
|
||||
// marked as used by earlier replicas.
|
||||
for (int i = 0; i < p.Replicas.Length; i++)
|
||||
{
|
||||
var sName = p.Replicas[i].Selector;
|
||||
|
||||
if (string.IsNullOrEmpty(sName) && !(p.Replicas.Length == 1 && p.Selectors.Count == 1))
|
||||
{
|
||||
var s = new FrostFsSelector(string.Empty)
|
||||
{
|
||||
Count = p.Replicas[i].CountNodes(),
|
||||
Filter = Context.mainFilterName
|
||||
};
|
||||
|
||||
var nodes = c.GetSelection(s);
|
||||
result[i].AddRange(FlattenNodes(nodes));
|
||||
|
||||
if (unique)
|
||||
{
|
||||
foreach (var n in result[i])
|
||||
{
|
||||
c.UsedNodes[n.Hash()] = true;
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (unique)
|
||||
{
|
||||
if (!c.ProcessedSelectors.TryGetValue(sName, out var s) || s == null)
|
||||
{
|
||||
throw new FrostFsException($"selector not found: {sName}");
|
||||
}
|
||||
|
||||
var nodes = c.GetSelection(c.ProcessedSelectors[sName]);
|
||||
|
||||
result[i].AddRange(FlattenNodes(nodes));
|
||||
|
||||
foreach (var n in result[i])
|
||||
{
|
||||
c.UsedNodes[n.Hash()] = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var nodes = c.Selections[sName];
|
||||
result[i].AddRange(FlattenNodes(nodes));
|
||||
}
|
||||
}
|
||||
|
||||
var collection = new FrostFsNodeInfo[result.Count][];
|
||||
for (int i = 0; i < result.Count; i++)
|
||||
{
|
||||
collection[i] = [.. result[i]];
|
||||
}
|
||||
|
||||
return collection;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,5 +1,9 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
|
||||
using FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
using FrostFS.SDK.Cryptography;
|
||||
|
||||
namespace FrostFS.SDK;
|
||||
|
||||
|
@ -8,11 +12,69 @@ public class FrostFsNodeInfo(
|
|||
NodeState state,
|
||||
IReadOnlyCollection<string> addresses,
|
||||
IReadOnlyDictionary<string, string> attributes,
|
||||
ReadOnlyMemory<byte> publicKey)
|
||||
ReadOnlyMemory<byte> publicKey) : IHasher
|
||||
{
|
||||
public NodeState State { get; private set; } = state;
|
||||
public FrostFsVersion Version { get; private set; } = version;
|
||||
public IReadOnlyCollection<string> Addresses { get; private set; } = addresses;
|
||||
public IReadOnlyDictionary<string, string> Attributes { get; private set; } = attributes;
|
||||
public ReadOnlyMemory<byte> PublicKey { get; private set; } = publicKey;
|
||||
}
|
||||
private ulong _hash;
|
||||
|
||||
// attrPrice is a key to the node attribute that indicates the
|
||||
// price in GAS tokens for storing one GB of data during one Epoch.
|
||||
internal const string AttrPrice = "Price";
|
||||
|
||||
// attrCapacity is a key to the node attribute that indicates the
|
||||
// total available disk space in Gigabytes.
|
||||
internal const string AttrCapacity = "Capacity";
|
||||
|
||||
// attrExternalAddr is a key for the attribute storing node external addresses.
|
||||
internal const string AttrExternalAddr = "ExternalAddr";
|
||||
|
||||
// sepExternalAddr is a separator for multi-value ExternalAddr attribute.
|
||||
internal const string SepExternalAddr = ",";
|
||||
|
||||
private ulong price = ulong.MaxValue;
|
||||
|
||||
public NodeState State { get; } = state;
|
||||
|
||||
public FrostFsVersion Version { get; } = version;
|
||||
|
||||
public IReadOnlyCollection<string> Addresses { get; } = addresses;
|
||||
|
||||
public IReadOnlyDictionary<string, string> Attributes { get; } = attributes;
|
||||
|
||||
public ReadOnlyMemory<byte> PublicKey { get; } = publicKey;
|
||||
|
||||
public ulong Hash()
|
||||
{
|
||||
if (_hash == 0)
|
||||
{
|
||||
using var murmur3 = new Murmur3(0);
|
||||
murmur3.Initialize();
|
||||
_hash = murmur3.GetCheckSum64(PublicKey.ToArray());
|
||||
}
|
||||
|
||||
return _hash;
|
||||
}
|
||||
|
||||
internal ulong GetCapacity()
|
||||
{
|
||||
if (!Attributes.TryGetValue(AttrCapacity, out var val))
|
||||
return 0;
|
||||
|
||||
return ulong.Parse(val, CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
internal ulong Price
|
||||
{
|
||||
get
|
||||
{
|
||||
if (price == ulong.MaxValue)
|
||||
{
|
||||
if (!Attributes.TryGetValue(AttrPrice, out var val))
|
||||
price = 0;
|
||||
else
|
||||
price = uint.Parse(val, CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
|
||||
using System;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Linq;
|
||||
|
||||
using FrostFS.Netmap;
|
||||
|
@ -7,18 +7,31 @@ using FrostFS.SDK.Client;
|
|||
|
||||
namespace FrostFS.SDK;
|
||||
|
||||
public struct FrostFsPlacementPolicy(bool unique, params FrostFsReplica[] replicas)
|
||||
public struct FrostFsPlacementPolicy(bool unique,
|
||||
uint backupFactor,
|
||||
Collection<FrostFsSelector> selectors,
|
||||
Collection<FrostFsFilter> filters,
|
||||
params FrostFsReplica[] replicas)
|
||||
: IEquatable<FrostFsPlacementPolicy>
|
||||
{
|
||||
private PlacementPolicy policy;
|
||||
|
||||
public FrostFsReplica[] Replicas { get; private set; } = replicas;
|
||||
public bool Unique { get; private set; } = unique;
|
||||
public FrostFsReplica[] Replicas { get; } = replicas;
|
||||
|
||||
public Collection<FrostFsSelector> Selectors { get; } = selectors;
|
||||
|
||||
public Collection<FrostFsFilter> Filters { get; } = filters;
|
||||
|
||||
public bool Unique { get; } = unique;
|
||||
|
||||
public uint BackupFactor { get; } = backupFactor;
|
||||
|
||||
public override readonly bool Equals(object obj)
|
||||
{
|
||||
if (obj is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var other = (FrostFsPlacementPolicy)obj;
|
||||
|
||||
|
@ -46,14 +59,10 @@ public struct FrostFsPlacementPolicy(bool unique, params FrostFsReplica[] replic
|
|||
return policy;
|
||||
}
|
||||
|
||||
//public static FrostFsPlacementPolicy ToModel(placementPolicy)
|
||||
//{
|
||||
// return new FrostFsPlacementPolicy(
|
||||
// placementPolicy.Unique,
|
||||
// placementPolicy.Replicas.Select(replica => replica.ToModel()).ToArray()
|
||||
// );
|
||||
//}
|
||||
|
||||
internal readonly bool IsUnique()
|
||||
{
|
||||
return Unique || Replicas.Any(r => r.EcDataCount != 0 || r.EcParityCount != 0);
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
|
@ -86,4 +95,4 @@ public struct FrostFsPlacementPolicy(bool unique, params FrostFsReplica[] replic
|
|||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,6 +6,8 @@ public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
|||
{
|
||||
public int Count { get; set; }
|
||||
public string Selector { get; set; }
|
||||
public uint EcDataCount { get; set; }
|
||||
public uint EcParityCount { get; set; }
|
||||
|
||||
public FrostFsReplica(int count, string? selector = null)
|
||||
{
|
||||
|
@ -18,16 +20,23 @@ public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
|||
public override readonly bool Equals(object obj)
|
||||
{
|
||||
if (obj is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var other = (FrostFsReplica)obj;
|
||||
|
||||
return Count == other.Count && Selector == other.Selector;
|
||||
}
|
||||
|
||||
public readonly uint CountNodes()
|
||||
{
|
||||
return Count != 0 ? (uint)Count : EcDataCount + EcParityCount;
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
return Count + Selector.GetHashCode();
|
||||
return (Count + Selector.GetHashCode()) ^ (int)EcDataCount ^ (int)EcParityCount;
|
||||
}
|
||||
|
||||
public static bool operator ==(FrostFsReplica left, FrostFsReplica right)
|
||||
|
@ -42,6 +51,9 @@ public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
|||
|
||||
public readonly bool Equals(FrostFsReplica other)
|
||||
{
|
||||
return Count == other.Count && Selector == other.Selector;
|
||||
return Count == other.Count
|
||||
&& Selector == other.Selector
|
||||
&& EcDataCount == other.EcDataCount
|
||||
&& EcParityCount == other.EcParityCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
10
src/FrostFS.SDK.Client/Models/Netmap/FrostFsSelector.cs
Normal file
10
src/FrostFS.SDK.Client/Models/Netmap/FrostFsSelector.cs
Normal file
|
@ -0,0 +1,10 @@
|
|||
namespace FrostFS.SDK;
|
||||
|
||||
public class FrostFsSelector(string name)
|
||||
{
|
||||
public string Name { get; set; } = name;
|
||||
public uint Count { get; set; }
|
||||
public int Clause { get; set; }
|
||||
public string? Attribute { get; set; }
|
||||
public string? Filter { get; set; }
|
||||
}
|
11
src/FrostFS.SDK.Client/Models/Netmap/IFrostFsFilter.cs
Normal file
11
src/FrostFS.SDK.Client/Models/Netmap/IFrostFsFilter.cs
Normal file
|
@ -0,0 +1,11 @@
|
|||
namespace FrostFS.SDK
|
||||
{
|
||||
public interface IFrostFsFilter
|
||||
{
|
||||
FrostFsFilter[] Filters { get; }
|
||||
string Key { get; }
|
||||
string Name { get; }
|
||||
int Operation { get; }
|
||||
string Value { get; }
|
||||
}
|
||||
}
|
7
src/FrostFS.SDK.Client/Models/Netmap/NodeAttrPair.cs
Normal file
7
src/FrostFS.SDK.Client/Models/Netmap/NodeAttrPair.cs
Normal file
|
@ -0,0 +1,7 @@
|
|||
namespace FrostFS.SDK;
|
||||
|
||||
struct NodeAttrPair
|
||||
{
|
||||
internal string attr;
|
||||
internal FrostFsNodeInfo[] nodes;
|
||||
}
|
8
src/FrostFS.SDK.Client/Models/Netmap/Placement/Clause.cs
Normal file
8
src/FrostFS.SDK.Client/Models/Netmap/Placement/Clause.cs
Normal file
|
@ -0,0 +1,8 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
public enum FrostFsClause
|
||||
{
|
||||
Unspecified = 0,
|
||||
Same,
|
||||
Distinct
|
||||
}
|
456
src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs
Normal file
456
src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs
Normal file
|
@ -0,0 +1,456 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
|
||||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal struct Context
|
||||
{
|
||||
private const string errInvalidFilterName = "filter name is invalid";
|
||||
private const string errInvalidFilterOp = "invalid filter operation";
|
||||
private const string errFilterNotFound = "filter not found";
|
||||
private const string errNonEmptyFilters = "simple filter contains sub-filters";
|
||||
private const string errNotEnoughNodes = "not enough nodes to SELECT from";
|
||||
private const string errUnnamedTopFilter = "unnamed top-level filter";
|
||||
|
||||
internal const string mainFilterName = "*";
|
||||
internal const string likeWildcard = "*";
|
||||
|
||||
// network map to operate on
|
||||
internal FrostFsNetmapSnapshot NetMap { get; }
|
||||
|
||||
// cache of processed filters
|
||||
internal Dictionary<string, FrostFsFilter> ProcessedFilters { get; } = [];
|
||||
|
||||
// cache of processed selectors
|
||||
internal Dictionary<string, FrostFsSelector> ProcessedSelectors { get; } = [];
|
||||
|
||||
// stores results of selector processing
|
||||
internal Dictionary<string, List<List<FrostFsNodeInfo>>> Selections { get; } = [];
|
||||
|
||||
// cache of parsed numeric values
|
||||
internal Dictionary<string, ulong> NumCache { get; } = [];
|
||||
|
||||
internal byte[]? HrwSeed { get; set; }
|
||||
|
||||
// hrw.Hash of hrwSeed
|
||||
internal ulong HrwSeedHash { get; set; }
|
||||
|
||||
// container backup factor
|
||||
internal uint Cbf { get; set; }
|
||||
|
||||
// nodes already used in previous selections, which is needed when the placement
|
||||
// policy uses the UNIQUE flag. Nodes marked as used are not used in subsequent
|
||||
// base selections.
|
||||
internal Dictionary<ulong, bool> UsedNodes { get; } = [];
|
||||
|
||||
// If true, returns an error when netmap does not contain enough nodes for selection.
|
||||
// By default best effort is taken.
|
||||
internal bool Strict { get; set; }
|
||||
|
||||
// weightFunc is a weighting function for determining node priority
|
||||
// which combines low price and high performance
|
||||
private readonly Func<FrostFsNodeInfo, double> weightFunc;
|
||||
|
||||
public Context(FrostFsNetmapSnapshot netMap)
|
||||
{
|
||||
NetMap = netMap;
|
||||
weightFunc = Tools.DefaultWeightFunc(NetMap.NodeInfoCollection);
|
||||
}
|
||||
|
||||
internal void ProcessFilters(FrostFsPlacementPolicy policy)
|
||||
{
|
||||
foreach (var filter in policy.Filters)
|
||||
{
|
||||
ProcessFilter(filter, true);
|
||||
}
|
||||
}
|
||||
|
||||
readonly void ProcessFilter(FrostFsFilter filter, bool top)
|
||||
{
|
||||
var filterName = filter.Name;
|
||||
if (filterName == mainFilterName)
|
||||
{
|
||||
throw new FrostFsException($"{errInvalidFilterName}: '{errInvalidFilterName}' is reserved");
|
||||
}
|
||||
|
||||
if (top && string.IsNullOrEmpty(filterName))
|
||||
{
|
||||
throw new FrostFsException(errUnnamedTopFilter);
|
||||
}
|
||||
|
||||
if (!top && !string.IsNullOrEmpty(filterName) && !ProcessedFilters.ContainsKey(filterName))
|
||||
{
|
||||
throw new FrostFsException(errFilterNotFound);
|
||||
}
|
||||
|
||||
if (filter.Operation == (int)Operation.AND ||
|
||||
filter.Operation == (int)Operation.OR ||
|
||||
filter.Operation == (int)Operation.NOT)
|
||||
{
|
||||
foreach (var f in filter.Filters)
|
||||
ProcessFilter(f, false);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (filter.Filters.Length != 0)
|
||||
{
|
||||
throw new FrostFsException(errNonEmptyFilters);
|
||||
}
|
||||
else if (!top && !string.IsNullOrEmpty(filterName))
|
||||
{
|
||||
// named reference
|
||||
return;
|
||||
}
|
||||
|
||||
switch (filter.Operation)
|
||||
{
|
||||
case (int)Operation.EQ:
|
||||
case (int)Operation.NE:
|
||||
case (int)Operation.LIKE:
|
||||
break;
|
||||
case (int)Operation.GT:
|
||||
case (int)Operation.GE:
|
||||
case (int)Operation.LT:
|
||||
case (int)Operation.LE:
|
||||
{
|
||||
var n = uint.Parse(filter.Value, CultureInfo.InvariantCulture);
|
||||
NumCache[filter.Value] = n;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new FrostFsException($"{errInvalidFilterOp}: {filter.Operation}");
|
||||
}
|
||||
}
|
||||
|
||||
if (top)
|
||||
{
|
||||
ProcessedFilters[filterName] = filter;
|
||||
}
|
||||
}
|
||||
|
||||
// processSelectors processes selectors and returns error is any of them is invalid.
|
||||
internal void ProcessSelectors(FrostFsPlacementPolicy policy)
|
||||
{
|
||||
foreach (var selector in policy.Selectors)
|
||||
{
|
||||
var filterName = selector.Filter;
|
||||
if (filterName != mainFilterName)
|
||||
{
|
||||
if (selector.Filter == null || !ProcessedFilters.ContainsKey(selector.Filter))
|
||||
{
|
||||
throw new FrostFsException($"{errFilterNotFound}: SELECT FROM '{filterName}'");
|
||||
}
|
||||
}
|
||||
|
||||
ProcessedSelectors[selector.Name] = selector;
|
||||
|
||||
var selection = GetSelection(selector);
|
||||
|
||||
Selections[selector.Name] = selection;
|
||||
}
|
||||
}
|
||||
|
||||
// calcNodesCount returns number of buckets and minimum number of nodes in every bucket
|
||||
// for the given selector.
|
||||
static (int bucketCount, int nodesInBucket) CalcNodesCount(FrostFsSelector selector)
|
||||
{
|
||||
return selector.Clause == (int)FrostFsClause.Same
|
||||
? (1, (int)selector.Count)
|
||||
: ((int)selector.Count, 1);
|
||||
}
|
||||
|
||||
// getSelectionBase returns nodes grouped by selector attribute.
|
||||
// It it guaranteed that each pair will contain at least one node.
|
||||
internal NodeAttrPair[] GetSelectionBase(FrostFsSelector selector)
|
||||
{
|
||||
var fName = selector.Filter ?? throw new FrostFsException("Filter name for selector is empty");
|
||||
|
||||
_ = ProcessedFilters.TryGetValue(fName, out var f);
|
||||
|
||||
var isMain = fName == mainFilterName;
|
||||
var result = new List<NodeAttrPair>();
|
||||
|
||||
var nodeMap = new Dictionary<string, List<FrostFsNodeInfo>>();
|
||||
var attr = selector.Attribute;
|
||||
|
||||
foreach (var node in NetMap.NodeInfoCollection)
|
||||
{
|
||||
if (UsedNodes.ContainsKey(node.Hash()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isMain || Match(f, node))
|
||||
{
|
||||
if (attr == null)
|
||||
{
|
||||
// Default attribute is transparent identifier which is different for every node.
|
||||
result.Add(new NodeAttrPair { attr = "", nodes = [node] });
|
||||
}
|
||||
else
|
||||
{
|
||||
var v = node.Attributes[attr];
|
||||
if (!nodeMap.TryGetValue(v, out var nodes) || nodes == null)
|
||||
{
|
||||
nodeMap[v] = [];
|
||||
}
|
||||
|
||||
nodeMap[v].Add(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(attr))
|
||||
{
|
||||
foreach (var v in nodeMap)
|
||||
{
|
||||
result.Add(new NodeAttrPair() { attr = v.Key, nodes = [.. v.Value] });
|
||||
}
|
||||
}
|
||||
|
||||
if (HrwSeed != null && HrwSeed.Length != 0)
|
||||
{
|
||||
double[] ws = [];
|
||||
|
||||
var sortedNodes = new NodeAttrPair[result.Count];
|
||||
|
||||
for (int i = 0; i < result.Count; i++)
|
||||
{
|
||||
var res = result[i];
|
||||
Tools.AppendWeightsTo(res.nodes, weightFunc, ref ws);
|
||||
sortedNodes[i].nodes = Tools.SortHasherSliceByWeightValue(res.nodes.ToList(), ws, HrwSeedHash).ToArray();
|
||||
sortedNodes[i].attr = result[i].attr;
|
||||
}
|
||||
|
||||
return sortedNodes;
|
||||
}
|
||||
return [.. result];
|
||||
}
|
||||
|
||||
static double CalcBucketWeight(List<FrostFsNodeInfo> ns, IAggregator a, Func<FrostFsNodeInfo, double> wf)
|
||||
{
|
||||
foreach (var node in ns)
|
||||
{
|
||||
a.Add(wf(node));
|
||||
}
|
||||
|
||||
return a.Compute();
|
||||
}
|
||||
|
||||
// getSelection returns nodes grouped by s.attribute.
|
||||
// Last argument specifies if more buckets can be used to fulfill CBF.
|
||||
internal List<List<FrostFsNodeInfo>> GetSelection(FrostFsSelector s)
|
||||
{
|
||||
var (bucketCount, nodesInBucket) = CalcNodesCount(s);
|
||||
|
||||
var buckets = GetSelectionBase(s);
|
||||
|
||||
if (Strict && buckets.Length < bucketCount)
|
||||
throw new FrostFsException($"errNotEnoughNodes: '{s.Name}'");
|
||||
|
||||
// We need deterministic output in case there is no pivot.
|
||||
// If pivot is set, buckets are sorted by HRW.
|
||||
// However, because initial order influences HRW order for buckets with equal weights,
|
||||
// we also need to have deterministic input to HRW sorting routine.
|
||||
if (HrwSeed == null || HrwSeed.Length == 0)
|
||||
{
|
||||
buckets = string.IsNullOrEmpty(s.Attribute)
|
||||
? [.. buckets.OrderBy(b => b.nodes[0].Hash())]
|
||||
: [.. buckets.OrderBy(b => b.attr)];
|
||||
}
|
||||
|
||||
var maxNodesInBucket = nodesInBucket * (int)Cbf;
|
||||
|
||||
var res = new List<List<FrostFsNodeInfo>>(buckets.Length);
|
||||
var fallback = new List<List<FrostFsNodeInfo>>(buckets.Length);
|
||||
|
||||
for (int i = 0; i < buckets.Length; i++)
|
||||
{
|
||||
var ns = buckets[i].nodes;
|
||||
if (ns.Length >= maxNodesInBucket)
|
||||
{
|
||||
res.Add(new List<FrostFsNodeInfo>(ns[..maxNodesInBucket]));
|
||||
}
|
||||
else if (ns.Length >= nodesInBucket)
|
||||
{
|
||||
fallback.Add(new List<FrostFsNodeInfo>(ns));
|
||||
}
|
||||
}
|
||||
|
||||
if (res.Count < bucketCount)
|
||||
{
|
||||
// Fallback to using minimum allowed backup factor (1).
|
||||
res.AddRange(fallback);
|
||||
|
||||
if (Strict && res.Count < bucketCount)
|
||||
{
|
||||
throw new FrostFsException($"{errNotEnoughNodes}: {s}");
|
||||
}
|
||||
}
|
||||
|
||||
if (HrwSeed != null && HrwSeed.Length != 0)
|
||||
{
|
||||
var weights = new double[res.Count];
|
||||
var a = new MeanIQRAgg();
|
||||
|
||||
for (int i = 0; i < res.Count; i++)
|
||||
{
|
||||
a.Clear();
|
||||
weights[i] = CalcBucketWeight(res[i], a, weightFunc);
|
||||
}
|
||||
|
||||
var hashers = res.Select(r => new HasherList(r)).ToList();
|
||||
hashers = Tools.SortHasherSliceByWeightValue(hashers, weights, HrwSeedHash);
|
||||
|
||||
for (int i = 0; i < res.Count; i++)
|
||||
{
|
||||
res[i] = hashers[i].Nodes;
|
||||
}
|
||||
}
|
||||
|
||||
if (res.Count < bucketCount)
|
||||
{
|
||||
if (Strict && res.Count == 0)
|
||||
{
|
||||
throw new FrostFsException(errNotEnoughNodes);
|
||||
}
|
||||
|
||||
bucketCount = res.Count;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(s.Attribute))
|
||||
{
|
||||
fallback = res.Skip(bucketCount).ToList();
|
||||
res = res.Take(bucketCount).ToList();
|
||||
|
||||
for (int i = 0; i < fallback.Count; i++)
|
||||
{
|
||||
var index = i % bucketCount;
|
||||
if (res[index].Count >= maxNodesInBucket)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
res[index].AddRange(fallback[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return res.Take(bucketCount).ToList();
|
||||
}
|
||||
|
||||
internal bool MatchKeyValue(FrostFsFilter f, FrostFsNodeInfo nodeInfo)
|
||||
{
|
||||
switch (f.Operation)
|
||||
{
|
||||
case (int)Operation.EQ:
|
||||
return nodeInfo.Attributes.TryGetValue(f.Key, out var val) && val == f.Value;
|
||||
case (int)Operation.LIKE:
|
||||
{
|
||||
var hasPrefix = f.Value.StartsWith(likeWildcard, StringComparison.Ordinal);
|
||||
var hasSuffix = f.Value.EndsWith(likeWildcard, StringComparison.Ordinal);
|
||||
|
||||
var start = hasPrefix ? likeWildcard.Length : 0;
|
||||
var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length;
|
||||
var str = f.Value[start..end];
|
||||
|
||||
if (hasPrefix && hasSuffix)
|
||||
return nodeInfo.Attributes[f.Key].Contains(str);
|
||||
|
||||
if (hasPrefix && !hasSuffix)
|
||||
return nodeInfo.Attributes[f.Key].EndsWith(str, StringComparison.Ordinal);
|
||||
|
||||
if (!hasPrefix && hasSuffix)
|
||||
return nodeInfo.Attributes[f.Key].StartsWith(str, StringComparison.Ordinal);
|
||||
|
||||
|
||||
return nodeInfo.Attributes[f.Key] == f.Value;
|
||||
}
|
||||
case (int)Operation.NE:
|
||||
return nodeInfo.Attributes[f.Key] != f.Value;
|
||||
default:
|
||||
{
|
||||
ulong attr;
|
||||
switch (f.Key)
|
||||
{
|
||||
case FrostFsNodeInfo.AttrPrice:
|
||||
attr = nodeInfo.Price;
|
||||
break;
|
||||
|
||||
case FrostFsNodeInfo.AttrCapacity:
|
||||
attr = nodeInfo.GetCapacity();
|
||||
break;
|
||||
default:
|
||||
if (!ulong.TryParse(nodeInfo.Attributes[f.Key], NumberStyles.Integer, CultureInfo.InvariantCulture, out attr))
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
|
||||
switch (f.Operation)
|
||||
{
|
||||
case (int)Operation.GT:
|
||||
return attr > NumCache[f.Value];
|
||||
case (int)Operation.GE:
|
||||
return attr >= NumCache[f.Value];
|
||||
case (int)Operation.LT:
|
||||
return attr < NumCache[f.Value];
|
||||
case (int)Operation.LE:
|
||||
return attr <= NumCache[f.Value];
|
||||
default:
|
||||
// do nothing and return false
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// will not happen if context was created from f (maybe panic?)
|
||||
return false;
|
||||
}
|
||||
|
||||
// match matches f against b. It returns no errors because
|
||||
// filter should have been parsed during context creation
|
||||
// and missing node properties are considered as a regular fail.
|
||||
internal bool Match(FrostFsFilter f, FrostFsNodeInfo nodeInfo)
|
||||
{
|
||||
switch (f.Operation)
|
||||
{
|
||||
case (int)Operation.NOT:
|
||||
{
|
||||
var inner = f.Filters;
|
||||
var fSub = inner[0];
|
||||
|
||||
if (!string.IsNullOrEmpty(inner[0].Name))
|
||||
{
|
||||
fSub = ProcessedFilters[inner[0].Name];
|
||||
}
|
||||
return !Match(fSub, nodeInfo);
|
||||
}
|
||||
case (int)Operation.AND:
|
||||
case (int)Operation.OR:
|
||||
{
|
||||
for (int i = 0; i < f.Filters.Length; i++)
|
||||
{
|
||||
var fSub = f.Filters[i];
|
||||
|
||||
if (!string.IsNullOrEmpty(f.Filters[i].Name))
|
||||
{
|
||||
fSub = ProcessedFilters[f.Filters[i].Name];
|
||||
}
|
||||
|
||||
var ok = Match(fSub, nodeInfo);
|
||||
|
||||
if (ok == (f.Operation == (int)Operation.OR))
|
||||
{
|
||||
return ok;
|
||||
}
|
||||
}
|
||||
|
||||
return f.Operation == (int)Operation.AND;
|
||||
}
|
||||
default:
|
||||
return MatchKeyValue(f, nodeInfo);
|
||||
}
|
||||
}
|
||||
}
|
26
src/FrostFS.SDK.Client/Models/Netmap/Placement/HasherList.cs
Normal file
26
src/FrostFS.SDK.Client/Models/Netmap/Placement/HasherList.cs
Normal file
|
@ -0,0 +1,26 @@
|
|||
using System.Collections.Generic;
|
||||
|
||||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal sealed class HasherList : IHasher
|
||||
{
|
||||
private readonly List<FrostFsNodeInfo> _nodes;
|
||||
|
||||
internal HasherList(List<FrostFsNodeInfo> nodes)
|
||||
{
|
||||
_nodes = nodes;
|
||||
}
|
||||
|
||||
internal List<FrostFsNodeInfo> Nodes
|
||||
{
|
||||
get
|
||||
{
|
||||
return _nodes;
|
||||
}
|
||||
}
|
||||
|
||||
public ulong Hash()
|
||||
{
|
||||
return _nodes.Count > 0 ? _nodes[0].Hash() : 0;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,8 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal interface IAggregator
|
||||
{
|
||||
void Add(double d);
|
||||
|
||||
double Compute();
|
||||
}
|
|
@ -0,0 +1,6 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal interface IHasher
|
||||
{
|
||||
ulong Hash();
|
||||
}
|
|
@ -0,0 +1,6 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
interface INormalizer
|
||||
{
|
||||
double Normalize(double w);
|
||||
}
|
20
src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanAgg.cs
Normal file
20
src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanAgg.cs
Normal file
|
@ -0,0 +1,20 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal struct MeanAgg
|
||||
{
|
||||
private double mean;
|
||||
private int count;
|
||||
|
||||
internal void Add(double n)
|
||||
{
|
||||
int c = count + 1;
|
||||
mean = mean * count / c + n / c;
|
||||
|
||||
count++;
|
||||
}
|
||||
|
||||
internal readonly double Compute()
|
||||
{
|
||||
return mean;
|
||||
}
|
||||
}
|
65
src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanIQRAgg.cs
Normal file
65
src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanIQRAgg.cs
Normal file
|
@ -0,0 +1,65 @@
|
|||
using System.Collections.ObjectModel;
|
||||
using System.Linq;
|
||||
|
||||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal struct MeanIQRAgg : IAggregator
|
||||
{
|
||||
private const int minLn = 4;
|
||||
internal Collection<double> arr = [];
|
||||
|
||||
public MeanIQRAgg()
|
||||
{
|
||||
}
|
||||
|
||||
public readonly void Add(double d)
|
||||
{
|
||||
arr.Add(d);
|
||||
}
|
||||
|
||||
public readonly double Compute()
|
||||
{
|
||||
var length = arr.Count;
|
||||
if (length == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var sorted = arr.OrderBy(p => p).ToArray();
|
||||
|
||||
double minV, maxV;
|
||||
|
||||
if (arr.Count < minLn)
|
||||
{
|
||||
minV = sorted[0];
|
||||
maxV = sorted[length - 1];
|
||||
}
|
||||
else
|
||||
{
|
||||
var start = length / minLn;
|
||||
var end = length * 3 / minLn - 1;
|
||||
|
||||
minV = sorted[start];
|
||||
maxV = sorted[end];
|
||||
}
|
||||
|
||||
var count = 0;
|
||||
double sum = 0;
|
||||
|
||||
foreach (var e in sorted)
|
||||
{
|
||||
if (e >= minV && e <= maxV)
|
||||
{
|
||||
sum += e;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
return sum / count;
|
||||
}
|
||||
|
||||
internal readonly void Clear()
|
||||
{
|
||||
arr.Clear();
|
||||
}
|
||||
}
|
27
src/FrostFS.SDK.Client/Models/Netmap/Placement/MinAgg.cs
Normal file
27
src/FrostFS.SDK.Client/Models/Netmap/Placement/MinAgg.cs
Normal file
|
@ -0,0 +1,27 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal struct MinAgg
|
||||
{
|
||||
private double min;
|
||||
private bool minFound;
|
||||
|
||||
internal void Add(double n)
|
||||
{
|
||||
if (!minFound)
|
||||
{
|
||||
min = n;
|
||||
minFound = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (n < min)
|
||||
{
|
||||
min = n;
|
||||
}
|
||||
}
|
||||
|
||||
internal readonly double Compute()
|
||||
{
|
||||
return min;
|
||||
}
|
||||
}
|
16
src/FrostFS.SDK.Client/Models/Netmap/Placement/Operation.cs
Normal file
16
src/FrostFS.SDK.Client/Models/Netmap/Placement/Operation.cs
Normal file
|
@ -0,0 +1,16 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
public enum Operation
|
||||
{
|
||||
Unspecified = 0,
|
||||
EQ,
|
||||
NE,
|
||||
GT,
|
||||
GE,
|
||||
LT,
|
||||
LE,
|
||||
OR,
|
||||
AND,
|
||||
NOT,
|
||||
LIKE
|
||||
}
|
|
@ -0,0 +1,8 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal struct ReverseMinNorm : INormalizer
|
||||
{
|
||||
internal double min;
|
||||
|
||||
public readonly double Normalize(double w) => (min + 1) / (w + 1);
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
using System.Collections.ObjectModel;
|
||||
|
||||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal struct SelectFilterExpr(uint cbf, FrostFsSelector selector, Collection<FrostFsFilter> filters)
|
||||
{
|
||||
internal uint Cbf { get; } = cbf;
|
||||
internal FrostFsSelector Selector { get; } = selector;
|
||||
internal Collection<FrostFsFilter> Filters { get; } = filters;
|
||||
}
|
|
@ -0,0 +1,23 @@
|
|||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
internal readonly struct SigmoidNorm : INormalizer
|
||||
{
|
||||
private readonly double _scale;
|
||||
|
||||
internal SigmoidNorm(double scale)
|
||||
{
|
||||
_scale = scale;
|
||||
}
|
||||
|
||||
public readonly double Normalize(double w)
|
||||
{
|
||||
if (_scale == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var x = w / _scale;
|
||||
|
||||
return x / (1 + x);
|
||||
}
|
||||
}
|
138
src/FrostFS.SDK.Client/Models/Netmap/Placement/Tools.cs
Normal file
138
src/FrostFS.SDK.Client/Models/Netmap/Placement/Tools.cs
Normal file
|
@ -0,0 +1,138 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
using static FrostFS.SDK.FrostFsNetmapSnapshot;
|
||||
|
||||
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
public static class Tools
|
||||
{
|
||||
internal static ulong Distance(ulong x, ulong y)
|
||||
{
|
||||
var acc = x ^ y;
|
||||
acc ^= acc >> 33;
|
||||
acc *= 0xff51afd7ed558ccd;
|
||||
acc ^= acc >> 33;
|
||||
acc *= 0xc4ceb9fe1a85ec53;
|
||||
acc ^= acc >> 33;
|
||||
|
||||
return acc;
|
||||
}
|
||||
|
||||
internal static double ReverceNormalize(double r, double w)
|
||||
{
|
||||
return (r + 1) / (w + 1);
|
||||
}
|
||||
|
||||
internal static double Normalize(double r, double w)
|
||||
{
|
||||
if (r == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var x = w / r;
|
||||
return x / (1 + x);
|
||||
}
|
||||
|
||||
internal static void AppendWeightsTo(FrostFsNodeInfo[] nodes, Func<FrostFsNodeInfo, double> wf, ref double[] weights)
|
||||
{
|
||||
if (weights.Length < nodes.Length)
|
||||
{
|
||||
weights = new double[nodes.Length];
|
||||
}
|
||||
|
||||
for (int i = 0; i < nodes.Length; i++)
|
||||
{
|
||||
weights[i] = wf(nodes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
internal static List<T> SortHasherSliceByWeightValue<T>(List<T> nodes, Span<double> weights, ulong hash) where T : IHasher
|
||||
{
|
||||
if (nodes.Count == 0)
|
||||
{
|
||||
return nodes;
|
||||
}
|
||||
|
||||
var allEquals = true;
|
||||
|
||||
if (weights.Length > 1)
|
||||
{
|
||||
for (int i = 1; i < weights.Length; i++)
|
||||
{
|
||||
if (weights[i] != weights[0])
|
||||
{
|
||||
allEquals = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var dist = new double[nodes.Count];
|
||||
|
||||
if (allEquals)
|
||||
{
|
||||
for (int i = 0; i < dist.Length; i++)
|
||||
{
|
||||
var x = nodes[i].Hash();
|
||||
dist[i] = Distance(x, hash);
|
||||
}
|
||||
|
||||
return SortHasherByDistance(nodes, dist, true);
|
||||
}
|
||||
|
||||
for (int i = 0; i < dist.Length; i++)
|
||||
{
|
||||
var d = Distance(nodes[i].Hash(), hash);
|
||||
dist[i] = (ulong.MaxValue - d) * weights[i];
|
||||
}
|
||||
|
||||
return SortHasherByDistance(nodes, dist, false);
|
||||
}
|
||||
|
||||
internal static List<T> SortHasherByDistance<T, N>(List<T> nodes, N[] dist, bool asc)
|
||||
{
|
||||
IndexedValue<T, N>[] indexes = new IndexedValue<T, N>[nodes.Count];
|
||||
for (int i = 0; i < dist.Length; i++)
|
||||
{
|
||||
indexes[i] = new IndexedValue<T, N>() { nodeInfo = nodes[i], dist = dist[i] };
|
||||
}
|
||||
|
||||
if (asc)
|
||||
{
|
||||
return new List<T>(indexes
|
||||
.OrderBy(x => x.dist)
|
||||
.Select(x => x.nodeInfo).ToArray());
|
||||
}
|
||||
else
|
||||
{
|
||||
return new List<T>(indexes
|
||||
.OrderByDescending(x => x.dist)
|
||||
.Select(x => x.nodeInfo));
|
||||
}
|
||||
}
|
||||
|
||||
internal static Func<FrostFsNodeInfo, double> DefaultWeightFunc(IReadOnlyList<FrostFsNodeInfo> nodes)
|
||||
{
|
||||
MeanAgg mean = new();
|
||||
MinAgg minV = new();
|
||||
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
mean.Add(node.GetCapacity());
|
||||
minV.Add(node.Price);
|
||||
}
|
||||
|
||||
return NewWeightFunc(
|
||||
NewSigmoidNorm(mean.Compute()),
|
||||
NewReverseMinNorm(minV.Compute()));
|
||||
}
|
||||
|
||||
private struct IndexedValue<T, N>
|
||||
{
|
||||
internal T nodeInfo;
|
||||
internal N dist;
|
||||
}
|
||||
}
|
|
@ -20,7 +20,9 @@ public class FrostFsSplitInfo
|
|||
|
||||
public SplitId SplitId => _splitId ??= new SplitId(_splitInfo.SplitId.ToUuid());
|
||||
|
||||
public FrostFsObjectId Link => _link ??= FrostFsObjectId.FromHash(_splitInfo.Link.Value.Span);
|
||||
public FrostFsObjectId? Link => _link ??= _splitInfo.Link == null
|
||||
? null : FrostFsObjectId.FromHash(_splitInfo.Link.Value.Span);
|
||||
|
||||
public FrostFsObjectId LastPart => _lastPart ??= FrostFsObjectId.FromHash(_splitInfo.LastPart.Value.Span);
|
||||
public FrostFsObjectId? LastPart => _lastPart ??= _splitInfo.LastPart == null
|
||||
? null : FrostFsObjectId.FromHash(_splitInfo.LastPart.Value.Span);
|
||||
}
|
||||
|
|
|
@ -39,6 +39,8 @@ public partial class Pool : IFrostFSClient
|
|||
|
||||
private OwnerID? _ownerId;
|
||||
|
||||
private FrostFsVersion _version;
|
||||
|
||||
private FrostFsOwner? _owner;
|
||||
|
||||
private FrostFsOwner Owner
|
||||
|
@ -94,6 +96,8 @@ public partial class Pool : IFrostFSClient
|
|||
throw new ArgumentException($"Missed required parameter {nameof(options.Key)}");
|
||||
}
|
||||
|
||||
_version = new FrostFsVersion(2, 13);
|
||||
|
||||
var nodesParams = AdjustNodeParams(options.NodeParams);
|
||||
|
||||
var cache = new SessionCache(options.SessionExpirationDuration);
|
||||
|
|
|
@ -268,7 +268,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
|||
|
||||
internal async Task<FrostFsObjectId> PutSingleObjectAsync(PrmSingleObjectPut args, CallContext ctx)
|
||||
{
|
||||
var grpcObject = ObjectTools.CreateObject(args.FrostFsObject, ClientContext);
|
||||
var grpcObject = ObjectTools.CreateSingleObject(args.FrostFsObject, ClientContext);
|
||||
|
||||
var request = new PutSingleRequest
|
||||
{
|
||||
|
@ -565,7 +565,7 @@ internal sealed class ObjectServiceProvider(ObjectService.ObjectServiceClient cl
|
|||
|
||||
if (header.Split != null)
|
||||
{
|
||||
ObjectTools.SetSplitValues(grpcHeader, header.Split, ClientContext);
|
||||
ObjectTools.SetSplitValues(grpcHeader, header.Split, ClientContext.Owner, ClientContext.Version, ClientContext.Key);
|
||||
}
|
||||
|
||||
var oid = new ObjectID { Value = grpcHeader.Sha256() };
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
using System;
|
||||
using System.Linq;
|
||||
|
||||
using FrostFS.Object;
|
||||
|
@ -9,9 +10,44 @@ using Google.Protobuf;
|
|||
|
||||
namespace FrostFS.SDK.Client;
|
||||
|
||||
internal static class ObjectTools
|
||||
public static class ObjectTools
|
||||
{
|
||||
internal static Object.Object CreateObject(FrostFsObject @object, ClientContext ctx)
|
||||
public static FrostFsObjectId CalculateObjectId(
|
||||
FrostFsObjectHeader header,
|
||||
ReadOnlyMemory<byte> payloadHash,
|
||||
FrostFsOwner owner,
|
||||
FrostFsVersion version,
|
||||
ClientKey key)
|
||||
{
|
||||
if (header is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(header));
|
||||
}
|
||||
|
||||
if (owner is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(owner));
|
||||
}
|
||||
|
||||
if (version is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(version));
|
||||
}
|
||||
|
||||
if (key is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(key));
|
||||
}
|
||||
|
||||
var grpcHeader = CreateHeader(header, payloadHash, owner, version);
|
||||
|
||||
if (header.Split != null)
|
||||
SetSplitValues(grpcHeader, header.Split, owner, version, key);
|
||||
|
||||
return new ObjectID { Value = grpcHeader.Sha256() }.ToModel();
|
||||
}
|
||||
|
||||
internal static Object.Object CreateSingleObject(FrostFsObject @object, ClientContext ctx)
|
||||
{
|
||||
@object.Header.OwnerId ??= ctx.Owner;
|
||||
@object.Header.Version ??= ctx.Version;
|
||||
|
@ -22,9 +58,10 @@ internal static class ObjectTools
|
|||
grpcHeader.PayloadHash = Sha256Checksum(@object.SingleObjectPayload);
|
||||
|
||||
var split = @object.Header.Split;
|
||||
|
||||
if (split != null)
|
||||
{
|
||||
SetSplitValues(grpcHeader, split, ctx);
|
||||
SetSplitValues(grpcHeader, split, ctx.Owner, ctx.Version, ctx.Key);
|
||||
}
|
||||
|
||||
var obj = new Object.Object
|
||||
|
@ -43,13 +80,22 @@ internal static class ObjectTools
|
|||
return obj;
|
||||
}
|
||||
|
||||
internal static void SetSplitValues(Header grpcHeader, FrostFsSplit split, ClientContext ctx)
|
||||
internal static void SetSplitValues(
|
||||
Header grpcHeader,
|
||||
FrostFsSplit split,
|
||||
FrostFsOwner owner,
|
||||
FrostFsVersion version,
|
||||
ClientKey key)
|
||||
{
|
||||
if (split == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (ctx.Key == null)
|
||||
throw new FrostFsInvalidObjectException(nameof(ctx.Key));
|
||||
if (key == null)
|
||||
{
|
||||
throw new FrostFsInvalidObjectException(nameof(key));
|
||||
}
|
||||
|
||||
grpcHeader.Split = new Header.Types.Split
|
||||
{
|
||||
|
@ -57,33 +103,38 @@ internal static class ObjectTools
|
|||
};
|
||||
|
||||
if (split.Children != null && split.Children.Count != 0)
|
||||
{
|
||||
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToMessage()));
|
||||
}
|
||||
|
||||
if (split.ParentHeader is not null)
|
||||
{
|
||||
var grpcParentHeader = CreateHeader(split.ParentHeader, [], ctx);
|
||||
var grpcParentHeader = CreateHeader(split.ParentHeader, Array.Empty<byte>().Sha256(), owner, version);
|
||||
|
||||
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
|
||||
grpcHeader.Split.ParentHeader = grpcParentHeader;
|
||||
grpcHeader.Split.ParentSignature = new Signature
|
||||
{
|
||||
Key = ctx.Key.PublicKeyProto,
|
||||
Sign = ctx.Key.ECDsaKey.SignData(grpcHeader.Split.Parent.ToByteArray()),
|
||||
Key = key.PublicKeyProto,
|
||||
Sign = key.ECDsaKey.SignData(grpcHeader.Split.Parent.ToByteArray()),
|
||||
};
|
||||
}
|
||||
|
||||
grpcHeader.Split.Previous = split.Previous?.ToMessage();
|
||||
}
|
||||
|
||||
internal static Header CreateHeader(FrostFsObjectHeader header, byte[]? payload, ClientContext ctx)
|
||||
internal static Header CreateHeader(
|
||||
FrostFsObjectHeader header,
|
||||
ReadOnlyMemory<byte> payloadChecksum,
|
||||
FrostFsOwner owner,
|
||||
FrostFsVersion version)
|
||||
{
|
||||
header.OwnerId ??= ctx.Owner;
|
||||
header.Version ??= ctx.Version;
|
||||
header.OwnerId ??= owner;
|
||||
header.Version ??= version;
|
||||
|
||||
var grpcHeader = header.GetHeader();
|
||||
|
||||
if (payload != null) // && payload.Length > 0
|
||||
grpcHeader.PayloadHash = Sha256Checksum(payload);
|
||||
grpcHeader.PayloadHash = ChecksumFromSha256(payloadChecksum);
|
||||
|
||||
return grpcHeader;
|
||||
}
|
||||
|
@ -96,4 +147,13 @@ internal static class ObjectTools
|
|||
Sum = ByteString.CopyFrom(data.Sha256())
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
internal static Checksum ChecksumFromSha256(ReadOnlyMemory<byte> dataHash)
|
||||
{
|
||||
return new Checksum
|
||||
{
|
||||
Type = ChecksumType.Sha256,
|
||||
Sum = UnsafeByteOperations.UnsafeWrap(dataHash)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ using System.Security.Cryptography;
|
|||
|
||||
namespace FrostFS.SDK.Cryptography;
|
||||
|
||||
internal class Murmur3_128 : HashAlgorithm
|
||||
public class Murmur3 : HashAlgorithm
|
||||
{
|
||||
private const ulong c1 = 0x87c37b91114253d5;
|
||||
private const ulong c2 = 0x4cf5ad432745937f;
|
||||
|
@ -17,14 +17,31 @@ internal class Murmur3_128 : HashAlgorithm
|
|||
private readonly uint seed;
|
||||
private int length;
|
||||
|
||||
public Murmur3_128(uint seed)
|
||||
public Murmur3(uint seed)
|
||||
{
|
||||
this.seed = seed;
|
||||
Initialize();
|
||||
}
|
||||
|
||||
public ulong GetCheckSum64(byte[] bytes)
|
||||
{
|
||||
if (bytes is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(bytes));
|
||||
}
|
||||
|
||||
Initialize();
|
||||
HashCore(bytes, 0, bytes.Length);
|
||||
return HashFinalUlong();
|
||||
}
|
||||
|
||||
protected override void HashCore(byte[] array, int ibStart, int cbSize)
|
||||
{
|
||||
if (array is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(array));
|
||||
}
|
||||
|
||||
length += cbSize;
|
||||
int remainder = cbSize & 15;
|
||||
int alignedLength = ibStart + (cbSize - remainder);
|
||||
|
@ -92,6 +109,11 @@ internal class Murmur3_128 : HashAlgorithm
|
|||
}
|
||||
|
||||
protected override byte[] HashFinal()
|
||||
{
|
||||
return BitConverter.GetBytes(HashFinalUlong());
|
||||
}
|
||||
|
||||
protected ulong HashFinalUlong()
|
||||
{
|
||||
h1 ^= (ulong)length;
|
||||
h2 ^= (ulong)length;
|
||||
|
@ -102,7 +124,7 @@ internal class Murmur3_128 : HashAlgorithm
|
|||
h1 += h2;
|
||||
h2 += h1;
|
||||
|
||||
return BitConverter.GetBytes(h1);
|
||||
return h1;
|
||||
}
|
||||
|
||||
public override void Initialize()
|
||||
|
|
|
@ -23,7 +23,7 @@ public abstract class ServiceBase(string key)
|
|||
public FrostFsPlacementPolicy PlacementPolicy { get; set; } = DefaultPlacementPolicy;
|
||||
|
||||
public static FrostFsVersion DefaultVersion { get; } = new(2, 13);
|
||||
public static FrostFsPlacementPolicy DefaultPlacementPolicy { get; } = new FrostFsPlacementPolicy(true, new FrostFsReplica(1));
|
||||
public static FrostFsPlacementPolicy DefaultPlacementPolicy { get; } = new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1));
|
||||
|
||||
#pragma warning disable CA2227 // this is specific object, should be treated as is
|
||||
public Metadata? Metadata { get; set; }
|
||||
|
|
|
@ -166,7 +166,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase
|
|||
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["key1", "value1"]);
|
||||
|
||||
|
@ -216,7 +216,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase
|
|||
await Cleanup(pool);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
lightWait);
|
||||
|
||||
var containerId = await pool.CreateContainerAsync(createContainerParam, default);
|
||||
|
@ -311,7 +311,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase
|
|||
await Cleanup(pool);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -396,7 +396,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase
|
|||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams);
|
||||
|
||||
var container = await pool.CreateContainerAsync(createContainerParam, ctx);
|
||||
|
@ -479,7 +479,7 @@ public class MultithreadPoolSmokeTests : SmokeTestsBase
|
|||
await Cleanup(pool);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
lightWait);
|
||||
|
||||
var containerId = await pool.CreateContainerAsync(createContainerParam, default);
|
||||
|
|
|
@ -99,7 +99,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase
|
|||
var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["key1", "value1"]);
|
||||
|
||||
|
@ -144,7 +144,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
lightWait);
|
||||
|
||||
var containerId = await client.CreateContainerAsync(createContainerParam, default);
|
||||
|
@ -237,7 +237,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase
|
|||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -308,7 +308,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -388,7 +388,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -442,7 +442,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -499,7 +499,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase
|
|||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams);
|
||||
|
||||
var container = await client.CreateContainerAsync(createContainerParam, ctx);
|
||||
|
@ -580,7 +580,7 @@ public class MultithreadSmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
lightWait);
|
||||
|
||||
var containerId = await client.CreateContainerAsync(createContainerParam, default);
|
||||
|
|
|
@ -164,7 +164,7 @@ public class PoolSmokeTests : SmokeTestsBase
|
|||
var token = await pool.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["key1", "value1"]);
|
||||
|
||||
|
@ -214,7 +214,7 @@ public class PoolSmokeTests : SmokeTestsBase
|
|||
await Cleanup(pool);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
lightWait);
|
||||
|
||||
var containerId = await pool.CreateContainerAsync(createContainerParam, default);
|
||||
|
@ -311,7 +311,7 @@ public class PoolSmokeTests : SmokeTestsBase
|
|||
var ctx = new CallContext(TimeSpan.Zero);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -396,7 +396,7 @@ public class PoolSmokeTests : SmokeTestsBase
|
|||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams);
|
||||
|
||||
var container = await pool.CreateContainerAsync(createContainerParam, ctx);
|
||||
|
@ -480,7 +480,7 @@ public class PoolSmokeTests : SmokeTestsBase
|
|||
await Cleanup(pool);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
lightWait);
|
||||
|
||||
var containerId = await pool.CreateContainerAsync(createContainerParam, default);
|
||||
|
|
|
@ -81,7 +81,7 @@ public class SmokeClientTests : SmokeTestsBase
|
|||
var token = await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["key1", "value1"]);
|
||||
|
||||
|
@ -126,7 +126,7 @@ public class SmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
lightWait);
|
||||
|
||||
var containerId = await client.CreateContainerAsync(createContainerParam, default);
|
||||
|
@ -219,7 +219,7 @@ public class SmokeClientTests : SmokeTestsBase
|
|||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -300,7 +300,7 @@ public class SmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -380,7 +380,7 @@ public class SmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -436,7 +436,7 @@ public class SmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams,
|
||||
xheaders: ["testKey", "testValue"]);
|
||||
|
||||
|
@ -493,7 +493,7 @@ public class SmokeClientTests : SmokeTestsBase
|
|||
var ctx = new CallContext(TimeSpan.FromSeconds(20));
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
PrmWait.DefaultParams);
|
||||
|
||||
var container = await client.CreateContainerAsync(createContainerParam, ctx);
|
||||
|
@ -573,7 +573,7 @@ public class SmokeClientTests : SmokeTestsBase
|
|||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
|
||||
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1))),
|
||||
lightWait);
|
||||
|
||||
var containerId = await client.CreateContainerAsync(createContainerParam, default);
|
||||
|
|
100
src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_default.json
Normal file
100
src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_default.json
Normal file
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
"name": "default CBF is 3",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "DE"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Berlin"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "FR"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Paris"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "set default CBF",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "EU"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "EU",
|
||||
"count": 1,
|
||||
"clause": "SAME",
|
||||
"attribute": "Location",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
101
src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_minimal.json
Normal file
101
src/FrostFS.SDK.Tests/TestData/PlacementTests/cbf_minimal.json
Normal file
|
@ -0,0 +1,101 @@
|
|||
{
|
||||
"name": "Real node count multiplier is in range [1, specified CBF]",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "DE"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "DE"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "DE"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "select 2, CBF is 2",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 2,
|
||||
"clause": "SAME",
|
||||
"attribute": "Country",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "select 3, CBF is 2",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 3,
|
||||
"clause": "SAME",
|
||||
"attribute": "Country",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,156 @@
|
|||
{
|
||||
"name": "CBF requirements",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Attr",
|
||||
"value": "Same"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Attr",
|
||||
"value": "Same"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Attr",
|
||||
"value": "Same"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Attr",
|
||||
"value": "Same"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "default CBF, no selector",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 2
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
3
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "explicit CBF, no selector",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 2
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 3,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
3
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "select distinct, weak CBF",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 2,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 3,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
3
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "select same, weak CBF",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 2,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 3,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 2,
|
||||
"clause": "SAME",
|
||||
"attribute": "Attr",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,345 @@
|
|||
{
|
||||
"name": "compound filter",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Storage",
|
||||
"value": "SSD"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "10"
|
||||
},
|
||||
{
|
||||
"key": "IntField",
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"key": "Param",
|
||||
"value": "Value1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "good",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "Storage",
|
||||
"op": "EQ",
|
||||
"value": "SSD",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"op": "AND",
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"op": "Unspecified",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "IntField",
|
||||
"op": "LT",
|
||||
"value": "123",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"op": "Unspecified",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"op": "OR",
|
||||
"filters": [
|
||||
{
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value1",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value2",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "bad storage type",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "Storage",
|
||||
"op": "EQ",
|
||||
"value": "HDD",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"op": "AND",
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"op": "Unspecified",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "IntField",
|
||||
"op": "LT",
|
||||
"value": "123",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"op": "Unspecified",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"op": "OR",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value1",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value2",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "bad rating",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "Storage",
|
||||
"op": "EQ",
|
||||
"value": "SSD",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "15",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"op": "AND",
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"op": "Unspecified",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "IntField",
|
||||
"op": "LT",
|
||||
"value": "123",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"op": "Unspecified",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"op": "OR",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value1",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value2",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "bad param",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "Storage",
|
||||
"op": "EQ",
|
||||
"value": "SSD",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"op": "AND",
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"op": "Unspecified",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "IntField",
|
||||
"op": "LT",
|
||||
"value": "123",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"op": "Unspecified",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"op": "OR",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value0",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value2",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,81 @@
|
|||
{
|
||||
"name": "invalid integer field",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "IntegerField",
|
||||
"value": "true"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "IntegerField",
|
||||
"value": "str"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "empty string is not casted to 0",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "IntegerField",
|
||||
"op": "LE",
|
||||
"value": "8",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "non-empty string is not casted to a number",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "IntegerField",
|
||||
"op": "GE",
|
||||
"value": "0",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
397
src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_simple.json
Normal file
397
src/FrostFS.SDK.Tests/TestData/PlacementTests/filter_simple.json
Normal file
|
@ -0,0 +1,397 @@
|
|||
{
|
||||
"name": "single-op filters",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "GE true",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "GE false",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "5",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "GT true",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "GT",
|
||||
"value": "3",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "GT false",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "GT",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "LE true",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "LE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "LE false",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "LE",
|
||||
"value": "3",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "LT true",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "LT",
|
||||
"value": "5",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "LT false",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "LT",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "EQ true",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Germany",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "EQ false",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "China",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "NE true",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Country",
|
||||
"op": "NE",
|
||||
"value": "France",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "NE false",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Country",
|
||||
"op": "NE",
|
||||
"value": "Germany",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
225
src/FrostFS.SDK.Tests/TestData/PlacementTests/hrw_sort.json
Normal file
225
src/FrostFS.SDK.Tests/TestData/PlacementTests/hrw_sort.json
Normal file
|
@ -0,0 +1,225 @@
|
|||
{
|
||||
"name": "HRW ordering",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "7"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name":"select 3 nodes in 3 distinct countries, same placement",
|
||||
"policy": {
|
||||
"containerBackupFactor": 1,
|
||||
"filters": [],
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "Main"
|
||||
}
|
||||
],
|
||||
"selectors": [
|
||||
{
|
||||
"attribute": "Country",
|
||||
"clause": "DISTINCT",
|
||||
"count": 3,
|
||||
"filter": "*",
|
||||
"name": "Main"
|
||||
}
|
||||
]
|
||||
},
|
||||
"pivot": "Y29udGFpbmVySUQ=",
|
||||
"result": [[
|
||||
5,
|
||||
0,
|
||||
7
|
||||
]],
|
||||
"placement": {
|
||||
"pivot": "b2JqZWN0SUQ=",
|
||||
"result": [[
|
||||
5,
|
||||
0,
|
||||
7
|
||||
]]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name":"select 6 nodes in 3 distinct countries, different placement",
|
||||
"policy": {
|
||||
"containerBackupFactor": 2,
|
||||
"filters": [
|
||||
],
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "Main"
|
||||
}
|
||||
],
|
||||
"selectors": [
|
||||
{
|
||||
"attribute": "Country",
|
||||
"clause": "DISTINCT",
|
||||
"count": 3,
|
||||
"filter": "*",
|
||||
"name": "Main"
|
||||
}
|
||||
]
|
||||
},
|
||||
"pivot": "Y29udGFpbmVySUQ=",
|
||||
"result": [[
|
||||
5,
|
||||
4,
|
||||
0,
|
||||
1,
|
||||
7,
|
||||
2]],
|
||||
|
||||
"placement": {
|
||||
"pivot": "b2JqZWN0SUQ=",
|
||||
"result": [[
|
||||
5,
|
||||
4,
|
||||
0,
|
||||
7,
|
||||
2,
|
||||
1]]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
107
src/FrostFS.SDK.Tests/TestData/PlacementTests/issue213.json
Normal file
107
src/FrostFS.SDK.Tests/TestData/PlacementTests/issue213.json
Normal file
|
@ -0,0 +1,107 @@
|
|||
{
|
||||
"name": "unnamed selector (nspcc-dev/neofs-api-go#213)",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Saint-Petersburg"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Sweden"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Stockholm"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Finalnd"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Helsinki"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "test",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 4
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "",
|
||||
"count": 4,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "LOC_EU"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "LOC_EU",
|
||||
"key": "Location",
|
||||
"op": "EQ",
|
||||
"value": "Europe",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
279
src/FrostFS.SDK.Tests/TestData/PlacementTests/many_selects.json
Normal file
279
src/FrostFS.SDK.Tests/TestData/PlacementTests/many_selects.json
Normal file
|
@ -0,0 +1,279 @@
|
|||
{
|
||||
"name": "single-op filters",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "SPB"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "5"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Berlin"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "6"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Paris"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Lyon"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "5"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "SPB"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "7"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Darmstadt"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "7"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Frankfurt"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "9"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "SPB"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "9"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "SPB"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "Select",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "SameRU"
|
||||
},
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "DistinctRU"
|
||||
},
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "Good"
|
||||
},
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "Main"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "SameRU",
|
||||
"count": 2,
|
||||
"clause": "SAME",
|
||||
"attribute": "City",
|
||||
"filter": "FromRU"
|
||||
},
|
||||
{
|
||||
"name": "DistinctRU",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "City",
|
||||
"filter": "FromRU"
|
||||
},
|
||||
{
|
||||
"name": "Good",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "Good"
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"count": 3,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromRU",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"name": "Good",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4"
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
5,
|
||||
9,
|
||||
10
|
||||
],
|
||||
[
|
||||
2,
|
||||
6,
|
||||
0,
|
||||
5
|
||||
],
|
||||
[
|
||||
1,
|
||||
8,
|
||||
2,
|
||||
5
|
||||
],
|
||||
[
|
||||
3,
|
||||
4,
|
||||
1,
|
||||
7,
|
||||
0,
|
||||
2
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,93 @@
|
|||
{
|
||||
"name": "multiple replicas (#215)",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Saint-Petersburg"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Berlin"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Paris"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "test",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "LOC_SPB_PLACE"
|
||||
},
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "LOC_MSK_PLACE"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "LOC_SPB_PLACE",
|
||||
"count": 1,
|
||||
"clause": "UNSPECIFIED",
|
||||
"filter": "LOC_SPB"
|
||||
},
|
||||
{
|
||||
"name": "LOC_MSK_PLACE",
|
||||
"count": 1,
|
||||
"clause": "UNSPECIFIED",
|
||||
"filter": "LOC_MSK"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "LOC_SPB",
|
||||
"key": "City",
|
||||
"op": "EQ",
|
||||
"value": "Saint-Petersburg",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "LOC_MSK",
|
||||
"key": "City",
|
||||
"op": "EQ",
|
||||
"value": "Moscow",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
],
|
||||
[
|
||||
1
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,328 @@
|
|||
{
|
||||
"name": "multiple REP, asymmetric",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "5"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "6"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "NA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "NewYork"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "7"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "AF"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Cairo"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "8"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "AF"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Cairo"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "9"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "SA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Lima"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "10"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "AF"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Cairo"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "11"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "NA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "NewYork"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "12"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "NA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "LosAngeles"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "13"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "SA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Lima"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "test",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "SPB"
|
||||
},
|
||||
{
|
||||
"count": 2,
|
||||
"selector": "Americas"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "SPB",
|
||||
"count": 1,
|
||||
"clause": "SAME",
|
||||
"attribute": "City",
|
||||
"filter": "SPBSSD"
|
||||
},
|
||||
{
|
||||
"name": "Americas",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "City",
|
||||
"filter": "Americas"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "SPBSSD",
|
||||
"op": "AND",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "RU",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "City",
|
||||
"op": "EQ",
|
||||
"value": "St.Petersburg",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "SSD",
|
||||
"op": "EQ",
|
||||
"value": "1",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Americas",
|
||||
"op": "OR",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Continent",
|
||||
"op": "EQ",
|
||||
"value": "NA",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Continent",
|
||||
"op": "EQ",
|
||||
"value": "SA",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
1,
|
||||
4
|
||||
],
|
||||
[
|
||||
8,
|
||||
12,
|
||||
5,
|
||||
10
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,97 @@
|
|||
{
|
||||
"name": "non-strict selections",
|
||||
"comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": []
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "not enough nodes (backup factor)",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "MyStore"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "MyStore",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "FromRU"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromRU",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Russia",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "not enough nodes (buckets)",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "MyStore"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "MyStore",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "FromRU"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromRU",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Russia",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
113
src/FrostFS.SDK.Tests/TestData/PlacementTests/rep_only.json
Normal file
113
src/FrostFS.SDK.Tests/TestData/PlacementTests/rep_only.json
Normal file
|
@ -0,0 +1,113 @@
|
|||
{
|
||||
"name": "REP X",
|
||||
"nodes": [
|
||||
{
|
||||
"publicKey": "",
|
||||
"addresses": [],
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Saint-Petersburg"
|
||||
}
|
||||
],
|
||||
"state": "Unspecified"
|
||||
},
|
||||
{
|
||||
"publicKey": "",
|
||||
"addresses": [],
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
],
|
||||
"state": "Unspecified"
|
||||
},
|
||||
{
|
||||
"publicKey": "",
|
||||
"addresses": [],
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Berlin"
|
||||
}
|
||||
],
|
||||
"state": "Unspecified"
|
||||
},
|
||||
{
|
||||
"publicKey": "",
|
||||
"addresses": [],
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Paris"
|
||||
}
|
||||
],
|
||||
"state": "Unspecified"
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "REP 1",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "REP 3",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 3
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
3,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "REP 5",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 5
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,116 @@
|
|||
{
|
||||
"name": "select with unspecified attribute",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "test",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 4,
|
||||
"clause": "DISTINCT",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,87 @@
|
|||
{
|
||||
"name": "invalid selections",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": []
|
||||
}
|
||||
],
|
||||
"tests": [
|
||||
{
|
||||
"name": "missing filter",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "MyStore"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "MyStore",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "FromNL"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromRU",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Russia",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "filter not found"
|
||||
},
|
||||
{
|
||||
"name": "not enough nodes (filter results in empty set)",
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "MyStore"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "MyStore",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "FromMoon"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromMoon",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Moon",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
|
@ -21,7 +21,7 @@ public abstract class ContainerTestsBase
|
|||
|
||||
Mocker = new ContainerMocker(key)
|
||||
{
|
||||
PlacementPolicy = new FrostFsPlacementPolicy(true, new FrostFsReplica(1)),
|
||||
PlacementPolicy = new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1)),
|
||||
Version = new FrostFsVersion(2, 13),
|
||||
ContainerGuid = Guid.NewGuid()
|
||||
};
|
||||
|
|
|
@ -30,7 +30,7 @@ public abstract class ObjectTestsBase
|
|||
|
||||
Mocker = new ObjectMocker(key)
|
||||
{
|
||||
PlacementPolicy = new FrostFsPlacementPolicy(true, new FrostFsReplica(1)),
|
||||
PlacementPolicy = new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1)),
|
||||
Version = new FrostFsVersion(2, 13),
|
||||
ContainerGuid = Guid.NewGuid()
|
||||
};
|
||||
|
|
275
src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs
Normal file
275
src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs
Normal file
|
@ -0,0 +1,275 @@
|
|||
using System.Collections.ObjectModel;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
using FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace FrostFS.SDK.Tests.Unit;
|
||||
|
||||
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
|
||||
public class PlacementVectorTests(ITestOutputHelper testOutputHelper)
|
||||
{
|
||||
private static readonly JsonSerializerOptions serializeOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
private readonly ITestOutputHelper _testOutputHelper = testOutputHelper;
|
||||
|
||||
[Fact]
|
||||
public void PlacementTest()
|
||||
{
|
||||
var path = ".\\..\\..\\..\\TestData\\PlacementTests";
|
||||
Assert.True(Directory.Exists(path));
|
||||
|
||||
var files = Directory.GetFiles(path);
|
||||
|
||||
FrostFsVersion v = new(2, 13);
|
||||
var addresses = new string[] { "localhost", "server1" };
|
||||
|
||||
foreach (var file in files.Where(f => f.EndsWith(".json", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
//if (!file.EndsWith("selector_invalid.json"))
|
||||
// continue;
|
||||
|
||||
var fileName = file[(file.LastIndexOf("..\\") + 3)..];
|
||||
_testOutputHelper.WriteLine($"Open file {fileName}");
|
||||
|
||||
var str = File.ReadAllText(file);
|
||||
Assert.False(string.IsNullOrEmpty(str));
|
||||
|
||||
var testCase = JsonSerializer.Deserialize<TestCase>(str, serializeOptions);
|
||||
|
||||
Assert.NotNull(testCase);
|
||||
Assert.NotNull(testCase.Nodes);
|
||||
Assert.True(testCase.Nodes.Length > 0);
|
||||
|
||||
_testOutputHelper.WriteLine($"Test case: \"{testCase.Name}\"");
|
||||
|
||||
var nodes = testCase.Nodes
|
||||
.Select(n => new FrostFsNodeInfo(v,
|
||||
n.State,
|
||||
addresses.AsReadOnly(),
|
||||
n.Attributes?.ToDictionary(x => x.Key, x => x.Value) ?? [],
|
||||
n.PublicKeyBytes
|
||||
)
|
||||
)
|
||||
.ToArray()
|
||||
.AsReadOnly();
|
||||
|
||||
var netmap = new FrostFsNetmapSnapshot(100, nodes);
|
||||
|
||||
Assert.NotNull(testCase.Tests);
|
||||
|
||||
foreach (var test in testCase.Tests)
|
||||
{
|
||||
_testOutputHelper.WriteLine($"Start test \"{test.Name}\"");
|
||||
|
||||
var policy = new FrostFsPlacementPolicy(
|
||||
test.Policy!.Unique,
|
||||
test.Policy.ContainerBackupFactor,
|
||||
new Collection<FrostFsSelector>(test.Policy.Selectors?.Select(s => s.Selector).ToList() ?? []),
|
||||
new Collection<FrostFsFilter>(test.Policy.Filters?.Select(f => f.Filter).ToList() ?? []),
|
||||
test.Policy.Replicas?.Select(r => new FrostFsReplica(r.Count, r.Selector)).ToArray() ?? []
|
||||
);
|
||||
|
||||
try
|
||||
{
|
||||
var result = netmap.ContainerNodes(policy, test.PivotBytes);
|
||||
|
||||
if (test.Result == null)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(test.Error))
|
||||
{
|
||||
Assert.Fail("Error is expected but has not been thrown");
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.NotNull(test.Policy?.Replicas);
|
||||
Assert.Equal(result.Length, test.Policy.Replicas.Length);
|
||||
|
||||
for (int i = 0; i < result.Length; i++)
|
||||
{
|
||||
Assert.Empty(result[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Equal(test.Result.Length, result.Length);
|
||||
|
||||
for (var i = 0; i < test.Result.Length; i++)
|
||||
{
|
||||
Assert.Equal(test.Result[i].Length, result[i].Length);
|
||||
for (var j = 0; j < test.Result[i].Length; j++)
|
||||
{
|
||||
CompareNodes(nodes[test.Result[i][j]].Attributes, result[i][j]);
|
||||
}
|
||||
}
|
||||
|
||||
if (test.Placement?.Result != null && test.Placement.PivotBytes != null)
|
||||
{
|
||||
var placementResult = netmap.PlacementVectors(result, test.Placement.PivotBytes);
|
||||
|
||||
Assert.Equal(test.Placement.Result.Length, placementResult.Length);
|
||||
|
||||
for (int i = 0; i < placementResult.Length; i++)
|
||||
{
|
||||
Assert.Equal(test.Placement.Result[i].Length, placementResult[i].Length);
|
||||
for (int j = 0; j < placementResult[i].Length; j++)
|
||||
{
|
||||
CompareNodes(nodes[test.Placement.Result[i][j]].Attributes, placementResult[i][j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(test.Error))
|
||||
{
|
||||
Assert.Contains(test.Error, ex.Message, StringComparison.InvariantCulture);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
_testOutputHelper.WriteLine($"Done");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static void CompareNodes(IReadOnlyDictionary<string, string> attrs, FrostFsNodeInfo nodeInfo)
|
||||
{
|
||||
Assert.Equal(attrs.Count, nodeInfo.Attributes.Count);
|
||||
Assert.True(attrs.OrderBy(k => k.Key).SequenceEqual(nodeInfo.Attributes.OrderBy(x => x.Key)));
|
||||
}
|
||||
}
|
||||
|
||||
public class TestCase
|
||||
{
|
||||
public string? Name { get; set; }
|
||||
|
||||
public Node[]? Nodes { get; set; }
|
||||
|
||||
public TestData[]? Tests { get; set; }
|
||||
}
|
||||
|
||||
|
||||
|
||||
public class Node
|
||||
{
|
||||
[JsonPropertyName("attributes")]
|
||||
public KeyValuePair<string, string>[]? Attributes { get; set; }
|
||||
|
||||
public string? PublicKey { get; set; }
|
||||
|
||||
internal byte[]? PublicKeyBytes => string.IsNullOrEmpty(PublicKey) ? [] : Convert.FromBase64String(PublicKey);
|
||||
|
||||
public string[]? Addresses { get; set; }
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<NodeState>))]
|
||||
public NodeState State { get; set; } = NodeState.Online;
|
||||
}
|
||||
|
||||
public class TestData
|
||||
{
|
||||
public string? Name { get; set; }
|
||||
|
||||
public PolicyDto? Policy { get; set; }
|
||||
|
||||
public string? Pivot { get; set; }
|
||||
|
||||
public int[][]? Result { get; set; }
|
||||
|
||||
public string? Error { get; set; }
|
||||
|
||||
internal byte[]? PivotBytes => Pivot != null ? Convert.FromBase64String(Pivot) : null;
|
||||
|
||||
public ResultData? Placement { get; set; }
|
||||
}
|
||||
|
||||
public class PolicyDto
|
||||
{
|
||||
public bool Unique { get; set; }
|
||||
|
||||
public uint ContainerBackupFactor { get; set; }
|
||||
|
||||
public FilterDto[]? Filters { get; set; }
|
||||
|
||||
public ReplicaDto[]? Replicas { get; set; }
|
||||
|
||||
public SelectorDto[]? Selectors { get; set; }
|
||||
}
|
||||
|
||||
public class SelectorDto()
|
||||
{
|
||||
public uint Count { get; set; }
|
||||
|
||||
public string? Name { get; set; }
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<ClauseValues>))]
|
||||
public ClauseValues Clause { get; set; }
|
||||
|
||||
public string? Attribute { get; set; }
|
||||
|
||||
public string? Filter { get; set; }
|
||||
|
||||
public FrostFsSelector Selector => new(Name ?? string.Empty)
|
||||
{
|
||||
Count = Count,
|
||||
Clause = (int)Clause,
|
||||
Filter = Filter,
|
||||
Attribute = Attribute
|
||||
};
|
||||
}
|
||||
|
||||
public class FilterDto
|
||||
{
|
||||
public string? Name { get; set; }
|
||||
|
||||
public string? Key { get; set; }
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<Operation>))]
|
||||
public Operation Op { get; set; }
|
||||
|
||||
public string? Value { get; set; }
|
||||
|
||||
public FilterDto[]? Filters { get; set; }
|
||||
|
||||
public FrostFsFilter Filter => new(
|
||||
Name ?? string.Empty,
|
||||
Key ?? string.Empty,
|
||||
(int)Op,
|
||||
Value ?? string.Empty,
|
||||
Filters != null ? Filters.Select(f => f.Filter).ToArray() : []);
|
||||
}
|
||||
|
||||
public class ReplicaDto
|
||||
{
|
||||
public int Count { get; set; }
|
||||
|
||||
public string? Selector { get; set; }
|
||||
}
|
||||
|
||||
public class ResultData
|
||||
{
|
||||
public string? Pivot { get; set; }
|
||||
|
||||
public int[][]? Result { get; set; }
|
||||
|
||||
internal byte[]? PivotBytes => Pivot != null ? Convert.FromBase64String(Pivot) : null;
|
||||
}
|
||||
|
||||
public enum ClauseValues
|
||||
{
|
||||
UNSPECIFIED = 0,
|
||||
SAME,
|
||||
DISTINCT
|
||||
}
|
|
@ -30,7 +30,7 @@ public abstract class SessionTestsBase
|
|||
|
||||
Mocker = new SessionMocker(key)
|
||||
{
|
||||
PlacementPolicy = new FrostFsPlacementPolicy(true, new FrostFsReplica(1)),
|
||||
PlacementPolicy = new FrostFsPlacementPolicy(true, 1, [], [], new FrostFsReplica(1)),
|
||||
Version = new FrostFsVersion(2, 13)
|
||||
};
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue