[#29] Client: Add object placement methods #40
25 changed files with 1382 additions and 32 deletions
10
src/FrostFS.SDK.Client/Models/Netmap/FrostFsFilter.cs
Normal file
10
src/FrostFS.SDK.Client/Models/Netmap/FrostFsFilter.cs
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
namespace FrostFS.SDK;
|
||||||
|
|
||||||
|
public class FrostFsFilter(string name, string key, int operation, string value, FrostFsFilter[] filters) : IFrostFsFilter
|
||||||
|
{
|
||||||
|
public string Name { get; } = name;
|
||||||
|
public string Key { get; } = key;
|
||||||
|
public int Operation { get; } = operation;
|
||||||
|
public string Value { get; } = value;
|
||||||
|
public FrostFsFilter[] Filters { get; } = filters;
|
||||||
|
}
|
|
@ -1,4 +1,11 @@
|
||||||
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.ObjectModel;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
using FrostFS.SDK.Client;
|
||||||
|
using FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
using FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
namespace FrostFS.SDK;
|
namespace FrostFS.SDK;
|
||||||
|
|
||||||
|
@ -7,4 +14,252 @@ public class FrostFsNetmapSnapshot(ulong epoch, IReadOnlyList<FrostFsNodeInfo> n
|
||||||
public ulong Epoch { get; private set; } = epoch;
|
public ulong Epoch { get; private set; } = epoch;
|
||||||
|
|
||||||
public IReadOnlyList<FrostFsNodeInfo> NodeInfoCollection { get; private set; } = nodeInfoCollection;
|
public IReadOnlyList<FrostFsNodeInfo> NodeInfoCollection { get; private set; } = nodeInfoCollection;
|
||||||
}
|
|
||||||
|
internal static INormalizer NewReverseMinNorm(double minV)
|
||||||
|
{
|
||||||
|
return new ReverseMinNorm { min = minV };
|
||||||
|
}
|
||||||
|
|
||||||
|
// newSigmoidNorm returns a normalizer which
|
||||||
|
// normalize values in range of 0.0 to 1.0 to a scaled sigmoid.
|
||||||
|
internal static INormalizer NewSigmoidNorm(double scale)
|
||||||
|
{
|
||||||
|
return new SigmoidNorm(scale);
|
||||||
|
}
|
||||||
|
|
||||||
|
// PlacementVectors sorts container nodes returned by ContainerNodes method
|
||||||
|
// and returns placement vectors for the entity identified by the given pivot.
|
||||||
|
// For example, in order to build node list to store the object, binary-encoded
|
||||||
|
// object identifier can be used as pivot. Result is deterministic for
|
||||||
|
// the fixed NetMap and parameters.
|
||||||
|
public FrostFsNodeInfo[][] PlacementVectors(FrostFsNodeInfo[][] vectors, byte[] pivot)
|
||||||
|
{
|
||||||
|
if (vectors is null)
|
||||||
|
{
|
||||||
|
throw new ArgumentNullException(nameof(vectors));
|
||||||
|
}
|
||||||
|
|
||||||
|
using var murmur3 = new Murmur3(0);
|
||||||
|
var hash = murmur3.GetCheckSum64(pivot);
|
||||||
|
|
||||||
|
var wf = Tools.DefaultWeightFunc(NodeInfoCollection.ToArray());
|
||||||
|
|
||||||
|
var result = new FrostFsNodeInfo[vectors.Length][];
|
||||||
|
var maxSize = vectors.Max(x => x.Length);
|
||||||
|
|
||||||
|
var spanWeigths = new double[maxSize];
|
||||||
|
|
||||||
|
for (int i = 0; i < vectors.Length; i++)
|
||||||
|
{
|
||||||
|
result[i] = new FrostFsNodeInfo[vectors[i].Length];
|
||||||
|
|
||||||
|
for (int j = 0; j < vectors[i].Length; j++)
|
||||||
|
{
|
||||||
|
result[i][j] = vectors[i][j];
|
||||||
|
}
|
||||||
|
|
||||||
|
Tools.AppendWeightsTo(result[i], wf, spanWeigths);
|
||||||
|
|
||||||
|
Tools.SortHasherSliceByWeightValue(result[i].ToList<FrostFsNodeInfo>(), spanWeigths, hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelectFilterNodes returns a two-dimensional list of nodes as a result of applying the
|
||||||
|
// given SelectFilterExpr to the NetMap.
|
||||||
|
// If the SelectFilterExpr contains only filters, the result contains a single row with the
|
||||||
|
// result of the last filter application.
|
||||||
|
// If the SelectFilterExpr contains only selectors, the result contains the selection rows
|
||||||
|
// of the last select application.
|
||||||
|
List<List<FrostFsNodeInfo>> SelectFilterNodes(SelectFilterExpr expr)
|
||||||
|
{
|
||||||
|
var policy = new FrostFsPlacementPolicy(false);
|
||||||
|
|
||||||
|
foreach (var f in expr.Filters)
|
||||||
|
policy.Filters.Add(f);
|
||||||
|
|
||||||
|
policy.Selectors.Add(expr.Selector);
|
||||||
|
|
||||||
|
var ctx = new Context(this)
|
||||||
|
{
|
||||||
|
Cbf = expr.Cbf
|
||||||
|
};
|
||||||
|
|
||||||
|
ctx.ProcessFilters(policy);
|
||||||
|
ctx.ProcessSelectors(policy);
|
||||||
|
|
||||||
|
var ret = new List<List<FrostFsNodeInfo>>();
|
||||||
|
|
||||||
|
if (expr.Selector == null)
|
||||||
|
{
|
||||||
|
var lastFilter = expr.Filters[^1];
|
||||||
|
|
||||||
|
var subCollestion = new List<FrostFsNodeInfo>();
|
||||||
|
ret.Add(subCollestion);
|
||||||
|
|
||||||
|
foreach (var nodeInfo in NodeInfoCollection)
|
||||||
|
{
|
||||||
|
if (ctx.Match(ctx.ProcessedFilters[lastFilter.Name], nodeInfo))
|
||||||
|
{
|
||||||
|
subCollestion.Add(nodeInfo);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (expr.Selector.Name != null)
|
||||||
|
{
|
||||||
|
var sel = ctx.GetSelection(ctx.ProcessedSelectors[expr.Selector.Name]);
|
||||||
|
|
||||||
|
foreach (var ns in sel)
|
||||||
|
{
|
||||||
|
var subCollestion = new List<FrostFsNodeInfo>();
|
||||||
|
ret.Add(subCollestion);
|
||||||
|
foreach (var n in ns)
|
||||||
|
{
|
||||||
|
subCollestion.Add(n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static Func<FrostFsNodeInfo, double> NewWeightFunc(INormalizer capNorm, INormalizer priceNorm)
|
||||||
|
{
|
||||||
|
return new Func<FrostFsNodeInfo, double>((FrostFsNodeInfo nodeInfo) =>
|
||||||
|
{
|
||||||
|
return capNorm.Normalize(nodeInfo.GetCapacity()) * priceNorm.Normalize(nodeInfo.Price);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FrostFsNodeInfo[] FlattenNodes(List<List<FrostFsNodeInfo>> nodes)
|
||||||
|
{
|
||||||
|
int sz = 0;
|
||||||
|
foreach (var ns in nodes)
|
||||||
|
{
|
||||||
|
sz += ns.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
var result = new FrostFsNodeInfo[sz];
|
||||||
|
|
||||||
|
int i = 0;
|
||||||
|
foreach (var ns in nodes)
|
||||||
|
{
|
||||||
|
foreach (var n in ns)
|
||||||
|
{
|
||||||
|
result[i++] = n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ContainerNodes returns two-dimensional list of nodes as a result of applying
|
||||||
|
// given PlacementPolicy to the NetMap. Each line of the list corresponds to a
|
||||||
|
// replica descriptor. Line order corresponds to order of ReplicaDescriptor list
|
||||||
|
// in the policy. Nodes are pre-filtered according to the Filter list from
|
||||||
|
// the policy, and then selected by Selector list. Result is deterministic for
|
||||||
|
// the fixed NetMap and parameters.
|
||||||
|
//
|
||||||
|
// Result can be used in PlacementVectors.
|
||||||
|
public FrostFsNodeInfo[][] ContainerNodes(FrostFsPlacementPolicy p, byte[]? pivot)
|
||||||
|
{
|
||||||
|
var c = new Context(this)
|
||||||
|
{
|
||||||
|
Cbf = p.BackupFactor
|
||||||
|
};
|
||||||
|
|
||||||
|
if (pivot != null && pivot.Length > 0)
|
||||||
|
{
|
||||||
|
c.HrwSeed = pivot;
|
||||||
|
|
||||||
|
using var murmur = new Murmur3(0);
|
||||||
|
c.HrwSeedHash = murmur.GetCheckSum64(pivot);
|
||||||
|
}
|
||||||
|
|
||||||
|
c.ProcessFilters(p);
|
||||||
|
c.ProcessSelectors(p);
|
||||||
|
|
||||||
|
var unique = p.IsUnique();
|
||||||
|
|
||||||
|
var result = new List<List<FrostFsNodeInfo>>(p.Replicas.Length);
|
||||||
|
for (int i = 0; i < p.Replicas.Length; i++)
|
||||||
|
{
|
||||||
|
result.Add([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Note that the cached selectors are not used when the policy contains the UNIQUE flag.
|
||||||
|
// This is necessary because each selection vector affects potentially the subsequent vectors
|
||||||
|
// and thus we call getSelection in such case, in order to take into account nodes previously
|
||||||
|
// marked as used by earlier replicas.
|
||||||
|
for (int i = 0; i < p.Replicas.Length; i++)
|
||||||
|
{
|
||||||
|
var sName = p.Replicas[i].Selector;
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(sName) && !(p.Replicas.Length == 1 && p.Selectors.Count == 1))
|
||||||
|
{
|
||||||
|
var s = new FrostFsSelector(Context.mainFilterName)
|
||||||
|
{
|
||||||
|
Count = p.Replicas[i].CountNodes()
|
||||||
|
};
|
||||||
|
|
||||||
|
var nodes = c.GetSelection(s);
|
||||||
|
|
||||||
|
var arg = new List<List<FrostFsNodeInfo>>(nodes.Count);
|
||||||
|
for (int j = 0; j < nodes.Count; j++)
|
||||||
|
{
|
||||||
|
arg[i] = nodes[j];
|
||||||
|
}
|
||||||
|
|
||||||
|
result[i].AddRange(FlattenNodes(arg));
|
||||||
|
|
||||||
|
if (unique)
|
||||||
|
{
|
||||||
|
foreach (var n in result[i])
|
||||||
|
{
|
||||||
|
c.UsedNodes[n.Hash()] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unique)
|
||||||
|
{
|
||||||
|
if (!c.ProcessedSelectors.TryGetValue(sName, out var s) || s == null)
|
||||||
|
{
|
||||||
|
throw new FrostFsException($"selector not found: {sName}");
|
||||||
|
}
|
||||||
|
|
||||||
|
var nodes = c.GetSelection(c.ProcessedSelectors[sName]);
|
||||||
|
|
||||||
|
result[i].AddRange(FlattenNodes(nodes));
|
||||||
|
|
||||||
|
foreach (var n in result[i])
|
||||||
|
{
|
||||||
|
c.UsedNodes[n.Hash()] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var nodes = c.Selections[sName];
|
||||||
|
|
||||||
|
var arg = new List<List<FrostFsNodeInfo>>(nodes.Count);
|
||||||
|
for (int j = 0; j < nodes.Count; j++)
|
||||||
|
{
|
||||||
|
arg[i] = nodes[j];
|
||||||
|
}
|
||||||
|
|
||||||
|
result[i].AddRange(FlattenNodes(arg));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var collection = new FrostFsNodeInfo[result.Count][];
|
||||||
|
for(int i =0; i < result.Count; i++)
|
||||||
|
{
|
||||||
|
collection[i] = [.. result[i]];
|
||||||
|
}
|
||||||
|
|
||||||
|
return collection;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Globalization;
|
||||||
|
|
||||||
|
using FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
using FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
namespace FrostFS.SDK;
|
namespace FrostFS.SDK;
|
||||||
|
|
||||||
|
@ -8,11 +12,69 @@ public class FrostFsNodeInfo(
|
||||||
NodeState state,
|
NodeState state,
|
||||||
IReadOnlyCollection<string> addresses,
|
IReadOnlyCollection<string> addresses,
|
||||||
IReadOnlyDictionary<string, string> attributes,
|
IReadOnlyDictionary<string, string> attributes,
|
||||||
ReadOnlyMemory<byte> publicKey)
|
ReadOnlyMemory<byte> publicKey) : IHasher
|
||||||
{
|
{
|
||||||
public NodeState State { get; private set; } = state;
|
private ulong _hash;
|
||||||
public FrostFsVersion Version { get; private set; } = version;
|
|
||||||
public IReadOnlyCollection<string> Addresses { get; private set; } = addresses;
|
// attrPrice is a key to the node attribute that indicates the
|
||||||
public IReadOnlyDictionary<string, string> Attributes { get; private set; } = attributes;
|
// price in GAS tokens for storing one GB of data during one Epoch.
|
||||||
public ReadOnlyMemory<byte> PublicKey { get; private set; } = publicKey;
|
internal const string AttrPrice = "Price";
|
||||||
}
|
|
||||||
|
// attrCapacity is a key to the node attribute that indicates the
|
||||||
|
// total available disk space in Gigabytes.
|
||||||
|
internal const string AttrCapacity = "Capacity";
|
||||||
|
|
||||||
|
// attrExternalAddr is a key for the attribute storing node external addresses.
|
||||||
|
internal const string AttrExternalAddr = "ExternalAddr";
|
||||||
|
|
||||||
|
// sepExternalAddr is a separator for multi-value ExternalAddr attribute.
|
||||||
|
internal const string SepExternalAddr = ",";
|
||||||
|
|
||||||
|
private ulong price = ulong.MaxValue;
|
||||||
|
|
||||||
|
public NodeState State { get; } = state;
|
||||||
|
|
||||||
|
public FrostFsVersion Version { get; } = version;
|
||||||
|
|
||||||
|
public IReadOnlyCollection<string> Addresses { get; } = addresses;
|
||||||
|
|
||||||
|
public IReadOnlyDictionary<string, string> Attributes { get; } = attributes;
|
||||||
|
|
||||||
|
public ReadOnlyMemory<byte> PublicKey { get; } = publicKey;
|
||||||
|
|
||||||
|
public ulong Hash()
|
||||||
|
{
|
||||||
|
if (_hash == 0)
|
||||||
|
{
|
||||||
|
using var murmur3 = new Murmur3(0);
|
||||||
|
murmur3.Initialize();
|
||||||
|
_hash = murmur3.GetCheckSum64(PublicKey.ToArray());
|
||||||
|
}
|
||||||
|
|
||||||
|
return _hash;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal ulong GetCapacity()
|
||||||
|
{
|
||||||
|
if (!Attributes.TryGetValue(AttrCapacity, out var val))
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
return ulong.Parse(val, CultureInfo.InvariantCulture);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal ulong Price
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
if (price == ulong.MaxValue)
|
||||||
|
{
|
||||||
|
if (!Attributes.TryGetValue(AttrPrice, out var val))
|
||||||
|
price = 0;
|
||||||
|
else
|
||||||
|
price = uint.Parse(val, CultureInfo.InvariantCulture);
|
||||||
|
}
|
||||||
|
|
||||||
|
return price;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
using System.Collections.ObjectModel;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
using FrostFS.Netmap;
|
using FrostFS.Netmap;
|
||||||
|
@ -13,12 +13,21 @@ public struct FrostFsPlacementPolicy(bool unique, params FrostFsReplica[] replic
|
||||||
private PlacementPolicy policy;
|
private PlacementPolicy policy;
|
||||||
|
|
||||||
public FrostFsReplica[] Replicas { get; private set; } = replicas;
|
public FrostFsReplica[] Replicas { get; private set; } = replicas;
|
||||||
|
|
||||||
|
public Collection<FrostFsSelector> Selectors { get; } = [];
|
||||||
|
|
||||||
|
public Collection<FrostFsFilter> Filters { get; } = [];
|
||||||
|
|
||||||
public bool Unique { get; private set; } = unique;
|
public bool Unique { get; private set; } = unique;
|
||||||
|
|
||||||
|
public uint BackupFactor { get; set; }
|
||||||
|
|
||||||
public override readonly bool Equals(object obj)
|
public override readonly bool Equals(object obj)
|
||||||
{
|
{
|
||||||
if (obj is null)
|
if (obj is null)
|
||||||
|
{
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
var other = (FrostFsPlacementPolicy)obj;
|
var other = (FrostFsPlacementPolicy)obj;
|
||||||
|
|
||||||
|
@ -46,14 +55,10 @@ public struct FrostFsPlacementPolicy(bool unique, params FrostFsReplica[] replic
|
||||||
return policy;
|
return policy;
|
||||||
}
|
}
|
||||||
|
|
||||||
//public static FrostFsPlacementPolicy ToModel(placementPolicy)
|
internal readonly bool IsUnique()
|
||||||
//{
|
{
|
||||||
// return new FrostFsPlacementPolicy(
|
return Unique || Replicas.Any(r => r.EcDataCount != 0 || r.EcParityCount != 0);
|
||||||
// placementPolicy.Unique,
|
}
|
||||||
// placementPolicy.Replicas.Select(replica => replica.ToModel()).ToArray()
|
|
||||||
// );
|
|
||||||
//}
|
|
||||||
|
|
||||||
|
|
||||||
public override readonly int GetHashCode()
|
public override readonly int GetHashCode()
|
||||||
{
|
{
|
||||||
|
@ -86,4 +91,4 @@ public struct FrostFsPlacementPolicy(bool unique, params FrostFsReplica[] replic
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,8 @@ public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
||||||
{
|
{
|
||||||
public int Count { get; set; }
|
public int Count { get; set; }
|
||||||
public string Selector { get; set; }
|
public string Selector { get; set; }
|
||||||
|
public uint EcDataCount { get; set; }
|
||||||
|
public uint EcParityCount { get; set; }
|
||||||
|
|
||||||
public FrostFsReplica(int count, string? selector = null)
|
public FrostFsReplica(int count, string? selector = null)
|
||||||
{
|
{
|
||||||
|
@ -18,16 +20,23 @@ public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
||||||
public override readonly bool Equals(object obj)
|
public override readonly bool Equals(object obj)
|
||||||
{
|
{
|
||||||
if (obj is null)
|
if (obj is null)
|
||||||
|
{
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
var other = (FrostFsReplica)obj;
|
var other = (FrostFsReplica)obj;
|
||||||
|
|
||||||
return Count == other.Count && Selector == other.Selector;
|
return Count == other.Count && Selector == other.Selector;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public readonly uint CountNodes()
|
||||||
|
{
|
||||||
|
return Count != 0 ? (uint)Count : EcDataCount + EcParityCount;
|
||||||
|
}
|
||||||
|
|
||||||
public override readonly int GetHashCode()
|
public override readonly int GetHashCode()
|
||||||
{
|
{
|
||||||
return Count + Selector.GetHashCode();
|
return (Count + Selector.GetHashCode()) ^ (int)EcDataCount ^ (int)EcParityCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static bool operator ==(FrostFsReplica left, FrostFsReplica right)
|
public static bool operator ==(FrostFsReplica left, FrostFsReplica right)
|
||||||
|
@ -42,6 +51,9 @@ public struct FrostFsReplica : IEquatable<FrostFsReplica>
|
||||||
|
|
||||||
public readonly bool Equals(FrostFsReplica other)
|
public readonly bool Equals(FrostFsReplica other)
|
||||||
{
|
{
|
||||||
return Count == other.Count && Selector == other.Selector;
|
return Count == other.Count
|
||||||
|
&& Selector == other.Selector
|
||||||
|
&& EcDataCount == other.EcDataCount
|
||||||
|
&& EcParityCount == other.EcParityCount;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
10
src/FrostFS.SDK.Client/Models/Netmap/FrostFsSelector.cs
Normal file
10
src/FrostFS.SDK.Client/Models/Netmap/FrostFsSelector.cs
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
namespace FrostFS.SDK;
|
||||||
|
|
||||||
|
public class FrostFsSelector(string name)
|
||||||
|
{
|
||||||
|
public string Name { get; } = name;
|
||||||
|
public uint Count { get; set; }
|
||||||
|
public uint Clause { get; set; }
|
||||||
|
public string? Attribute { get; set; }
|
||||||
|
public string? Filter { get; set; }
|
||||||
|
}
|
11
src/FrostFS.SDK.Client/Models/Netmap/IFrostFsFilter.cs
Normal file
11
src/FrostFS.SDK.Client/Models/Netmap/IFrostFsFilter.cs
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
namespace FrostFS.SDK
|
||||||
|
{
|
||||||
|
public interface IFrostFsFilter
|
||||||
|
{
|
||||||
|
FrostFsFilter[] Filters { get; }
|
||||||
|
string Key { get; }
|
||||||
|
string Name { get; }
|
||||||
|
int Operation { get; }
|
||||||
|
string Value { get; }
|
||||||
|
}
|
||||||
|
}
|
7
src/FrostFS.SDK.Client/Models/Netmap/NodeAttrPair.cs
Normal file
7
src/FrostFS.SDK.Client/Models/Netmap/NodeAttrPair.cs
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
namespace FrostFS.SDK;
|
||||||
|
|
||||||
|
struct NodeAttrPair
|
||||||
|
{
|
||||||
|
internal string attr;
|
||||||
|
internal FrostFsNodeInfo[] nodes;
|
||||||
|
}
|
8
src/FrostFS.SDK.Client/Models/Netmap/Placement/Clause.cs
Normal file
8
src/FrostFS.SDK.Client/Models/Netmap/Placement/Clause.cs
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
public enum FrostFsClause
|
||||||
|
{
|
||||||
|
Unspecified = 0,
|
||||||
|
Same,
|
||||||
|
Distinct
|
||||||
|
}
|
433
src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs
Normal file
433
src/FrostFS.SDK.Client/Models/Netmap/Placement/Context.cs
Normal file
|
@ -0,0 +1,433 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Globalization;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal struct Context
|
||||||
|
{
|
||||||
|
private const string errInvalidFilterName = "filter name is invalid";
|
||||||
|
private const string errInvalidFilterOp = "invalid filter operation";
|
||||||
|
private const string errFilterNotFound = "filter not found";
|
||||||
|
private const string errNonEmptyFilters = "simple filter contains sub-filters";
|
||||||
|
private const string errNotEnoughNodes = "not enough nodes to SELECT from";
|
||||||
|
private const string errUnnamedTopFilter = "unnamed top-level filter";
|
||||||
|
|
||||||
|
internal const string mainFilterName = "*";
|
||||||
|
internal const string likeWildcard = "*";
|
||||||
|
|
||||||
|
// network map to operate on
|
||||||
|
internal FrostFsNetmapSnapshot NetMap { get; }
|
||||||
|
|
||||||
|
// cache of processed filters
|
||||||
|
internal Dictionary<string, FrostFsFilter> ProcessedFilters { get; } = [];
|
||||||
|
|
||||||
|
// cache of processed selectors
|
||||||
|
internal Dictionary<string, FrostFsSelector> ProcessedSelectors { get; } = [];
|
||||||
|
|
||||||
|
// stores results of selector processing
|
||||||
|
internal Dictionary<string, List<List<FrostFsNodeInfo>>> Selections { get; } = [];
|
||||||
|
|
||||||
|
// cache of parsed numeric values
|
||||||
|
internal Dictionary<string, ulong> NumCache { get; } = [];
|
||||||
|
|
||||||
|
internal byte[]? HrwSeed { get; set; }
|
||||||
|
|
||||||
|
// hrw.Hash of hrwSeed
|
||||||
|
internal ulong HrwSeedHash { get; set; }
|
||||||
|
|
||||||
|
// container backup factor
|
||||||
|
internal uint Cbf { get; set; }
|
||||||
|
|
||||||
|
// nodes already used in previous selections, which is needed when the placement
|
||||||
|
// policy uses the UNIQUE flag. Nodes marked as used are not used in subsequent
|
||||||
|
// base selections.
|
||||||
|
internal Dictionary<ulong, bool> UsedNodes { get; } = [];
|
||||||
|
|
||||||
|
// If true, returns an error when netmap does not contain enough nodes for selection.
|
||||||
|
// By default best effort is taken.
|
||||||
|
internal bool Strict { get; set; }
|
||||||
|
|
||||||
|
// weightFunc is a weighting function for determining node priority
|
||||||
|
// which combines low price and high performance
|
||||||
|
private readonly Func<FrostFsNodeInfo, double> weightFunc;
|
||||||
|
|
||||||
|
public Context(FrostFsNetmapSnapshot netMap)
|
||||||
|
{
|
||||||
|
NetMap = netMap;
|
||||||
|
weightFunc = Tools.DefaultWeightFunc(NetMap.NodeInfoCollection);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal void ProcessFilters(FrostFsPlacementPolicy policy)
|
||||||
|
{
|
||||||
|
foreach (var filter in policy.Filters)
|
||||||
|
{
|
||||||
|
ProcessFilter(filter, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
readonly void ProcessFilter(FrostFsFilter filter, bool top)
|
||||||
|
{
|
||||||
|
var filterName = filter.Name;
|
||||||
|
if (filterName == mainFilterName)
|
||||||
|
{
|
||||||
|
throw new FrostFsException($"{errInvalidFilterName}: '{errInvalidFilterName}' is reserved");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (top && string.IsNullOrEmpty(filterName))
|
||||||
|
{
|
||||||
|
throw new FrostFsException(errUnnamedTopFilter);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!top && !string.IsNullOrEmpty(filterName) && !ProcessedFilters.ContainsKey(filterName))
|
||||||
|
{
|
||||||
|
throw new FrostFsException(errFilterNotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (filter.Operation == (int)Operation.AND ||
|
||||||
|
filter.Operation == (int)Operation.OR ||
|
||||||
|
filter.Operation == (int)Operation.NOT)
|
||||||
|
{
|
||||||
|
foreach (var f in filter.Filters)
|
||||||
|
ProcessFilter(f, false);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (filter.Filters.Length != 0)
|
||||||
|
{
|
||||||
|
throw new FrostFsException(errNonEmptyFilters);
|
||||||
|
}
|
||||||
|
else if (!top && !string.IsNullOrEmpty(filterName))
|
||||||
|
{
|
||||||
|
// named reference
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (filter.Operation == (int)Operation.EQ ||
|
||||||
|
filter.Operation == (int)Operation.NE ||
|
||||||
|
filter.Operation == (int)Operation.LIKE ||
|
||||||
|
filter.Operation == (int)Operation.GT ||
|
||||||
|
filter.Operation == (int)Operation.GE ||
|
||||||
|
filter.Operation == (int)Operation.LT ||
|
||||||
|
filter.Operation == (int)Operation.LE)
|
||||||
|
{
|
||||||
|
var n = uint.Parse(filter.Value, CultureInfo.InvariantCulture);
|
||||||
|
NumCache[filter.Value] = n;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw new FrostFsException($"{errInvalidFilterOp}: {filter.Operation}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (top)
|
||||||
|
{
|
||||||
|
ProcessedFilters[filterName] = filter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// processSelectors processes selectors and returns error is any of them is invalid.
|
||||||
|
internal void ProcessSelectors(FrostFsPlacementPolicy policy)
|
||||||
|
{
|
||||||
|
foreach (var selector in policy.Selectors)
|
||||||
|
{
|
||||||
|
var filterName = selector.Filter;
|
||||||
|
if (filterName != mainFilterName)
|
||||||
|
{
|
||||||
|
if (selector.Filter == null || !ProcessedFilters.ContainsKey(selector.Filter))
|
||||||
|
{
|
||||||
|
throw new FrostFsException($"{errFilterNotFound}: SELECT FROM '{filterName}'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ProcessedSelectors[selector.Name] = selector;
|
||||||
|
|
||||||
|
var selection = GetSelection(selector);
|
||||||
|
|
||||||
|
Selections[selector.Name] = selection;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// calcNodesCount returns number of buckets and minimum number of nodes in every bucket
|
||||||
|
// for the given selector.
|
||||||
|
static (int bucketCount, int nodesInBucket) CalcNodesCount(FrostFsSelector selector)
|
||||||
|
{
|
||||||
|
return selector.Clause == (uint)FrostFsClause.Same
|
||||||
|
? (1, (int)selector.Count)
|
||||||
|
: ((int)selector.Count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// getSelectionBase returns nodes grouped by selector attribute.
|
||||||
|
// It it guaranteed that each pair will contain at least one node.
|
||||||
|
internal NodeAttrPair[] GetSelectionBase(FrostFsSelector selector)
|
||||||
|
{
|
||||||
|
var fName = selector.Filter ?? throw new FrostFsException("Filter name for selector is empty");
|
||||||
|
|
||||||
|
_ = ProcessedFilters.TryGetValue(fName, out var f);
|
||||||
|
|
||||||
|
var isMain = fName == mainFilterName;
|
||||||
|
var result = new List<NodeAttrPair>();
|
||||||
|
|
||||||
|
var nodeMap = new Dictionary<string, List<FrostFsNodeInfo>>();
|
||||||
|
var attr = selector.Attribute;
|
||||||
|
|
||||||
|
foreach (var node in NetMap.NodeInfoCollection)
|
||||||
|
{
|
||||||
|
if (UsedNodes.ContainsKey(node.Hash()))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isMain || Match(f, node))
|
||||||
|
{
|
||||||
|
if (attr == null)
|
||||||
|
{
|
||||||
|
// Default attribute is transparent identifier which is different for every node.
|
||||||
|
result.Add(new NodeAttrPair { attr = "", nodes = [node] });
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var v = node.Attributes[attr];
|
||||||
|
if (!nodeMap.TryGetValue(v, out var nodes) || nodes == null)
|
||||||
|
{
|
||||||
|
nodeMap[v] = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
nodeMap[v].Add(node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(attr))
|
||||||
|
{
|
||||||
|
foreach (var v in nodeMap)
|
||||||
|
{
|
||||||
|
result.Add(new NodeAttrPair() { attr = v.Key, nodes = [.. v.Value] });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (HrwSeed != null && HrwSeed.Length != 0)
|
||||||
|
{
|
||||||
|
double[] ws = [];
|
||||||
|
|
||||||
|
foreach (var res in result)
|
||||||
|
{
|
||||||
|
Tools.AppendWeightsTo(res.nodes, weightFunc, ws);
|
||||||
|
Tools.SortHasherSliceByWeightValue(res.nodes.ToList(), ws, HrwSeedHash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [.. result];
|
||||||
|
}
|
||||||
|
|
||||||
|
static double CalcBucketWeight(List<FrostFsNodeInfo> ns, IAggregator a, Func<FrostFsNodeInfo, double> wf)
|
||||||
|
{
|
||||||
|
foreach (var node in ns)
|
||||||
|
{
|
||||||
|
a.Add(wf(node));
|
||||||
|
}
|
||||||
|
|
||||||
|
return a.Compute();
|
||||||
|
}
|
||||||
|
|
||||||
|
// getSelection returns nodes grouped by s.attribute.
|
||||||
|
// Last argument specifies if more buckets can be used to fulfill CBF.
|
||||||
|
internal List<List<FrostFsNodeInfo>> GetSelection(FrostFsSelector s)
|
||||||
|
{
|
||||||
|
var (bucketCount, nodesInBucket) = CalcNodesCount(s);
|
||||||
|
|
||||||
|
var buckets = GetSelectionBase(s);
|
||||||
|
|
||||||
|
if (Strict && buckets.Length < bucketCount)
|
||||||
|
throw new FrostFsException($"errNotEnoughNodes: '{s.Name}'");
|
||||||
|
|
||||||
|
// We need deterministic output in case there is no pivot.
|
||||||
|
// If pivot is set, buckets are sorted by HRW.
|
||||||
|
// However, because initial order influences HRW order for buckets with equal weights,
|
||||||
|
// we also need to have deterministic input to HRW sorting routine.
|
||||||
|
if (HrwSeed == null || HrwSeed.Length == 0)
|
||||||
|
{
|
||||||
|
buckets = string.IsNullOrEmpty(s.Attribute)
|
||||||
|
? [.. buckets.OrderBy(b => b.nodes[0].Hash())]
|
||||||
|
: [.. buckets.OrderBy(b => b.attr)];
|
||||||
|
}
|
||||||
|
|
||||||
|
var maxNodesInBucket = nodesInBucket * (int)Cbf;
|
||||||
|
|
||||||
|
var res = new List<List<FrostFsNodeInfo>>(buckets.Length);
|
||||||
|
var fallback = new List<List<FrostFsNodeInfo>>(buckets.Length);
|
||||||
|
|
||||||
|
for (int i = 0; i < buckets.Length; i++)
|
||||||
|
{
|
||||||
|
var ns = buckets[i].nodes;
|
||||||
|
if (ns.Length >= maxNodesInBucket)
|
||||||
|
{
|
||||||
|
res.Add(new List<FrostFsNodeInfo>(ns[..maxNodesInBucket]));
|
||||||
|
}
|
||||||
|
else if (ns.Length >= nodesInBucket)
|
||||||
|
{
|
||||||
|
fallback.Add(new List<FrostFsNodeInfo>(ns));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (res.Count < bucketCount)
|
||||||
|
{
|
||||||
|
// Fallback to using minimum allowed backup factor (1).
|
||||||
|
res = fallback;
|
||||||
|
|
||||||
|
if (Strict && res.Count < bucketCount)
|
||||||
|
{
|
||||||
|
throw new FrostFsException($"{errNotEnoughNodes}: {s}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (HrwSeed != null && HrwSeed.Length != 0)
|
||||||
|
{
|
||||||
|
var weights = new double[res.Count];
|
||||||
|
var a = new MeanIQRAgg();
|
||||||
|
|
||||||
|
for (int i = 0; i < res.Count; i++)
|
||||||
|
{
|
||||||
|
a.Clear();
|
||||||
|
weights[i] = CalcBucketWeight(res[i], a, weightFunc);
|
||||||
|
}
|
||||||
|
|
||||||
|
var hashers = res.Select(r => new HasherList(r)).ToList();
|
||||||
|
Tools.SortHasherSliceByWeightValue(hashers, weights, HrwSeedHash);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (res.Count < bucketCount)
|
||||||
|
{
|
||||||
|
if (Strict && res.Count == 0)
|
||||||
|
{
|
||||||
|
throw new FrostFsException(errNotEnoughNodes);
|
||||||
|
}
|
||||||
|
|
||||||
|
bucketCount = res.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(s.Attribute))
|
||||||
|
{
|
||||||
|
fallback = res.Skip(bucketCount).ToList();
|
||||||
|
res = res.Take(bucketCount).ToList();
|
||||||
|
|
||||||
|
for (int i = 0; i < fallback.Count; i++)
|
||||||
|
{
|
||||||
|
var index = i % bucketCount;
|
||||||
|
if (res[index].Count >= maxNodesInBucket)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
res[index].AddRange(fallback[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.Take(bucketCount).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
internal bool MatchKeyValue(FrostFsFilter f, FrostFsNodeInfo nodeInfo)
|
||||||
|
{
|
||||||
|
switch (f.Operation)
|
||||||
|
{
|
||||||
|
case (int)Operation.EQ:
|
||||||
|
return nodeInfo.Attributes[f.Key] == f.Value;
|
||||||
|
case (int)Operation.LIKE:
|
||||||
|
{
|
||||||
|
var hasPrefix = f.Value.StartsWith(likeWildcard, StringComparison.Ordinal);
|
||||||
|
var hasSuffix = f.Value.EndsWith(likeWildcard, StringComparison.Ordinal);
|
||||||
|
|
||||||
|
var start = hasPrefix ? likeWildcard.Length : 0;
|
||||||
|
var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length;
|
||||||
|
var str = f.Value[start..end];
|
||||||
|
|
||||||
|
if (hasPrefix && hasSuffix)
|
||||||
|
return nodeInfo.Attributes[f.Key].Contains(str);
|
||||||
|
|
||||||
|
if (hasPrefix && !hasSuffix)
|
||||||
|
return nodeInfo.Attributes[f.Key].EndsWith(str, StringComparison.Ordinal);
|
||||||
|
|
||||||
|
if (!hasPrefix && hasSuffix)
|
||||||
|
return nodeInfo.Attributes[f.Key].StartsWith(str, StringComparison.Ordinal);
|
||||||
|
|
||||||
|
|
||||||
|
return nodeInfo.Attributes[f.Key] == f.Value;
|
||||||
|
}
|
||||||
|
case (int)Operation.NE:
|
||||||
|
return nodeInfo.Attributes[f.Key] != f.Value;
|
||||||
|
default:
|
||||||
|
{
|
||||||
|
var attr = f.Key switch
|
||||||
|
{
|
||||||
|
FrostFsNodeInfo.AttrPrice => nodeInfo.Price,
|
||||||
|
FrostFsNodeInfo.AttrCapacity => nodeInfo.GetCapacity(),
|
||||||
|
_ => uint.Parse(nodeInfo.Attributes[f.Key], CultureInfo.InvariantCulture),
|
||||||
|
};
|
||||||
|
|
||||||
|
switch (f.Operation)
|
||||||
|
{
|
||||||
|
case (int)Operation.GT:
|
||||||
|
return attr > NumCache[f.Value];
|
||||||
|
case (int)Operation.GE:
|
||||||
|
return attr >= NumCache[f.Value];
|
||||||
|
case (int)Operation.LT:
|
||||||
|
return attr < NumCache[f.Value];
|
||||||
|
case (int)Operation.LE:
|
||||||
|
return attr <= NumCache[f.Value];
|
||||||
|
default:
|
||||||
|
// do nothing and return false
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// will not happen if context was created from f (maybe panic?)
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// match matches f against b. It returns no errors because
|
||||||
|
// filter should have been parsed during context creation
|
||||||
|
// and missing node properties are considered as a regular fail.
|
||||||
|
internal bool Match(FrostFsFilter f, FrostFsNodeInfo nodeInfo)
|
||||||
|
{
|
||||||
|
switch (f.Operation)
|
||||||
|
{
|
||||||
|
case (int)Operation.NOT:
|
||||||
|
{
|
||||||
|
var inner = f.Filters;
|
||||||
|
var fSub = inner[0];
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(inner[0].Name))
|
||||||
|
{
|
||||||
|
fSub = ProcessedFilters[inner[0].Name];
|
||||||
|
}
|
||||||
|
return !Match(fSub, nodeInfo);
|
||||||
|
}
|
||||||
|
case (int)Operation.AND:
|
||||||
|
case (int)Operation.OR:
|
||||||
|
{
|
||||||
|
for (int i = 0; i < f.Filters.Length; i++)
|
||||||
|
{
|
||||||
|
var fSub = f.Filters[i];
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(f.Filters[i].Name))
|
||||||
|
{
|
||||||
|
fSub = ProcessedFilters[f.Filters[i].Name];
|
||||||
|
}
|
||||||
|
|
||||||
|
var ok = Match(fSub, nodeInfo);
|
||||||
|
|
||||||
|
if (ok == (f.Operation == (int)Operation.OR))
|
||||||
|
{
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return f.Operation == (int)Operation.AND;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return MatchKeyValue(f, nodeInfo);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
18
src/FrostFS.SDK.Client/Models/Netmap/Placement/HasherList.cs
Normal file
18
src/FrostFS.SDK.Client/Models/Netmap/Placement/HasherList.cs
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal sealed class HasherList : IHasher
|
||||||
|
{
|
||||||
|
private readonly List<FrostFsNodeInfo> _nodes;
|
||||||
|
|
||||||
|
internal HasherList(List<FrostFsNodeInfo> nodes)
|
||||||
|
{
|
||||||
|
_nodes = nodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ulong Hash()
|
||||||
|
{
|
||||||
|
return _nodes.Count > 0 ? _nodes[0].Hash() : 0;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,8 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal interface IAggregator
|
||||||
|
{
|
||||||
|
void Add(double d);
|
||||||
|
|
||||||
|
double Compute();
|
||||||
|
}
|
|
@ -0,0 +1,6 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal interface IHasher
|
||||||
|
{
|
||||||
|
ulong Hash();
|
||||||
|
}
|
|
@ -0,0 +1,6 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
interface INormalizer
|
||||||
|
{
|
||||||
|
double Normalize(double w);
|
||||||
|
}
|
20
src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanAgg.cs
Normal file
20
src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanAgg.cs
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal struct MeanAgg
|
||||||
|
{
|
||||||
|
private double mean;
|
||||||
|
private int count;
|
||||||
|
|
||||||
|
internal void Add(double n)
|
||||||
|
{
|
||||||
|
int c = count + 1;
|
||||||
|
mean *= (double)count / c + n / c;
|
||||||
|
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal readonly double Compute()
|
||||||
|
{
|
||||||
|
return mean;
|
||||||
|
}
|
||||||
|
}
|
65
src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanIQRAgg.cs
Normal file
65
src/FrostFS.SDK.Client/Models/Netmap/Placement/MeanIQRAgg.cs
Normal file
|
@ -0,0 +1,65 @@
|
||||||
|
using System.Collections.ObjectModel;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal struct MeanIQRAgg : IAggregator
|
||||||
|
{
|
||||||
|
private const int minLn = 4;
|
||||||
|
internal Collection<double> arr = [];
|
||||||
|
|
||||||
|
public MeanIQRAgg()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public readonly void Add(double d)
|
||||||
|
{
|
||||||
|
arr.Add(d);
|
||||||
|
}
|
||||||
|
|
||||||
|
public readonly double Compute()
|
||||||
|
{
|
||||||
|
var length = arr.Count;
|
||||||
|
if (length == 0)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sorted = arr.OrderBy(p => p).ToArray();
|
||||||
|
|
||||||
|
double minV, maxV;
|
||||||
|
|
||||||
|
if (arr.Count < minLn)
|
||||||
|
{
|
||||||
|
minV = sorted[0];
|
||||||
|
maxV = sorted[length - 1];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var start = length / minLn;
|
||||||
|
var end = length * 3 / minLn - 1;
|
||||||
|
|
||||||
|
minV = sorted[start];
|
||||||
|
maxV = sorted[end];
|
||||||
|
}
|
||||||
|
|
||||||
|
var count = 0;
|
||||||
|
double sum = 0;
|
||||||
|
|
||||||
|
foreach (var e in sorted)
|
||||||
|
{
|
||||||
|
if (e >= minV && e <= maxV)
|
||||||
|
{
|
||||||
|
sum += e;
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sum / count;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal readonly void Clear()
|
||||||
|
{
|
||||||
|
arr.Clear();
|
||||||
|
}
|
||||||
|
}
|
27
src/FrostFS.SDK.Client/Models/Netmap/Placement/MinAgg.cs
Normal file
27
src/FrostFS.SDK.Client/Models/Netmap/Placement/MinAgg.cs
Normal file
|
@ -0,0 +1,27 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal struct MinAgg
|
||||||
|
{
|
||||||
|
private double min;
|
||||||
|
private bool minFound;
|
||||||
|
|
||||||
|
internal void Add(double n)
|
||||||
|
{
|
||||||
|
if (!minFound)
|
||||||
|
{
|
||||||
|
min = n;
|
||||||
|
minFound = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (n < min)
|
||||||
|
{
|
||||||
|
min = n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal readonly double Compute()
|
||||||
|
{
|
||||||
|
return min;
|
||||||
|
}
|
||||||
|
}
|
16
src/FrostFS.SDK.Client/Models/Netmap/Placement/Operation.cs
Normal file
16
src/FrostFS.SDK.Client/Models/Netmap/Placement/Operation.cs
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
public enum Operation
|
||||||
|
{
|
||||||
|
Unspecified = 0,
|
||||||
|
EQ,
|
||||||
|
NE,
|
||||||
|
GT,
|
||||||
|
GE,
|
||||||
|
LT,
|
||||||
|
LE,
|
||||||
|
OR,
|
||||||
|
AND,
|
||||||
|
NOT,
|
||||||
|
LIKE
|
||||||
|
}
|
|
@ -0,0 +1,8 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal struct ReverseMinNorm : INormalizer
|
||||||
|
{
|
||||||
|
internal double min;
|
||||||
|
|
||||||
|
public readonly double Normalize(double w) => (min + 1) / (w + 1);
|
||||||
|
}
|
|
@ -0,0 +1,10 @@
|
||||||
|
using System.Collections.ObjectModel;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal struct SelectFilterExpr(uint cbf, FrostFsSelector selector, Collection<FrostFsFilter> filters)
|
||||||
|
{
|
||||||
|
internal uint Cbf { get; } = cbf;
|
||||||
|
internal FrostFsSelector Selector { get; } = selector;
|
||||||
|
internal Collection<FrostFsFilter> Filters { get; } = filters;
|
||||||
|
}
|
|
@ -0,0 +1,23 @@
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
internal readonly struct SigmoidNorm : INormalizer
|
||||||
|
{
|
||||||
|
private readonly double _scale;
|
||||||
|
|
||||||
|
internal SigmoidNorm(double scale)
|
||||||
|
{
|
||||||
|
_scale = scale;
|
||||||
|
}
|
||||||
|
|
||||||
|
public readonly double Normalize(double w)
|
||||||
|
{
|
||||||
|
if (_scale == 0)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
var x = w / _scale;
|
||||||
|
|
||||||
|
return x / (1 + x);
|
||||||
|
}
|
||||||
|
}
|
140
src/FrostFS.SDK.Client/Models/Netmap/Placement/Tools.cs
Normal file
140
src/FrostFS.SDK.Client/Models/Netmap/Placement/Tools.cs
Normal file
|
@ -0,0 +1,140 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
using static FrostFS.SDK.FrostFsNetmapSnapshot;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
public static class Tools
|
||||||
|
{
|
||||||
|
internal static ulong Distance(ulong x, ulong y)
|
||||||
|
{
|
||||||
|
var acc = x ^ y;
|
||||||
|
acc ^= acc >> 33;
|
||||||
|
acc *= 0xff51afd7ed558ccd;
|
||||||
|
acc ^= acc >> 33;
|
||||||
|
acc *= 0xc4ceb9fe1a85ec53;
|
||||||
|
acc ^= acc >> 33;
|
||||||
|
|
||||||
|
return acc;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static double ReverceNormalize(double r, double w)
|
||||||
|
{
|
||||||
|
return (r + 1) / (w + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static double Normalize(double r, double w)
|
||||||
|
{
|
||||||
|
if (r == 0)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
var x = w / r;
|
||||||
|
return x / (1 + x);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static void AppendWeightsTo(FrostFsNodeInfo[] nodes, Func<FrostFsNodeInfo, double> wf, double[] weights)
|
||||||
|
{
|
||||||
|
if (weights.Length < nodes.Length)
|
||||||
|
{
|
||||||
|
weights = new double[nodes.Length];
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < nodes.Length; i++)
|
||||||
|
{
|
||||||
|
weights[i] = wf(nodes[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static void SortHasherSliceByWeightValue<T>(List<T> nodes, Span<double> weights, ulong hash) where T : IHasher
|
||||||
|
{
|
||||||
|
if (nodes.Count == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var allEquals = true;
|
||||||
|
|
||||||
|
if (weights.Length > 1)
|
||||||
|
{
|
||||||
|
for (int i = 1; i < weights.Length; i++)
|
||||||
|
{
|
||||||
|
if (weights[i] != weights[0])
|
||||||
|
{
|
||||||
|
allEquals = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var dist = new ulong[nodes.Count];
|
||||||
|
|
||||||
|
if (allEquals)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < dist.Length; i++)
|
||||||
|
{
|
||||||
|
var x = nodes[i].Hash();
|
||||||
|
dist[i] = Distance(x, hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
SortHasherByDistance(nodes, dist, true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < dist.Length; i++)
|
||||||
|
{
|
||||||
|
var d = Distance(nodes[i].Hash(), hash);
|
||||||
|
dist[i] = ulong.MaxValue - (ulong)(d * weights[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
SortHasherByDistance(nodes, dist, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static void SortHasherByDistance<T, N>(List<T> nodes, N[] dist, bool asc)
|
||||||
|
{
|
||||||
|
IndexedValue<T, N>[] indexes = new IndexedValue<T, N>[nodes.Count];
|
||||||
|
for (int i = 0; i < dist.Length; i++)
|
||||||
|
{
|
||||||
|
indexes[i] = new IndexedValue<T, N>() { nodeInfo = nodes[i], dist = dist[i] };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (asc)
|
||||||
|
{
|
||||||
|
nodes = new List<T>(indexes
|
||||||
|
.OrderBy(x => x.dist)
|
||||||
|
.Select(x => x.nodeInfo).ToArray());
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
nodes = new List<T>(indexes
|
||||||
|
.OrderByDescending(x => x.dist)
|
||||||
|
.Select(x => x.nodeInfo)
|
||||||
|
.ToArray());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static Func<FrostFsNodeInfo, double> DefaultWeightFunc(IReadOnlyList<FrostFsNodeInfo> nodes)
|
||||||
|
{
|
||||||
|
MeanAgg mean = new();
|
||||||
|
MinAgg minV = new();
|
||||||
|
|
||||||
|
foreach (var node in nodes)
|
||||||
|
{
|
||||||
|
mean.Add(node.GetCapacity());
|
||||||
|
minV.Add(node.Price);
|
||||||
|
}
|
||||||
|
|
||||||
|
return NewWeightFunc(
|
||||||
|
NewSigmoidNorm(mean.Compute()),
|
||||||
|
NewReverseMinNorm(minV.Compute()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private struct IndexedValue<T, N>
|
||||||
|
{
|
||||||
|
internal T nodeInfo;
|
||||||
|
internal N dist;
|
||||||
|
}
|
||||||
|
}
|
|
@ -13,7 +13,7 @@ namespace FrostFS.SDK.Client;
|
||||||
public static class ObjectTools
|
public static class ObjectTools
|
||||||
{
|
{
|
||||||
public static FrostFsObjectId CalculateObjectId(
|
public static FrostFsObjectId CalculateObjectId(
|
||||||
FrostFsObjectHeader header,
|
FrostFsObjectHeader header,
|
||||||
ReadOnlyMemory<byte> payloadHash,
|
ReadOnlyMemory<byte> payloadHash,
|
||||||
FrostFsOwner owner,
|
FrostFsOwner owner,
|
||||||
FrostFsVersion version,
|
FrostFsVersion version,
|
||||||
|
@ -58,6 +58,7 @@ public static class ObjectTools
|
||||||
grpcHeader.PayloadHash = Sha256Checksum(@object.SingleObjectPayload);
|
grpcHeader.PayloadHash = Sha256Checksum(@object.SingleObjectPayload);
|
||||||
|
|
||||||
var split = @object.Header.Split;
|
var split = @object.Header.Split;
|
||||||
|
|
||||||
if (split != null)
|
if (split != null)
|
||||||
{
|
{
|
||||||
SetSplitValues(grpcHeader, split, ctx.Owner, ctx.Version, ctx.Key);
|
SetSplitValues(grpcHeader, split, ctx.Owner, ctx.Version, ctx.Key);
|
||||||
|
@ -80,17 +81,21 @@ public static class ObjectTools
|
||||||
}
|
}
|
||||||
|
|
||||||
internal static void SetSplitValues(
|
internal static void SetSplitValues(
|
||||||
Header grpcHeader,
|
Header grpcHeader,
|
||||||
FrostFsSplit split,
|
FrostFsSplit split,
|
||||||
FrostFsOwner owner,
|
FrostFsOwner owner,
|
||||||
FrostFsVersion version,
|
FrostFsVersion version,
|
||||||
ClientKey key)
|
ClientKey key)
|
||||||
{
|
{
|
||||||
if (split == null)
|
if (split == null)
|
||||||
|
{
|
||||||
return;
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (key == null)
|
if (key == null)
|
||||||
|
{
|
||||||
throw new FrostFsInvalidObjectException(nameof(key));
|
throw new FrostFsInvalidObjectException(nameof(key));
|
||||||
|
}
|
||||||
|
|
||||||
grpcHeader.Split = new Header.Types.Split
|
grpcHeader.Split = new Header.Types.Split
|
||||||
{
|
{
|
||||||
|
@ -98,7 +103,9 @@ public static class ObjectTools
|
||||||
};
|
};
|
||||||
|
|
||||||
if (split.Children != null && split.Children.Count != 0)
|
if (split.Children != null && split.Children.Count != 0)
|
||||||
|
{
|
||||||
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToMessage()));
|
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToMessage()));
|
||||||
|
}
|
||||||
|
|
||||||
if (split.ParentHeader is not null)
|
if (split.ParentHeader is not null)
|
||||||
{
|
{
|
||||||
|
@ -118,8 +125,8 @@ public static class ObjectTools
|
||||||
|
|
||||||
internal static Header CreateHeader(
|
internal static Header CreateHeader(
|
||||||
FrostFsObjectHeader header,
|
FrostFsObjectHeader header,
|
||||||
ReadOnlyMemory<byte> payloadChecksum,
|
ReadOnlyMemory<byte> payloadChecksum,
|
||||||
FrostFsOwner owner,
|
FrostFsOwner owner,
|
||||||
FrostFsVersion version)
|
FrostFsVersion version)
|
||||||
{
|
{
|
||||||
header.OwnerId ??= owner;
|
header.OwnerId ??= owner;
|
||||||
|
@ -128,7 +135,7 @@ public static class ObjectTools
|
||||||
var grpcHeader = header.GetHeader();
|
var grpcHeader = header.GetHeader();
|
||||||
|
|
||||||
grpcHeader.PayloadHash = ChecksumFromSha256(payloadChecksum);
|
grpcHeader.PayloadHash = ChecksumFromSha256(payloadChecksum);
|
||||||
|
|
||||||
return grpcHeader;
|
return grpcHeader;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -149,4 +156,4 @@ public static class ObjectTools
|
||||||
Sum = UnsafeByteOperations.UnsafeWrap(dataHash)
|
Sum = UnsafeByteOperations.UnsafeWrap(dataHash)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,7 @@ using System.Security.Cryptography;
|
||||||
|
|
||||||
namespace FrostFS.SDK.Cryptography;
|
namespace FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
internal class Murmur3_128 : HashAlgorithm
|
public class Murmur3 : HashAlgorithm
|
||||||
{
|
{
|
||||||
private const ulong c1 = 0x87c37b91114253d5;
|
private const ulong c1 = 0x87c37b91114253d5;
|
||||||
private const ulong c2 = 0x4cf5ad432745937f;
|
private const ulong c2 = 0x4cf5ad432745937f;
|
||||||
|
@ -17,14 +17,31 @@ internal class Murmur3_128 : HashAlgorithm
|
||||||
private readonly uint seed;
|
private readonly uint seed;
|
||||||
private int length;
|
private int length;
|
||||||
|
|
||||||
public Murmur3_128(uint seed)
|
public Murmur3(uint seed)
|
||||||
{
|
{
|
||||||
this.seed = seed;
|
this.seed = seed;
|
||||||
Initialize();
|
Initialize();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ulong GetCheckSum64(byte[] bytes)
|
||||||
|
{
|
||||||
|
if (bytes is null)
|
||||||
|
{
|
||||||
|
throw new ArgumentNullException(nameof(bytes));
|
||||||
|
}
|
||||||
|
|
||||||
|
Initialize();
|
||||||
|
HashCore(bytes, 0, bytes.Length);
|
||||||
|
return HashFinalUlong();
|
||||||
|
}
|
||||||
|
|
||||||
protected override void HashCore(byte[] array, int ibStart, int cbSize)
|
protected override void HashCore(byte[] array, int ibStart, int cbSize)
|
||||||
{
|
{
|
||||||
|
if (array is null)
|
||||||
|
{
|
||||||
|
throw new ArgumentNullException(nameof(array));
|
||||||
|
}
|
||||||
|
|
||||||
length += cbSize;
|
length += cbSize;
|
||||||
int remainder = cbSize & 15;
|
int remainder = cbSize & 15;
|
||||||
int alignedLength = ibStart + (cbSize - remainder);
|
int alignedLength = ibStart + (cbSize - remainder);
|
||||||
|
@ -92,6 +109,11 @@ internal class Murmur3_128 : HashAlgorithm
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override byte[] HashFinal()
|
protected override byte[] HashFinal()
|
||||||
|
{
|
||||||
|
return BitConverter.GetBytes(HashFinalUlong());
|
||||||
|
}
|
||||||
|
|
||||||
|
protected ulong HashFinalUlong()
|
||||||
{
|
{
|
||||||
h1 ^= (ulong)length;
|
h1 ^= (ulong)length;
|
||||||
h2 ^= (ulong)length;
|
h2 ^= (ulong)length;
|
||||||
|
@ -102,7 +124,7 @@ internal class Murmur3_128 : HashAlgorithm
|
||||||
h1 += h2;
|
h1 += h2;
|
||||||
h2 += h1;
|
h2 += h1;
|
||||||
|
|
||||||
return BitConverter.GetBytes(h1);
|
return h1;
|
||||||
}
|
}
|
||||||
|
|
||||||
public override void Initialize()
|
public override void Initialize()
|
||||||
|
|
161
src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs
Normal file
161
src/FrostFS.SDK.Tests/Unit/PlacementVectorTests.cs
Normal file
|
@ -0,0 +1,161 @@
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
|
||||||
|
using FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.Tests.Unit;
|
||||||
|
|
||||||
|
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
|
||||||
|
public class PlacementVectorTests
|
||||||
|
{
|
||||||
|
Dictionary<string, string>[] attribs;
|
||||||
|
|
||||||
|
public PlacementVectorTests()
|
||||||
|
{
|
||||||
|
var attribs1 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "Germany" },
|
||||||
|
{"Price", "2" },
|
||||||
|
{"Capacity", "10000"}
|
||||||
|
};
|
||||||
|
|
||||||
|
var attribs2 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "Germany" },
|
||||||
|
{"Price", "4" },
|
||||||
|
{"Capacity", "1"}
|
||||||
|
};
|
||||||
|
|
||||||
|
var attribs3 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "France" },
|
||||||
|
{"Price", "3" },
|
||||||
|
{"Capacity", "10"}
|
||||||
|
};
|
||||||
|
|
||||||
|
var attribs4 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "Russia" },
|
||||||
|
{"Price", "2" },
|
||||||
|
{"Capacity", "10000"}
|
||||||
|
};
|
||||||
|
|
||||||
|
var attribs5 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "Russia" },
|
||||||
|
{"Price", "1" },
|
||||||
|
{"Capacity", "10000"}
|
||||||
|
};
|
||||||
|
|
||||||
|
var attribs6 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "Russia" },
|
||||||
|
{"Capacity", "10000"}
|
||||||
|
};
|
||||||
|
var attribs7 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "France" },
|
||||||
|
{"Price", "100" },
|
||||||
|
{"Capacity", "10000"}
|
||||||
|
};
|
||||||
|
var attribs8 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "France" },
|
||||||
|
{"Price", "7" },
|
||||||
|
{"Capacity", "10000"}
|
||||||
|
};
|
||||||
|
var attribs9 = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
{"Country", "Russia" },
|
||||||
|
{"Price", "2" },
|
||||||
|
{"Capacity", "1"}
|
||||||
|
};
|
||||||
|
|
||||||
|
attribs = [attribs1, attribs2, attribs3, attribs4, attribs5, attribs6, attribs7, attribs8, attribs9];
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void PlacementVectorTest()
|
||||||
|
{
|
||||||
|
FrostFsVersion v = new(2, 13);
|
||||||
|
var addresses = new string[] { "localhost", "server1" };
|
||||||
|
var key1 = new byte[] { 1 };
|
||||||
|
var key2 = new byte[] { 2 };
|
||||||
|
var key3 = new byte[] { 3 };
|
||||||
|
|
||||||
|
var nodes = new List<FrostFsNodeInfo>
|
||||||
|
{
|
||||||
|
new(v, NodeState.Online, addresses.AsReadOnly(), attribs[5].AsReadOnly(), key1),
|
||||||
|
new(v, NodeState.Online, addresses.AsReadOnly(), attribs[0].AsReadOnly(), key2),
|
||||||
|
new(v, NodeState.Online, addresses.AsReadOnly(), attribs[8].AsReadOnly(), key3)
|
||||||
|
};
|
||||||
|
|
||||||
|
var netmap = new FrostFsNetmapSnapshot(100, nodes.AsReadOnly());
|
||||||
|
|
||||||
|
var arg = new FrostFsNodeInfo[1][];
|
||||||
|
var pivot = "objectID"u8.ToArray();
|
||||||
|
|
||||||
|
arg[0] = [.. nodes];
|
||||||
|
var result = netmap.PlacementVectors(arg, pivot);
|
||||||
|
|
||||||
|
Assert.Single(result);
|
||||||
|
Assert.Equal(3, result[0].Length);
|
||||||
|
Assert.Equal(key1, result[0][0].PublicKey);
|
||||||
|
Assert.Equal(key2, result[0][1].PublicKey);
|
||||||
|
Assert.Equal(key3, result[0][2].PublicKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void TestPlacementPolicyUnique()
|
||||||
|
{
|
||||||
|
FrostFsVersion version = new(2, 13);
|
||||||
|
var p = new FrostFsPlacementPolicy(true, [new FrostFsReplica(1, "S"), new FrostFsReplica(1, "S")])
|
||||||
|
{
|
||||||
|
BackupFactor = 2
|
||||||
|
};
|
||||||
|
p.Selectors.Add(new FrostFsSelector("S")
|
||||||
|
{
|
||||||
|
Attribute = "City",
|
||||||
|
Count = 1,
|
||||||
|
Filter = "*",
|
||||||
|
Clause = (int)FrostFsClause.Same
|
||||||
|
});
|
||||||
|
|
||||||
|
List<FrostFsNodeInfo> nodes = [];
|
||||||
|
|
||||||
|
var cities = new string[] { "Moscow", "Berlin", "Shenzhen" };
|
||||||
|
for (int i = 0; i < 3; i++)
|
||||||
|
{
|
||||||
|
for (int j = 0; j < 3; j++)
|
||||||
|
{
|
||||||
|
var attr = new Dictionary<string, string> { { "City", cities[i] } };
|
||||||
|
var key = new byte[] { (byte)(i * 4 + j) };
|
||||||
|
var node = new FrostFsNodeInfo(version, NodeState.Online, [], attr, key);
|
||||||
|
|
||||||
|
nodes.Add(node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var netMap = new FrostFsNetmapSnapshot(100, nodes.AsReadOnly());
|
||||||
|
|
||||||
|
var v = netMap.ContainerNodes(p, null);
|
||||||
|
|
||||||
|
Assert.Equal(2, v.Length);
|
||||||
|
Assert.Equal(2, v[0].Length);
|
||||||
|
Assert.Equal(2, v[1].Length);
|
||||||
|
|
||||||
|
|
||||||
|
for (int i = 0; i < v.Length; i++)
|
||||||
|
{
|
||||||
|
foreach (var ni in v[i])
|
||||||
|
{
|
||||||
|
for (int j = 0; j < i; j++)
|
||||||
|
{
|
||||||
|
foreach (var nj in v[j])
|
||||||
|
{
|
||||||
|
Assert.NotEqual(ni.Hash, nj.Hash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue