456 lines
15 KiB
C#
456 lines
15 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Globalization;
|
|
using System.Linq;
|
|
|
|
namespace FrostFS.SDK.Client.Models.Netmap.Placement;
|
|
|
|
internal struct Context
|
|
{
|
|
private const string errInvalidFilterName = "filter name is invalid";
|
|
private const string errInvalidFilterOp = "invalid filter operation";
|
|
private const string errFilterNotFound = "filter not found";
|
|
private const string errNonEmptyFilters = "simple filter contains sub-filters";
|
|
private const string errNotEnoughNodes = "not enough nodes to SELECT from";
|
|
private const string errUnnamedTopFilter = "unnamed top-level filter";
|
|
|
|
internal const string mainFilterName = "*";
|
|
internal const string likeWildcard = "*";
|
|
|
|
// network map to operate on
|
|
internal FrostFsNetmapSnapshot NetMap { get; }
|
|
|
|
// cache of processed filters
|
|
internal Dictionary<string, FrostFsFilter> ProcessedFilters { get; } = [];
|
|
|
|
// cache of processed selectors
|
|
internal Dictionary<string, FrostFsSelector> ProcessedSelectors { get; } = [];
|
|
|
|
// stores results of selector processing
|
|
internal Dictionary<string, List<List<FrostFsNodeInfo>>> Selections { get; } = [];
|
|
|
|
// cache of parsed numeric values
|
|
internal Dictionary<string, ulong> NumCache { get; } = [];
|
|
|
|
internal byte[]? HrwSeed { get; set; }
|
|
|
|
// hrw.Hash of hrwSeed
|
|
internal ulong HrwSeedHash { get; set; }
|
|
|
|
// container backup factor
|
|
internal uint Cbf { get; set; }
|
|
|
|
// nodes already used in previous selections, which is needed when the placement
|
|
// policy uses the UNIQUE flag. Nodes marked as used are not used in subsequent
|
|
// base selections.
|
|
internal Dictionary<ulong, bool> UsedNodes { get; } = [];
|
|
|
|
// If true, returns an error when netmap does not contain enough nodes for selection.
|
|
// By default best effort is taken.
|
|
internal bool Strict { get; set; }
|
|
|
|
// weightFunc is a weighting function for determining node priority
|
|
// which combines low price and high performance
|
|
private readonly Func<FrostFsNodeInfo, double> weightFunc;
|
|
|
|
public Context(FrostFsNetmapSnapshot netMap)
|
|
{
|
|
NetMap = netMap;
|
|
weightFunc = Tools.DefaultWeightFunc(NetMap.NodeInfoCollection);
|
|
}
|
|
|
|
internal void ProcessFilters(FrostFsPlacementPolicy policy)
|
|
{
|
|
foreach (var filter in policy.Filters)
|
|
{
|
|
ProcessFilter(filter, true);
|
|
}
|
|
}
|
|
|
|
readonly void ProcessFilter(FrostFsFilter filter, bool top)
|
|
{
|
|
var filterName = filter.Name;
|
|
if (filterName == mainFilterName)
|
|
{
|
|
throw new FrostFsException($"{errInvalidFilterName}: '{errInvalidFilterName}' is reserved");
|
|
}
|
|
|
|
if (top && string.IsNullOrEmpty(filterName))
|
|
{
|
|
throw new FrostFsException(errUnnamedTopFilter);
|
|
}
|
|
|
|
if (!top && !string.IsNullOrEmpty(filterName) && !ProcessedFilters.ContainsKey(filterName))
|
|
{
|
|
throw new FrostFsException(errFilterNotFound);
|
|
}
|
|
|
|
if (filter.Operation == (int)Operation.AND ||
|
|
filter.Operation == (int)Operation.OR ||
|
|
filter.Operation == (int)Operation.NOT)
|
|
{
|
|
foreach (var f in filter.Filters)
|
|
ProcessFilter(f, false);
|
|
}
|
|
else
|
|
{
|
|
if (filter.Filters.Length != 0)
|
|
{
|
|
throw new FrostFsException(errNonEmptyFilters);
|
|
}
|
|
else if (!top && !string.IsNullOrEmpty(filterName))
|
|
{
|
|
// named reference
|
|
return;
|
|
}
|
|
|
|
switch (filter.Operation)
|
|
{
|
|
case (int)Operation.EQ:
|
|
case (int)Operation.NE:
|
|
case (int)Operation.LIKE:
|
|
break;
|
|
case (int)Operation.GT:
|
|
case (int)Operation.GE:
|
|
case (int)Operation.LT:
|
|
case (int)Operation.LE:
|
|
{
|
|
var n = uint.Parse(filter.Value, CultureInfo.InvariantCulture);
|
|
NumCache[filter.Value] = n;
|
|
break;
|
|
}
|
|
default:
|
|
throw new FrostFsException($"{errInvalidFilterOp}: {filter.Operation}");
|
|
}
|
|
}
|
|
|
|
if (top)
|
|
{
|
|
ProcessedFilters[filterName] = filter;
|
|
}
|
|
}
|
|
|
|
// processSelectors processes selectors and returns error is any of them is invalid.
|
|
internal void ProcessSelectors(FrostFsPlacementPolicy policy)
|
|
{
|
|
foreach (var selector in policy.Selectors)
|
|
{
|
|
var filterName = selector.Filter;
|
|
if (filterName != mainFilterName)
|
|
{
|
|
if (selector.Filter == null || !ProcessedFilters.ContainsKey(selector.Filter))
|
|
{
|
|
throw new FrostFsException($"{errFilterNotFound}: SELECT FROM '{filterName}'");
|
|
}
|
|
}
|
|
|
|
ProcessedSelectors[selector.Name] = selector;
|
|
|
|
var selection = GetSelection(selector);
|
|
|
|
Selections[selector.Name] = selection;
|
|
}
|
|
}
|
|
|
|
// calcNodesCount returns number of buckets and minimum number of nodes in every bucket
|
|
// for the given selector.
|
|
static (int bucketCount, int nodesInBucket) CalcNodesCount(FrostFsSelector selector)
|
|
{
|
|
return selector.Clause == (int)FrostFsClause.Same
|
|
? (1, (int)selector.Count)
|
|
: ((int)selector.Count, 1);
|
|
}
|
|
|
|
// getSelectionBase returns nodes grouped by selector attribute.
|
|
// It it guaranteed that each pair will contain at least one node.
|
|
internal NodeAttrPair[] GetSelectionBase(FrostFsSelector selector)
|
|
{
|
|
var fName = selector.Filter ?? throw new FrostFsException("Filter name for selector is empty");
|
|
|
|
_ = ProcessedFilters.TryGetValue(fName, out var f);
|
|
|
|
var isMain = fName == mainFilterName;
|
|
var result = new List<NodeAttrPair>();
|
|
|
|
var nodeMap = new Dictionary<string, List<FrostFsNodeInfo>>();
|
|
var attr = selector.Attribute;
|
|
|
|
foreach (var node in NetMap.NodeInfoCollection)
|
|
{
|
|
if (UsedNodes.ContainsKey(node.Hash()))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (isMain || Match(f, node))
|
|
{
|
|
if (attr == null)
|
|
{
|
|
// Default attribute is transparent identifier which is different for every node.
|
|
result.Add(new NodeAttrPair { attr = "", nodes = [node] });
|
|
}
|
|
else
|
|
{
|
|
var v = node.Attributes[attr];
|
|
if (!nodeMap.TryGetValue(v, out var nodes) || nodes == null)
|
|
{
|
|
nodeMap[v] = [];
|
|
}
|
|
|
|
nodeMap[v].Add(node);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(attr))
|
|
{
|
|
foreach (var v in nodeMap)
|
|
{
|
|
result.Add(new NodeAttrPair() { attr = v.Key, nodes = [.. v.Value] });
|
|
}
|
|
}
|
|
|
|
if (HrwSeed != null && HrwSeed.Length != 0)
|
|
{
|
|
double[] ws = [];
|
|
|
|
var sortedNodes = new NodeAttrPair[result.Count];
|
|
|
|
for (int i = 0; i < result.Count; i++)
|
|
{
|
|
var res = result[i];
|
|
Tools.AppendWeightsTo(res.nodes, weightFunc, ref ws);
|
|
sortedNodes[i].nodes = Tools.SortHasherSliceByWeightValue(res.nodes.ToList(), ws, HrwSeedHash).ToArray();
|
|
sortedNodes[i].attr = result[i].attr;
|
|
}
|
|
|
|
return sortedNodes;
|
|
}
|
|
return [.. result];
|
|
}
|
|
|
|
static double CalcBucketWeight(List<FrostFsNodeInfo> ns, MeanIQRAgg a, Func<FrostFsNodeInfo, double> wf)
|
|
{
|
|
foreach (var node in ns)
|
|
{
|
|
a.Add(wf(node));
|
|
}
|
|
|
|
return a.Compute();
|
|
}
|
|
|
|
// getSelection returns nodes grouped by s.attribute.
|
|
// Last argument specifies if more buckets can be used to fulfill CBF.
|
|
internal List<List<FrostFsNodeInfo>> GetSelection(FrostFsSelector s)
|
|
{
|
|
var (bucketCount, nodesInBucket) = CalcNodesCount(s);
|
|
|
|
var buckets = GetSelectionBase(s);
|
|
|
|
if (Strict && buckets.Length < bucketCount)
|
|
throw new FrostFsException($"errNotEnoughNodes: '{s.Name}'");
|
|
|
|
// We need deterministic output in case there is no pivot.
|
|
// If pivot is set, buckets are sorted by HRW.
|
|
// However, because initial order influences HRW order for buckets with equal weights,
|
|
// we also need to have deterministic input to HRW sorting routine.
|
|
if (HrwSeed == null || HrwSeed.Length == 0)
|
|
{
|
|
buckets = string.IsNullOrEmpty(s.Attribute)
|
|
? [.. buckets.OrderBy(b => b.nodes[0].Hash())]
|
|
: [.. buckets.OrderBy(b => b.attr)];
|
|
}
|
|
|
|
var maxNodesInBucket = nodesInBucket * (int)Cbf;
|
|
|
|
var res = new List<List<FrostFsNodeInfo>>(buckets.Length);
|
|
var fallback = new List<List<FrostFsNodeInfo>>(buckets.Length);
|
|
|
|
for (int i = 0; i < buckets.Length; i++)
|
|
{
|
|
var ns = buckets[i].nodes;
|
|
if (ns.Length >= maxNodesInBucket)
|
|
{
|
|
res.Add(ns.Take(maxNodesInBucket).ToList());
|
|
}
|
|
else if (ns.Length >= nodesInBucket)
|
|
{
|
|
fallback.Add(new List<FrostFsNodeInfo>(ns));
|
|
}
|
|
}
|
|
|
|
if (res.Count < bucketCount)
|
|
{
|
|
// Fallback to using minimum allowed backup factor (1).
|
|
res.AddRange(fallback);
|
|
|
|
if (Strict && res.Count < bucketCount)
|
|
{
|
|
throw new FrostFsException($"{errNotEnoughNodes}: {s}");
|
|
}
|
|
}
|
|
|
|
if (HrwSeed != null && HrwSeed.Length != 0)
|
|
{
|
|
var weights = new double[res.Count];
|
|
var a = new MeanIQRAgg();
|
|
|
|
for (int i = 0; i < res.Count; i++)
|
|
{
|
|
a.Clear();
|
|
weights[i] = CalcBucketWeight(res[i], a, weightFunc);
|
|
}
|
|
|
|
var hashers = res.Select(r => new HasherList(r)).ToList();
|
|
hashers = Tools.SortHasherSliceByWeightValue(hashers, weights, HrwSeedHash);
|
|
|
|
for (int i = 0; i < res.Count; i++)
|
|
{
|
|
res[i] = hashers[i].Nodes;
|
|
}
|
|
}
|
|
|
|
if (res.Count < bucketCount)
|
|
{
|
|
if (Strict && res.Count == 0)
|
|
{
|
|
throw new FrostFsException(errNotEnoughNodes);
|
|
}
|
|
|
|
bucketCount = res.Count;
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(s.Attribute))
|
|
{
|
|
fallback = res.Skip(bucketCount).ToList();
|
|
res = res.Take(bucketCount).ToList();
|
|
|
|
for (int i = 0; i < fallback.Count; i++)
|
|
{
|
|
var index = i % bucketCount;
|
|
if (res[index].Count >= maxNodesInBucket)
|
|
{
|
|
break;
|
|
}
|
|
|
|
res[index].AddRange(fallback[i]);
|
|
}
|
|
}
|
|
|
|
return res.Take(bucketCount).ToList();
|
|
}
|
|
|
|
internal bool MatchKeyValue(FrostFsFilter f, FrostFsNodeInfo nodeInfo)
|
|
{
|
|
switch (f.Operation)
|
|
{
|
|
case (int)Operation.EQ:
|
|
return nodeInfo.Attributes.TryGetValue(f.Key, out var val) && val == f.Value;
|
|
case (int)Operation.LIKE:
|
|
{
|
|
var hasPrefix = f.Value.StartsWith(likeWildcard, StringComparison.Ordinal);
|
|
var hasSuffix = f.Value.EndsWith(likeWildcard, StringComparison.Ordinal);
|
|
|
|
var start = hasPrefix ? likeWildcard.Length : 0;
|
|
var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length;
|
|
var str = f.Value.Substring(start, end-start);
|
|
|
|
if (hasPrefix && hasSuffix)
|
|
return nodeInfo.Attributes[f.Key].Contains(str);
|
|
|
|
if (hasPrefix && !hasSuffix)
|
|
return nodeInfo.Attributes[f.Key].EndsWith(str, StringComparison.Ordinal);
|
|
|
|
if (!hasPrefix && hasSuffix)
|
|
return nodeInfo.Attributes[f.Key].StartsWith(str, StringComparison.Ordinal);
|
|
|
|
|
|
return nodeInfo.Attributes[f.Key] == f.Value;
|
|
}
|
|
case (int)Operation.NE:
|
|
return nodeInfo.Attributes[f.Key] != f.Value;
|
|
default:
|
|
{
|
|
ulong attr;
|
|
switch (f.Key)
|
|
{
|
|
case FrostFsNodeInfo.AttrPrice:
|
|
attr = nodeInfo.Price;
|
|
break;
|
|
|
|
case FrostFsNodeInfo.AttrCapacity:
|
|
attr = nodeInfo.GetCapacity();
|
|
break;
|
|
default:
|
|
if (!ulong.TryParse(nodeInfo.Attributes[f.Key], NumberStyles.Integer, CultureInfo.InvariantCulture, out attr))
|
|
return false;
|
|
break;
|
|
}
|
|
|
|
switch (f.Operation)
|
|
{
|
|
case (int)Operation.GT:
|
|
return attr > NumCache[f.Value];
|
|
case (int)Operation.GE:
|
|
return attr >= NumCache[f.Value];
|
|
case (int)Operation.LT:
|
|
return attr < NumCache[f.Value];
|
|
case (int)Operation.LE:
|
|
return attr <= NumCache[f.Value];
|
|
default:
|
|
// do nothing and return false
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
// will not happen if context was created from f (maybe panic?)
|
|
return false;
|
|
}
|
|
|
|
// match matches f against b. It returns no errors because
|
|
// filter should have been parsed during context creation
|
|
// and missing node properties are considered as a regular fail.
|
|
internal bool Match(FrostFsFilter f, FrostFsNodeInfo nodeInfo)
|
|
{
|
|
switch (f.Operation)
|
|
{
|
|
case (int)Operation.NOT:
|
|
{
|
|
var inner = f.Filters;
|
|
var fSub = inner[0];
|
|
|
|
if (!string.IsNullOrEmpty(inner[0].Name))
|
|
{
|
|
fSub = ProcessedFilters[inner[0].Name];
|
|
}
|
|
return !Match(fSub, nodeInfo);
|
|
}
|
|
case (int)Operation.AND:
|
|
case (int)Operation.OR:
|
|
{
|
|
for (int i = 0; i < f.Filters.Length; i++)
|
|
{
|
|
var fSub = f.Filters[i];
|
|
|
|
if (!string.IsNullOrEmpty(f.Filters[i].Name))
|
|
{
|
|
fSub = ProcessedFilters[f.Filters[i].Name];
|
|
}
|
|
|
|
var ok = Match(fSub, nodeInfo);
|
|
|
|
if (ok == (f.Operation == (int)Operation.OR))
|
|
{
|
|
return ok;
|
|
}
|
|
}
|
|
|
|
return f.Operation == (int)Operation.AND;
|
|
}
|
|
default:
|
|
return MatchKeyValue(f, nodeInfo);
|
|
}
|
|
}
|
|
}
|