using System; using System.Collections.Generic; using System.Globalization; using System.Linq; namespace FrostFS.SDK.Client.Models.Netmap.Placement; internal struct Context { private const string errInvalidFilterName = "filter name is invalid"; private const string errInvalidFilterOp = "invalid filter operation"; private const string errFilterNotFound = "filter not found"; private const string errNonEmptyFilters = "simple filter contains sub-filters"; private const string errNotEnoughNodes = "not enough nodes to SELECT from"; private const string errUnnamedTopFilter = "unnamed top-level filter"; internal const string mainFilterName = "*"; internal const string likeWildcard = "*"; // network map to operate on internal FrostFsNetmapSnapshot NetMap { get; } // cache of processed filters internal Dictionary ProcessedFilters { get; } = []; // cache of processed selectors internal Dictionary ProcessedSelectors { get; } = []; // stores results of selector processing internal Dictionary>> Selections { get; } = []; // cache of parsed numeric values internal Dictionary NumCache { get; } = []; internal byte[]? HrwSeed { get; set; } // hrw.Hash of hrwSeed internal ulong HrwSeedHash { get; set; } // container backup factor internal uint Cbf { get; set; } // nodes already used in previous selections, which is needed when the placement // policy uses the UNIQUE flag. Nodes marked as used are not used in subsequent // base selections. internal Dictionary UsedNodes { get; } = []; // If true, returns an error when netmap does not contain enough nodes for selection. // By default best effort is taken. internal bool Strict { get; set; } // weightFunc is a weighting function for determining node priority // which combines low price and high performance private readonly Func weightFunc; public Context(FrostFsNetmapSnapshot netMap) { NetMap = netMap; weightFunc = Tools.DefaultWeightFunc(NetMap.NodeInfoCollection); } internal void ProcessFilters(FrostFsPlacementPolicy policy) { foreach (var filter in policy.Filters) { ProcessFilter(filter, true); } } readonly void ProcessFilter(FrostFsFilter filter, bool top) { var filterName = filter.Name; if (filterName == mainFilterName) { throw new FrostFsException($"{errInvalidFilterName}: '{errInvalidFilterName}' is reserved"); } if (top && string.IsNullOrEmpty(filterName)) { throw new FrostFsException(errUnnamedTopFilter); } if (!top && !string.IsNullOrEmpty(filterName) && !ProcessedFilters.ContainsKey(filterName)) { throw new FrostFsException(errFilterNotFound); } if (filter.Operation == (int)Operation.AND || filter.Operation == (int)Operation.OR || filter.Operation == (int)Operation.NOT) { foreach (var f in filter.Filters) ProcessFilter(f, false); } else { if (filter.Filters.Length != 0) { throw new FrostFsException(errNonEmptyFilters); } else if (!top && !string.IsNullOrEmpty(filterName)) { // named reference return; } switch (filter.Operation) { case (int)Operation.EQ: case (int)Operation.NE: case (int)Operation.LIKE: break; case (int)Operation.GT: case (int)Operation.GE: case (int)Operation.LT: case (int)Operation.LE: { var n = uint.Parse(filter.Value, CultureInfo.InvariantCulture); NumCache[filter.Value] = n; break; } default: throw new FrostFsException($"{errInvalidFilterOp}: {filter.Operation}"); } } if (top) { ProcessedFilters[filterName] = filter; } } // processSelectors processes selectors and returns error is any of them is invalid. internal void ProcessSelectors(FrostFsPlacementPolicy policy) { foreach (var selector in policy.Selectors) { var filterName = selector.Filter; if (filterName != mainFilterName) { if (selector.Filter == null || !ProcessedFilters.ContainsKey(selector.Filter)) { throw new FrostFsException($"{errFilterNotFound}: SELECT FROM '{filterName}'"); } } ProcessedSelectors[selector.Name] = selector; var selection = GetSelection(selector); Selections[selector.Name] = selection; } } // calcNodesCount returns number of buckets and minimum number of nodes in every bucket // for the given selector. static (int bucketCount, int nodesInBucket) CalcNodesCount(FrostFsSelector selector) { return selector.Clause == (int)FrostFsClause.Same ? (1, (int)selector.Count) : ((int)selector.Count, 1); } // getSelectionBase returns nodes grouped by selector attribute. // It it guaranteed that each pair will contain at least one node. internal NodeAttrPair[] GetSelectionBase(FrostFsSelector selector) { var fName = selector.Filter ?? throw new FrostFsException("Filter name for selector is empty"); _ = ProcessedFilters.TryGetValue(fName, out var f); var isMain = fName == mainFilterName; var result = new List(); var nodeMap = new Dictionary>(); var attr = selector.Attribute; foreach (var node in NetMap.NodeInfoCollection) { if (UsedNodes.ContainsKey(node.Hash())) { continue; } if (isMain || Match(f, node)) { if (attr == null) { // Default attribute is transparent identifier which is different for every node. result.Add(new NodeAttrPair { attr = "", nodes = [node] }); } else { var v = node.Attributes[attr]; if (!nodeMap.TryGetValue(v, out var nodes) || nodes == null) { nodeMap[v] = []; } nodeMap[v].Add(node); } } } if (!string.IsNullOrEmpty(attr)) { foreach (var v in nodeMap) { result.Add(new NodeAttrPair() { attr = v.Key, nodes = [.. v.Value] }); } } if (HrwSeed != null && HrwSeed.Length != 0) { double[] ws = []; var sortedNodes = new NodeAttrPair[result.Count]; for (int i = 0; i < result.Count; i++) { var res = result[i]; Tools.AppendWeightsTo(res.nodes, weightFunc, ref ws); sortedNodes[i].nodes = Tools.SortHasherSliceByWeightValue(res.nodes.ToList(), ws, HrwSeedHash).ToArray(); sortedNodes[i].attr = result[i].attr; } return sortedNodes; } return [.. result]; } static double CalcBucketWeight(List ns, MeanIQRAgg a, Func wf) { foreach (var node in ns) { a.Add(wf(node)); } return a.Compute(); } // getSelection returns nodes grouped by s.attribute. // Last argument specifies if more buckets can be used to fulfill CBF. internal List> GetSelection(FrostFsSelector s) { var (bucketCount, nodesInBucket) = CalcNodesCount(s); var buckets = GetSelectionBase(s); if (Strict && buckets.Length < bucketCount) throw new FrostFsException($"errNotEnoughNodes: '{s.Name}'"); // We need deterministic output in case there is no pivot. // If pivot is set, buckets are sorted by HRW. // However, because initial order influences HRW order for buckets with equal weights, // we also need to have deterministic input to HRW sorting routine. if (HrwSeed == null || HrwSeed.Length == 0) { buckets = string.IsNullOrEmpty(s.Attribute) ? [.. buckets.OrderBy(b => b.nodes[0].Hash())] : [.. buckets.OrderBy(b => b.attr)]; } var maxNodesInBucket = nodesInBucket * (int)Cbf; var res = new List>(buckets.Length); var fallback = new List>(buckets.Length); for (int i = 0; i < buckets.Length; i++) { var ns = buckets[i].nodes; if (ns.Length >= maxNodesInBucket) { res.Add(ns.Take(maxNodesInBucket).ToList()); } else if (ns.Length >= nodesInBucket) { fallback.Add(new List(ns)); } } if (res.Count < bucketCount) { // Fallback to using minimum allowed backup factor (1). res.AddRange(fallback); if (Strict && res.Count < bucketCount) { throw new FrostFsException($"{errNotEnoughNodes}: {s}"); } } if (HrwSeed != null && HrwSeed.Length != 0) { var weights = new double[res.Count]; var a = new MeanIQRAgg(); for (int i = 0; i < res.Count; i++) { a.Clear(); weights[i] = CalcBucketWeight(res[i], a, weightFunc); } var hashers = res.Select(r => new HasherList(r)).ToList(); hashers = Tools.SortHasherSliceByWeightValue(hashers, weights, HrwSeedHash); for (int i = 0; i < res.Count; i++) { res[i] = hashers[i].Nodes; } } if (res.Count < bucketCount) { if (Strict && res.Count == 0) { throw new FrostFsException(errNotEnoughNodes); } bucketCount = res.Count; } if (string.IsNullOrEmpty(s.Attribute)) { fallback = res.Skip(bucketCount).ToList(); res = res.Take(bucketCount).ToList(); for (int i = 0; i < fallback.Count; i++) { var index = i % bucketCount; if (res[index].Count >= maxNodesInBucket) { break; } res[index].AddRange(fallback[i]); } } return res.Take(bucketCount).ToList(); } internal bool MatchKeyValue(FrostFsFilter f, FrostFsNodeInfo nodeInfo) { switch (f.Operation) { case (int)Operation.EQ: return nodeInfo.Attributes.TryGetValue(f.Key, out var val) && val == f.Value; case (int)Operation.LIKE: { var hasPrefix = f.Value.StartsWith(likeWildcard, StringComparison.Ordinal); var hasSuffix = f.Value.EndsWith(likeWildcard, StringComparison.Ordinal); var start = hasPrefix ? likeWildcard.Length : 0; var end = hasSuffix ? f.Value.Length - likeWildcard.Length : f.Value.Length; var str = f.Value.Substring(start, end-start); if (hasPrefix && hasSuffix) return nodeInfo.Attributes[f.Key].Contains(str); if (hasPrefix && !hasSuffix) return nodeInfo.Attributes[f.Key].EndsWith(str, StringComparison.Ordinal); if (!hasPrefix && hasSuffix) return nodeInfo.Attributes[f.Key].StartsWith(str, StringComparison.Ordinal); return nodeInfo.Attributes[f.Key] == f.Value; } case (int)Operation.NE: return nodeInfo.Attributes[f.Key] != f.Value; default: { ulong attr; switch (f.Key) { case FrostFsNodeInfo.AttrPrice: attr = nodeInfo.Price; break; case FrostFsNodeInfo.AttrCapacity: attr = nodeInfo.GetCapacity(); break; default: if (!ulong.TryParse(nodeInfo.Attributes[f.Key], NumberStyles.Integer, CultureInfo.InvariantCulture, out attr)) return false; break; } switch (f.Operation) { case (int)Operation.GT: return attr > NumCache[f.Value]; case (int)Operation.GE: return attr >= NumCache[f.Value]; case (int)Operation.LT: return attr < NumCache[f.Value]; case (int)Operation.LE: return attr <= NumCache[f.Value]; default: // do nothing and return false break; } } break; } // will not happen if context was created from f (maybe panic?) return false; } // match matches f against b. It returns no errors because // filter should have been parsed during context creation // and missing node properties are considered as a regular fail. internal bool Match(FrostFsFilter f, FrostFsNodeInfo nodeInfo) { switch (f.Operation) { case (int)Operation.NOT: { var inner = f.Filters; var fSub = inner[0]; if (!string.IsNullOrEmpty(inner[0].Name)) { fSub = ProcessedFilters[inner[0].Name]; } return !Match(fSub, nodeInfo); } case (int)Operation.AND: case (int)Operation.OR: { for (int i = 0; i < f.Filters.Length; i++) { var fSub = f.Filters[i]; if (!string.IsNullOrEmpty(f.Filters[i].Name)) { fSub = ProcessedFilters[f.Filters[i].Name]; } var ok = Match(fSub, nodeInfo); if (ok == (f.Operation == (int)Operation.OR)) { return ok; } } return f.Operation == (int)Operation.AND; } default: return MatchKeyValue(f, nodeInfo); } } }