From 555ccc63b25501f4286fbebd6baccb31b4f8b076 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 15 Sep 2023 14:29:49 +0300 Subject: [PATCH] [#167] netmap: Allow to select insufficient number of nodes Signed-off-by: Evgenii Stratonikov --- netmap/context.go | 4 ++ netmap/json_tests/non_strict.json | 95 +++++++++++++++++++++++++ netmap/json_tests/rep_only.json | 9 ++- netmap/json_tests/selector_invalid.json | 38 ++-------- netmap/selector.go | 11 ++- 5 files changed, 120 insertions(+), 37 deletions(-) create mode 100644 netmap/json_tests/non_strict.json diff --git a/netmap/context.go b/netmap/context.go index 00942e5d..df3c85ba 100644 --- a/netmap/context.go +++ b/netmap/context.go @@ -40,6 +40,10 @@ type context struct { // policy uses the UNIQUE flag. Nodes marked as used are not used in subsequent // base selections. usedNodes map[uint64]bool + + // If true, returns an error when netmap does not contain enough nodes for selection. + // By default best effort is taken. + strict bool } // Various validation errors. diff --git a/netmap/json_tests/non_strict.json b/netmap/json_tests/non_strict.json new file mode 100644 index 00000000..2d564d78 --- /dev/null +++ b/netmap/json_tests/non_strict.json @@ -0,0 +1,95 @@ +{ + "name": "non-strict selections", + "comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.", + "nodes": [ + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + } + ] + }, + { + "attributes": [ ] + } + ], + "tests": { + "not enough nodes (backup factor)": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromRU" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [ ] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + "not enough nodes (buckets)": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromRU" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [ ] + } + ] + }, + "result": [ + [ + 0 + ] + ] + } + } +} diff --git a/netmap/json_tests/rep_only.json b/netmap/json_tests/rep_only.json index 91683e90..db37e625 100644 --- a/netmap/json_tests/rep_only.json +++ b/netmap/json_tests/rep_only.json @@ -104,7 +104,14 @@ "selectors": [], "filters": [] }, - "error": "not enough nodes" + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] } } } diff --git a/netmap/json_tests/selector_invalid.json b/netmap/json_tests/selector_invalid.json index 32d0a09f..ef307a70 100644 --- a/netmap/json_tests/selector_invalid.json +++ b/netmap/json_tests/selector_invalid.json @@ -52,7 +52,7 @@ }, "error": "filter not found" }, - "not enough nodes (backup factor)": { + "not enough nodes (filter results in empty set)": { "policy": { "replicas": [ { @@ -67,45 +67,15 @@ "count": 2, "clause": "DISTINCT", "attribute": "Country", - "filter": "FromRU" + "filter": "FromMoon" } ], "filters": [ { - "name": "FromRU", + "name": "FromMoon", "key": "Country", "op": "EQ", - "value": "Russia", - "filters": [] - } - ] - }, - "error": "not enough nodes" - }, - "not enough nodes (buckets)": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "MyStore" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "MyStore", - "count": 2, - "clause": "DISTINCT", - "attribute": "Country", - "filter": "FromRU" - } - ], - "filters": [ - { - "name": "FromRU", - "key": "Country", - "op": "EQ", - "value": "Russia", + "value": "Moon", "filters": [] } ] diff --git a/netmap/selector.go b/netmap/selector.go index d8c767a2..f9247ff7 100644 --- a/netmap/selector.go +++ b/netmap/selector.go @@ -60,7 +60,7 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) { bucketCount, nodesInBucket := calcNodesCount(s) buckets := c.getSelectionBase(s) - if len(buckets) < bucketCount { + if c.strict && len(buckets) < bucketCount { return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName()) } @@ -96,7 +96,7 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) { if len(res) < bucketCount { // Fallback to using minimum allowed backup factor (1). res = append(res, fallback...) - if len(res) < bucketCount { + if c.strict && len(res) < bucketCount { return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName()) } } @@ -110,6 +110,13 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) { hrw.SortHasherSliceByWeightValue(res, weights, c.hrwSeedHash) } + if len(res) < bucketCount { + if len(res) == 0 { + return nil, errNotEnoughNodes + } + bucketCount = len(res) + } + if s.GetAttribute() == "" { res, fallback = res[:bucketCount], res[bucketCount:] for i := range fallback {