[#167] netmap: Allow to select insufficient number of nodes

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

netmap/context.go

@@ -40,6 +40,10 @@ type context struct {
 	// policy uses the UNIQUE flag. Nodes marked as used are not used in subsequent
 	// base selections.
 	usedNodes map[uint64]bool
+
+	// If true, returns an error when netmap does not contain enough nodes for selection.
+	// By default best effort is taken.
+	strict bool
 }
 
 // Various validation errors.

netmap/json_tests/non_strict.json

@@ -0,0 +1,95 @@
+{
+  "name": "non-strict selections",
+  "comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.",
+  "nodes": [
+    {
+      "attributes": [
+        {
+          "key": "Country",
+          "value": "Russia"
+        }
+      ]
+    },
+    {
+      "attributes": [
+        {
+          "key": "Country",
+          "value": "Germany"
+        }
+      ]
+    },
+    {
+      "attributes": [ ]
+    }
+  ],
+  "tests": {
+    "not enough nodes (backup factor)": {
+      "policy": {
+        "replicas": [
+          {
+            "count": 1,
+            "selector": "MyStore"
+          }
+        ],
+        "containerBackupFactor": 2,
+        "selectors": [
+          {
+            "name": "MyStore",
+            "count": 2,
+            "clause": "DISTINCT",
+            "attribute": "Country",
+            "filter": "FromRU"
+          }
+        ],
+        "filters": [
+          {
+            "name": "FromRU",
+            "key": "Country",
+            "op": "EQ",
+            "value": "Russia",
+            "filters": [ ]
+          }
+        ]
+      },
+      "result": [
+        [
+          0
+        ]
+      ]
+    },
+    "not enough nodes (buckets)": {
+      "policy": {
+        "replicas": [
+          {
+            "count": 1,
+            "selector": "MyStore"
+          }
+        ],
+        "containerBackupFactor": 1,
+        "selectors": [
+          {
+            "name": "MyStore",
+            "count": 2,
+            "clause": "DISTINCT",
+            "attribute": "Country",
+            "filter": "FromRU"
+          }
+        ],
+        "filters": [
+          {
+            "name": "FromRU",
+            "key": "Country",
+            "op": "EQ",
+            "value": "Russia",
+            "filters": [ ]
+          }
+        ]
+      },
+      "result": [
+        [
+          0
+        ]
+      ]
+    }
+  }
+}

netmap/json_tests/rep_only.json

@@ -104,7 +104,14 @@
         "selectors": [],
         "filters": []
       },
-      "error": "not enough nodes"
+      "result": [
+        [
+          0,
+          1,
+          2,
+          3
+        ]
+      ]
     }
   }
 }

netmap/json_tests/selector_invalid.json

@@ -52,7 +52,7 @@
       },
       "error": "filter not found"
     },
-    "not enough nodes (backup factor)": {
+    "not enough nodes (filter results in empty set)": {
       "policy": {
         "replicas": [
           {
@@ -67,45 +67,15 @@
             "count": 2,
             "clause": "DISTINCT",
             "attribute": "Country",
-            "filter": "FromRU"
+            "filter": "FromMoon"
           }
         ],
         "filters": [
           {
-            "name": "FromRU",
+            "name": "FromMoon",
             "key": "Country",
             "op": "EQ",
-            "value": "Russia",
+            "value": "Moon",
-            "filters": []
-          }
-        ]
-      },
-      "error": "not enough nodes"
-    },
-    "not enough nodes (buckets)": {
-      "policy": {
-        "replicas": [
-          {
-            "count": 1,
-            "selector": "MyStore"
-          }
-        ],
-        "containerBackupFactor": 1,
-        "selectors": [
-          {
-            "name": "MyStore",
-            "count": 2,
-            "clause": "DISTINCT",
-            "attribute": "Country",
-            "filter": "FromRU"
-          }
-        ],
-        "filters": [
-          {
-            "name": "FromRU",
-            "key": "Country",
-            "op": "EQ",
-            "value": "Russia",
             "filters": []
           }
         ]

netmap/selector.go

@@ -60,7 +60,7 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) {
 	bucketCount, nodesInBucket := calcNodesCount(s)
 	buckets := c.getSelectionBase(s)
 
-	if len(buckets) < bucketCount {
+	if c.strict && len(buckets) < bucketCount {
 		return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName())
 	}
 
@@ -96,7 +96,7 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) {
 	if len(res) < bucketCount {
 		// Fallback to using minimum allowed backup factor (1).
 		res = append(res, fallback...)
-		if len(res) < bucketCount {
+		if c.strict && len(res) < bucketCount {
 			return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName())
 		}
 	}
@@ -110,6 +110,13 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) {
 		hrw.SortHasherSliceByWeightValue(res, weights, c.hrwSeedHash)
 	}
 
+	if len(res) < bucketCount {
+		if len(res) == 0 {
+			return nil, errNotEnoughNodes
+		}
+		bucketCount = len(res)
+	}
+
 	if s.GetAttribute() == "" {
 		res, fallback = res[:bucketCount], res[bucketCount:]
 		for i := range fallback {