diff --git a/netmap/context.go b/netmap/context.go index 00942e5..df3c85b 100644 --- a/netmap/context.go +++ b/netmap/context.go @@ -40,6 +40,10 @@ type context struct { // policy uses the UNIQUE flag. Nodes marked as used are not used in subsequent // base selections. usedNodes map[uint64]bool + + // If true, returns an error when netmap does not contain enough nodes for selection. + // By default best effort is taken. + strict bool } // Various validation errors. diff --git a/netmap/json_test.go b/netmap/json_test.go index b8749d9..1f63852 100644 --- a/netmap/json_test.go +++ b/netmap/json_test.go @@ -2,6 +2,7 @@ package netmap import ( "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -47,7 +48,8 @@ func TestPlacementPolicy_Interopability(t *testing.T) { require.NoError(t, err) for i := range ds { - bs, err := os.ReadFile(filepath.Join(testsDir, ds[i].Name())) + filename := filepath.Join(testsDir, ds[i].Name()) + bs, err := os.ReadFile(filename) require.NoError(t, err) var tc TestCase @@ -56,7 +58,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) { srcNodes := make([]NodeInfo, len(tc.Nodes)) copy(srcNodes, tc.Nodes) - t.Run(tc.Name, func(t *testing.T) { + t.Run(fmt.Sprintf("%s:%s", filename, tc.Name), func(t *testing.T) { var nm NetMap nm.SetNodes(tc.Nodes) diff --git a/netmap/json_tests/non_strict.json b/netmap/json_tests/non_strict.json new file mode 100644 index 0000000..2d564d7 --- /dev/null +++ b/netmap/json_tests/non_strict.json @@ -0,0 +1,95 @@ +{ + "name": "non-strict selections", + "comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.", + "nodes": [ + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + } + ] + }, + { + "attributes": [ ] + } + ], + "tests": { + "not enough nodes (backup factor)": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromRU" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [ ] + } + ] + }, + "result": [ + [ + 0 + ] + ] + }, + "not enough nodes (buckets)": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromRU" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [ ] + } + ] + }, + "result": [ + [ + 0 + ] + ] + } + } +} diff --git a/netmap/json_tests/rep_only.json b/netmap/json_tests/rep_only.json index 91683e9..db37e62 100644 --- a/netmap/json_tests/rep_only.json +++ b/netmap/json_tests/rep_only.json @@ -104,7 +104,14 @@ "selectors": [], "filters": [] }, - "error": "not enough nodes" + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] } } } diff --git a/netmap/json_tests/selector_invalid.json b/netmap/json_tests/selector_invalid.json index 9a37ba2..ef307a7 100644 --- a/netmap/json_tests/selector_invalid.json +++ b/netmap/json_tests/selector_invalid.json @@ -24,7 +24,12 @@ "tests": { "missing filter": { "policy": { - "replicas": [], + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], "containerBackupFactor": 1, "selectors": [ { @@ -47,9 +52,14 @@ }, "error": "filter not found" }, - "not enough nodes (backup factor)": { + "not enough nodes (filter results in empty set)": { "policy": { - "replicas": [], + "replicas": [ + { + "count": 1, + "selector": "MyStore" + } + ], "containerBackupFactor": 2, "selectors": [ { @@ -57,40 +67,15 @@ "count": 2, "clause": "DISTINCT", "attribute": "Country", - "filter": "FromRU" + "filter": "FromMoon" } ], "filters": [ { - "name": "FromRU", + "name": "FromMoon", "key": "Country", "op": "EQ", - "value": "Russia", - "filters": [] - } - ] - }, - "error": "not enough nodes" - }, - "not enough nodes (buckets)": { - "policy": { - "replicas": [], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "MyStore", - "count": 2, - "clause": "DISTINCT", - "attribute": "Country", - "filter": "FromRU" - } - ], - "filters": [ - { - "name": "FromRU", - "key": "Country", - "op": "EQ", - "value": "Russia", + "value": "Moon", "filters": [] } ] diff --git a/netmap/selector.go b/netmap/selector.go index d8c767a..f9247ff 100644 --- a/netmap/selector.go +++ b/netmap/selector.go @@ -60,7 +60,7 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) { bucketCount, nodesInBucket := calcNodesCount(s) buckets := c.getSelectionBase(s) - if len(buckets) < bucketCount { + if c.strict && len(buckets) < bucketCount { return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName()) } @@ -96,7 +96,7 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) { if len(res) < bucketCount { // Fallback to using minimum allowed backup factor (1). res = append(res, fallback...) - if len(res) < bucketCount { + if c.strict && len(res) < bucketCount { return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName()) } } @@ -110,6 +110,13 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) { hrw.SortHasherSliceByWeightValue(res, weights, c.hrwSeedHash) } + if len(res) < bucketCount { + if len(res) == 0 { + return nil, errNotEnoughNodes + } + bucketCount = len(res) + } + if s.GetAttribute() == "" { res, fallback = res[:bucketCount], res[bucketCount:] for i := range fallback {