TrueCloudLab
/
frostfs-sdk-go
28
0
Fork 18
Code Issues 27 Pull Requests 2 Actions Releases Activity

[#167] netmap: Allow to select insufficient number of nodes
DCO / DCO (pull_request) Successful in 57s Details
Tests and linters / Lint (pull_request) Successful in 2m31s Details
Tests and linters / Tests (1.19) (pull_request) Successful in 3m22s Details
Tests and linters / Tests (1.20) (pull_request) Successful in 5m5s Details 

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
pull/167/head
Evgenii Stratonikov 2023-09-15 14:29:49 +03:00
parent 0550438b53
commit 555ccc63b2
5 changed files with 120 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
netmap/context.go
View File

@ -40,6 +40,10 @@ type context struct {
// policy uses the UNIQUE flag. Nodes marked as used are not used in subsequent
// base selections.
usedNodes map[uint64]bool
// If true, returns an error when netmap does not contain enough nodes for selection.
// By default best effort is taken.
strict bool
}
// Various validation errors.

95
netmap/json_tests/non_strict.json 100644
View File

@ -0,0 +1,95 @@
{
"name": "non-strict selections",
"comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Russia"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
}
]
},
{
"attributes": [ ]
}
],
"tests": {
"not enough nodes (backup factor)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": [ ]
}
]
},
"result": [
[
0
]
]
},
"not enough nodes (buckets)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": [ ]
}
]
},
"result": [
[
0
]
]
}
}
}

9
netmap/json_tests/rep_only.json
View File

@ -104,7 +104,14 @@
"selectors": [],
"filters": []
},
"error": "not enough nodes"
"result": [
[
0,
1,
2,
3
]
]
}
}
}

38
netmap/json_tests/selector_invalid.json
View File

@ -52,7 +52,7 @@
},
"error": "filter not found"
},
"not enough nodes (backup factor)": {
"not enough nodes (filter results in empty set)": {
"policy": {
"replicas": [
{
@ -67,45 +67,15 @@
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
"filter": "FromMoon"
}
],
"filters": [
{
"name": "FromRU",
"name": "FromMoon",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": []
}
]
},
"error": "not enough nodes"
},
"not enough nodes (buckets)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"value": "Moon",
"filters": []
}
]

11
netmap/selector.go
View File

@ -60,7 +60,7 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) {
bucketCount, nodesInBucket := calcNodesCount(s)
buckets := c.getSelectionBase(s)
if len(buckets) < bucketCount {
if c.strict && len(buckets) < bucketCount {
return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName())
}
@ -96,7 +96,7 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) {
if len(res) < bucketCount {
// Fallback to using minimum allowed backup factor (1).
res = append(res, fallback...)
if len(res) < bucketCount {
if c.strict && len(res) < bucketCount {
return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName())
}
}
@ -110,6 +110,13 @@ func (c *context) getSelection(s netmap.Selector) ([]nodes, error) {
hrw.SortHasherSliceByWeightValue(res, weights, c.hrwSeedHash)
}
if len(res) < bucketCount {
if len(res) == 0 {
return nil, errNotEnoughNodes
}
bucketCount = len(res)
}
if s.GetAttribute() == "" {
res, fallback = res[:bucketCount], res[bucketCount:]
for i := range fallback {