elebedeva/frostfs-sdk-go
1
0
Fork 0
forked from TrueCloudLab/frostfs-sdk-go
Code Pull requests Activity

[#109] netmap: convert tests to the json format

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2022-01-10 11:34:14 +03:00 committed by Alex Vanin
parent 49a17a7159
commit 2f446c8e42
18 changed files with 2622 additions and 609 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
netmap/filter_test.go
View file

@ -85,121 +85,20 @@ func TestContext_ProcessFiltersInvalid(t *testing.T) {
} }
} }
func TestFilter_MatchSimple(t *testing.T) { func TestFilter_MatchSimple_InvalidOp(t *testing.T) {
b := &Node{AttrMap: map[string]string{ b := &Node{AttrMap: map[string]string{
"Rating": "4", "Rating": "4",
"Country": "Germany", "Country": "Germany",
}} }}
testCases := []struct {
name string
ok bool
f *Filter
}{
{
"GE_true", true,
newFilter("Main", "Rating", "4", OpGE),
},
{
"GE_false", false,
newFilter("Main", "Rating", "5", OpGE),
},
{
"GT_true", true,
newFilter("Main", "Rating", "3", OpGT),
},
{
"GT_false", false,
newFilter("Main", "Rating", "4", OpGT),
},
{
"LE_true", true,
newFilter("Main", "Rating", "4", OpLE),
},
{
"LE_false", false,
newFilter("Main", "Rating", "3", OpLE),
},
{
"LT_true", true,
newFilter("Main", "Rating", "5", OpLT),
},
{
"LT_false", false,
newFilter("Main", "Rating", "4", OpLT),
},
{
"EQ_true", true,
newFilter("Main", "Country", "Germany", OpEQ),
},
{
"EQ_false", false,
newFilter("Main", "Country", "China", OpEQ),
},
{
"NE_true", true,
newFilter("Main", "Country", "France", OpNE),
},
{
"NE_false", false,
newFilter("Main", "Country", "Germany", OpNE),
},
}
for _, tc := range testCases {
c := NewContext(new(Netmap))
p := newPlacementPolicy(1, nil, nil, []*Filter{tc.f})
require.NoError(t, c.processFilters(p))
require.Equal(t, tc.ok, c.match(tc.f, b))
}
t.Run("InvalidOp", func(t *testing.T) { f := newFilter("Main", "Rating", "5", OpEQ)
f := newFilter("Main", "Rating", "5", OpEQ)
c := NewContext(new(Netmap))
p := newPlacementPolicy(1, nil, nil, []*Filter{f})
require.NoError(t, c.processFilters(p))
// just for the coverage
f.SetOperation(0)
require.False(t, c.match(f, b))
})
}
func TestFilter_Match(t *testing.T) {
fs := []*Filter{
newFilter("StorageSSD", "Storage", "SSD", OpEQ),
newFilter("GoodRating", "Rating", "4", OpGE),
newFilter("Main", "", "", OpAND,
newFilter("StorageSSD", "", "", 0),
newFilter("", "IntField", "123", OpLT),
newFilter("GoodRating", "", "", 0),
newFilter("", "", "", OpOR,
newFilter("", "Param", "Value1", OpEQ),
newFilter("", "Param", "Value2", OpEQ),
)),
}
c := NewContext(new(Netmap)) c := NewContext(new(Netmap))
p := newPlacementPolicy(1, nil, nil, fs) p := newPlacementPolicy(1, nil, nil, []*Filter{f})
require.NoError(t, c.processFilters(p)) require.NoError(t, c.processFilters(p))
t.Run("Good", func(t *testing.T) { // just for the coverage
n := getTestNode("Storage", "SSD", "Rating", "10", "IntField", "100", "Param", "Value1") f.SetOperation(0)
require.True(t, c.applyFilter("Main", n)) require.False(t, c.match(f, b))
})
t.Run("InvalidStorage", func(t *testing.T) {
n := getTestNode("Storage", "HDD", "Rating", "10", "IntField", "100", "Param", "Value1")
require.False(t, c.applyFilter("Main", n))
})
t.Run("InvalidRating", func(t *testing.T) {
n := getTestNode("Storage", "SSD", "Rating", "3", "IntField", "100", "Param", "Value1")
require.False(t, c.applyFilter("Main", n))
})
t.Run("InvalidIntField", func(t *testing.T) {
n := getTestNode("Storage", "SSD", "Rating", "3", "IntField", "str", "Param", "Value1")
require.False(t, c.applyFilter("Main", n))
})
t.Run("InvalidParam", func(t *testing.T) {
n := getTestNode("Storage", "SSD", "Rating", "3", "IntField", "100", "Param", "NotValue")
require.False(t, c.applyFilter("Main", n))
})
} }
func testFilter() *Filter { func testFilter() *Filter {

82
netmap/json_test.go Normal file
View file

@ -0,0 +1,82 @@
package netmap
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
// TestCase represents collection of placement policy tests for a single node set.
type TestCase struct {
Name string `json:"name"`
Nodes []NodeInfo `json:"nodes"`
Tests map[string]struct {
Policy PlacementPolicy `json:"policy"`
Pivot []byte `json:"pivot,omitempty"`
Result [][]int `json:"result,omitempty"`
Error string `json:"error,omitempty"`
Placement struct {
Pivot []byte
Result [][]int
} `json:"placement,omitempty"`
}
}
func compareNodes(t *testing.T, expected [][]int, nodes Nodes, actual []Nodes) {
require.Equal(t, len(expected), len(actual))
for i := range expected {
require.Equal(t, len(expected[i]), len(actual[i]))
for j, index := range expected[i] {
require.Equal(t, nodes[index], actual[i][j])
}
}
}
func TestPlacementPolicy_Interopability(t *testing.T) {
const testsDir = "./json_tests"
f, err := os.Open(testsDir)
require.NoError(t, err)
ds, err := f.ReadDir(0)
require.NoError(t, err)
for i := range ds {
bs, err := ioutil.ReadFile(filepath.Join(testsDir, ds[i].Name()))
require.NoError(t, err)
var tc TestCase
require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
t.Run(tc.Name, func(t *testing.T) {
nodes := NodesFromInfo(tc.Nodes)
nm, err := NewNetmap(nodes)
require.NoError(t, err)
for name, tt := range tc.Tests {
t.Run(name, func(t *testing.T) {
v, err := nm.GetContainerNodes(&tt.Policy, tt.Pivot)
if tt.Result == nil {
require.Error(t, err)
require.Contains(t, err.Error(), tt.Error)
} else {
require.NoError(t, err)
res := v.Replicas()
compareNodes(t, tt.Result, nodes, res)
if tt.Placement.Result != nil {
res, err := nm.GetPlacementVectors(v, tt.Placement.Pivot)
require.NoError(t, err)
compareNodes(t, tt.Placement.Result, nodes, res)
}
}
})
}
})
}
}

100
netmap/json_tests/cbf_default.json Normal file
View file

@ -0,0 +1,100 @@
{
"name": "default CBF is 3",
"nodes": [
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "DE"
},
{
"key": "City",
"value": "Berlin"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "FR"
},
{
"key": "City",
"value": "Paris"
}
]
}
],
"tests": {
"set default CBF": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "EU"
}
],
"containerBackupFactor": 0,
"selectors": [
{
"name": "EU",
"count": 1,
"clause": "SAME",
"attribute": "Location",
"filter": "*"
}
],
"filters": [],
"subnetId": null
},
"result": [
[
0,
1,
2
]
]
}
}
}

101
netmap/json_tests/cbf_minimal.json Normal file
View file

@ -0,0 +1,101 @@
{
"name": "Real node count multiplier is in range [1, specified CBF]",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "DE"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "DE"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "DE"
}
]
}
],
"tests": {
"select 2, CBF is 2": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "SAME",
"attribute": "Country",
"filter": "*"
}
],
"filters": [],
"subnetId": null
},
"result": [
[
0,
1,
2
]
]
},
"select 3, CBF is 2": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "X",
"count": 3,
"clause": "SAME",
"attribute": "Country",
"filter": "*"
}
],
"filters": [],
"subnetId": null
},
"result": [
[
0,
1,
2
]
]
}
}
}

159
netmap/json_tests/cbf_requirements.json Normal file
View file

@ -0,0 +1,159 @@
{
"name": "CBF requirements",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Attr",
"value": "Same"
}
]
}
],
"tests": {
"default CBF, no selector": {
"policy": {
"replicas": [
{
"count": 2,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": [],
"subnetId": null
},
"result": [
[
0,
2,
1,
3
]
]
},
"explicit CBF, no selector": {
"policy": {
"replicas": [
{
"count": 2,
"selector": ""
}
],
"containerBackupFactor": 3,
"selectors": [],
"filters": [],
"subnetId": null
},
"result": [
[
0,
2,
1,
3
]
]
},
"select distinct, weak CBF": {
"policy": {
"replicas": [
{
"count": 2,
"selector": "X"
}
],
"containerBackupFactor": 3,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "DISTINCT",
"attribute": "",
"filter": "*"
}
],
"filters": [],
"subnetId": null
},
"result": [
[
0,
2,
1,
3
]
]
},
"select same, weak CBF": {
"policy": {
"replicas": [
{
"count": 2,
"selector": "X"
}
],
"containerBackupFactor": 3,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "SAME",
"attribute": "Attr",
"filter": "*"
}
],
"filters": [],
"subnetId": null
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

387
netmap/json_tests/filter_complex.json Normal file
View file

@ -0,0 +1,387 @@
{
"name": "compound filter",
"nodes": [
{
"attributes": [
{
"key": "Storage",
"value": "SSD"
},
{
"key": "Rating",
"value": "10"
},
{
"key": "IntField",
"value": "100"
},
{
"key": "Param",
"value": "Value1"
}
]
}
],
"tests": {
"good": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
],
"subnetId": null
},
"result": [
[
0
]
]
},
"bad storage type": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "HDD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"bad rating": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "15",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"bad param": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value0",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
],
"subnetId": null
},
"error": "not enough nodes"
}
}
}

83
netmap/json_tests/filter_invalid_integer.json Normal file
View file

@ -0,0 +1,83 @@
{
"name": "invalid integer field",
"nodes": [
{
"attributes": [
{
"key": "IntegerField",
"value": ""
}
]
},
{
"attributes": [
{
"key": "IntegerField",
"value": "str"
}
]
}
],
"tests": {
"empty string is not casted to 0": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "IntegerField",
"op": "LE",
"value": "8",
"filters": []
}
]
},
"error": "not enough nodes"
},
"non-empty string is not casted to a number": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "IntegerField",
"op": "GE",
"value": "0",
"filters": []
}
]
},
"error": "not enough nodes"
}
}
}

415
netmap/json_tests/filter_simple.json Normal file
View file

@ -0,0 +1,415 @@
{
"name": "single-op filters",
"nodes": [
{
"attributes": [
{
"key": "Rating",
"value": "4"
},
{
"key": "Country",
"value": "Germany"
}
]
}
],
"tests": {
"GE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
}
],
"subnetId": null
},
"result": [
[
0
]
]
},
"GE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GE",
"value": "5",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"GT true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GT",
"value": "3",
"filters": []
}
],
"subnetId": null
},
"result": [
[
0
]
]
},
"GT false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GT",
"value": "4",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"LE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LE",
"value": "4",
"filters": []
}
],
"subnetId": null
},
"result": [
[
0
]
]
},
"LE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LE",
"value": "3",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"LT true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LT",
"value": "5",
"filters": []
}
],
"subnetId": null
},
"result": [
[
0
]
]
},
"LT false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LT",
"value": "4",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"EQ true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "EQ",
"value": "Germany",
"filters": []
}
],
"subnetId": null
},
"result": [
[
0
]
]
},
"EQ false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "EQ",
"value": "China",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"NE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "NE",
"value": "France",
"filters": []
}
],
"subnetId": null
},
"result": [
[
0
]
]
},
"NE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "NE",
"value": "Germany",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
}
}
}

165
netmap/json_tests/hrw_sort.json Normal file
View file

@ -0,0 +1,165 @@
{
"name": "HRW ordering",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Price",
"value": "4"
},
{
"key": "Capacity",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "3"
},
{
"key": "Capacity",
"value": "10"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "1"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "100"
},
{
"key": "Capacity",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "7"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "1"
}
]
}
],
"tests": {
"select 3 nodes in 3 distinct countries, same placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[],"subnetId":null},
"pivot": "Y29udGFpbmVySUQ=",
"result": [[4, 0, 7]],
"placement": {
"pivot": "b2JqZWN0SUQ=",
"result": [[4, 0, 7]]
}
},
"select 6 nodes in 3 distinct countries, different placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[],"subnetId":null},
"pivot": "Y29udGFpbmVySUQ=",
"result": [[4, 3, 0, 1, 7, 2]],
"placement": {
"pivot": "b2JqZWN0SUQ=",
"result": [[4, 3, 0, 7, 2, 1]]
}
}
}
}

109
netmap/json_tests/issue213.json Normal file
View file

@ -0,0 +1,109 @@
{
"name": "unnamed selector (nspcc-dev/neofs-api-go#213)",
"nodes": [
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Russia"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Russia"
},
{
"key": "City",
"value": "Saint-Petersburg"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Sweden"
},
{
"key": "City",
"value": "Stockholm"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Finalnd"
},
{
"key": "City",
"value": "Helsinki"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 4,
"selector": ""
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "",
"count": 4,
"clause": "DISTINCT",
"attribute": "",
"filter": "LOC_EU"
}
],
"filters": [
{
"name": "LOC_EU",
"key": "Location",
"op": "EQ",
"value": "Europe",
"filters": []
}
],
"subnetId": null
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

95
netmap/json_tests/multiple_rep.json Normal file
View file

@ -0,0 +1,95 @@
{
"name": "multiple replicas (#215)",
"nodes": [
{
"attributes": [
{
"key": "City",
"value": "Saint-Petersburg"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Berlin"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Paris"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "LOC_SPB_PLACE"
},
{
"count": 1,
"selector": "LOC_MSK_PLACE"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "LOC_SPB_PLACE",
"count": 1,
"clause": "CLAUSE_UNSPECIFIED",
"attribute": "",
"filter": "LOC_SPB"
},
{
"name": "LOC_MSK_PLACE",
"count": 1,
"clause": "CLAUSE_UNSPECIFIED",
"attribute": "",
"filter": "LOC_MSK"
}
],
"filters": [
{
"name": "LOC_SPB",
"key": "City",
"op": "EQ",
"value": "Saint-Petersburg",
"filters": []
},
{
"name": "LOC_MSK",
"key": "City",
"op": "EQ",
"value": "Moscow",
"filters": []
}
],
"subnetId": null
},
"result": [
[
0
],
[
1
]
]
}
}
}

332
netmap/json_tests/multiple_rep_asymmetric.json Normal file
View file

@ -0,0 +1,332 @@
{
"name": "multiple REP, asymmetric",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "0"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "5"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "6"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "NewYork"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "7"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "8"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "9"
},
{
"key": "Continent",
"value": "SA"
},
{
"key": "City",
"value": "Lima"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "10"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "11"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "NewYork"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "12"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "LosAngeles"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "13"
},
{
"key": "Continent",
"value": "SA"
},
{
"key": "City",
"value": "Lima"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "SPB"
},
{
"count": 2,
"selector": "Americas"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "SPB",
"count": 1,
"clause": "SAME",
"attribute": "City",
"filter": "SPBSSD"
},
{
"name": "Americas",
"count": 2,
"clause": "DISTINCT",
"attribute": "City",
"filter": "Americas"
}
],
"filters": [
{
"name": "SPBSSD",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "",
"key": "Country",
"op": "EQ",
"value": "RU",
"filters": []
},
{
"name": "",
"key": "City",
"op": "EQ",
"value": "St.Petersburg",
"filters": []
},
{
"name": "",
"key": "SSD",
"op": "EQ",
"value": "1",
"filters": []
}
]
},
{
"name": "Americas",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Continent",
"op": "EQ",
"value": "NA",
"filters": []
},
{
"name": "",
"key": "Continent",
"op": "EQ",
"value": "SA",
"filters": []
}
]
}
],
"subnetId": null
},
"result": [
[
1,
4
],
[
8,
12,
5,
10
]
]
}
}
}

113
netmap/json_tests/rep_only.json Normal file
View file

@ -0,0 +1,113 @@
{
"name": "REP X",
"nodes": [
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Saint-Petersburg",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Moscow",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Berlin",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Paris",
"parents": []
}
],
"state": "UNSPECIFIED"
}
],
"tests": {
"REP 1": {
"policy": {
"replicas": [
{
"count": 1,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": [],
"subnetId": null
},
"result": [
[
0,
1,
2
]
]
},
"REP 3": {
"policy": {
"replicas": [
{
"count": 3,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": [],
"subnetId": null
},
"result": [
[
0,
3,
1,
2
]
]
},
"REP 5": {
"policy": {
"replicas": [
{
"count": 5,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": [],
"subnetId": null
},
"error": "not enough nodes"
}
}
}

117
netmap/json_tests/select_no_attribute.json Normal file
View file

@ -0,0 +1,117 @@
{
"name": "select with unspecified attribute",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "0"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "X",
"count": 4,
"clause": "DISTINCT",
"attribute": "",
"filter": "*"
}
],
"filters": [],
"subnetId": null
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

104
netmap/json_tests/selector_invalid.json Normal file
View file

@ -0,0 +1,104 @@
{
"name": "invalid selections",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Russia"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
}
]
},
{
"attributes": []
}
],
"tests": {
"missing filter": {
"policy": {
"replicas": [],
"containerBackupFactor": 1,
"selectors": [
{
"name": "MyStore",
"count": 1,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromNL"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": []
}
],
"subnetId": null
},
"error": "filter not found"
},
"not enough nodes (backup factor)": {
"policy": {
"replicas": [],
"containerBackupFactor": 2,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"not enough nodes (buckets)": {
"policy": {
"replicas": [],
"containerBackupFactor": 1,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
}
}
}

254
netmap/json_tests/subnet.json Normal file
View file

@ -0,0 +1,254 @@
{
"name": "subnet tests",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "0"
},
{
"key": "City",
"value": "Paris"
},
{
"key": "__NEOFS__SUBNET_0",
"value": "False"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "City",
"value": "Paris"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "City",
"value": "London"
},
{
"key": "__NEOFS__SUBNET_1",
"value": "True"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "City",
"value": "London"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "City",
"value": "Toronto"
},
{
"key": "__NEOFS__SUBNET_1",
"value": "True"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "5"
},
{
"key": "City",
"value": "Toronto"
},
{
"key": "__NEOFS__SUBNET_2",
"value": "True"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "6"
},
{
"key": "City",
"value": "Tokyo"
},
{
"key": "__NEOFS__SUBNET_2",
"value": "True"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "7"
},
{
"key": "City",
"value": "Tokyo"
},
{
"key": "__NEOFS__SUBNET_2",
"value": "True"
}
],
"state": "UNSPECIFIED"
}
],
"tests": {
"select from default subnet, fail": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 0,
"selectors": [
{
"name": "S",
"count": 2,
"clause": "SAME",
"attribute": "City",
"filter": "F"
}
],
"filters": [
{
"name": "F",
"key": "City",
"op": "EQ",
"value": "Paris",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"select from default subnet, success": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 0,
"selectors": [
{
"name": "S",
"count": 2,
"clause": "SAME",
"attribute": "City",
"filter": "F"
}
],
"filters": [
{
"name": "F",
"key": "City",
"op": "EQ",
"value": "Toronto",
"filters": []
}
],
"subnetId": null
},
"result": [
[
4,
5
]
]
},
"select from non-default subnet, success": {
"policy": {
"replicas": [
{
"count": 3,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": [],
"subnetId": {
"value": 2
}
},
"result": [
[
5,
6,
7
]
]
},
"select subnet via filters": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "SAME",
"attribute": "City",
"filter": "F"
}
],
"filters": [
{
"name": "F",
"key": "__NEOFS_SUBNET.2.ENABLED",
"op": "EQ",
"value": "True"
}
]
},
"error": "not enough nodes"
}
}
}

157
netmap/policy_test.go
View file

@ -1,169 +1,12 @@
package netmap package netmap
import ( import (
"errors"
"strconv"
"testing" "testing"
"github.com/nspcc-dev/neofs-api-go/v2/netmap" "github.com/nspcc-dev/neofs-api-go/v2/netmap"
subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
const subnetAttrPrefix = "__NEOFS_SUBNET"
func subnetAttrName(subnet uint32) string {
return subnetAttrPrefix + "." +
strconv.FormatUint(uint64(subnet), 10) + ".ENABLED"
}
func TestPlacementPolicy_Subnet(t *testing.T) {
nodes := []NodeInfo{
nodeInfoFromAttributes("ID", "0", "City", "Paris"),
nodeInfoFromAttributes("ID", "1", "City", "Paris"),
nodeInfoFromAttributes("ID", "2", "City", "London"),
nodeInfoFromAttributes("ID", "3", "City", "London"),
nodeInfoFromAttributes("ID", "4", "City", "Toronto"),
nodeInfoFromAttributes("ID", "5", "City", "Toronto"),
nodeInfoFromAttributes("ID", "6", "City", "Tokyo"),
nodeInfoFromAttributes("ID", "7", "City", "Tokyo"),
}
var id subnetid.ID
nodes[0].ExitSubnet(id)
id.SetNumber(1)
nodes[2].EnterSubnet(id)
nodes[4].EnterSubnet(id)
id.SetNumber(2)
nodes[5].EnterSubnet(id)
nodes[6].EnterSubnet(id)
nodes[7].EnterSubnet(id)
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
t.Run("select 2 nodes from the default subnet in Paris", func(t *testing.T) {
p := newPlacementPolicy(0,
[]*Replica{newReplica(1, "S")},
[]*Selector{newSelector("S", "City", ClauseSame, 2, "F")},
[]*Filter{newFilter("F", "City", "Paris", OpEQ)})
_, err := nm.GetContainerNodes(p, nil)
require.True(t, errors.Is(err, ErrNotEnoughNodes), "got: %v", err)
})
t.Run("select 2 nodes from the default subnet in London", func(t *testing.T) {
p := newPlacementPolicy(0,
[]*Replica{newReplica(1, "S")},
[]*Selector{newSelector("S", "City", ClauseSame, 2, "F")},
[]*Filter{newFilter("F", "City", "London", OpEQ)})
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
nodes := v.Flatten()
require.Equal(t, 2, len(nodes))
for _, n := range v.Flatten() {
id := n.Attribute("ID")
require.Contains(t, []string{"2", "3"}, id)
}
})
t.Run("select 2 nodes from the default subnet in Toronto", func(t *testing.T) {
p := newPlacementPolicy(0,
[]*Replica{newReplica(1, "S")},
[]*Selector{newSelector("S", "City", ClauseSame, 2, "F")},
[]*Filter{newFilter("F", "City", "Toronto", OpEQ)})
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
nodes := v.Flatten()
require.Equal(t, 2, len(nodes))
for _, n := range v.Flatten() {
id := n.Attribute("ID")
require.Contains(t, []string{"4", "5"}, id)
}
})
t.Run("select 3 nodes from the non-default subnet", func(t *testing.T) {
p := newPlacementPolicy(0,
[]*Replica{newReplica(3, "")},
nil, nil)
p.SetSubnetID(newSubnetID(2))
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
nodes := v.Flatten()
require.Equal(t, 3, len(nodes))
for _, n := range v.Flatten() {
id := n.Attribute("ID")
require.Contains(t, []string{"5", "6", "7"}, id)
}
})
t.Run("select nodes from the subnet via filter", func(t *testing.T) {
p := newPlacementPolicy(0,
[]*Replica{newReplica(1, "")},
nil,
[]*Filter{newFilter(MainFilterName, subnetAttrName(2), "True", OpEQ, nil)})
_, err := nm.GetContainerNodes(p, nil)
require.Error(t, err)
})
}
func TestPlacementPolicy_CBFWithEmptySelector(t *testing.T) {
nodes := []NodeInfo{
nodeInfoFromAttributes("ID", "1", "Attr", "Same"),
nodeInfoFromAttributes("ID", "2", "Attr", "Same"),
nodeInfoFromAttributes("ID", "3", "Attr", "Same"),
nodeInfoFromAttributes("ID", "4", "Attr", "Same"),
}
p1 := newPlacementPolicy(0,
[]*Replica{newReplica(2, "")},
nil, // selectors
nil, // filters
)
p2 := newPlacementPolicy(3,
[]*Replica{newReplica(2, "")},
nil, // selectors
nil, // filters
)
p3 := newPlacementPolicy(3,
[]*Replica{newReplica(2, "X")},
[]*Selector{newSelector("X", "", ClauseDistinct, 2, "*")},
nil, // filters
)
p4 := newPlacementPolicy(3,
[]*Replica{newReplica(2, "X")},
[]*Selector{newSelector("X", "Attr", ClauseSame, 2, "*")},
nil, // filters
)
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
v, err := nm.GetContainerNodes(p1, nil)
require.NoError(t, err)
assert.Len(t, v.Flatten(), 4)
v, err = nm.GetContainerNodes(p2, nil)
require.NoError(t, err)
assert.Len(t, v.Flatten(), 4)
v, err = nm.GetContainerNodes(p3, nil)
require.NoError(t, err)
assert.Len(t, v.Flatten(), 4)
v, err = nm.GetContainerNodes(p4, nil)
require.NoError(t, err)
assert.Len(t, v.Flatten(), 4)
}
func TestPlacementPolicyFromV2(t *testing.T) { func TestPlacementPolicyFromV2(t *testing.T) {
pV2 := new(netmap.PlacementPolicy) pV2 := new(netmap.PlacementPolicy)

345
netmap/selector_test.go
View file

@ -1,7 +1,6 @@
package netmap package netmap
import ( import (
"errors"
"fmt" "fmt"
"math/rand" "math/rand"
"sort" "sort"
@ -187,242 +186,6 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
} }
} }
func TestPlacementPolicy_UnspecifiedClause(t *testing.T) {
p := newPlacementPolicy(1,
[]*Replica{newReplica(1, "X")},
[]*Selector{
newSelector("X", "", ClauseDistinct, 4, "*"),
},
nil,
)
nodes := []NodeInfo{
nodeInfoFromAttributes("ID", "1", "Country", "RU", "City", "St.Petersburg", "SSD", "0"),
nodeInfoFromAttributes("ID", "2", "Country", "RU", "City", "St.Petersburg", "SSD", "1"),
nodeInfoFromAttributes("ID", "3", "Country", "RU", "City", "Moscow", "SSD", "1"),
nodeInfoFromAttributes("ID", "4", "Country", "RU", "City", "Moscow", "SSD", "1"),
}
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
require.Equal(t, 4, len(v.Flatten()))
}
func TestPlacementPolicy_Minimal(t *testing.T) {
nodes := []NodeInfo{
nodeInfoFromAttributes("City", "Saint-Petersburg"),
nodeInfoFromAttributes("City", "Moscow"),
nodeInfoFromAttributes("City", "Berlin"),
nodeInfoFromAttributes("City", "Paris"),
}
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
runTest := func(t *testing.T, rep uint32, expectError bool) {
p := newPlacementPolicy(0,
[]*Replica{newReplica(rep, "")},
nil, nil)
v, err := nm.GetContainerNodes(p, nil)
if expectError {
require.Error(t, err)
return
}
require.NoError(t, err)
count := int(rep * defaultCBF)
if count > len(nm.Nodes) {
count = len(nm.Nodes)
}
require.EqualValues(t, count, len(v.Flatten()))
}
t.Run("REP 1", func(t *testing.T) {
runTest(t, 1, false)
})
t.Run("REP 3", func(t *testing.T) {
runTest(t, 3, false)
})
t.Run("REP 5", func(t *testing.T) {
runTest(t, 5, true)
})
}
// Issue #215.
func TestPlacementPolicy_MultipleREP(t *testing.T) {
p := newPlacementPolicy(1,
[]*Replica{
newReplica(1, "LOC_SPB_PLACE"),
newReplica(1, "LOC_MSK_PLACE"),
},
[]*Selector{
newSelector("LOC_SPB_PLACE", "", ClauseUnspecified, 1, "LOC_SPB"),
newSelector("LOC_MSK_PLACE", "", ClauseUnspecified, 1, "LOC_MSK"),
},
[]*Filter{
newFilter("LOC_SPB", "City", "Saint-Petersburg", OpEQ),
newFilter("LOC_MSK", "City", "Moscow", OpEQ),
},
)
nodes := []NodeInfo{
nodeInfoFromAttributes("City", "Saint-Petersburg"),
nodeInfoFromAttributes("City", "Moscow"),
nodeInfoFromAttributes("City", "Berlin"),
nodeInfoFromAttributes("City", "Paris"),
}
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
rs := v.Replicas()
require.Equal(t, 2, len(rs))
require.Equal(t, 1, len(rs[0]))
require.Equal(t, "Saint-Petersburg", rs[0][0].Attribute("City"))
require.Equal(t, 1, len(rs[1]))
require.Equal(t, "Moscow", rs[1][0].Attribute("City"))
}
func TestPlacementPolicy_DefaultCBF(t *testing.T) {
p := newPlacementPolicy(0,
[]*Replica{
newReplica(1, "EU"),
},
[]*Selector{
newSelector("EU", "Location", ClauseSame, 1, "*"),
},
nil)
nodes := []NodeInfo{
nodeInfoFromAttributes("Location", "Europe", "Country", "RU", "City", "St.Petersburg"),
nodeInfoFromAttributes("Location", "Europe", "Country", "RU", "City", "Moscow"),
nodeInfoFromAttributes("Location", "Europe", "Country", "DE", "City", "Berlin"),
nodeInfoFromAttributes("Location", "Europe", "Country", "FR", "City", "Paris"),
}
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
require.Equal(t, defaultCBF, len(v.Flatten()))
}
func TestPlacementPolicy_GetPlacementVectors(t *testing.T) {
p := newPlacementPolicy(2,
[]*Replica{
newReplica(1, "SPB"),
newReplica(2, "Americas"),
},
[]*Selector{
newSelector("SPB", "City", ClauseSame, 1, "SPBSSD"),
newSelector("Americas", "City", ClauseDistinct, 2, "Americas"),
},
[]*Filter{
newFilter("SPBSSD", "", "", OpAND,
newFilter("", "Country", "RU", OpEQ),
newFilter("", "City", "St.Petersburg", OpEQ),
newFilter("", "SSD", "1", OpEQ)),
newFilter("Americas", "", "", OpOR,
newFilter("", "Continent", "NA", OpEQ),
newFilter("", "Continent", "SA", OpEQ)),
})
nodes := []NodeInfo{
nodeInfoFromAttributes("ID", "1", "Country", "RU", "City", "St.Petersburg", "SSD", "0"),
nodeInfoFromAttributes("ID", "2", "Country", "RU", "City", "St.Petersburg", "SSD", "1"),
nodeInfoFromAttributes("ID", "3", "Country", "RU", "City", "Moscow", "SSD", "1"),
nodeInfoFromAttributes("ID", "4", "Country", "RU", "City", "Moscow", "SSD", "1"),
nodeInfoFromAttributes("ID", "5", "Country", "RU", "City", "St.Petersburg", "SSD", "1"),
nodeInfoFromAttributes("ID", "6", "Continent", "NA", "City", "NewYork"),
nodeInfoFromAttributes("ID", "7", "Continent", "AF", "City", "Cairo"),
nodeInfoFromAttributes("ID", "8", "Continent", "AF", "City", "Cairo"),
nodeInfoFromAttributes("ID", "9", "Continent", "SA", "City", "Lima"),
nodeInfoFromAttributes("ID", "10", "Continent", "AF", "City", "Cairo"),
nodeInfoFromAttributes("ID", "11", "Continent", "NA", "City", "NewYork"),
nodeInfoFromAttributes("ID", "12", "Continent", "NA", "City", "LosAngeles"),
nodeInfoFromAttributes("ID", "13", "Continent", "SA", "City", "Lima"),
}
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
require.Equal(t, 2, len(v.Replicas()))
require.Equal(t, 6, len(v.Flatten()))
require.Equal(t, 2, len(v.Replicas()[0]))
ids := map[string]struct{}{}
for _, ni := range v.Replicas()[0] {
require.Equal(t, "RU", ni.Attribute("Country"))
require.Equal(t, "St.Petersburg", ni.Attribute("City"))
require.Equal(t, "1", ni.Attribute("SSD"))
ids[ni.Attribute("ID")] = struct{}{}
}
require.Equal(t, len(v.Replicas()[0]), len(ids), "not all nodes we distinct")
require.Equal(t, 4, len(v.Replicas()[1])) // 2 cities * 2 HRWB
ids = map[string]struct{}{}
for _, ni := range v.Replicas()[1] {
require.Contains(t, []string{"NA", "SA"}, ni.Attribute("Continent"))
ids[ni.Attribute("ID")] = struct{}{}
}
require.Equal(t, len(v.Replicas()[1]), len(ids), "not all nodes we distinct")
}
func TestPlacementPolicy_LowerBound(t *testing.T) {
p := newPlacementPolicy(
2, // backup factor
[]*Replica{
newReplica(1, "X"),
},
[]*Selector{
newSelector("X", "Country", ClauseSame, 2, "*"),
},
nil, // filters
)
nodes := []NodeInfo{
nodeInfoFromAttributes("ID", "1", "Country", "DE"),
nodeInfoFromAttributes("ID", "2", "Country", "DE"),
nodeInfoFromAttributes("ID", "3", "Country", "DE"),
}
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
require.Equal(t, 3, len(v.Flatten()))
}
func TestIssue213(t *testing.T) {
p := newPlacementPolicy(1,
[]*Replica{
newReplica(4, ""),
},
[]*Selector{
newSelector("", "", ClauseDistinct, 4, "LOC_EU"),
},
[]*Filter{
newFilter("LOC_EU", "Location", "Europe", OpEQ),
})
nodes := []NodeInfo{
nodeInfoFromAttributes("Location", "Europe", "Country", "Russia", "City", "Moscow"),
nodeInfoFromAttributes("Location", "Europe", "Country", "Russia", "City", "Saint-Petersburg"),
nodeInfoFromAttributes("Location", "Europe", "Country", "Sweden", "City", "Stockholm"),
nodeInfoFromAttributes("Location", "Europe", "Country", "Finalnd", "City", "Helsinki"),
}
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
v, err := nm.GetContainerNodes(p, nil)
require.NoError(t, err)
require.Equal(t, 4, len(v.Flatten()))
}
func TestPlacementPolicy_ProcessSelectors(t *testing.T) { func TestPlacementPolicy_ProcessSelectors(t *testing.T) {
p := newPlacementPolicy(2, nil, p := newPlacementPolicy(2, nil,
[]*Selector{ []*Selector{
@ -472,114 +235,6 @@ func TestPlacementPolicy_ProcessSelectors(t *testing.T) {
} }
} }
func TestPlacementPolicy_ProcessSelectorsHRW(t *testing.T) {
p := newPlacementPolicy(1, nil,
[]*Selector{
newSelector("Main", "Country", ClauseDistinct, 3, "*"),
}, nil)
// bucket weight order: RU > DE > FR
nodes := []NodeInfo{
nodeInfoFromAttributes("Country", "Germany", AttrPrice, "2", AttrCapacity, "10000"),
nodeInfoFromAttributes("Country", "Germany", AttrPrice, "4", AttrCapacity, "1"),
nodeInfoFromAttributes("Country", "France", AttrPrice, "3", AttrCapacity, "10"),
nodeInfoFromAttributes("Country", "Russia", AttrPrice, "2", AttrCapacity, "10000"),
nodeInfoFromAttributes("Country", "Russia", AttrPrice, "1", AttrCapacity, "10000"),
nodeInfoFromAttributes("Country", "Russia", AttrCapacity, "10000"),
nodeInfoFromAttributes("Country", "France", AttrPrice, "100", AttrCapacity, "1"),
nodeInfoFromAttributes("Country", "France", AttrPrice, "7", AttrCapacity, "10000"),
nodeInfoFromAttributes("Country", "Russia", AttrPrice, "2", AttrCapacity, "1"),
}
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
c := NewContext(nm)
c.setPivot([]byte("containerID"))
c.setCBF(p.ContainerBackupFactor())
c.weightFunc = newWeightFunc(newMaxNorm(10000), newReverseMinNorm(1))
c.aggregator = func() aggregator {
return new(maxAgg)
}
require.NoError(t, c.processFilters(p))
require.NoError(t, c.processSelectors(p))
cnt := c.Selections["Main"]
expected := []Nodes{
{{Index: 4, Capacity: 10000, Price: 1}}, // best RU
{{Index: 0, Capacity: 10000, Price: 2}}, // best DE
{{Index: 7, Capacity: 10000, Price: 7}}, // best FR
}
require.Equal(t, len(expected), len(cnt))
for i := range expected {
require.Equal(t, len(expected[i]), len(cnt[i]))
require.Equal(t, expected[i][0].Index, cnt[i][0].Index)
require.Equal(t, expected[i][0].Capacity, cnt[i][0].Capacity)
require.Equal(t, expected[i][0].Price, cnt[i][0].Price)
}
res, err := nm.GetPlacementVectors(containerNodes(cnt), []byte("objectID"))
require.NoError(t, err)
require.Equal(t, res, cnt)
}
func newMaxNorm(max float64) normalizer {
return &maxNorm{max: max}
}
func TestPlacementPolicy_ProcessSelectorsInvalid(t *testing.T) {
testCases := []struct {
name string
p *PlacementPolicy
err error
}{
{
"MissingSelector",
newPlacementPolicy(2, nil,
[]*Selector{nil},
[]*Filter{}),
ErrMissingField,
},
{
"InvalidFilterReference",
newPlacementPolicy(1, nil,
[]*Selector{newSelector("MyStore", "Country", ClauseDistinct, 1, "FromNL")},
[]*Filter{newFilter("FromRU", "Country", "Russia", OpEQ)}),
ErrFilterNotFound,
},
{
"NotEnoughNodes (backup factor)",
newPlacementPolicy(2, nil,
[]*Selector{newSelector("MyStore", "Country", ClauseDistinct, 2, "FromRU")},
[]*Filter{newFilter("FromRU", "Country", "Russia", OpEQ)}),
ErrNotEnoughNodes,
},
{
"NotEnoughNodes (buckets)",
newPlacementPolicy(1, nil,
[]*Selector{newSelector("MyStore", "Country", ClauseDistinct, 2, "FromRU")},
[]*Filter{newFilter("FromRU", "Country", "Russia", OpEQ)}),
ErrNotEnoughNodes,
},
}
nodes := []NodeInfo{
nodeInfoFromAttributes("Country", "Russia"),
nodeInfoFromAttributes("Country", "Germany"),
nodeInfoFromAttributes(),
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
nm, err := NewNetmap(NodesFromInfo(nodes))
require.NoError(t, err)
c := NewContext(nm)
c.setCBF(tc.p.ContainerBackupFactor())
require.NoError(t, c.processFilters(tc.p))
err = c.processSelectors(tc.p)
require.True(t, errors.Is(err, tc.err), "got: %v", err)
})
}
}
func testSelector() *Selector { func testSelector() *Selector {
s := new(Selector) s := new(Selector)
s.SetName("name") s.SetName("name")