diff --git a/netmap/filter_test.go b/netmap/filter_test.go index c9103e12..17c442b7 100644 --- a/netmap/filter_test.go +++ b/netmap/filter_test.go @@ -85,121 +85,20 @@ func TestContext_ProcessFiltersInvalid(t *testing.T) { } } -func TestFilter_MatchSimple(t *testing.T) { +func TestFilter_MatchSimple_InvalidOp(t *testing.T) { b := &Node{AttrMap: map[string]string{ "Rating": "4", "Country": "Germany", }} - testCases := []struct { - name string - ok bool - f *Filter - }{ - { - "GE_true", true, - newFilter("Main", "Rating", "4", OpGE), - }, - { - "GE_false", false, - newFilter("Main", "Rating", "5", OpGE), - }, - { - "GT_true", true, - newFilter("Main", "Rating", "3", OpGT), - }, - { - "GT_false", false, - newFilter("Main", "Rating", "4", OpGT), - }, - { - "LE_true", true, - newFilter("Main", "Rating", "4", OpLE), - }, - { - "LE_false", false, - newFilter("Main", "Rating", "3", OpLE), - }, - { - "LT_true", true, - newFilter("Main", "Rating", "5", OpLT), - }, - { - "LT_false", false, - newFilter("Main", "Rating", "4", OpLT), - }, - { - "EQ_true", true, - newFilter("Main", "Country", "Germany", OpEQ), - }, - { - "EQ_false", false, - newFilter("Main", "Country", "China", OpEQ), - }, - { - "NE_true", true, - newFilter("Main", "Country", "France", OpNE), - }, - { - "NE_false", false, - newFilter("Main", "Country", "Germany", OpNE), - }, - } - for _, tc := range testCases { - c := NewContext(new(Netmap)) - p := newPlacementPolicy(1, nil, nil, []*Filter{tc.f}) - require.NoError(t, c.processFilters(p)) - require.Equal(t, tc.ok, c.match(tc.f, b)) - } - t.Run("InvalidOp", func(t *testing.T) { - f := newFilter("Main", "Rating", "5", OpEQ) - c := NewContext(new(Netmap)) - p := newPlacementPolicy(1, nil, nil, []*Filter{f}) - require.NoError(t, c.processFilters(p)) - - // just for the coverage - f.SetOperation(0) - require.False(t, c.match(f, b)) - }) -} - -func TestFilter_Match(t *testing.T) { - fs := []*Filter{ - newFilter("StorageSSD", "Storage", "SSD", OpEQ), - newFilter("GoodRating", "Rating", "4", OpGE), - newFilter("Main", "", "", OpAND, - newFilter("StorageSSD", "", "", 0), - newFilter("", "IntField", "123", OpLT), - newFilter("GoodRating", "", "", 0), - newFilter("", "", "", OpOR, - newFilter("", "Param", "Value1", OpEQ), - newFilter("", "Param", "Value2", OpEQ), - )), - } + f := newFilter("Main", "Rating", "5", OpEQ) c := NewContext(new(Netmap)) - p := newPlacementPolicy(1, nil, nil, fs) + p := newPlacementPolicy(1, nil, nil, []*Filter{f}) require.NoError(t, c.processFilters(p)) - t.Run("Good", func(t *testing.T) { - n := getTestNode("Storage", "SSD", "Rating", "10", "IntField", "100", "Param", "Value1") - require.True(t, c.applyFilter("Main", n)) - }) - t.Run("InvalidStorage", func(t *testing.T) { - n := getTestNode("Storage", "HDD", "Rating", "10", "IntField", "100", "Param", "Value1") - require.False(t, c.applyFilter("Main", n)) - }) - t.Run("InvalidRating", func(t *testing.T) { - n := getTestNode("Storage", "SSD", "Rating", "3", "IntField", "100", "Param", "Value1") - require.False(t, c.applyFilter("Main", n)) - }) - t.Run("InvalidIntField", func(t *testing.T) { - n := getTestNode("Storage", "SSD", "Rating", "3", "IntField", "str", "Param", "Value1") - require.False(t, c.applyFilter("Main", n)) - }) - t.Run("InvalidParam", func(t *testing.T) { - n := getTestNode("Storage", "SSD", "Rating", "3", "IntField", "100", "Param", "NotValue") - require.False(t, c.applyFilter("Main", n)) - }) + // just for the coverage + f.SetOperation(0) + require.False(t, c.match(f, b)) } func testFilter() *Filter { diff --git a/netmap/json_test.go b/netmap/json_test.go new file mode 100644 index 00000000..9ca0263c --- /dev/null +++ b/netmap/json_test.go @@ -0,0 +1,82 @@ +package netmap + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestCase represents collection of placement policy tests for a single node set. +type TestCase struct { + Name string `json:"name"` + Nodes []NodeInfo `json:"nodes"` + Tests map[string]struct { + Policy PlacementPolicy `json:"policy"` + Pivot []byte `json:"pivot,omitempty"` + Result [][]int `json:"result,omitempty"` + Error string `json:"error,omitempty"` + Placement struct { + Pivot []byte + Result [][]int + } `json:"placement,omitempty"` + } +} + +func compareNodes(t *testing.T, expected [][]int, nodes Nodes, actual []Nodes) { + require.Equal(t, len(expected), len(actual)) + for i := range expected { + require.Equal(t, len(expected[i]), len(actual[i])) + for j, index := range expected[i] { + require.Equal(t, nodes[index], actual[i][j]) + } + } +} + +func TestPlacementPolicy_Interopability(t *testing.T) { + const testsDir = "./json_tests" + + f, err := os.Open(testsDir) + require.NoError(t, err) + + ds, err := f.ReadDir(0) + require.NoError(t, err) + + for i := range ds { + bs, err := ioutil.ReadFile(filepath.Join(testsDir, ds[i].Name())) + require.NoError(t, err) + + var tc TestCase + require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) + + t.Run(tc.Name, func(t *testing.T) { + nodes := NodesFromInfo(tc.Nodes) + nm, err := NewNetmap(nodes) + require.NoError(t, err) + + for name, tt := range tc.Tests { + t.Run(name, func(t *testing.T) { + v, err := nm.GetContainerNodes(&tt.Policy, tt.Pivot) + if tt.Result == nil { + require.Error(t, err) + require.Contains(t, err.Error(), tt.Error) + } else { + require.NoError(t, err) + + res := v.Replicas() + compareNodes(t, tt.Result, nodes, res) + + if tt.Placement.Result != nil { + res, err := nm.GetPlacementVectors(v, tt.Placement.Pivot) + require.NoError(t, err) + compareNodes(t, tt.Placement.Result, nodes, res) + } + } + }) + } + }) + } +} diff --git a/netmap/json_tests/cbf_default.json b/netmap/json_tests/cbf_default.json new file mode 100644 index 00000000..73e79a5b --- /dev/null +++ b/netmap/json_tests/cbf_default.json @@ -0,0 +1,100 @@ +{ + "name": "default CBF is 3", + "nodes": [ + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "DE" + }, + { + "key": "City", + "value": "Berlin" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "FR" + }, + { + "key": "City", + "value": "Paris" + } + ] + } + ], + "tests": { + "set default CBF": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "EU" + } + ], + "containerBackupFactor": 0, + "selectors": [ + { + "name": "EU", + "count": 1, + "clause": "SAME", + "attribute": "Location", + "filter": "*" + } + ], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 1, + 2 + ] + ] + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/cbf_minimal.json b/netmap/json_tests/cbf_minimal.json new file mode 100644 index 00000000..477e7c51 --- /dev/null +++ b/netmap/json_tests/cbf_minimal.json @@ -0,0 +1,101 @@ +{ + "name": "Real node count multiplier is in range [1, specified CBF]", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "Country", + "value": "DE" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "Country", + "value": "DE" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "Country", + "value": "DE" + } + ] + } + ], + "tests": { + "select 2, CBF is 2": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "X" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "X", + "count": 2, + "clause": "SAME", + "attribute": "Country", + "filter": "*" + } + ], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 1, + 2 + ] + ] + }, + "select 3, CBF is 2": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "X" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "X", + "count": 3, + "clause": "SAME", + "attribute": "Country", + "filter": "*" + } + ], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 1, + 2 + ] + ] + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/cbf_requirements.json b/netmap/json_tests/cbf_requirements.json new file mode 100644 index 00000000..bb541d67 --- /dev/null +++ b/netmap/json_tests/cbf_requirements.json @@ -0,0 +1,159 @@ +{ + "name": "CBF requirements", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "Attr", + "value": "Same" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "Attr", + "value": "Same" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "Attr", + "value": "Same" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "4" + }, + { + "key": "Attr", + "value": "Same" + } + ] + } + ], + "tests": { + "default CBF, no selector": { + "policy": { + "replicas": [ + { + "count": 2, + "selector": "" + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 2, + 1, + 3 + ] + ] + }, + "explicit CBF, no selector": { + "policy": { + "replicas": [ + { + "count": 2, + "selector": "" + } + ], + "containerBackupFactor": 3, + "selectors": [], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 2, + 1, + 3 + ] + ] + }, + "select distinct, weak CBF": { + "policy": { + "replicas": [ + { + "count": 2, + "selector": "X" + } + ], + "containerBackupFactor": 3, + "selectors": [ + { + "name": "X", + "count": 2, + "clause": "DISTINCT", + "attribute": "", + "filter": "*" + } + ], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 2, + 1, + 3 + ] + ] + }, + "select same, weak CBF": { + "policy": { + "replicas": [ + { + "count": 2, + "selector": "X" + } + ], + "containerBackupFactor": 3, + "selectors": [ + { + "name": "X", + "count": 2, + "clause": "SAME", + "attribute": "Attr", + "filter": "*" + } + ], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/filter_complex.json b/netmap/json_tests/filter_complex.json new file mode 100644 index 00000000..988f7000 --- /dev/null +++ b/netmap/json_tests/filter_complex.json @@ -0,0 +1,387 @@ +{ + "name": "compound filter", + "nodes": [ + { + "attributes": [ + { + "key": "Storage", + "value": "SSD" + }, + { + "key": "Rating", + "value": "10" + }, + { + "key": "IntField", + "value": "100" + }, + { + "key": "Param", + "value": "Value1" + } + ] + } + ], + "tests": { + "good": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "StorageSSD", + "key": "Storage", + "op": "EQ", + "value": "SSD", + "filters": [] + }, + { + "name": "GoodRating", + "key": "Rating", + "op": "GE", + "value": "4", + "filters": [] + }, + { + "name": "Main", + "key": "", + "op": "AND", + "value": "", + "filters": [ + { + "name": "StorageSSD", + "key": "", + "op": "OPERATION_UNSPECIFIED", + "value": "", + "filters": [] + }, + { + "name": "", + "key": "IntField", + "op": "LT", + "value": "123", + "filters": [] + }, + { + "name": "GoodRating", + "key": "", + "op": "OPERATION_UNSPECIFIED", + "value": "", + "filters": [] + }, + { + "name": "", + "key": "", + "op": "OR", + "value": "", + "filters": [ + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value1", + "filters": [] + }, + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value2", + "filters": [] + } + ] + } + ] + } + ], + "subnetId": null + }, + "result": [ + [ + 0 + ] + ] + }, + "bad storage type": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "StorageSSD", + "key": "Storage", + "op": "EQ", + "value": "HDD", + "filters": [] + }, + { + "name": "GoodRating", + "key": "Rating", + "op": "GE", + "value": "4", + "filters": [] + }, + { + "name": "Main", + "key": "", + "op": "AND", + "value": "", + "filters": [ + { + "name": "StorageSSD", + "key": "", + "op": "OPERATION_UNSPECIFIED", + "value": "", + "filters": [] + }, + { + "name": "", + "key": "IntField", + "op": "LT", + "value": "123", + "filters": [] + }, + { + "name": "GoodRating", + "key": "", + "op": "OPERATION_UNSPECIFIED", + "value": "", + "filters": [] + }, + { + "name": "", + "key": "", + "op": "OR", + "value": "", + "filters": [ + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value1", + "filters": [] + }, + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value2", + "filters": [] + } + ] + } + ] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "bad rating": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "StorageSSD", + "key": "Storage", + "op": "EQ", + "value": "SSD", + "filters": [] + }, + { + "name": "GoodRating", + "key": "Rating", + "op": "GE", + "value": "15", + "filters": [] + }, + { + "name": "Main", + "key": "", + "op": "AND", + "value": "", + "filters": [ + { + "name": "StorageSSD", + "key": "", + "op": "OPERATION_UNSPECIFIED", + "value": "", + "filters": [] + }, + { + "name": "", + "key": "IntField", + "op": "LT", + "value": "123", + "filters": [] + }, + { + "name": "GoodRating", + "key": "", + "op": "OPERATION_UNSPECIFIED", + "value": "", + "filters": [] + }, + { + "name": "", + "key": "", + "op": "OR", + "value": "", + "filters": [ + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value1", + "filters": [] + }, + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value2", + "filters": [] + } + ] + } + ] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "bad param": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "StorageSSD", + "key": "Storage", + "op": "EQ", + "value": "SSD", + "filters": [] + }, + { + "name": "GoodRating", + "key": "Rating", + "op": "GE", + "value": "4", + "filters": [] + }, + { + "name": "Main", + "key": "", + "op": "AND", + "value": "", + "filters": [ + { + "name": "StorageSSD", + "key": "", + "op": "OPERATION_UNSPECIFIED", + "value": "", + "filters": [] + }, + { + "name": "", + "key": "IntField", + "op": "LT", + "value": "123", + "filters": [] + }, + { + "name": "GoodRating", + "key": "", + "op": "OPERATION_UNSPECIFIED", + "value": "", + "filters": [] + }, + { + "name": "", + "key": "", + "op": "OR", + "value": "", + "filters": [ + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value0", + "filters": [] + }, + { + "name": "", + "key": "Param", + "op": "EQ", + "value": "Value2", + "filters": [] + } + ] + } + ] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/filter_invalid_integer.json b/netmap/json_tests/filter_invalid_integer.json new file mode 100644 index 00000000..e04cbb37 --- /dev/null +++ b/netmap/json_tests/filter_invalid_integer.json @@ -0,0 +1,83 @@ +{ + "name": "invalid integer field", + "nodes": [ + { + "attributes": [ + { + "key": "IntegerField", + "value": "" + } + ] + }, + { + "attributes": [ + { + "key": "IntegerField", + "value": "str" + } + ] + } + ], + "tests": { + "empty string is not casted to 0": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "IntegerField", + "op": "LE", + "value": "8", + "filters": [] + } + ] + }, + "error": "not enough nodes" + }, + "non-empty string is not casted to a number": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "IntegerField", + "op": "GE", + "value": "0", + "filters": [] + } + ] + }, + "error": "not enough nodes" + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/filter_simple.json b/netmap/json_tests/filter_simple.json new file mode 100644 index 00000000..3e3eaaf6 --- /dev/null +++ b/netmap/json_tests/filter_simple.json @@ -0,0 +1,415 @@ +{ + "name": "single-op filters", + "nodes": [ + { + "attributes": [ + { + "key": "Rating", + "value": "4" + }, + { + "key": "Country", + "value": "Germany" + } + ] + } + ], + "tests": { + "GE true": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "GE", + "value": "4", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 0 + ] + ] + }, + "GE false": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "GE", + "value": "5", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "GT true": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "GT", + "value": "3", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 0 + ] + ] + }, + "GT false": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "GT", + "value": "4", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "LE true": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "LE", + "value": "4", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 0 + ] + ] + }, + "LE false": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "LE", + "value": "3", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "LT true": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "LT", + "value": "5", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 0 + ] + ] + }, + "LT false": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Rating", + "op": "LT", + "value": "4", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "EQ true": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Country", + "op": "EQ", + "value": "Germany", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 0 + ] + ] + }, + "EQ false": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Country", + "op": "EQ", + "value": "China", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "NE true": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Country", + "op": "NE", + "value": "France", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 0 + ] + ] + }, + "NE false": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "DISTINCT", + "attribute": "", + "filter": "Main" + } + ], + "filters": [ + { + "name": "Main", + "key": "Country", + "op": "NE", + "value": "Germany", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/hrw_sort.json b/netmap/json_tests/hrw_sort.json new file mode 100644 index 00000000..521a852e --- /dev/null +++ b/netmap/json_tests/hrw_sort.json @@ -0,0 +1,165 @@ +{ + "name": "HRW ordering", + "nodes": [ + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + }, + { + "key": "Price", + "value": "2" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + }, + { + "key": "Price", + "value": "4" + }, + { + "key": "Capacity", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "France" + }, + { + "key": "Price", + "value": "3" + }, + { + "key": "Capacity", + "value": "10" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Price", + "value": "2" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Price", + "value": "1" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "France" + }, + { + "key": "Price", + "value": "100" + }, + { + "key": "Capacity", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "France" + }, + { + "key": "Price", + "value": "7" + }, + { + "key": "Capacity", + "value": "10000" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + }, + { + "key": "Price", + "value": "2" + }, + { + "key": "Capacity", + "value": "1" + } + ] + } + ], + "tests": { + "select 3 nodes in 3 distinct countries, same placement": { + "policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[],"subnetId":null}, + "pivot": "Y29udGFpbmVySUQ=", + "result": [[4, 0, 7]], + "placement": { + "pivot": "b2JqZWN0SUQ=", + "result": [[4, 0, 7]] + } + }, + "select 6 nodes in 3 distinct countries, different placement": { + "policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[],"subnetId":null}, + "pivot": "Y29udGFpbmVySUQ=", + "result": [[4, 3, 0, 1, 7, 2]], + "placement": { + "pivot": "b2JqZWN0SUQ=", + "result": [[4, 3, 0, 7, 2, 1]] + } + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/issue213.json b/netmap/json_tests/issue213.json new file mode 100644 index 00000000..513e39d8 --- /dev/null +++ b/netmap/json_tests/issue213.json @@ -0,0 +1,109 @@ +{ + "name": "unnamed selector (nspcc-dev/neofs-api-go#213)", + "nodes": [ + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "Russia" + }, + { + "key": "City", + "value": "Moscow" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "Russia" + }, + { + "key": "City", + "value": "Saint-Petersburg" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "Sweden" + }, + { + "key": "City", + "value": "Stockholm" + } + ] + }, + { + "attributes": [ + { + "key": "Location", + "value": "Europe" + }, + { + "key": "Country", + "value": "Finalnd" + }, + { + "key": "City", + "value": "Helsinki" + } + ] + } + ], + "tests": { + "test": { + "policy": { + "replicas": [ + { + "count": 4, + "selector": "" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "", + "count": 4, + "clause": "DISTINCT", + "attribute": "", + "filter": "LOC_EU" + } + ], + "filters": [ + { + "name": "LOC_EU", + "key": "Location", + "op": "EQ", + "value": "Europe", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/multiple_rep.json b/netmap/json_tests/multiple_rep.json new file mode 100644 index 00000000..8abb6cd5 --- /dev/null +++ b/netmap/json_tests/multiple_rep.json @@ -0,0 +1,95 @@ +{ + "name": "multiple replicas (#215)", + "nodes": [ + { + "attributes": [ + { + "key": "City", + "value": "Saint-Petersburg" + } + ] + }, + { + "attributes": [ + { + "key": "City", + "value": "Moscow" + } + ] + }, + { + "attributes": [ + { + "key": "City", + "value": "Berlin" + } + ] + }, + { + "attributes": [ + { + "key": "City", + "value": "Paris" + } + ] + } + ], + "tests": { + "test": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "LOC_SPB_PLACE" + }, + { + "count": 1, + "selector": "LOC_MSK_PLACE" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "LOC_SPB_PLACE", + "count": 1, + "clause": "CLAUSE_UNSPECIFIED", + "attribute": "", + "filter": "LOC_SPB" + }, + { + "name": "LOC_MSK_PLACE", + "count": 1, + "clause": "CLAUSE_UNSPECIFIED", + "attribute": "", + "filter": "LOC_MSK" + } + ], + "filters": [ + { + "name": "LOC_SPB", + "key": "City", + "op": "EQ", + "value": "Saint-Petersburg", + "filters": [] + }, + { + "name": "LOC_MSK", + "key": "City", + "op": "EQ", + "value": "Moscow", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 0 + ], + [ + 1 + ] + ] + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/multiple_rep_asymmetric.json b/netmap/json_tests/multiple_rep_asymmetric.json new file mode 100644 index 00000000..0e6e7964 --- /dev/null +++ b/netmap/json_tests/multiple_rep_asymmetric.json @@ -0,0 +1,332 @@ +{ + "name": "multiple REP, asymmetric", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "0" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "4" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "5" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "6" + }, + { + "key": "Continent", + "value": "NA" + }, + { + "key": "City", + "value": "NewYork" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "7" + }, + { + "key": "Continent", + "value": "AF" + }, + { + "key": "City", + "value": "Cairo" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "8" + }, + { + "key": "Continent", + "value": "AF" + }, + { + "key": "City", + "value": "Cairo" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "9" + }, + { + "key": "Continent", + "value": "SA" + }, + { + "key": "City", + "value": "Lima" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "10" + }, + { + "key": "Continent", + "value": "AF" + }, + { + "key": "City", + "value": "Cairo" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "11" + }, + { + "key": "Continent", + "value": "NA" + }, + { + "key": "City", + "value": "NewYork" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "12" + }, + { + "key": "Continent", + "value": "NA" + }, + { + "key": "City", + "value": "LosAngeles" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "13" + }, + { + "key": "Continent", + "value": "SA" + }, + { + "key": "City", + "value": "Lima" + } + ] + } + ], + "tests": { + "test": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "SPB" + }, + { + "count": 2, + "selector": "Americas" + } + ], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "SPB", + "count": 1, + "clause": "SAME", + "attribute": "City", + "filter": "SPBSSD" + }, + { + "name": "Americas", + "count": 2, + "clause": "DISTINCT", + "attribute": "City", + "filter": "Americas" + } + ], + "filters": [ + { + "name": "SPBSSD", + "key": "", + "op": "AND", + "value": "", + "filters": [ + { + "name": "", + "key": "Country", + "op": "EQ", + "value": "RU", + "filters": [] + }, + { + "name": "", + "key": "City", + "op": "EQ", + "value": "St.Petersburg", + "filters": [] + }, + { + "name": "", + "key": "SSD", + "op": "EQ", + "value": "1", + "filters": [] + } + ] + }, + { + "name": "Americas", + "key": "", + "op": "OR", + "value": "", + "filters": [ + { + "name": "", + "key": "Continent", + "op": "EQ", + "value": "NA", + "filters": [] + }, + { + "name": "", + "key": "Continent", + "op": "EQ", + "value": "SA", + "filters": [] + } + ] + } + ], + "subnetId": null + }, + "result": [ + [ + 1, + 4 + ], + [ + 8, + 12, + 5, + 10 + ] + ] + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/rep_only.json b/netmap/json_tests/rep_only.json new file mode 100644 index 00000000..b69ba5e3 --- /dev/null +++ b/netmap/json_tests/rep_only.json @@ -0,0 +1,113 @@ +{ + "name": "REP X", + "nodes": [ + { + "publicKey": "", + "addresses": [], + "attributes": [ + { + "key": "City", + "value": "Saint-Petersburg", + "parents": [] + } + ], + "state": "UNSPECIFIED" + }, + { + "publicKey": "", + "addresses": [], + "attributes": [ + { + "key": "City", + "value": "Moscow", + "parents": [] + } + ], + "state": "UNSPECIFIED" + }, + { + "publicKey": "", + "addresses": [], + "attributes": [ + { + "key": "City", + "value": "Berlin", + "parents": [] + } + ], + "state": "UNSPECIFIED" + }, + { + "publicKey": "", + "addresses": [], + "attributes": [ + { + "key": "City", + "value": "Paris", + "parents": [] + } + ], + "state": "UNSPECIFIED" + } + ], + "tests": { + "REP 1": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "" + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 1, + 2 + ] + ] + }, + "REP 3": { + "policy": { + "replicas": [ + { + "count": 3, + "selector": "" + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 3, + 1, + 2 + ] + ] + }, + "REP 5": { + "policy": { + "replicas": [ + { + "count": 5, + "selector": "" + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [], + "subnetId": null + }, + "error": "not enough nodes" + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/select_no_attribute.json b/netmap/json_tests/select_no_attribute.json new file mode 100644 index 00000000..2dd39312 --- /dev/null +++ b/netmap/json_tests/select_no_attribute.json @@ -0,0 +1,117 @@ +{ + "name": "select with unspecified attribute", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "0" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "St.Petersburg" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + }, + { + "key": "SSD", + "value": "1" + } + ] + }, + { + "attributes": [ + { + "key": "ID", + "value": "4" + }, + { + "key": "Country", + "value": "RU" + }, + { + "key": "City", + "value": "Moscow" + }, + { + "key": "SSD", + "value": "1" + } + ] + } + ], + "tests": { + "test": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "X" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "X", + "count": 4, + "clause": "DISTINCT", + "attribute": "", + "filter": "*" + } + ], + "filters": [], + "subnetId": null + }, + "result": [ + [ + 0, + 1, + 2, + 3 + ] + ] + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/selector_invalid.json b/netmap/json_tests/selector_invalid.json new file mode 100644 index 00000000..9d709fdf --- /dev/null +++ b/netmap/json_tests/selector_invalid.json @@ -0,0 +1,104 @@ +{ + "name": "invalid selections", + "nodes": [ + { + "attributes": [ + { + "key": "Country", + "value": "Russia" + } + ] + }, + { + "attributes": [ + { + "key": "Country", + "value": "Germany" + } + ] + }, + { + "attributes": [] + } + ], + "tests": { + "missing filter": { + "policy": { + "replicas": [], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "MyStore", + "count": 1, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromNL" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [] + } + ], + "subnetId": null + }, + "error": "filter not found" + }, + "not enough nodes (backup factor)": { + "policy": { + "replicas": [], + "containerBackupFactor": 2, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromRU" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "not enough nodes (buckets)": { + "policy": { + "replicas": [], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "MyStore", + "count": 2, + "clause": "DISTINCT", + "attribute": "Country", + "filter": "FromRU" + } + ], + "filters": [ + { + "name": "FromRU", + "key": "Country", + "op": "EQ", + "value": "Russia", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + } + } +} \ No newline at end of file diff --git a/netmap/json_tests/subnet.json b/netmap/json_tests/subnet.json new file mode 100644 index 00000000..b75dccbd --- /dev/null +++ b/netmap/json_tests/subnet.json @@ -0,0 +1,254 @@ +{ + "name": "subnet tests", + "nodes": [ + { + "attributes": [ + { + "key": "ID", + "value": "0" + }, + { + "key": "City", + "value": "Paris" + }, + { + "key": "__NEOFS__SUBNET_0", + "value": "False" + } + ], + "state": "UNSPECIFIED" + }, + { + "attributes": [ + { + "key": "ID", + "value": "1" + }, + { + "key": "City", + "value": "Paris" + } + ], + "state": "UNSPECIFIED" + }, + { + "attributes": [ + { + "key": "ID", + "value": "2" + }, + { + "key": "City", + "value": "London" + }, + { + "key": "__NEOFS__SUBNET_1", + "value": "True" + } + ], + "state": "UNSPECIFIED" + }, + { + "attributes": [ + { + "key": "ID", + "value": "3" + }, + { + "key": "City", + "value": "London" + } + ], + "state": "UNSPECIFIED" + }, + { + "attributes": [ + { + "key": "ID", + "value": "4" + }, + { + "key": "City", + "value": "Toronto" + }, + { + "key": "__NEOFS__SUBNET_1", + "value": "True" + } + ], + "state": "UNSPECIFIED" + }, + { + "attributes": [ + { + "key": "ID", + "value": "5" + }, + { + "key": "City", + "value": "Toronto" + }, + { + "key": "__NEOFS__SUBNET_2", + "value": "True" + } + ], + "state": "UNSPECIFIED" + }, + { + "attributes": [ + { + "key": "ID", + "value": "6" + }, + { + "key": "City", + "value": "Tokyo" + }, + { + "key": "__NEOFS__SUBNET_2", + "value": "True" + } + ], + "state": "UNSPECIFIED" + }, + { + "attributes": [ + { + "key": "ID", + "value": "7" + }, + { + "key": "City", + "value": "Tokyo" + }, + { + "key": "__NEOFS__SUBNET_2", + "value": "True" + } + ], + "state": "UNSPECIFIED" + } + ], + "tests": { + "select from default subnet, fail": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 0, + "selectors": [ + { + "name": "S", + "count": 2, + "clause": "SAME", + "attribute": "City", + "filter": "F" + } + ], + "filters": [ + { + "name": "F", + "key": "City", + "op": "EQ", + "value": "Paris", + "filters": [] + } + ], + "subnetId": null + }, + "error": "not enough nodes" + }, + "select from default subnet, success": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 0, + "selectors": [ + { + "name": "S", + "count": 2, + "clause": "SAME", + "attribute": "City", + "filter": "F" + } + ], + "filters": [ + { + "name": "F", + "key": "City", + "op": "EQ", + "value": "Toronto", + "filters": [] + } + ], + "subnetId": null + }, + "result": [ + [ + 4, + 5 + ] + ] + }, + "select from non-default subnet, success": { + "policy": { + "replicas": [ + { + "count": 3, + "selector": "" + } + ], + "containerBackupFactor": 0, + "selectors": [], + "filters": [], + "subnetId": { + "value": 2 + } + }, + "result": [ + [ + 5, + 6, + 7 + ] + ] + }, + "select subnet via filters": { + "policy": { + "replicas": [ + { + "count": 1, + "selector": "S" + } + ], + "containerBackupFactor": 1, + "selectors": [ + { + "name": "S", + "count": 1, + "clause": "SAME", + "attribute": "City", + "filter": "F" + } + ], + "filters": [ + { + "name": "F", + "key": "__NEOFS_SUBNET.2.ENABLED", + "op": "EQ", + "value": "True" + } + ] + }, + "error": "not enough nodes" + } + } +} \ No newline at end of file diff --git a/netmap/policy_test.go b/netmap/policy_test.go index 908a46a4..546bc3b2 100644 --- a/netmap/policy_test.go +++ b/netmap/policy_test.go @@ -1,169 +1,12 @@ package netmap import ( - "errors" - "strconv" "testing" "github.com/nspcc-dev/neofs-api-go/v2/netmap" - subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -const subnetAttrPrefix = "__NEOFS_SUBNET" - -func subnetAttrName(subnet uint32) string { - return subnetAttrPrefix + "." + - strconv.FormatUint(uint64(subnet), 10) + ".ENABLED" -} - -func TestPlacementPolicy_Subnet(t *testing.T) { - nodes := []NodeInfo{ - nodeInfoFromAttributes("ID", "0", "City", "Paris"), - nodeInfoFromAttributes("ID", "1", "City", "Paris"), - nodeInfoFromAttributes("ID", "2", "City", "London"), - nodeInfoFromAttributes("ID", "3", "City", "London"), - nodeInfoFromAttributes("ID", "4", "City", "Toronto"), - nodeInfoFromAttributes("ID", "5", "City", "Toronto"), - nodeInfoFromAttributes("ID", "6", "City", "Tokyo"), - nodeInfoFromAttributes("ID", "7", "City", "Tokyo"), - } - var id subnetid.ID - nodes[0].ExitSubnet(id) - - id.SetNumber(1) - nodes[2].EnterSubnet(id) - nodes[4].EnterSubnet(id) - - id.SetNumber(2) - nodes[5].EnterSubnet(id) - nodes[6].EnterSubnet(id) - nodes[7].EnterSubnet(id) - - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - - t.Run("select 2 nodes from the default subnet in Paris", func(t *testing.T) { - p := newPlacementPolicy(0, - []*Replica{newReplica(1, "S")}, - []*Selector{newSelector("S", "City", ClauseSame, 2, "F")}, - []*Filter{newFilter("F", "City", "Paris", OpEQ)}) - - _, err := nm.GetContainerNodes(p, nil) - require.True(t, errors.Is(err, ErrNotEnoughNodes), "got: %v", err) - }) - t.Run("select 2 nodes from the default subnet in London", func(t *testing.T) { - p := newPlacementPolicy(0, - []*Replica{newReplica(1, "S")}, - []*Selector{newSelector("S", "City", ClauseSame, 2, "F")}, - []*Filter{newFilter("F", "City", "London", OpEQ)}) - - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - - nodes := v.Flatten() - require.Equal(t, 2, len(nodes)) - for _, n := range v.Flatten() { - id := n.Attribute("ID") - require.Contains(t, []string{"2", "3"}, id) - } - }) - t.Run("select 2 nodes from the default subnet in Toronto", func(t *testing.T) { - p := newPlacementPolicy(0, - []*Replica{newReplica(1, "S")}, - []*Selector{newSelector("S", "City", ClauseSame, 2, "F")}, - []*Filter{newFilter("F", "City", "Toronto", OpEQ)}) - - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - - nodes := v.Flatten() - require.Equal(t, 2, len(nodes)) - for _, n := range v.Flatten() { - id := n.Attribute("ID") - require.Contains(t, []string{"4", "5"}, id) - } - }) - t.Run("select 3 nodes from the non-default subnet", func(t *testing.T) { - p := newPlacementPolicy(0, - []*Replica{newReplica(3, "")}, - nil, nil) - p.SetSubnetID(newSubnetID(2)) - - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - - nodes := v.Flatten() - require.Equal(t, 3, len(nodes)) - for _, n := range v.Flatten() { - id := n.Attribute("ID") - require.Contains(t, []string{"5", "6", "7"}, id) - } - }) - t.Run("select nodes from the subnet via filter", func(t *testing.T) { - p := newPlacementPolicy(0, - []*Replica{newReplica(1, "")}, - nil, - []*Filter{newFilter(MainFilterName, subnetAttrName(2), "True", OpEQ, nil)}) - - _, err := nm.GetContainerNodes(p, nil) - require.Error(t, err) - }) -} - -func TestPlacementPolicy_CBFWithEmptySelector(t *testing.T) { - nodes := []NodeInfo{ - nodeInfoFromAttributes("ID", "1", "Attr", "Same"), - nodeInfoFromAttributes("ID", "2", "Attr", "Same"), - nodeInfoFromAttributes("ID", "3", "Attr", "Same"), - nodeInfoFromAttributes("ID", "4", "Attr", "Same"), - } - - p1 := newPlacementPolicy(0, - []*Replica{newReplica(2, "")}, - nil, // selectors - nil, // filters - ) - - p2 := newPlacementPolicy(3, - []*Replica{newReplica(2, "")}, - nil, // selectors - nil, // filters - ) - - p3 := newPlacementPolicy(3, - []*Replica{newReplica(2, "X")}, - []*Selector{newSelector("X", "", ClauseDistinct, 2, "*")}, - nil, // filters - ) - - p4 := newPlacementPolicy(3, - []*Replica{newReplica(2, "X")}, - []*Selector{newSelector("X", "Attr", ClauseSame, 2, "*")}, - nil, // filters - ) - - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - - v, err := nm.GetContainerNodes(p1, nil) - require.NoError(t, err) - assert.Len(t, v.Flatten(), 4) - - v, err = nm.GetContainerNodes(p2, nil) - require.NoError(t, err) - assert.Len(t, v.Flatten(), 4) - - v, err = nm.GetContainerNodes(p3, nil) - require.NoError(t, err) - assert.Len(t, v.Flatten(), 4) - - v, err = nm.GetContainerNodes(p4, nil) - require.NoError(t, err) - assert.Len(t, v.Flatten(), 4) -} - func TestPlacementPolicyFromV2(t *testing.T) { pV2 := new(netmap.PlacementPolicy) diff --git a/netmap/selector_test.go b/netmap/selector_test.go index b84cb1d2..8a55303e 100644 --- a/netmap/selector_test.go +++ b/netmap/selector_test.go @@ -1,7 +1,6 @@ package netmap import ( - "errors" "fmt" "math/rand" "sort" @@ -187,242 +186,6 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) { } } -func TestPlacementPolicy_UnspecifiedClause(t *testing.T) { - p := newPlacementPolicy(1, - []*Replica{newReplica(1, "X")}, - []*Selector{ - newSelector("X", "", ClauseDistinct, 4, "*"), - }, - nil, - ) - nodes := []NodeInfo{ - nodeInfoFromAttributes("ID", "1", "Country", "RU", "City", "St.Petersburg", "SSD", "0"), - nodeInfoFromAttributes("ID", "2", "Country", "RU", "City", "St.Petersburg", "SSD", "1"), - nodeInfoFromAttributes("ID", "3", "Country", "RU", "City", "Moscow", "SSD", "1"), - nodeInfoFromAttributes("ID", "4", "Country", "RU", "City", "Moscow", "SSD", "1"), - } - - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - require.Equal(t, 4, len(v.Flatten())) -} - -func TestPlacementPolicy_Minimal(t *testing.T) { - nodes := []NodeInfo{ - nodeInfoFromAttributes("City", "Saint-Petersburg"), - nodeInfoFromAttributes("City", "Moscow"), - nodeInfoFromAttributes("City", "Berlin"), - nodeInfoFromAttributes("City", "Paris"), - } - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - - runTest := func(t *testing.T, rep uint32, expectError bool) { - p := newPlacementPolicy(0, - []*Replica{newReplica(rep, "")}, - nil, nil) - - v, err := nm.GetContainerNodes(p, nil) - - if expectError { - require.Error(t, err) - return - } - - require.NoError(t, err) - count := int(rep * defaultCBF) - if count > len(nm.Nodes) { - count = len(nm.Nodes) - } - require.EqualValues(t, count, len(v.Flatten())) - } - - t.Run("REP 1", func(t *testing.T) { - runTest(t, 1, false) - }) - t.Run("REP 3", func(t *testing.T) { - runTest(t, 3, false) - }) - t.Run("REP 5", func(t *testing.T) { - runTest(t, 5, true) - }) -} - -// Issue #215. -func TestPlacementPolicy_MultipleREP(t *testing.T) { - p := newPlacementPolicy(1, - []*Replica{ - newReplica(1, "LOC_SPB_PLACE"), - newReplica(1, "LOC_MSK_PLACE"), - }, - []*Selector{ - newSelector("LOC_SPB_PLACE", "", ClauseUnspecified, 1, "LOC_SPB"), - newSelector("LOC_MSK_PLACE", "", ClauseUnspecified, 1, "LOC_MSK"), - }, - []*Filter{ - newFilter("LOC_SPB", "City", "Saint-Petersburg", OpEQ), - newFilter("LOC_MSK", "City", "Moscow", OpEQ), - }, - ) - nodes := []NodeInfo{ - nodeInfoFromAttributes("City", "Saint-Petersburg"), - nodeInfoFromAttributes("City", "Moscow"), - nodeInfoFromAttributes("City", "Berlin"), - nodeInfoFromAttributes("City", "Paris"), - } - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - - rs := v.Replicas() - require.Equal(t, 2, len(rs)) - require.Equal(t, 1, len(rs[0])) - require.Equal(t, "Saint-Petersburg", rs[0][0].Attribute("City")) - require.Equal(t, 1, len(rs[1])) - require.Equal(t, "Moscow", rs[1][0].Attribute("City")) -} - -func TestPlacementPolicy_DefaultCBF(t *testing.T) { - p := newPlacementPolicy(0, - []*Replica{ - newReplica(1, "EU"), - }, - []*Selector{ - newSelector("EU", "Location", ClauseSame, 1, "*"), - }, - nil) - nodes := []NodeInfo{ - nodeInfoFromAttributes("Location", "Europe", "Country", "RU", "City", "St.Petersburg"), - nodeInfoFromAttributes("Location", "Europe", "Country", "RU", "City", "Moscow"), - nodeInfoFromAttributes("Location", "Europe", "Country", "DE", "City", "Berlin"), - nodeInfoFromAttributes("Location", "Europe", "Country", "FR", "City", "Paris"), - } - - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - require.Equal(t, defaultCBF, len(v.Flatten())) -} - -func TestPlacementPolicy_GetPlacementVectors(t *testing.T) { - p := newPlacementPolicy(2, - []*Replica{ - newReplica(1, "SPB"), - newReplica(2, "Americas"), - }, - []*Selector{ - newSelector("SPB", "City", ClauseSame, 1, "SPBSSD"), - newSelector("Americas", "City", ClauseDistinct, 2, "Americas"), - }, - []*Filter{ - newFilter("SPBSSD", "", "", OpAND, - newFilter("", "Country", "RU", OpEQ), - newFilter("", "City", "St.Petersburg", OpEQ), - newFilter("", "SSD", "1", OpEQ)), - newFilter("Americas", "", "", OpOR, - newFilter("", "Continent", "NA", OpEQ), - newFilter("", "Continent", "SA", OpEQ)), - }) - nodes := []NodeInfo{ - nodeInfoFromAttributes("ID", "1", "Country", "RU", "City", "St.Petersburg", "SSD", "0"), - nodeInfoFromAttributes("ID", "2", "Country", "RU", "City", "St.Petersburg", "SSD", "1"), - nodeInfoFromAttributes("ID", "3", "Country", "RU", "City", "Moscow", "SSD", "1"), - nodeInfoFromAttributes("ID", "4", "Country", "RU", "City", "Moscow", "SSD", "1"), - nodeInfoFromAttributes("ID", "5", "Country", "RU", "City", "St.Petersburg", "SSD", "1"), - nodeInfoFromAttributes("ID", "6", "Continent", "NA", "City", "NewYork"), - nodeInfoFromAttributes("ID", "7", "Continent", "AF", "City", "Cairo"), - nodeInfoFromAttributes("ID", "8", "Continent", "AF", "City", "Cairo"), - nodeInfoFromAttributes("ID", "9", "Continent", "SA", "City", "Lima"), - nodeInfoFromAttributes("ID", "10", "Continent", "AF", "City", "Cairo"), - nodeInfoFromAttributes("ID", "11", "Continent", "NA", "City", "NewYork"), - nodeInfoFromAttributes("ID", "12", "Continent", "NA", "City", "LosAngeles"), - nodeInfoFromAttributes("ID", "13", "Continent", "SA", "City", "Lima"), - } - - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - require.Equal(t, 2, len(v.Replicas())) - require.Equal(t, 6, len(v.Flatten())) - - require.Equal(t, 2, len(v.Replicas()[0])) - ids := map[string]struct{}{} - for _, ni := range v.Replicas()[0] { - require.Equal(t, "RU", ni.Attribute("Country")) - require.Equal(t, "St.Petersburg", ni.Attribute("City")) - require.Equal(t, "1", ni.Attribute("SSD")) - ids[ni.Attribute("ID")] = struct{}{} - } - require.Equal(t, len(v.Replicas()[0]), len(ids), "not all nodes we distinct") - - require.Equal(t, 4, len(v.Replicas()[1])) // 2 cities * 2 HRWB - ids = map[string]struct{}{} - for _, ni := range v.Replicas()[1] { - require.Contains(t, []string{"NA", "SA"}, ni.Attribute("Continent")) - ids[ni.Attribute("ID")] = struct{}{} - } - require.Equal(t, len(v.Replicas()[1]), len(ids), "not all nodes we distinct") -} - -func TestPlacementPolicy_LowerBound(t *testing.T) { - p := newPlacementPolicy( - 2, // backup factor - []*Replica{ - newReplica(1, "X"), - }, - []*Selector{ - newSelector("X", "Country", ClauseSame, 2, "*"), - }, - nil, // filters - ) - - nodes := []NodeInfo{ - nodeInfoFromAttributes("ID", "1", "Country", "DE"), - nodeInfoFromAttributes("ID", "2", "Country", "DE"), - nodeInfoFromAttributes("ID", "3", "Country", "DE"), - } - - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - - require.Equal(t, 3, len(v.Flatten())) -} - -func TestIssue213(t *testing.T) { - p := newPlacementPolicy(1, - []*Replica{ - newReplica(4, ""), - }, - []*Selector{ - newSelector("", "", ClauseDistinct, 4, "LOC_EU"), - }, - []*Filter{ - newFilter("LOC_EU", "Location", "Europe", OpEQ), - }) - nodes := []NodeInfo{ - nodeInfoFromAttributes("Location", "Europe", "Country", "Russia", "City", "Moscow"), - nodeInfoFromAttributes("Location", "Europe", "Country", "Russia", "City", "Saint-Petersburg"), - nodeInfoFromAttributes("Location", "Europe", "Country", "Sweden", "City", "Stockholm"), - nodeInfoFromAttributes("Location", "Europe", "Country", "Finalnd", "City", "Helsinki"), - } - - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - - v, err := nm.GetContainerNodes(p, nil) - require.NoError(t, err) - require.Equal(t, 4, len(v.Flatten())) -} - func TestPlacementPolicy_ProcessSelectors(t *testing.T) { p := newPlacementPolicy(2, nil, []*Selector{ @@ -472,114 +235,6 @@ func TestPlacementPolicy_ProcessSelectors(t *testing.T) { } } -func TestPlacementPolicy_ProcessSelectorsHRW(t *testing.T) { - p := newPlacementPolicy(1, nil, - []*Selector{ - newSelector("Main", "Country", ClauseDistinct, 3, "*"), - }, nil) - - // bucket weight order: RU > DE > FR - nodes := []NodeInfo{ - nodeInfoFromAttributes("Country", "Germany", AttrPrice, "2", AttrCapacity, "10000"), - nodeInfoFromAttributes("Country", "Germany", AttrPrice, "4", AttrCapacity, "1"), - nodeInfoFromAttributes("Country", "France", AttrPrice, "3", AttrCapacity, "10"), - nodeInfoFromAttributes("Country", "Russia", AttrPrice, "2", AttrCapacity, "10000"), - nodeInfoFromAttributes("Country", "Russia", AttrPrice, "1", AttrCapacity, "10000"), - nodeInfoFromAttributes("Country", "Russia", AttrCapacity, "10000"), - nodeInfoFromAttributes("Country", "France", AttrPrice, "100", AttrCapacity, "1"), - nodeInfoFromAttributes("Country", "France", AttrPrice, "7", AttrCapacity, "10000"), - nodeInfoFromAttributes("Country", "Russia", AttrPrice, "2", AttrCapacity, "1"), - } - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - c := NewContext(nm) - c.setPivot([]byte("containerID")) - c.setCBF(p.ContainerBackupFactor()) - c.weightFunc = newWeightFunc(newMaxNorm(10000), newReverseMinNorm(1)) - c.aggregator = func() aggregator { - return new(maxAgg) - } - - require.NoError(t, c.processFilters(p)) - require.NoError(t, c.processSelectors(p)) - - cnt := c.Selections["Main"] - expected := []Nodes{ - {{Index: 4, Capacity: 10000, Price: 1}}, // best RU - {{Index: 0, Capacity: 10000, Price: 2}}, // best DE - {{Index: 7, Capacity: 10000, Price: 7}}, // best FR - } - require.Equal(t, len(expected), len(cnt)) - for i := range expected { - require.Equal(t, len(expected[i]), len(cnt[i])) - require.Equal(t, expected[i][0].Index, cnt[i][0].Index) - require.Equal(t, expected[i][0].Capacity, cnt[i][0].Capacity) - require.Equal(t, expected[i][0].Price, cnt[i][0].Price) - } - - res, err := nm.GetPlacementVectors(containerNodes(cnt), []byte("objectID")) - require.NoError(t, err) - require.Equal(t, res, cnt) -} - -func newMaxNorm(max float64) normalizer { - return &maxNorm{max: max} -} - -func TestPlacementPolicy_ProcessSelectorsInvalid(t *testing.T) { - testCases := []struct { - name string - p *PlacementPolicy - err error - }{ - { - "MissingSelector", - newPlacementPolicy(2, nil, - []*Selector{nil}, - []*Filter{}), - ErrMissingField, - }, - { - "InvalidFilterReference", - newPlacementPolicy(1, nil, - []*Selector{newSelector("MyStore", "Country", ClauseDistinct, 1, "FromNL")}, - []*Filter{newFilter("FromRU", "Country", "Russia", OpEQ)}), - ErrFilterNotFound, - }, - { - "NotEnoughNodes (backup factor)", - newPlacementPolicy(2, nil, - []*Selector{newSelector("MyStore", "Country", ClauseDistinct, 2, "FromRU")}, - []*Filter{newFilter("FromRU", "Country", "Russia", OpEQ)}), - ErrNotEnoughNodes, - }, - { - "NotEnoughNodes (buckets)", - newPlacementPolicy(1, nil, - []*Selector{newSelector("MyStore", "Country", ClauseDistinct, 2, "FromRU")}, - []*Filter{newFilter("FromRU", "Country", "Russia", OpEQ)}), - ErrNotEnoughNodes, - }, - } - nodes := []NodeInfo{ - nodeInfoFromAttributes("Country", "Russia"), - nodeInfoFromAttributes("Country", "Germany"), - nodeInfoFromAttributes(), - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - nm, err := NewNetmap(NodesFromInfo(nodes)) - require.NoError(t, err) - c := NewContext(nm) - c.setCBF(tc.p.ContainerBackupFactor()) - require.NoError(t, c.processFilters(tc.p)) - - err = c.processSelectors(tc.p) - require.True(t, errors.Is(err, tc.err), "got: %v", err) - }) - } -} - func testSelector() *Selector { s := new(Selector) s.SetName("name")