TrueCloudLab/frostfs-sdk-go
33
0
Fork 21
Code Issues 34 Pull requests 1 Releases Activity Actions

[#166] netmap: tests moved from JSON to YML #192

Merged
fyrchik merged 2 commits from achuprov/frostfs-sdk-go:yml_tests into master 2024-09-04 19:51:15 +00:00
Conversation 2 Commits 2 Files changed 33 +1588 -2541
33 changed files with 1588 additions and 2541 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
netmap/json_tests/cbf_default.json
View file

@ -1,99 +0,0 @@
{
"name": "default CBF is 3",
"nodes": [
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "DE"
},
{
"key": "City",
"value": "Berlin"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "FR"
},
{
"key": "City",
"value": "Paris"
}
]
}
],
"tests": {
"set default CBF": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "EU"
}
],
"containerBackupFactor": 0,
"selectors": [
{
"name": "EU",
"count": 1,
"clause": "SAME",
"attribute": "Location",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2
]
]
}
}
}

99
netmap/json_tests/cbf_minimal.json
View file

@ -1,99 +0,0 @@
{
"name": "Real node count multiplier is in range [1, specified CBF]",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "DE"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "DE"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "DE"
}
]
}
],
"tests": {
"select 2, CBF is 2": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "SAME",
"attribute": "Country",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2
]
]
},
"select 3, CBF is 2": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "X",
"count": 3,
"clause": "SAME",
"attribute": "Country",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2
]
]
}
}
}

155
netmap/json_tests/cbf_requirements.json
View file

@ -1,155 +0,0 @@
{
"name": "CBF requirements",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Attr",
"value": "Same"
}
]
}
],
"tests": {
"default CBF, no selector": {
"policy": {
"replicas": [
{
"count": 2,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": []
},
"result": [
[
0,
2,
1,
3
]
]
},
"explicit CBF, no selector": {
"policy": {
"replicas": [
{
"count": 2,
"selector": ""
}
],
"containerBackupFactor": 3,
"selectors": [],
"filters": []
},
"result": [
[
0,
2,
1,
3
]
]
},
"select distinct, weak CBF": {
"policy": {
"replicas": [
{
"count": 2,
"selector": "X"
}
],
"containerBackupFactor": 3,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "DISTINCT",
"attribute": "",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
2,
1,
3
]
]
},
"select same, weak CBF": {
"policy": {
"replicas": [
{
"count": 2,
"selector": "X"
}
],
"containerBackupFactor": 3,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "SAME",
"attribute": "Attr",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

383
netmap/json_tests/filter_complex.json
View file

@ -1,383 +0,0 @@
{
"name": "compound filter",
"nodes": [
{
"attributes": [
{
"key": "Storage",
"value": "SSD"
},
{
"key": "Rating",
"value": "10"
},
{
"key": "IntField",
"value": "100"
},
{
"key": "Param",
"value": "Value1"
}
]
}
],
"tests": {
"good": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
]
},
"result": [
[
0
]
]
},
"bad storage type": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "HDD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
]
},
"error": "not enough nodes"
},
"bad rating": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "15",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
]
},
"error": "not enough nodes"
},
"bad param": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value0",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
]
},
"error": "not enough nodes"
}
}
}

83
netmap/json_tests/filter_invalid_integer.json
View file

@ -1,83 +0,0 @@
{
"name": "invalid integer field",
"nodes": [
{
"attributes": [
{
"key": "IntegerField",
"value": "true"
}
]
},
{
"attributes": [
{
"key": "IntegerField",
"value": "str"
}
]
}
],
"tests": {
"empty string is not casted to 0": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "IntegerField",
"op": "LE",
"value": "8",
"filters": []
}
]
},
"error": "not enough nodes"
},
"non-empty string is not casted to a number": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "IntegerField",
"op": "GE",
"value": "0",
"filters": []
}
]
},
"error": "not enough nodes"
}
}
}

403
netmap/json_tests/filter_simple.json
View file

@ -1,403 +0,0 @@
{
"name": "single-op filters",
"nodes": [
{
"attributes": [
{
"key": "Rating",
"value": "4"
},
{
"key": "Country",
"value": "Germany"
}
]
}
],
"tests": {
"GE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"GE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GE",
"value": "5",
"filters": []
}
]
},
"error": "not enough nodes"
},
"GT true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GT",
"value": "3",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"GT false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GT",
"value": "4",
"filters": []
}
]
},
"error": "not enough nodes"
},
"LE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LE",
"value": "4",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"LE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LE",
"value": "3",
"filters": []
}
]
},
"error": "not enough nodes"
},
"LT true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LT",
"value": "5",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"LT false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LT",
"value": "4",
"filters": []
}
]
},
"error": "not enough nodes"
},
"EQ true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "EQ",
"value": "Germany",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"EQ false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "EQ",
"value": "China",
"filters": []
}
]
},
"error": "not enough nodes"
},
"NE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "NE",
"value": "France",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"NE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "NE",
"value": "Germany",
"filters": []
}
]
},
"error": "not enough nodes"
}
}
}

165
netmap/json_tests/hrw_sort.json
View file

@ -1,165 +0,0 @@
{
"name": "HRW ordering",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Price",
"value": "4"
},
{
"key": "Capacity",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "3"
},
{
"key": "Capacity",
"value": "10"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "1"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "100"
},
{
"key": "Capacity",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "7"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "1"
}
]
}
],
"tests": {
"select 3 nodes in 3 distinct countries, same placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
"pivot": "Y29udGFpbmVySUQ=",
"result": [ [ 5, 0, 7 ] ],
"placement": {
"pivot": "b2JqZWN0SUQ=",
"result": [ [ 5, 0, 7 ] ]
}
},
"select 6 nodes in 3 distinct countries, different placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
"pivot": "Y29udGFpbmVySUQ=",
"result": [ [ 5, 4, 0, 1, 7, 2 ] ],
"placement": {
"pivot": "b2JqZWN0SUQ=",
"result": [ [ 5, 4, 0, 7, 2, 1 ] ]
}
}
}
}

108
netmap/json_tests/issue213.json
View file

@ -1,108 +0,0 @@
{
"name": "unnamed selector (nspcc-dev/neofs-api-go#213)",
"nodes": [
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Russia"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Russia"
},
{
"key": "City",
"value": "Saint-Petersburg"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Sweden"
},
{
"key": "City",
"value": "Stockholm"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Finalnd"
},
{
"key": "City",
"value": "Helsinki"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 4,
"selector": ""
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "",
"count": 4,
"clause": "DISTINCT",
"attribute": "",
"filter": "LOC_EU"
}
],
"filters": [
{
"name": "LOC_EU",
"key": "Location",
"op": "EQ",
"value": "Europe",
"filters": []
}
]
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

192
netmap/json_tests/many_selects.json
View file

@ -1,192 +0,0 @@
{
"name": "single-op filters",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "1"
},
{
"key": "City",
"value": "SPB"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Rating",
"value": "5"
},
{
"key": "City",
"value": "Berlin"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "6"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Rating",
"value": "4"
},
{
"key": "City",
"value": "Paris"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Rating",
"value": "1"
},
{
"key": "City",
"value": "Lyon"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "5"
},
{
"key": "City",
"value": "SPB"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "7"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Rating",
"value": "3"
},
{
"key": "City",
"value": "Darmstadt"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Rating",
"value": "7"
},
{
"key": "City",
"value": "Frankfurt"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "9"
},
{
"key": "City",
"value": "SPB"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "9"
},
{
"key": "City",
"value": "SPB"
}
]
}
],
"tests": {
"Select": {
"policy": {"replicas":[{"count":1,"selector":"SameRU"},{"count":1,"selector":"DistinctRU"},{"count":1,"selector":"Good"},{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"SameRU","count":2,"clause":"SAME","attribute":"City","filter":"FromRU"},{"name":"DistinctRU","count":2,"clause":"DISTINCT","attribute":"City","filter":"FromRU"},{"name":"Good","count":2,"clause":"DISTINCT","attribute":"Country","filter":"Good"},{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[{"name":"FromRU","key":"Country","op":"EQ","value":"Russia"},{"name":"Good","key":"Rating","op":"GE","value":"4"}]},
"result": [
[0, 5, 9, 10],
[2, 6, 0, 5],
[1, 8, 2, 5],
[3, 4, 1, 7, 0, 2]
]
}
}
}

94
netmap/json_tests/multiple_rep.json
View file

@ -1,94 +0,0 @@
{
"name": "multiple replicas (#215)",
"nodes": [
{
"attributes": [
{
"key": "City",
"value": "Saint-Petersburg"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Berlin"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Paris"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "LOC_SPB_PLACE"
},
{
"count": 1,
"selector": "LOC_MSK_PLACE"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "LOC_SPB_PLACE",
"count": 1,
"clause": "CLAUSE_UNSPECIFIED",
"attribute": "",
"filter": "LOC_SPB"
},
{
"name": "LOC_MSK_PLACE",
"count": 1,
"clause": "CLAUSE_UNSPECIFIED",
"attribute": "",
"filter": "LOC_MSK"
}
],
"filters": [
{
"name": "LOC_SPB",
"key": "City",
"op": "EQ",
"value": "Saint-Petersburg",
"filters": []
},
{
"name": "LOC_MSK",
"key": "City",
"op": "EQ",
"value": "Moscow",
"filters": []
}
]
},
"result": [
[
0
],
[
1
]
]
}
}
}

331
netmap/json_tests/multiple_rep_asymmetric.json
View file

@ -1,331 +0,0 @@
{
"name": "multiple REP, asymmetric",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "0"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "5"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "6"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "NewYork"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "7"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "8"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "9"
},
{
"key": "Continent",
"value": "SA"
},
{
"key": "City",
"value": "Lima"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "10"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "11"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "NewYork"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "12"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "LosAngeles"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "13"
},
{
"key": "Continent",
"value": "SA"
},
{
"key": "City",
"value": "Lima"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "SPB"
},
{
"count": 2,
"selector": "Americas"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "SPB",
"count": 1,
"clause": "SAME",
"attribute": "City",
"filter": "SPBSSD"
},
{
"name": "Americas",
"count": 2,
"clause": "DISTINCT",
"attribute": "City",
"filter": "Americas"
}
],
"filters": [
{
"name": "SPBSSD",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "",
"key": "Country",
"op": "EQ",
"value": "RU",
"filters": []
},
{
"name": "",
"key": "City",
"op": "EQ",
"value": "St.Petersburg",
"filters": []
},
{
"name": "",
"key": "SSD",
"op": "EQ",
"value": "1",
"filters": []
}
]
},
{
"name": "Americas",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Continent",
"op": "EQ",
"value": "NA",
"filters": []
},
{
"name": "",
"key": "Continent",
"op": "EQ",
"value": "SA",
"filters": []
}
]
}
]
},
"result": [
[
1,
4
],
[
8,
12,
5,
10
]
]
}
}
}

95
netmap/json_tests/non_strict.json
View file

@ -1,95 +0,0 @@
{
"name": "non-strict selections",
"comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Russia"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
}
]
},
{
"attributes": [ ]
}
],
"tests": {
"not enough nodes (backup factor)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": [ ]
}
]
},
"result": [
[
0
]
]
},
"not enough nodes (buckets)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": [ ]
}
]
},
"result": [
[
0
]
]
}
}
}

117
netmap/json_tests/rep_only.json
View file

@ -1,117 +0,0 @@
{
"name": "REP X",
"nodes": [
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Saint-Petersburg",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Moscow",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Berlin",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Paris",
"parents": []
}
],
"state": "UNSPECIFIED"
}
],
"tests": {
"REP 1": {
"policy": {
"replicas": [
{
"count": 1,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": []
},
"result": [
[
0,
1,
2
]
]
},
"REP 3": {
"policy": {
"replicas": [
{
"count": 3,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": []
},
"result": [
[
0,
3,
1,
2
]
]
},
"REP 5": {
"policy": {
"replicas": [
{
"count": 5,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": []
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

116
netmap/json_tests/select_no_attribute.json
View file

@ -1,116 +0,0 @@
{
"name": "select with unspecified attribute",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "0"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "X",
"count": 4,
"clause": "DISTINCT",
"attribute": "",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

86
netmap/json_tests/selector_invalid.json
View file

@ -1,86 +0,0 @@
{
"name": "invalid selections",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Russia"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
}
]
},
{
"attributes": []
}
],
"tests": {
"missing filter": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "MyStore",
"count": 1,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromNL"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": []
}
]
},
"error": "filter not found"
},
"not enough nodes (filter results in empty set)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromMoon"
}
],
"filters": [
{
"name": "FromMoon",
"key": "Country",
"op": "EQ",
"value": "Moon",
"filters": []
}
]
},
"error": "not enough nodes"
}
}
}

5
netmap/policy.go
View file

@ -217,6 +217,11 @@ func (s *Selector) SelectByBucketAttribute(bucket string) {
s.m.SetAttribute(bucket) s.m.SetAttribute(bucket)
} }
// SetClause sets the clause for the Selector.
func (s *Selector) SetClause(clause netmap.Clause) {
s.m.SetClause(clause)
}
// SelectSame makes selection algorithm to select only nodes having the same values // SelectSame makes selection algorithm to select only nodes having the same values
// of the bucket attribute. // of the bucket attribute.
// //

48
netmap/json_test.go → netmap/yml_test.go
View file

@ -1,6 +1,7 @@
package netmap package netmap
import ( import (
"encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
@ -8,24 +9,41 @@ import (
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
) )
// TestCase represents collection of placement policy tests for a single node set. // TestCase represents collection of placement policy tests for a single node set.
type TestCase struct { type TestCase struct {
Name string `json:"name"` Name string `json:"name" yaml:"name"`
Nodes []NodeInfo `json:"nodes"` Nodes []NodeInfo `json:"nodes" yaml:"nodes"`
Tests map[string]struct { Tests map[string]struct {
Policy PlacementPolicy `json:"policy"` Policy PlacementPolicy
Pivot []byte `json:"pivot,omitempty"` Pivot Base64
Result [][]int `json:"result,omitempty"` Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty" yaml:"error,omitempty"`
Placement struct { Placement struct {
Pivot []byte Pivot Base64 `json:"pivot" yaml:"pivot"`
Result [][]int Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
} `json:"placement,omitempty"` } `json:"placement,omitempty" yaml:"placement,omitempty"`
} }
} }
// Base64 is a type that will hold the decoded Base64 data.
type Base64 []byte
func (b *Base64) UnmarshalYAML(unmarshal func(interface{}) error) error {
var base64Str string
if err := unmarshal(&base64Str); err != nil {
return err
}
decodedBytes, err := base64.StdEncoding.DecodeString(base64Str)
if err != nil {
return err
}
*b = decodedBytes
return nil
}
var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy) var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy)
func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) { func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
@ -39,7 +57,7 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn
} }
func TestPlacementPolicy_Interopability(t *testing.T) { func TestPlacementPolicy_Interopability(t *testing.T) {
const testsDir = "./json_tests" const testsDir = "./yml_tests"
f, err := os.Open(testsDir) f, err := os.Open(testsDir)
require.NoError(t, err) require.NoError(t, err)
@ -53,7 +71,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
var tc TestCase var tc TestCase
require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) require.NoError(t, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
srcNodes := make([]NodeInfo, len(tc.Nodes)) srcNodes := make([]NodeInfo, len(tc.Nodes))
copy(srcNodes, tc.Nodes) copy(srcNodes, tc.Nodes)
@ -88,7 +106,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
} }
func BenchmarkPlacementPolicyInteropability(b *testing.B) { func BenchmarkPlacementPolicyInteropability(b *testing.B) {
const testsDir = "./json_tests" const testsDir = "./yml_tests"
f, err := os.Open(testsDir) f, err := os.Open(testsDir)
require.NoError(b, err) require.NoError(b, err)
@ -101,7 +119,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
require.NoError(b, err) require.NoError(b, err)
var tc TestCase var tc TestCase
require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) require.NoError(b, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
b.Run(tc.Name, func(b *testing.B) { b.Run(tc.Name, func(b *testing.B) {
var nm NetMap var nm NetMap
@ -140,12 +158,12 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
} }
func BenchmarkManySelects(b *testing.B) { func BenchmarkManySelects(b *testing.B) {
testsFile := filepath.Join("json_tests", "many_selects.json") testsFile := filepath.Join("yml_tests", "many_selects.yml")
bs, err := os.ReadFile(testsFile) bs, err := os.ReadFile(testsFile)
require.NoError(b, err) require.NoError(b, err)
var tc TestCase var tc TestCase
require.NoError(b, json.Unmarshal(bs, &tc)) require.NoError(b, yaml.Unmarshal(bs, &tc))
tt, ok := tc.Tests["Select"] tt, ok := tc.Tests["Select"]
require.True(b, ok) require.True(b, ok)

48
netmap/yml_tests/cbf_default.yml Normal file
View file

@ -0,0 +1,48 @@
name: default CBF is 3
nodes:
- attributes:
- key: Location
value: Europe
- key: Country
value: RU
- key: City
value: St.Petersburg
- attributes:
- key: Location
value: Europe
- key: Country
value: RU
- key: City
value: Moscow
- attributes:
- key: Location
value: Europe
- key: Country
value: DE
- key: City
value: Berlin
- attributes:
- key: Location
value: Europe
- key: Country
value: FR
- key: City
value: Paris
tests:
set default CBF:
policy:
replicas:
- count: 1
selector: EU
containerBackupFactor: 0
selectors:
- name: EU
count: 1
clause: SAME
attribute: Location
filter: '*'
filters: []
result:
- - 0
- 1
- 2

52
netmap/yml_tests/cbf_minimal.yml Normal file
View file

@ -0,0 +1,52 @@
name: Real node count multiplier is in range [1, specified CBF]
nodes:
- attributes:
- key: ID
value: '1'
- key: Country
value: DE
- attributes:
- key: ID
value: '2'
- key: Country
value: DE
- attributes:
- key: ID
value: '3'
- key: Country
value: DE
tests:
select 2, CBF is 2:
policy:
replicas:
- count: 1
selector: X
containerBackupFactor: 2
selectors:
- name: X
count: 2
clause: SAME
attribute: Country
filter: '*'
filters: []
result:
- - 0
- 1
- 2
select 3, CBF is 2:
policy:
replicas:
- count: 1
selector: X
containerBackupFactor: 2
selectors:
- name: X
count: 3
clause: SAME
attribute: Country
filter: '*'
filters: []
result:
- - 0
- 1
- 2

82
netmap/yml_tests/cbf_requirements.yml Normal file
View file

@ -0,0 +1,82 @@
name: CBF requirements
nodes:
- attributes:
- key: ID
value: '1'
- key: Attr
value: Same
- attributes:
- key: ID
value: '2'
- key: Attr
value: Same
- attributes:
- key: ID
value: '3'
- key: Attr
value: Same
- attributes:
- key: ID
value: '4'
- key: Attr
value: Same
tests:
default CBF, no selector:
policy:
replicas:
- count: 2
containerBackupFactor: 0
selectors: []
filters: []
result:
- - 0
- 2
- 1
- 3
explicit CBF, no selector:
policy:
replicas:
- count: 2
containerBackupFactor: 3
selectors: []
filters: []
result:
- - 0
- 2
- 1
- 3
select distinct, weak CBF:
policy:
replicas:
- count: 2
selector: X
containerBackupFactor: 3
selectors:
- name: X
count: 2
clause: DISTINCT
filter: '*'
filters: []
result:
- - 0
- 2
- 1
- 3
select same, weak CBF:
policy:
replicas:
- count: 2
selector: X
containerBackupFactor: 3
selectors:
- name: X
count: 2
clause: SAME
attribute: Attr
filter: '*'
filters: []
result:
- - 0
- 1
- 2
- 3

210
netmap/yml_tests/filter_complex.yml Normal file
View file

@ -0,0 +1,210 @@
name: compound filter
nodes:
- attributes:
- key: Storage
value: SSD
- key: Rating
value: '10'
- key: IntField
value: '100'
- key: Param
value: Value1
tests:
good:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: StorageSSD
key: Storage
op: EQ
value: SSD
filters: []
- name: GoodRating
key: Rating
op: GE
value: '4'
filters: []
- name: Main
op: AND
filters:
- name: StorageSSD
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
key: IntField
op: LT
value: '123'
filters: []
- name: GoodRating
op: OPERATION_UNSPECIFIED
filters: []
- op: OR
filters:
- key: Param
op: EQ
value: Value1
filters: []
- key: Param
op: EQ
value: Value2
filters: []
result:
- - 0
bad storage type:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: StorageSSD
key: Storage
op: EQ
value: HDD
filters: []
- name: GoodRating
key: Rating
op: GE
value: '4'
filters: []
- name: Main
op: AND
filters:
- name: StorageSSD
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
key: IntField
op: LT
value: '123'
filters: []
- name: GoodRating
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
op: OR
filters:
- name: ''
key: Param
op: EQ
value: Value1
filters: []
- name: ''
key: Param
op: EQ
value: Value2
filters: []
error: not enough nodes
bad rating:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: StorageSSD
key: Storage
op: EQ
value: SSD
filters: []
- name: GoodRating
key: Rating
op: GE
value: '15'
filters: []
- name: Main
op: AND
filters:
- name: StorageSSD
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
key: IntField
op: LT
value: '123'
filters: []
- name: GoodRating
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
op: OR
filters:
- name: ''
key: Param
op: EQ
value: Value1
filters: []
- name: ''
key: Param
op: EQ
value: Value2
filters: []
error: not enough nodes
bad param:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: StorageSSD
key: Storage
op: EQ
value: SSD
filters: []
- name: GoodRating
key: Rating
op: GE
value: '4'
filters: []
- name: Main
op: AND
filters:
- name: StorageSSD
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
key: IntField
op: LT
value: '123'
filters: []
- name: GoodRating
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
op: OR
filters:
- name: ''
key: Param
op: EQ
value: Value0
filters: []
- name: ''
key: Param
op: EQ
value: Value2
filters: []
error: not enough nodes

45
netmap/yml_tests/filter_invalid_integer.yml Normal file
View file

@ -0,0 +1,45 @@
name: invalid integer field
nodes:
- attributes:
- key: IntegerField
value: 'true'
- attributes:
- key: IntegerField
value: str
tests:
empty string is not casted to 0:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: IntegerField
op: LE
value: '8'
filters: []
error: not enough nodes
non-empty string is not casted to a number:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: IntegerField
op: GE
value: '0'
filters: []
error: not enough nodes

230
netmap/yml_tests/filter_simple.yml Normal file
View file

@ -0,0 +1,230 @@
name: single-op filters
nodes:
- attributes:
- key: Rating
value: '4'
- key: Country
value: Germany
tests:
GE true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: GE
value: '4'
filters: []
result:
- - 0
GE false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: GE
value: '5'
filters: []
error: not enough nodes
GT true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: GT
value: '3'
filters: []
result:
- - 0
GT false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: GT
value: '4'
filters: []
error: not enough nodes
LE true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: LE
value: '4'
filters: []
result:
- - 0
LE false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: LE
value: '3'
filters: []
error: not enough nodes
LT true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: LT
value: '5'
filters: []
result:
- - 0
LT false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: LT
value: '4'
filters: []
error: not enough nodes
EQ true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Country
op: EQ
value: Germany
filters: []
result:
- - 0
EQ false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Country
op: EQ
value: China
filters: []
error: not enough nodes
NE true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Country
op: NE
value: France
filters: []
result:
- - 0
NE false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Country
op: NE
value: Germany
filters: []
error: not enough nodes

118
netmap/yml_tests/hrw_sort.yml Normal file
View file

@ -0,0 +1,118 @@
name: HRW ordering
nodes:
- attributes:
- key: Country
value: Germany
- key: Price
value: '2'
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: Germany
- key: Price
value: '4'
- key: Capacity
value: '1'
- attributes:
- key: Country
value: France
- key: Price
value: '3'
- key: Capacity
value: '10'
- attributes:
- key: Country
value: Russia
- key: Price
value: '2'
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: Russia
- key: Price
value: '1'
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: Russia
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: France
- key: Price
value: '100'
- key: Capacity
value: '1'
- attributes:
- key: Country
value: France
- key: Price
value: '7'
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: Russia
- key: Price
value: '2'
- key: Capacity
value: '1'
tests:
select 3 nodes in 3 distinct countries, same placement:
policy:
replicas:
- count: 1
selector: Main
containerBackupFactor: 1
selectors:
- name: Main
count: 3
clause: DISTINCT
attribute: Country
filter: '*'
filters: []
pivot: Y29udGFpbmVySUQ=
result:
- - 5
- 0
- 7
placement:
pivot: b2JqZWN0SUQ=
result:
- - 5
- 0
- 7
select 6 nodes in 3 distinct countries, different placement:
policy:
replicas:
- count: 1
selector: Main
containerBackupFactor: 2
selectors:
- name: Main
count: 3
clause: DISTINCT
attribute: Country
filter: '*'
filters: []
pivot: Y29udGFpbmVySUQ=
result:
- - 5
- 4
- 0
- 1
- 7
- 2
placement:
pivot: b2JqZWN0SUQ=
result:
- - 5
- 4
- 0
- 7
- 2
- 1

52
netmap/yml_tests/issue213.yml Normal file
View file

@ -0,0 +1,52 @@
name: unnamed selector (nspcc-dev/neofs-api-go#213)
nodes:
- attributes:
- key: Location
value: Europe
- key: Country
value: Russia
- key: City
value: Moscow
- attributes:
- key: Location
value: Europe
- key: Country
value: Russia
- key: City
value: Saint-Petersburg
- attributes:
- key: Location
value: Europe
- key: Country
value: Sweden
- key: City
value: Stockholm
- attributes:
- key: Location
value: Europe
- key: Country
value: Finalnd
- key: City
value: Helsinki
tests:
test:
policy:
replicas:
- count: 4
containerBackupFactor: 1
selectors:
- name: ''
count: 4
clause: DISTINCT
filter: LOC_EU
filters:
- name: LOC_EU
key: Location
op: EQ
value: Europe
filters: []
result:
- - 0
- 1
- 2
- 3

141
netmap/yml_tests/many_selects.yml Normal file
View file

@ -0,0 +1,141 @@
name: single-op filters
nodes:
- attributes:
- key: Country
value: Russia
- key: Rating
value: '1'
- key: City
value: SPB
- attributes:
- key: Country
value: Germany
- key: Rating
value: '5'
- key: City
value: Berlin
- attributes:
- key: Country
value: Russia
- key: Rating
value: '6'
- key: City
value: Moscow
- attributes:
- key: Country
value: France
- key: Rating
value: '4'
- key: City
value: Paris
- attributes:
- key: Country
value: France
- key: Rating
value: '1'
- key: City
value: Lyon
- attributes:
- key: Country
value: Russia
- key: Rating
value: '5'
- key: City
value: SPB
- attributes:
- key: Country
value: Russia
- key: Rating
value: '7'
- key: City
value: Moscow
- attributes:
- key: Country
value: Germany
- key: Rating
value: '3'
- key: City
value: Darmstadt
- attributes:
- key: Country
value: Germany
- key: Rating
value: '7'
- key: City
value: Frankfurt
- attributes:
- key: Country
value: Russia
- key: Rating
value: '9'
- key: City
value: SPB
- attributes:
- key: Country
value: Russia
- key: Rating
value: '9'
- key: City
value: SPB
tests:
Select:
policy:
replicas:
- count: 1
selector: SameRU
- count: 1
selector: DistinctRU
- count: 1
selector: Good
- count: 1
selector: Main
containerBackupFactor: 2
selectors:
- name: SameRU
count: 2
clause: SAME
attribute: City
filter: FromRU
- name: DistinctRU
count: 2
clause: DISTINCT
attribute: City
filter: FromRU
- name: Good
count: 2
clause: DISTINCT
attribute: Country
filter: Good
- name: Main
count: 3
clause: DISTINCT
attribute: Country
filter: '*'
filters:
- name: FromRU
key: Country
op: EQ
value: Russia
- name: Good
key: Rating
op: GE
value: '4'
result:
- - 0
- 5
- 9
- 10
- - 2
- 6
- 0
- 5
- - 1
- 8
- 2
- 5
- - 3
- 4
- 1
- 7
- 0
- 2

46
netmap/yml_tests/multiple_rep.yml Normal file
View file

@ -0,0 +1,46 @@
name: multiple replicas (#215)
nodes:
- attributes:
- key: City
value: Saint-Petersburg
- attributes:
- key: City
value: Moscow
- attributes:
- key: City
value: Berlin
- attributes:
- key: City
value: Paris
tests:
test:
policy:
replicas:
- count: 1
selector: LOC_SPB_PLACE
- count: 1
selector: LOC_MSK_PLACE
containerBackupFactor: 1
selectors:
- name: LOC_SPB_PLACE
count: 1
clause: CLAUSE_UNSPECIFIED
filter: LOC_SPB
- name: LOC_MSK_PLACE
count: 1
clause: CLAUSE_UNSPECIFIED
filter: LOC_MSK
filters:
- name: LOC_SPB
key: City
op: EQ
value: Saint-Petersburg
filters: []
- name: LOC_MSK
key: City
op: EQ
value: Moscow
filters: []
result:
- - 0
- - 1

162
netmap/yml_tests/multiple_rep_asymmetric.yml Normal file
View file

@ -0,0 +1,162 @@
name: multiple REP, asymmetric
nodes:
- attributes:
- key: ID
value: '1'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '0'
- attributes:
- key: ID
value: '2'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '1'
- attributes:
- key: ID
value: '3'
- key: Country
value: RU
- key: City
value: Moscow
- key: SSD
value: '1'
- attributes:
- key: ID
value: '4'
- key: Country
value: RU
- key: City
value: Moscow
- key: SSD
value: '1'
- attributes:
- key: ID
value: '5'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '1'
- attributes:
- key: ID
value: '6'
- key: Continent
value: NA
- key: City
value: NewYork
- attributes:
- key: ID
value: '7'
- key: Continent
value: AF
- key: City
value: Cairo
- attributes:
- key: ID
value: '8'
- key: Continent
value: AF
- key: City
value: Cairo
- attributes:
- key: ID
value: '9'
- key: Continent
value: SA
- key: City
value: Lima
- attributes:
- key: ID
value: '10'
- key: Continent
value: AF
- key: City
value: Cairo
- attributes:
- key: ID
value: '11'
- key: Continent
value: NA
- key: City
value: NewYork
- attributes:
- key: ID
value: '12'
- key: Continent
value: NA
- key: City
value: LosAngeles
- attributes:
- key: ID
value: '13'
- key: Continent
value: SA
- key: City
value: Lima
tests:
test:
policy:
replicas:
- count: 1
selector: SPB
- count: 2
selector: Americas
containerBackupFactor: 2
selectors:
- name: SPB
count: 1
clause: SAME
attribute: City
filter: SPBSSD
- name: Americas
count: 2
clause: DISTINCT
attribute: City
filter: Americas
filters:
- name: SPBSSD
op: AND
filters:
- name: ''
key: Country
op: EQ
value: RU
filters: []
- name: ''
key: City
op: EQ
value: St.Petersburg
filters: []
- name: ''
key: SSD
op: EQ
value: '1'
filters: []
- name: Americas
op: OR
filters:
- name: ''
key: Continent
op: EQ
value: NA
filters: []
- name: ''
key: Continent
op: EQ
value: SA
filters: []
result:
- - 1
- 4
- - 8
- 12
- 5
- 10

52
netmap/yml_tests/non_strict.yml Normal file
View file

@ -0,0 +1,52 @@
name: non-strict selections
comment: These test specify loose selection behaviour, to allow fetching already PUT
objects even when there is not enough nodes to select from.
nodes:
- attributes:
- key: Country
value: Russia
- attributes:
- key: Country
value: Germany
- attributes: []
tests:
not enough nodes (backup factor):
policy:
replicas:
- count: 1
selector: MyStore
containerBackupFactor: 2
selectors:
- name: MyStore
count: 2
clause: DISTINCT
attribute: Country
filter: FromRU
filters:
- name: FromRU
key: Country
op: EQ
value: Russia
filters: []
result:
- - 0
not enough nodes (buckets):
policy:
replicas:
- count: 1
selector: MyStore
containerBackupFactor: 1
selectors:
- name: MyStore
count: 2
clause: DISTINCT
attribute: Country
filter: FromRU
filters:
- name: FromRU
key: Country
op: EQ
value: Russia
filters: []
result:
- - 0

66
netmap/yml_tests/rep_only.yml Normal file
View file

@ -0,0 +1,66 @@
name: REP X
nodes:
- publicKey: ''
addresses: []
attributes:
- key: City
value: Saint-Petersburg
parents: []
state: UNSPECIFIED
- publicKey: ''
addresses: []
attributes:
- key: City
value: Moscow
parents: []
state: UNSPECIFIED
- publicKey: ''
addresses: []
attributes:
- key: City
value: Berlin
parents: []
state: UNSPECIFIED
- publicKey: ''
addresses: []
attributes:
- key: City
value: Paris
parents: []
state: UNSPECIFIED
tests:
REP 1:
policy:
replicas:
- count: 1
containerBackupFactor: 0
selectors: []
filters: []
result:
- - 0
- 1
- 2
REP 3:
policy:
replicas:
- count: 3
containerBackupFactor: 0
selectors: []
filters: []
result:
- - 0
- 3
- 1
- 2
REP 5:
policy:
replicas:
- count: 5
containerBackupFactor: 0
selectors: []
filters: []
result:
- - 0
- 1
- 2
- 3

56
netmap/yml_tests/select_no_attribute.yml Normal file
View file

@ -0,0 +1,56 @@
name: select with unspecified attribute
nodes:
- attributes:
- key: ID
value: '1'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '0'
- attributes:
- key: ID
value: '2'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '1'
- attributes:
- key: ID
value: '3'
- key: Country
value: RU
- key: City
value: Moscow
- key: SSD
value: '1'
- attributes:
- key: ID
value: '4'
- key: Country
value: RU
- key: City
value: Moscow
- key: SSD
value: '1'
tests:
test:
policy:
replicas:
- count: 1
selector: X
containerBackupFactor: 1
selectors:
- name: X
count: 4
clause: DISTINCT
filter: '*'
filters: []
result:
- - 0
- 1
- 2
- 3

48
netmap/yml_tests/selector_invalid.yml Normal file
View file

@ -0,0 +1,48 @@
name: invalid selections
nodes:
- attributes:
- key: Country
value: Russia
- attributes:
- key: Country
value: Germany
- attributes: []
tests:
missing filter:
policy:
replicas:
- count: 1
selector: MyStore
containerBackupFactor: 1
selectors:
- name: MyStore
count: 1
clause: DISTINCT
attribute: Country
filter: FromNL
filters:
- name: FromRU
key: Country
op: EQ
value: Russia
filters: []
error: filter not found
not enough nodes (filter results in empty set):
policy:
replicas:
- count: 1
selector: MyStore
containerBackupFactor: 2
selectors:
- name: MyStore
count: 2
clause: DISTINCT
attribute: Country
filter: FromMoon
filters:
- name: FromMoon
key: Country
op: EQ
value: Moon
filters: []
error: not enough nodes

142
netmap/yml_unmarshal.go Normal file
View file

@ -0,0 +1,142 @@
package netmap
import (
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
)
type tempPlacementPolicy struct {
BackupFactor uint32 `yaml:"containerBackupFactor"`
Filters []tempFilter `yaml:"filters"`
Selectors []tempSelector `yaml:"selectors"`
Replicas []tempReplica `yaml:"replicas"`
Unique bool `yaml:"unique"`
}
type tempFilter struct {
Name string `yaml:"name"`
Key string `yaml:"key"`
Op string `yaml:"op"`
Value string `yaml:"value"`
Filters []tempFilter `yaml:"filters"`
}
type tempSelector struct {
Name string `yaml:"name"`
Count uint32 `yaml:"count"`
Clause string `yaml:"clause"`
Attribute string `yaml:"attribute"`
Filter string `yaml:"filter"`
}
type tempReplica struct {
Count uint32 `yaml:"count"`
Selector string `yaml:"selector"`
}
func convertNFilters(temp []tempFilter) []netmap.Filter {
var filters []netmap.Filter
for _, tf := range temp {
filters = append(filters, convertNFilter(tf))
}
return filters
}
var stringToOperationMap = map[string]netmap.Operation{
"EQ": netmap.EQ,
"NE": netmap.NE,
"GT": netmap.GT,
"GE": netmap.GE,
"LT": netmap.LT,
"LE": netmap.LE,
"OR": netmap.OR,
"AND": netmap.AND,
"NOT": netmap.NOT,
}
func convertStringToOperation(opStr string) netmap.Operation {
opStr = strings.ToUpper(opStr)
if op, exists := stringToOperationMap[opStr]; exists {
return op
}
return netmap.UnspecifiedOperation
}
func convertStringToClause(clauseStr string) netmap.Clause {
switch strings.ToUpper(clauseStr) {
case "DISTINCT":
return netmap.Distinct
default:
return netmap.Same
}
}
func convertNFilter(temp tempFilter) netmap.Filter {
filter := netmap.Filter{}
filter.SetKey(temp.Key)
filter.SetName(temp.Name)
filter.SetValue(temp.Value)
filter.SetOp(convertStringToOperation(temp.Op))
if temp.Filters != nil {
filter.SetFilters(convertNFilters(temp.Filters))
}
return filter
}
func (p *PlacementPolicy) UnmarshalYAML(unmarshal func(interface{}) error) error {
var temp tempPlacementPolicy
if err := unmarshal(&temp); err != nil {
return err
}
for _, ts := range temp.Filters {
netmapFilters := convertNFilter(ts)
p.AddFilters(Filter{m: netmapFilters})
}
for _, ts := range temp.Selectors {
selector := Selector{}
selector.SetName(ts.Name)
selector.SetNumberOfNodes(ts.Count)
selector.SetClause(convertStringToClause(ts.Clause))
selector.SelectByBucketAttribute(ts.Attribute)
selector.SetFilterName(ts.Filter)
p.AddSelectors(selector)
}
for _, tr := range temp.Replicas {
replica := ReplicaDescriptor{}
replica.SetSelectorName(tr.Selector)
replica.m.SetCount(tr.Count)
p.AddReplicas(replica)
}
p.SetContainerBackupFactor(temp.BackupFactor)
p.SetUnique(temp.Unique)
return nil
}
type Attribute struct {
Key string `yaml:"key"`
Value string `yaml:"value"`
}
type tempNode struct {
Attributes []Attribute `yaml:"attributes"`
}
func (x *NodeInfo) UnmarshalYAML(unmarshal func(interface{}) error) error {
var temp tempNode
if err := unmarshal(&temp); err != nil {
return err
}
for _, atr := range temp.Attributes {
x.SetAttribute(atr.Key, atr.Value)
}
return nil
}