TrueCloudLab/frostfs-sdk-go
33
0
Fork 21
Code Issues 34 Pull requests 1 Releases Activity Actions

[#166] netmap: tests moved from JSON to YML #192

Merged
fyrchik merged 2 commits from achuprov/frostfs-sdk-go:yml_tests into master 2024-09-04 19:51:15 +00:00
Conversation 2 Commits 2 Files changed 33 +1588 -2541
33 changed files with 1588 additions and 2541 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
netmap/json_tests/cbf_default.json
View file

@ -1,99 +0,0 @@
{
"name": "default CBF is 3",
"nodes": [
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "DE"
},
{
"key": "City",
"value": "Berlin"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "FR"
},
{
"key": "City",
"value": "Paris"
}
]
}
],
"tests": {
"set default CBF": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "EU"
}
],
"containerBackupFactor": 0,
"selectors": [
{
"name": "EU",
"count": 1,
"clause": "SAME",
"attribute": "Location",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2
]
]
}
}
}

99
netmap/json_tests/cbf_minimal.json
View file

@ -1,99 +0,0 @@
{
"name": "Real node count multiplier is in range [1, specified CBF]",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "DE"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "DE"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "DE"
}
]
}
],
"tests": {
"select 2, CBF is 2": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "SAME",
"attribute": "Country",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2
]
]
},
"select 3, CBF is 2": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "X",
"count": 3,
"clause": "SAME",
"attribute": "Country",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2
]
]
}
}
}

155
netmap/json_tests/cbf_requirements.json
View file

@ -1,155 +0,0 @@
{
"name": "CBF requirements",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Attr",
"value": "Same"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Attr",
"value": "Same"
}
]
}
],
"tests": {
"default CBF, no selector": {
"policy": {
"replicas": [
{
"count": 2,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": []
},
"result": [
[
0,
2,
1,
3
]
]
},
"explicit CBF, no selector": {
"policy": {
"replicas": [
{
"count": 2,
"selector": ""
}
],
"containerBackupFactor": 3,
"selectors": [],
"filters": []
},
"result": [
[
0,
2,
1,
3
]
]
},
"select distinct, weak CBF": {
"policy": {
"replicas": [
{
"count": 2,
"selector": "X"
}
],
"containerBackupFactor": 3,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "DISTINCT",
"attribute": "",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
2,
1,
3
]
]
},
"select same, weak CBF": {
"policy": {
"replicas": [
{
"count": 2,
"selector": "X"
}
],
"containerBackupFactor": 3,
"selectors": [
{
"name": "X",
"count": 2,
"clause": "SAME",
"attribute": "Attr",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

383
netmap/json_tests/filter_complex.json
View file

@ -1,383 +0,0 @@
{
"name": "compound filter",
"nodes": [
{
"attributes": [
{
"key": "Storage",
"value": "SSD"
},
{
"key": "Rating",
"value": "10"
},
{
"key": "IntField",
"value": "100"
},
{
"key": "Param",
"value": "Value1"
}
]
}
],
"tests": {
"good": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
]
},
"result": [
[
0
]
]
},
"bad storage type": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "HDD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
]
},
"error": "not enough nodes"
},
"bad rating": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "15",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value1",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
]
},
"error": "not enough nodes"
},
"bad param": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "StorageSSD",
"key": "Storage",
"op": "EQ",
"value": "SSD",
"filters": []
},
{
"name": "GoodRating",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
},
{
"name": "Main",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "StorageSSD",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "IntField",
"op": "LT",
"value": "123",
"filters": []
},
{
"name": "GoodRating",
"key": "",
"op": "OPERATION_UNSPECIFIED",
"value": "",
"filters": []
},
{
"name": "",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value0",
"filters": []
},
{
"name": "",
"key": "Param",
"op": "EQ",
"value": "Value2",
"filters": []
}
]
}
]
}
]
},
"error": "not enough nodes"
}
}
}

83
netmap/json_tests/filter_invalid_integer.json
View file

@ -1,83 +0,0 @@
{
"name": "invalid integer field",
"nodes": [
{
"attributes": [
{
"key": "IntegerField",
"value": "true"
}
]
},
{
"attributes": [
{
"key": "IntegerField",
"value": "str"
}
]
}
],
"tests": {
"empty string is not casted to 0": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "IntegerField",
"op": "LE",
"value": "8",
"filters": []
}
]
},
"error": "not enough nodes"
},
"non-empty string is not casted to a number": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "IntegerField",
"op": "GE",
"value": "0",
"filters": []
}
]
},
"error": "not enough nodes"
}
}
}

403
netmap/json_tests/filter_simple.json
View file

@ -1,403 +0,0 @@
{
"name": "single-op filters",
"nodes": [
{
"attributes": [
{
"key": "Rating",
"value": "4"
},
{
"key": "Country",
"value": "Germany"
}
]
}
],
"tests": {
"GE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GE",
"value": "4",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"GE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GE",
"value": "5",
"filters": []
}
]
},
"error": "not enough nodes"
},
"GT true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GT",
"value": "3",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"GT false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "GT",
"value": "4",
"filters": []
}
]
},
"error": "not enough nodes"
},
"LE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LE",
"value": "4",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"LE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LE",
"value": "3",
"filters": []
}
]
},
"error": "not enough nodes"
},
"LT true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LT",
"value": "5",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"LT false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Rating",
"op": "LT",
"value": "4",
"filters": []
}
]
},
"error": "not enough nodes"
},
"EQ true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "EQ",
"value": "Germany",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"EQ false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "EQ",
"value": "China",
"filters": []
}
]
},
"error": "not enough nodes"
},
"NE true": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "NE",
"value": "France",
"filters": []
}
]
},
"result": [
[
0
]
]
},
"NE false": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "DISTINCT",
"attribute": "",
"filter": "Main"
}
],
"filters": [
{
"name": "Main",
"key": "Country",
"op": "NE",
"value": "Germany",
"filters": []
}
]
},
"error": "not enough nodes"
}
}
}

165
netmap/json_tests/hrw_sort.json
View file

@ -1,165 +0,0 @@
{
"name": "HRW ordering",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Price",
"value": "4"
},
{
"key": "Capacity",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "3"
},
{
"key": "Capacity",
"value": "10"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "1"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "100"
},
{
"key": "Capacity",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Price",
"value": "7"
},
{
"key": "Capacity",
"value": "10000"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Price",
"value": "2"
},
{
"key": "Capacity",
"value": "1"
}
]
}
],
"tests": {
"select 3 nodes in 3 distinct countries, same placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
"pivot": "Y29udGFpbmVySUQ=",
"result": [ [ 5, 0, 7 ] ],
"placement": {
"pivot": "b2JqZWN0SUQ=",
"result": [ [ 5, 0, 7 ] ]
}
},
"select 6 nodes in 3 distinct countries, different placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
"pivot": "Y29udGFpbmVySUQ=",
"result": [ [ 5, 4, 0, 1, 7, 2 ] ],
"placement": {
"pivot": "b2JqZWN0SUQ=",
"result": [ [ 5, 4, 0, 7, 2, 1 ] ]
}
}
}
}

108
netmap/json_tests/issue213.json
View file

@ -1,108 +0,0 @@
{
"name": "unnamed selector (nspcc-dev/neofs-api-go#213)",
"nodes": [
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Russia"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Russia"
},
{
"key": "City",
"value": "Saint-Petersburg"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Sweden"
},
{
"key": "City",
"value": "Stockholm"
}
]
},
{
"attributes": [
{
"key": "Location",
"value": "Europe"
},
{
"key": "Country",
"value": "Finalnd"
},
{
"key": "City",
"value": "Helsinki"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 4,
"selector": ""
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "",
"count": 4,
"clause": "DISTINCT",
"attribute": "",
"filter": "LOC_EU"
}
],
"filters": [
{
"name": "LOC_EU",
"key": "Location",
"op": "EQ",
"value": "Europe",
"filters": []
}
]
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

192
netmap/json_tests/many_selects.json
View file

@ -1,192 +0,0 @@
{
"name": "single-op filters",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "1"
},
{
"key": "City",
"value": "SPB"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Rating",
"value": "5"
},
{
"key": "City",
"value": "Berlin"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "6"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Rating",
"value": "4"
},
{
"key": "City",
"value": "Paris"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "France"
},
{
"key": "Rating",
"value": "1"
},
{
"key": "City",
"value": "Lyon"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "5"
},
{
"key": "City",
"value": "SPB"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "7"
},
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Rating",
"value": "3"
},
{
"key": "City",
"value": "Darmstadt"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
},
{
"key": "Rating",
"value": "7"
},
{
"key": "City",
"value": "Frankfurt"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "9"
},
{
"key": "City",
"value": "SPB"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Russia"
},
{
"key": "Rating",
"value": "9"
},
{
"key": "City",
"value": "SPB"
}
]
}
],
"tests": {
"Select": {
"policy": {"replicas":[{"count":1,"selector":"SameRU"},{"count":1,"selector":"DistinctRU"},{"count":1,"selector":"Good"},{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"SameRU","count":2,"clause":"SAME","attribute":"City","filter":"FromRU"},{"name":"DistinctRU","count":2,"clause":"DISTINCT","attribute":"City","filter":"FromRU"},{"name":"Good","count":2,"clause":"DISTINCT","attribute":"Country","filter":"Good"},{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[{"name":"FromRU","key":"Country","op":"EQ","value":"Russia"},{"name":"Good","key":"Rating","op":"GE","value":"4"}]},
"result": [
[0, 5, 9, 10],
[2, 6, 0, 5],
[1, 8, 2, 5],
[3, 4, 1, 7, 0, 2]
]
}
}
}

94
netmap/json_tests/multiple_rep.json
View file

@ -1,94 +0,0 @@
{
"name": "multiple replicas (#215)",
"nodes": [
{
"attributes": [
{
"key": "City",
"value": "Saint-Petersburg"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Moscow"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Berlin"
}
]
},
{
"attributes": [
{
"key": "City",
"value": "Paris"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "LOC_SPB_PLACE"
},
{
"count": 1,
"selector": "LOC_MSK_PLACE"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "LOC_SPB_PLACE",
"count": 1,
"clause": "CLAUSE_UNSPECIFIED",
"attribute": "",
"filter": "LOC_SPB"
},
{
"name": "LOC_MSK_PLACE",
"count": 1,
"clause": "CLAUSE_UNSPECIFIED",
"attribute": "",
"filter": "LOC_MSK"
}
],
"filters": [
{
"name": "LOC_SPB",
"key": "City",
"op": "EQ",
"value": "Saint-Petersburg",
"filters": []
},
{
"name": "LOC_MSK",
"key": "City",
"op": "EQ",
"value": "Moscow",
"filters": []
}
]
},
"result": [
[
0
],
[
1
]
]
}
}
}

331
netmap/json_tests/multiple_rep_asymmetric.json
View file

@ -1,331 +0,0 @@
{
"name": "multiple REP, asymmetric",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "0"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "5"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "6"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "NewYork"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "7"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "8"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "9"
},
{
"key": "Continent",
"value": "SA"
},
{
"key": "City",
"value": "Lima"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "10"
},
{
"key": "Continent",
"value": "AF"
},
{
"key": "City",
"value": "Cairo"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "11"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "NewYork"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "12"
},
{
"key": "Continent",
"value": "NA"
},
{
"key": "City",
"value": "LosAngeles"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "13"
},
{
"key": "Continent",
"value": "SA"
},
{
"key": "City",
"value": "Lima"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "SPB"
},
{
"count": 2,
"selector": "Americas"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "SPB",
"count": 1,
"clause": "SAME",
"attribute": "City",
"filter": "SPBSSD"
},
{
"name": "Americas",
"count": 2,
"clause": "DISTINCT",
"attribute": "City",
"filter": "Americas"
}
],
"filters": [
{
"name": "SPBSSD",
"key": "",
"op": "AND",
"value": "",
"filters": [
{
"name": "",
"key": "Country",
"op": "EQ",
"value": "RU",
"filters": []
},
{
"name": "",
"key": "City",
"op": "EQ",
"value": "St.Petersburg",
"filters": []
},
{
"name": "",
"key": "SSD",
"op": "EQ",
"value": "1",
"filters": []
}
]
},
{
"name": "Americas",
"key": "",
"op": "OR",
"value": "",
"filters": [
{
"name": "",
"key": "Continent",
"op": "EQ",
"value": "NA",
"filters": []
},
{
"name": "",
"key": "Continent",
"op": "EQ",
"value": "SA",
"filters": []
}
]
}
]
},
"result": [
[
1,
4
],
[
8,
12,
5,
10
]
]
}
}
}

95
netmap/json_tests/non_strict.json
View file

@ -1,95 +0,0 @@
{
"name": "non-strict selections",
"comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Russia"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
}
]
},
{
"attributes": [ ]
}
],
"tests": {
"not enough nodes (backup factor)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": [ ]
}
]
},
"result": [
[
0
]
]
},
"not enough nodes (buckets)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromRU"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": [ ]
}
]
},
"result": [
[
0
]
]
}
}
}

117
netmap/json_tests/rep_only.json
View file

@ -1,117 +0,0 @@
{
"name": "REP X",
"nodes": [
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Saint-Petersburg",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Moscow",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Berlin",
"parents": []
}
],
"state": "UNSPECIFIED"
},
{
"publicKey": "",
"addresses": [],
"attributes": [
{
"key": "City",
"value": "Paris",
"parents": []
}
],
"state": "UNSPECIFIED"
}
],
"tests": {
"REP 1": {
"policy": {
"replicas": [
{
"count": 1,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": []
},
"result": [
[
0,
1,
2
]
]
},
"REP 3": {
"policy": {
"replicas": [
{
"count": 3,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": []
},
"result": [
[
0,
3,
1,
2
]
]
},
"REP 5": {
"policy": {
"replicas": [
{
"count": 5,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": []
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

116
netmap/json_tests/select_no_attribute.json
View file

@ -1,116 +0,0 @@
{
"name": "select with unspecified attribute",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "0"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "St.Petersburg"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "Country",
"value": "RU"
},
{
"key": "City",
"value": "Moscow"
},
{
"key": "SSD",
"value": "1"
}
]
}
],
"tests": {
"test": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "X"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "X",
"count": 4,
"clause": "DISTINCT",
"attribute": "",
"filter": "*"
}
],
"filters": []
},
"result": [
[
0,
1,
2,
3
]
]
}
}
}

86
netmap/json_tests/selector_invalid.json
View file

@ -1,86 +0,0 @@
{
"name": "invalid selections",
"nodes": [
{
"attributes": [
{
"key": "Country",
"value": "Russia"
}
]
},
{
"attributes": [
{
"key": "Country",
"value": "Germany"
}
]
},
{
"attributes": []
}
],
"tests": {
"missing filter": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "MyStore",
"count": 1,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromNL"
}
],
"filters": [
{
"name": "FromRU",
"key": "Country",
"op": "EQ",
"value": "Russia",
"filters": []
}
]
},
"error": "filter not found"
},
"not enough nodes (filter results in empty set)": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "MyStore"
}
],
"containerBackupFactor": 2,
"selectors": [
{
"name": "MyStore",
"count": 2,
"clause": "DISTINCT",
"attribute": "Country",
"filter": "FromMoon"
}
],
"filters": [
{
"name": "FromMoon",
"key": "Country",
"op": "EQ",
"value": "Moon",
"filters": []
}
]
},
"error": "not enough nodes"
}
}
}

5
netmap/policy.go
View file

@ -217,6 +217,11 @@ func (s *Selector) SelectByBucketAttribute(bucket string) {
s.m.SetAttribute(bucket)
}
// SetClause sets the clause for the Selector.
func (s *Selector) SetClause(clause netmap.Clause) {
s.m.SetClause(clause)
}
// SelectSame makes selection algorithm to select only nodes having the same values
// of the bucket attribute.
//

48
netmap/json_test.go → netmap/yml_test.go
View file

@ -1,6 +1,7 @@
package netmap
import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
@ -8,24 +9,41 @@ import (
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
// TestCase represents collection of placement policy tests for a single node set.
type TestCase struct {
Name string `json:"name"`
Nodes []NodeInfo `json:"nodes"`
Name string `json:"name" yaml:"name"`
Nodes []NodeInfo `json:"nodes" yaml:"nodes"`
Tests map[string]struct {
Policy PlacementPolicy `json:"policy"`
Pivot []byte `json:"pivot,omitempty"`
Result [][]int `json:"result,omitempty"`
Error string `json:"error,omitempty"`
Policy PlacementPolicy
Pivot Base64
Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
Error string `json:"error,omitempty" yaml:"error,omitempty"`
Placement struct {
Pivot []byte
Result [][]int
} `json:"placement,omitempty"`
Pivot Base64 `json:"pivot" yaml:"pivot"`
Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
} `json:"placement,omitempty" yaml:"placement,omitempty"`
}
}
// Base64 is a type that will hold the decoded Base64 data.
type Base64 []byte
func (b *Base64) UnmarshalYAML(unmarshal func(interface{}) error) error {
var base64Str string
if err := unmarshal(&base64Str); err != nil {
return err
}
decodedBytes, err := base64.StdEncoding.DecodeString(base64Str)
if err != nil {
return err
}
*b = decodedBytes
return nil
}
var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy)
func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
@ -39,7 +57,7 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn
}
func TestPlacementPolicy_Interopability(t *testing.T) {
const testsDir = "./json_tests"
const testsDir = "./yml_tests"
f, err := os.Open(testsDir)
require.NoError(t, err)
@ -53,7 +71,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
require.NoError(t, err)
var tc TestCase
require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
require.NoError(t, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
srcNodes := make([]NodeInfo, len(tc.Nodes))
copy(srcNodes, tc.Nodes)
@ -88,7 +106,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
}
func BenchmarkPlacementPolicyInteropability(b *testing.B) {
const testsDir = "./json_tests"
const testsDir = "./yml_tests"
f, err := os.Open(testsDir)
require.NoError(b, err)
@ -101,7 +119,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
require.NoError(b, err)
var tc TestCase
require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
require.NoError(b, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
b.Run(tc.Name, func(b *testing.B) {
var nm NetMap
@ -140,12 +158,12 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
}
func BenchmarkManySelects(b *testing.B) {
testsFile := filepath.Join("json_tests", "many_selects.json")
testsFile := filepath.Join("yml_tests", "many_selects.yml")
bs, err := os.ReadFile(testsFile)
require.NoError(b, err)
var tc TestCase
require.NoError(b, json.Unmarshal(bs, &tc))
require.NoError(b, yaml.Unmarshal(bs, &tc))
tt, ok := tc.Tests["Select"]
require.True(b, ok)

48
netmap/yml_tests/cbf_default.yml Normal file
View file

@ -0,0 +1,48 @@
name: default CBF is 3
nodes:
- attributes:
- key: Location
value: Europe
- key: Country
value: RU
- key: City
value: St.Petersburg
- attributes:
- key: Location
value: Europe
- key: Country
value: RU
- key: City
value: Moscow
- attributes:
- key: Location
value: Europe
- key: Country
value: DE
- key: City
value: Berlin
- attributes:
- key: Location
value: Europe
- key: Country
value: FR
- key: City
value: Paris
tests:
set default CBF:
policy:
replicas:
- count: 1
selector: EU
containerBackupFactor: 0
selectors:
- name: EU
count: 1
clause: SAME
attribute: Location
filter: '*'
filters: []
result:
- - 0
- 1
- 2

52
netmap/yml_tests/cbf_minimal.yml Normal file
View file

@ -0,0 +1,52 @@
name: Real node count multiplier is in range [1, specified CBF]
nodes:
- attributes:
- key: ID
value: '1'
- key: Country
value: DE
- attributes:
- key: ID
value: '2'
- key: Country
value: DE
- attributes:
- key: ID
value: '3'
- key: Country
value: DE
tests:
select 2, CBF is 2:
policy:
replicas:
- count: 1
selector: X
containerBackupFactor: 2
selectors:
- name: X
count: 2
clause: SAME
attribute: Country
filter: '*'
filters: []
result:
- - 0
- 1
- 2
select 3, CBF is 2:
policy:
replicas:
- count: 1
selector: X
containerBackupFactor: 2
selectors:
- name: X
count: 3
clause: SAME
attribute: Country
filter: '*'
filters: []
result:
- - 0
- 1
- 2

82
netmap/yml_tests/cbf_requirements.yml Normal file
View file

@ -0,0 +1,82 @@
name: CBF requirements
nodes:
- attributes:
- key: ID
value: '1'
- key: Attr
value: Same
- attributes:
- key: ID
value: '2'
- key: Attr
value: Same
- attributes:
- key: ID
value: '3'
- key: Attr
value: Same
- attributes:
- key: ID
value: '4'
- key: Attr
value: Same
tests:
default CBF, no selector:
policy:
replicas:
- count: 2
containerBackupFactor: 0
selectors: []
filters: []
result:
- - 0
- 2
- 1
- 3
explicit CBF, no selector:
policy:
replicas:
- count: 2
containerBackupFactor: 3
selectors: []
filters: []
result:
- - 0
- 2
- 1
- 3
select distinct, weak CBF:
policy:
replicas:
- count: 2
selector: X
containerBackupFactor: 3
selectors:
- name: X
count: 2
clause: DISTINCT
filter: '*'
filters: []
result:
- - 0
- 2
- 1
- 3
select same, weak CBF:
policy:
replicas:
- count: 2
selector: X
containerBackupFactor: 3
selectors:
- name: X
count: 2
clause: SAME
attribute: Attr
filter: '*'
filters: []
result:
- - 0
- 1
- 2
- 3

210
netmap/yml_tests/filter_complex.yml Normal file
View file

@ -0,0 +1,210 @@
name: compound filter
nodes:
- attributes:
- key: Storage
value: SSD
- key: Rating
value: '10'
- key: IntField
value: '100'
- key: Param
value: Value1
tests:
good:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: StorageSSD
key: Storage
op: EQ
value: SSD
filters: []
- name: GoodRating
key: Rating
op: GE
value: '4'
filters: []
- name: Main
op: AND
filters:
- name: StorageSSD
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
key: IntField
fyrchik marked this conversation as resolved Outdated
fyrchik commented 2023-11-16 09:00:11 +00:00
Outdated
Review
Copy link

Can empty values be omitted?

Can empty values be omitted?
fyrchik commented 2023-11-16 09:00:12 +00:00
Outdated
Review
Copy link

Can empty values be omitted?

Can empty values be omitted?
achuprov commented 2023-11-16 12:32:07 +00:00
Outdated
Review
Copy link

fixed

fixed
op: LT
value: '123'
filters: []
- name: GoodRating
op: OPERATION_UNSPECIFIED
filters: []
- op: OR
filters:
- key: Param
op: EQ
value: Value1
filters: []
- key: Param
op: EQ
value: Value2
filters: []
result:
- - 0
bad storage type:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: StorageSSD
key: Storage
op: EQ
value: HDD
filters: []
- name: GoodRating
key: Rating
op: GE
value: '4'
filters: []
- name: Main
op: AND
filters:
- name: StorageSSD
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
key: IntField
op: LT
value: '123'
filters: []
- name: GoodRating
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
op: OR
filters:
- name: ''
key: Param
op: EQ
value: Value1
filters: []
- name: ''
key: Param
op: EQ
value: Value2
filters: []
error: not enough nodes
bad rating:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: StorageSSD
key: Storage
op: EQ
value: SSD
filters: []
- name: GoodRating
key: Rating
op: GE
value: '15'
filters: []
- name: Main
op: AND
filters:
- name: StorageSSD
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
key: IntField
op: LT
value: '123'
filters: []
- name: GoodRating
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
op: OR
filters:
- name: ''
key: Param
op: EQ
value: Value1
filters: []
- name: ''
key: Param
op: EQ
value: Value2
filters: []
error: not enough nodes
bad param:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: StorageSSD
key: Storage
op: EQ
value: SSD
filters: []
- name: GoodRating
key: Rating
op: GE
value: '4'
filters: []
- name: Main
op: AND
filters:
- name: StorageSSD
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
key: IntField
op: LT
value: '123'
filters: []
- name: GoodRating
op: OPERATION_UNSPECIFIED
filters: []
- name: ''
op: OR
filters:
- name: ''
key: Param
op: EQ
value: Value0
filters: []
- name: ''
key: Param
op: EQ
value: Value2
filters: []
error: not enough nodes

45
netmap/yml_tests/filter_invalid_integer.yml Normal file
View file

@ -0,0 +1,45 @@
name: invalid integer field
nodes:
- attributes:
- key: IntegerField
value: 'true'
- attributes:
- key: IntegerField
value: str
tests:
empty string is not casted to 0:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: IntegerField
op: LE
value: '8'
filters: []
error: not enough nodes
non-empty string is not casted to a number:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: IntegerField
op: GE
value: '0'
filters: []
error: not enough nodes

230
netmap/yml_tests/filter_simple.yml Normal file
View file

@ -0,0 +1,230 @@
name: single-op filters
nodes:
- attributes:
- key: Rating
value: '4'
- key: Country
value: Germany
tests:
GE true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: GE
value: '4'
filters: []
result:
- - 0
GE false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: GE
value: '5'
filters: []
error: not enough nodes
GT true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: GT
value: '3'
filters: []
result:
- - 0
GT false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: GT
value: '4'
filters: []
error: not enough nodes
LE true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: LE
value: '4'
filters: []
result:
- - 0
LE false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: LE
value: '3'
filters: []
error: not enough nodes
LT true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: LT
value: '5'
filters: []
result:
- - 0
LT false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Rating
op: LT
value: '4'
filters: []
error: not enough nodes
EQ true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Country
op: EQ
value: Germany
filters: []
result:
- - 0
EQ false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Country
op: EQ
value: China
filters: []
error: not enough nodes
NE true:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Country
op: NE
value: France
filters: []
result:
- - 0
NE false:
policy:
replicas:
- count: 1
selector: S
containerBackupFactor: 1
selectors:
- name: S
count: 1
clause: DISTINCT
filter: Main
filters:
- name: Main
key: Country
op: NE
value: Germany
filters: []
error: not enough nodes

118
netmap/yml_tests/hrw_sort.yml Normal file
View file

@ -0,0 +1,118 @@
name: HRW ordering
nodes:
- attributes:
- key: Country
value: Germany
- key: Price
value: '2'
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: Germany
- key: Price
value: '4'
- key: Capacity
value: '1'
- attributes:
- key: Country
value: France
- key: Price
value: '3'
- key: Capacity
value: '10'
- attributes:
- key: Country
value: Russia
- key: Price
value: '2'
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: Russia
- key: Price
value: '1'
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: Russia
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: France
- key: Price
value: '100'
- key: Capacity
value: '1'
- attributes:
- key: Country
value: France
- key: Price
value: '7'
- key: Capacity
value: '10000'
- attributes:
- key: Country
value: Russia
- key: Price
value: '2'
- key: Capacity
value: '1'
tests:
select 3 nodes in 3 distinct countries, same placement:
policy:
replicas:
- count: 1
selector: Main
containerBackupFactor: 1
selectors:
- name: Main
count: 3
clause: DISTINCT
attribute: Country
filter: '*'
filters: []
pivot: Y29udGFpbmVySUQ=
result:
- - 5
- 0
- 7
placement:
pivot: b2JqZWN0SUQ=
result:
- - 5
- 0
- 7
select 6 nodes in 3 distinct countries, different placement:
policy:
replicas:
- count: 1
selector: Main
containerBackupFactor: 2
selectors:
- name: Main
count: 3
clause: DISTINCT
attribute: Country
filter: '*'
filters: []
pivot: Y29udGFpbmVySUQ=
result:
- - 5
- 4
- 0
- 1
- 7
- 2
placement:
pivot: b2JqZWN0SUQ=
result:
- - 5
- 4
- 0
- 7
- 2
- 1

52
netmap/yml_tests/issue213.yml Normal file
View file

@ -0,0 +1,52 @@
name: unnamed selector (nspcc-dev/neofs-api-go#213)
nodes:
- attributes:
- key: Location
value: Europe
- key: Country
value: Russia
- key: City
value: Moscow
- attributes:
- key: Location
value: Europe
- key: Country
value: Russia
- key: City
value: Saint-Petersburg
- attributes:
- key: Location
value: Europe
- key: Country
value: Sweden
- key: City
value: Stockholm
- attributes:
- key: Location
value: Europe
- key: Country
value: Finalnd
- key: City
value: Helsinki
tests:
test:
policy:
replicas:
- count: 4
containerBackupFactor: 1
selectors:
- name: ''
count: 4
clause: DISTINCT
filter: LOC_EU
filters:
- name: LOC_EU
key: Location
op: EQ
value: Europe
filters: []
result:
- - 0
- 1
- 2
- 3

141
netmap/yml_tests/many_selects.yml Normal file
View file

@ -0,0 +1,141 @@
name: single-op filters
nodes:
- attributes:
- key: Country
value: Russia
- key: Rating
value: '1'
- key: City
value: SPB
- attributes:
- key: Country
value: Germany
- key: Rating
value: '5'
- key: City
value: Berlin
- attributes:
- key: Country
value: Russia
- key: Rating
value: '6'
- key: City
value: Moscow
- attributes:
- key: Country
value: France
- key: Rating
value: '4'
- key: City
value: Paris
- attributes:
- key: Country
value: France
- key: Rating
value: '1'
- key: City
value: Lyon
- attributes:
- key: Country
value: Russia
- key: Rating
value: '5'
- key: City
value: SPB
- attributes:
- key: Country
value: Russia
- key: Rating
value: '7'
- key: City
value: Moscow
- attributes:
- key: Country
value: Germany
- key: Rating
value: '3'
- key: City
value: Darmstadt
- attributes:
- key: Country
value: Germany
- key: Rating
value: '7'
- key: City
value: Frankfurt
- attributes:
- key: Country
value: Russia
- key: Rating
value: '9'
- key: City
value: SPB
- attributes:
- key: Country
value: Russia
- key: Rating
value: '9'
- key: City
value: SPB
tests:
Select:
policy:
replicas:
- count: 1
selector: SameRU
- count: 1
selector: DistinctRU
- count: 1
selector: Good
- count: 1
selector: Main
containerBackupFactor: 2
selectors:
- name: SameRU
count: 2
clause: SAME
attribute: City
filter: FromRU
- name: DistinctRU
count: 2
clause: DISTINCT
attribute: City
filter: FromRU
- name: Good
count: 2
clause: DISTINCT
attribute: Country
filter: Good
- name: Main
count: 3
clause: DISTINCT
attribute: Country
filter: '*'
filters:
- name: FromRU
key: Country
op: EQ
value: Russia
- name: Good
key: Rating
op: GE
value: '4'
result:
- - 0
- 5
- 9
- 10
- - 2
- 6
- 0
- 5
- - 1
- 8
- 2
- 5
- - 3
- 4
- 1
- 7
- 0
- 2

46
netmap/yml_tests/multiple_rep.yml Normal file
View file

@ -0,0 +1,46 @@
name: multiple replicas (#215)
nodes:
- attributes:
- key: City
value: Saint-Petersburg
- attributes:
- key: City
value: Moscow
- attributes:
- key: City
value: Berlin
- attributes:
- key: City
value: Paris
tests:
test:
policy:
replicas:
- count: 1
selector: LOC_SPB_PLACE
- count: 1
selector: LOC_MSK_PLACE
containerBackupFactor: 1
selectors:
- name: LOC_SPB_PLACE
count: 1
clause: CLAUSE_UNSPECIFIED
filter: LOC_SPB
- name: LOC_MSK_PLACE
count: 1
clause: CLAUSE_UNSPECIFIED
filter: LOC_MSK
filters:
- name: LOC_SPB
key: City
op: EQ
value: Saint-Petersburg
filters: []
- name: LOC_MSK
key: City
op: EQ
value: Moscow
filters: []
result:
- - 0
- - 1

162
netmap/yml_tests/multiple_rep_asymmetric.yml Normal file
View file

@ -0,0 +1,162 @@
name: multiple REP, asymmetric
nodes:
- attributes:
- key: ID
value: '1'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '0'
- attributes:
- key: ID
value: '2'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '1'
- attributes:
- key: ID
value: '3'
- key: Country
value: RU
- key: City
value: Moscow
- key: SSD
value: '1'
- attributes:
- key: ID
value: '4'
- key: Country
value: RU
- key: City
value: Moscow
- key: SSD
value: '1'
- attributes:
- key: ID
value: '5'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '1'
- attributes:
- key: ID
value: '6'
- key: Continent
value: NA
- key: City
value: NewYork
- attributes:
- key: ID
value: '7'
- key: Continent
value: AF
- key: City
value: Cairo
- attributes:
- key: ID
value: '8'
- key: Continent
value: AF
- key: City
value: Cairo
- attributes:
- key: ID
value: '9'
- key: Continent
value: SA
- key: City
value: Lima
- attributes:
- key: ID
value: '10'
- key: Continent
value: AF
- key: City
value: Cairo
- attributes:
- key: ID
value: '11'
- key: Continent
value: NA
- key: City
value: NewYork
- attributes:
- key: ID
value: '12'
- key: Continent
value: NA
- key: City
value: LosAngeles
- attributes:
- key: ID
value: '13'
- key: Continent
value: SA
- key: City
value: Lima
tests:
test:
policy:
replicas:
- count: 1
selector: SPB
- count: 2
selector: Americas
containerBackupFactor: 2
selectors:
- name: SPB
count: 1
clause: SAME
attribute: City
filter: SPBSSD
- name: Americas
count: 2
clause: DISTINCT
attribute: City
filter: Americas
filters:
- name: SPBSSD
op: AND
filters:
- name: ''
key: Country
op: EQ
value: RU
filters: []
- name: ''
key: City
op: EQ
value: St.Petersburg
filters: []
- name: ''
key: SSD
op: EQ
value: '1'
filters: []
- name: Americas
op: OR
filters:
- name: ''
key: Continent
op: EQ
value: NA
filters: []
- name: ''
key: Continent
op: EQ
value: SA
filters: []
result:
- - 1
- 4
- - 8
- 12
- 5
- 10

52
netmap/yml_tests/non_strict.yml Normal file
View file

@ -0,0 +1,52 @@
name: non-strict selections
comment: These test specify loose selection behaviour, to allow fetching already PUT
objects even when there is not enough nodes to select from.
nodes:
- attributes:
- key: Country
value: Russia
- attributes:
- key: Country
value: Germany
- attributes: []
tests:
not enough nodes (backup factor):
policy:
replicas:
- count: 1
selector: MyStore
containerBackupFactor: 2
selectors:
- name: MyStore
count: 2
clause: DISTINCT
attribute: Country
filter: FromRU
filters:
- name: FromRU
key: Country
op: EQ
value: Russia
filters: []
result:
- - 0
not enough nodes (buckets):
policy:
replicas:
- count: 1
selector: MyStore
containerBackupFactor: 1
selectors:
- name: MyStore
count: 2
clause: DISTINCT
attribute: Country
filter: FromRU
filters:
- name: FromRU
key: Country
op: EQ
value: Russia
filters: []
result:
- - 0

66
netmap/yml_tests/rep_only.yml Normal file
View file

@ -0,0 +1,66 @@
name: REP X
nodes:
- publicKey: ''
addresses: []
attributes:
- key: City
value: Saint-Petersburg
parents: []
state: UNSPECIFIED
- publicKey: ''
addresses: []
attributes:
- key: City
value: Moscow
parents: []
state: UNSPECIFIED
- publicKey: ''
addresses: []
attributes:
- key: City
value: Berlin
parents: []
state: UNSPECIFIED
- publicKey: ''
addresses: []
attributes:
- key: City
value: Paris
parents: []
state: UNSPECIFIED
tests:
REP 1:
policy:
replicas:
- count: 1
containerBackupFactor: 0
selectors: []
filters: []
result:
- - 0
- 1
- 2
REP 3:
policy:
replicas:
- count: 3
containerBackupFactor: 0
selectors: []
filters: []
result:
- - 0
- 3
- 1
- 2
REP 5:
policy:
replicas:
- count: 5
containerBackupFactor: 0
selectors: []
filters: []
result:
- - 0
- 1
- 2
- 3

56
netmap/yml_tests/select_no_attribute.yml Normal file
View file

@ -0,0 +1,56 @@
name: select with unspecified attribute
nodes:
- attributes:
- key: ID
value: '1'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '0'
- attributes:
- key: ID
value: '2'
- key: Country
value: RU
- key: City
value: St.Petersburg
- key: SSD
value: '1'
- attributes:
- key: ID
value: '3'
- key: Country
value: RU
- key: City
value: Moscow
- key: SSD
value: '1'
- attributes:
- key: ID
value: '4'
- key: Country
value: RU
- key: City
value: Moscow
- key: SSD
value: '1'
tests:
test:
policy:
replicas:
- count: 1
selector: X
containerBackupFactor: 1
selectors:
- name: X
count: 4
clause: DISTINCT
filter: '*'
filters: []
result:
- - 0
- 1
- 2
- 3

48
netmap/yml_tests/selector_invalid.yml Normal file
View file

@ -0,0 +1,48 @@
name: invalid selections
nodes:
- attributes:
- key: Country
value: Russia
- attributes:
- key: Country
value: Germany
- attributes: []
tests:
missing filter:
policy:
replicas:
- count: 1
selector: MyStore
containerBackupFactor: 1
selectors:
- name: MyStore
count: 1
clause: DISTINCT
attribute: Country
filter: FromNL
filters:
- name: FromRU
key: Country
op: EQ
value: Russia
filters: []
error: filter not found
not enough nodes (filter results in empty set):
policy:
replicas:
- count: 1
selector: MyStore
containerBackupFactor: 2
selectors:
- name: MyStore
count: 2
clause: DISTINCT
attribute: Country
filter: FromMoon
filters:
- name: FromMoon
key: Country
op: EQ
value: Moon
filters: []
error: not enough nodes

142
netmap/yml_unmarshal.go Normal file
View file

@ -0,0 +1,142 @@
package netmap
import (
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
)
type tempPlacementPolicy struct {
BackupFactor uint32 `yaml:"containerBackupFactor"`
Filters []tempFilter `yaml:"filters"`
Selectors []tempSelector `yaml:"selectors"`
Replicas []tempReplica `yaml:"replicas"`
Unique bool `yaml:"unique"`
}
type tempFilter struct {
Name string `yaml:"name"`
Key string `yaml:"key"`
Op string `yaml:"op"`
Value string `yaml:"value"`
Filters []tempFilter `yaml:"filters"`
}
type tempSelector struct {
Name string `yaml:"name"`
Count uint32 `yaml:"count"`
Clause string `yaml:"clause"`
Attribute string `yaml:"attribute"`
Filter string `yaml:"filter"`
}
type tempReplica struct {
Count uint32 `yaml:"count"`
Selector string `yaml:"selector"`
}
func convertNFilters(temp []tempFilter) []netmap.Filter {
var filters []netmap.Filter
for _, tf := range temp {
filters = append(filters, convertNFilter(tf))
}
return filters
}
var stringToOperationMap = map[string]netmap.Operation{
"EQ": netmap.EQ,
"NE": netmap.NE,
acid-ant commented 2023-11-15 10:42:13 +00:00
Outdated
Review
Copy link

How about to initialize map out of the function and just take the value from it?

How about to initialize map out of the function and just take the value from it?
"GT": netmap.GT,
"GE": netmap.GE,
"LT": netmap.LT,
"LE": netmap.LE,
"OR": netmap.OR,
"AND": netmap.AND,
"NOT": netmap.NOT,
}
func convertStringToOperation(opStr string) netmap.Operation {
opStr = strings.ToUpper(opStr)
if op, exists := stringToOperationMap[opStr]; exists {
return op
}
return netmap.UnspecifiedOperation
}
func convertStringToClause(clauseStr string) netmap.Clause {
switch strings.ToUpper(clauseStr) {
case "DISTINCT":
return netmap.Distinct
default:
return netmap.Same
}
}
func convertNFilter(temp tempFilter) netmap.Filter {
filter := netmap.Filter{}
filter.SetKey(temp.Key)
filter.SetName(temp.Name)
filter.SetValue(temp.Value)
filter.SetOp(convertStringToOperation(temp.Op))
if temp.Filters != nil {
filter.SetFilters(convertNFilters(temp.Filters))
}
return filter
}
func (p *PlacementPolicy) UnmarshalYAML(unmarshal func(interface{}) error) error {
var temp tempPlacementPolicy
if err := unmarshal(&temp); err != nil {
return err
}
for _, ts := range temp.Filters {
netmapFilters := convertNFilter(ts)
p.AddFilters(Filter{m: netmapFilters})
}
for _, ts := range temp.Selectors {
selector := Selector{}
selector.SetName(ts.Name)
selector.SetNumberOfNodes(ts.Count)
selector.SetClause(convertStringToClause(ts.Clause))
selector.SelectByBucketAttribute(ts.Attribute)
selector.SetFilterName(ts.Filter)
p.AddSelectors(selector)
}
for _, tr := range temp.Replicas {
replica := ReplicaDescriptor{}
replica.SetSelectorName(tr.Selector)
replica.m.SetCount(tr.Count)
p.AddReplicas(replica)
}
p.SetContainerBackupFactor(temp.BackupFactor)
p.SetUnique(temp.Unique)
return nil
}
type Attribute struct {
Key string `yaml:"key"`
Value string `yaml:"value"`
}
type tempNode struct {
Attributes []Attribute `yaml:"attributes"`
}
func (x *NodeInfo) UnmarshalYAML(unmarshal func(interface{}) error) error {
var temp tempNode
if err := unmarshal(&temp); err != nil {
return err
}
for _, atr := range temp.Attributes {
x.SetAttribute(atr.Key, atr.Value)
}
return nil
}