From dea8759762d094fe84ce78e1b4601b19c4b87ab1 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 16 Nov 2023 15:14:25 +0300 Subject: [PATCH 1/2] [#166] netmap: Move tests from JSON to YML Signed-off-by: Alexander Chuprov --- netmap/json_tests/cbf_default.json | 99 ----- netmap/json_tests/cbf_minimal.json | 99 ----- netmap/json_tests/cbf_requirements.json | 155 ------- netmap/json_tests/filter_complex.json | 383 ----------------- netmap/json_tests/filter_invalid_integer.json | 83 ---- netmap/json_tests/filter_simple.json | 403 ------------------ netmap/json_tests/hrw_sort.json | 165 ------- netmap/json_tests/issue213.json | 108 ----- netmap/json_tests/many_selects.json | 192 --------- netmap/json_tests/multiple_rep.json | 94 ---- .../json_tests/multiple_rep_asymmetric.json | 331 -------------- netmap/json_tests/non_strict.json | 95 ----- netmap/json_tests/rep_only.json | 117 ----- netmap/json_tests/select_no_attribute.json | 116 ----- netmap/json_tests/selector_invalid.json | 86 ---- netmap/yml_tests/cbf_default.yml | 48 +++ netmap/yml_tests/cbf_minimal.yml | 52 +++ netmap/yml_tests/cbf_requirements.yml | 82 ++++ netmap/yml_tests/filter_complex.yml | 210 +++++++++ netmap/yml_tests/filter_invalid_integer.yml | 45 ++ netmap/yml_tests/filter_simple.yml | 230 ++++++++++ netmap/yml_tests/hrw_sort.yml | 118 +++++ netmap/yml_tests/issue213.yml | 52 +++ netmap/yml_tests/many_selects.yml | 141 ++++++ netmap/yml_tests/multiple_rep.yml | 46 ++ netmap/yml_tests/multiple_rep_asymmetric.yml | 162 +++++++ netmap/yml_tests/non_strict.yml | 52 +++ netmap/yml_tests/rep_only.yml | 66 +++ netmap/yml_tests/select_no_attribute.yml | 56 +++ netmap/yml_tests/selector_invalid.yml | 48 +++ 30 files changed, 1408 insertions(+), 2526 deletions(-) delete mode 100644 netmap/json_tests/cbf_default.json delete mode 100644 netmap/json_tests/cbf_minimal.json delete mode 100644 netmap/json_tests/cbf_requirements.json delete mode 100644 netmap/json_tests/filter_complex.json delete mode 100644 netmap/json_tests/filter_invalid_integer.json delete mode 100644 netmap/json_tests/filter_simple.json delete mode 100644 netmap/json_tests/hrw_sort.json delete mode 100644 netmap/json_tests/issue213.json delete mode 100644 netmap/json_tests/many_selects.json delete mode 100644 netmap/json_tests/multiple_rep.json delete mode 100644 netmap/json_tests/multiple_rep_asymmetric.json delete mode 100644 netmap/json_tests/non_strict.json delete mode 100644 netmap/json_tests/rep_only.json delete mode 100644 netmap/json_tests/select_no_attribute.json delete mode 100644 netmap/json_tests/selector_invalid.json create mode 100644 netmap/yml_tests/cbf_default.yml create mode 100644 netmap/yml_tests/cbf_minimal.yml create mode 100644 netmap/yml_tests/cbf_requirements.yml create mode 100644 netmap/yml_tests/filter_complex.yml create mode 100644 netmap/yml_tests/filter_invalid_integer.yml create mode 100644 netmap/yml_tests/filter_simple.yml create mode 100644 netmap/yml_tests/hrw_sort.yml create mode 100644 netmap/yml_tests/issue213.yml create mode 100644 netmap/yml_tests/many_selects.yml create mode 100644 netmap/yml_tests/multiple_rep.yml create mode 100644 netmap/yml_tests/multiple_rep_asymmetric.yml create mode 100644 netmap/yml_tests/non_strict.yml create mode 100644 netmap/yml_tests/rep_only.yml create mode 100644 netmap/yml_tests/select_no_attribute.yml create mode 100644 netmap/yml_tests/selector_invalid.yml diff --git a/netmap/json_tests/cbf_default.json b/netmap/json_tests/cbf_default.json deleted file mode 100644 index 5a5dbc0..0000000 --- a/netmap/json_tests/cbf_default.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "name": "default CBF is 3", - "nodes": [ - { - "attributes": [ - { - "key": "Location", - "value": "Europe" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "St.Petersburg" - } - ] - }, - { - "attributes": [ - { - "key": "Location", - "value": "Europe" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "Moscow" - } - ] - }, - { - "attributes": [ - { - "key": "Location", - "value": "Europe" - }, - { - "key": "Country", - "value": "DE" - }, - { - "key": "City", - "value": "Berlin" - } - ] - }, - { - "attributes": [ - { - "key": "Location", - "value": "Europe" - }, - { - "key": "Country", - "value": "FR" - }, - { - "key": "City", - "value": "Paris" - } - ] - } - ], - "tests": { - "set default CBF": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "EU" - } - ], - "containerBackupFactor": 0, - "selectors": [ - { - "name": "EU", - "count": 1, - "clause": "SAME", - "attribute": "Location", - "filter": "*" - } - ], - "filters": [] - }, - "result": [ - [ - 0, - 1, - 2 - ] - ] - } - } -} diff --git a/netmap/json_tests/cbf_minimal.json b/netmap/json_tests/cbf_minimal.json deleted file mode 100644 index 9041b72..0000000 --- a/netmap/json_tests/cbf_minimal.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "name": "Real node count multiplier is in range [1, specified CBF]", - "nodes": [ - { - "attributes": [ - { - "key": "ID", - "value": "1" - }, - { - "key": "Country", - "value": "DE" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "2" - }, - { - "key": "Country", - "value": "DE" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "3" - }, - { - "key": "Country", - "value": "DE" - } - ] - } - ], - "tests": { - "select 2, CBF is 2": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "X" - } - ], - "containerBackupFactor": 2, - "selectors": [ - { - "name": "X", - "count": 2, - "clause": "SAME", - "attribute": "Country", - "filter": "*" - } - ], - "filters": [] - }, - "result": [ - [ - 0, - 1, - 2 - ] - ] - }, - "select 3, CBF is 2": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "X" - } - ], - "containerBackupFactor": 2, - "selectors": [ - { - "name": "X", - "count": 3, - "clause": "SAME", - "attribute": "Country", - "filter": "*" - } - ], - "filters": [] - }, - "result": [ - [ - 0, - 1, - 2 - ] - ] - } - } -} diff --git a/netmap/json_tests/cbf_requirements.json b/netmap/json_tests/cbf_requirements.json deleted file mode 100644 index c63358f..0000000 --- a/netmap/json_tests/cbf_requirements.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "name": "CBF requirements", - "nodes": [ - { - "attributes": [ - { - "key": "ID", - "value": "1" - }, - { - "key": "Attr", - "value": "Same" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "2" - }, - { - "key": "Attr", - "value": "Same" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "3" - }, - { - "key": "Attr", - "value": "Same" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "4" - }, - { - "key": "Attr", - "value": "Same" - } - ] - } - ], - "tests": { - "default CBF, no selector": { - "policy": { - "replicas": [ - { - "count": 2, - "selector": "" - } - ], - "containerBackupFactor": 0, - "selectors": [], - "filters": [] - }, - "result": [ - [ - 0, - 2, - 1, - 3 - ] - ] - }, - "explicit CBF, no selector": { - "policy": { - "replicas": [ - { - "count": 2, - "selector": "" - } - ], - "containerBackupFactor": 3, - "selectors": [], - "filters": [] - }, - "result": [ - [ - 0, - 2, - 1, - 3 - ] - ] - }, - "select distinct, weak CBF": { - "policy": { - "replicas": [ - { - "count": 2, - "selector": "X" - } - ], - "containerBackupFactor": 3, - "selectors": [ - { - "name": "X", - "count": 2, - "clause": "DISTINCT", - "attribute": "", - "filter": "*" - } - ], - "filters": [] - }, - "result": [ - [ - 0, - 2, - 1, - 3 - ] - ] - }, - "select same, weak CBF": { - "policy": { - "replicas": [ - { - "count": 2, - "selector": "X" - } - ], - "containerBackupFactor": 3, - "selectors": [ - { - "name": "X", - "count": 2, - "clause": "SAME", - "attribute": "Attr", - "filter": "*" - } - ], - "filters": [] - }, - "result": [ - [ - 0, - 1, - 2, - 3 - ] - ] - } - } -} diff --git a/netmap/json_tests/filter_complex.json b/netmap/json_tests/filter_complex.json deleted file mode 100644 index 2d0583c..0000000 --- a/netmap/json_tests/filter_complex.json +++ /dev/null @@ -1,383 +0,0 @@ -{ - "name": "compound filter", - "nodes": [ - { - "attributes": [ - { - "key": "Storage", - "value": "SSD" - }, - { - "key": "Rating", - "value": "10" - }, - { - "key": "IntField", - "value": "100" - }, - { - "key": "Param", - "value": "Value1" - } - ] - } - ], - "tests": { - "good": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "StorageSSD", - "key": "Storage", - "op": "EQ", - "value": "SSD", - "filters": [] - }, - { - "name": "GoodRating", - "key": "Rating", - "op": "GE", - "value": "4", - "filters": [] - }, - { - "name": "Main", - "key": "", - "op": "AND", - "value": "", - "filters": [ - { - "name": "StorageSSD", - "key": "", - "op": "OPERATION_UNSPECIFIED", - "value": "", - "filters": [] - }, - { - "name": "", - "key": "IntField", - "op": "LT", - "value": "123", - "filters": [] - }, - { - "name": "GoodRating", - "key": "", - "op": "OPERATION_UNSPECIFIED", - "value": "", - "filters": [] - }, - { - "name": "", - "key": "", - "op": "OR", - "value": "", - "filters": [ - { - "name": "", - "key": "Param", - "op": "EQ", - "value": "Value1", - "filters": [] - }, - { - "name": "", - "key": "Param", - "op": "EQ", - "value": "Value2", - "filters": [] - } - ] - } - ] - } - ] - }, - "result": [ - [ - 0 - ] - ] - }, - "bad storage type": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "StorageSSD", - "key": "Storage", - "op": "EQ", - "value": "HDD", - "filters": [] - }, - { - "name": "GoodRating", - "key": "Rating", - "op": "GE", - "value": "4", - "filters": [] - }, - { - "name": "Main", - "key": "", - "op": "AND", - "value": "", - "filters": [ - { - "name": "StorageSSD", - "key": "", - "op": "OPERATION_UNSPECIFIED", - "value": "", - "filters": [] - }, - { - "name": "", - "key": "IntField", - "op": "LT", - "value": "123", - "filters": [] - }, - { - "name": "GoodRating", - "key": "", - "op": "OPERATION_UNSPECIFIED", - "value": "", - "filters": [] - }, - { - "name": "", - "key": "", - "op": "OR", - "value": "", - "filters": [ - { - "name": "", - "key": "Param", - "op": "EQ", - "value": "Value1", - "filters": [] - }, - { - "name": "", - "key": "Param", - "op": "EQ", - "value": "Value2", - "filters": [] - } - ] - } - ] - } - ] - }, - "error": "not enough nodes" - }, - "bad rating": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "StorageSSD", - "key": "Storage", - "op": "EQ", - "value": "SSD", - "filters": [] - }, - { - "name": "GoodRating", - "key": "Rating", - "op": "GE", - "value": "15", - "filters": [] - }, - { - "name": "Main", - "key": "", - "op": "AND", - "value": "", - "filters": [ - { - "name": "StorageSSD", - "key": "", - "op": "OPERATION_UNSPECIFIED", - "value": "", - "filters": [] - }, - { - "name": "", - "key": "IntField", - "op": "LT", - "value": "123", - "filters": [] - }, - { - "name": "GoodRating", - "key": "", - "op": "OPERATION_UNSPECIFIED", - "value": "", - "filters": [] - }, - { - "name": "", - "key": "", - "op": "OR", - "value": "", - "filters": [ - { - "name": "", - "key": "Param", - "op": "EQ", - "value": "Value1", - "filters": [] - }, - { - "name": "", - "key": "Param", - "op": "EQ", - "value": "Value2", - "filters": [] - } - ] - } - ] - } - ] - }, - "error": "not enough nodes" - }, - "bad param": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "StorageSSD", - "key": "Storage", - "op": "EQ", - "value": "SSD", - "filters": [] - }, - { - "name": "GoodRating", - "key": "Rating", - "op": "GE", - "value": "4", - "filters": [] - }, - { - "name": "Main", - "key": "", - "op": "AND", - "value": "", - "filters": [ - { - "name": "StorageSSD", - "key": "", - "op": "OPERATION_UNSPECIFIED", - "value": "", - "filters": [] - }, - { - "name": "", - "key": "IntField", - "op": "LT", - "value": "123", - "filters": [] - }, - { - "name": "GoodRating", - "key": "", - "op": "OPERATION_UNSPECIFIED", - "value": "", - "filters": [] - }, - { - "name": "", - "key": "", - "op": "OR", - "value": "", - "filters": [ - { - "name": "", - "key": "Param", - "op": "EQ", - "value": "Value0", - "filters": [] - }, - { - "name": "", - "key": "Param", - "op": "EQ", - "value": "Value2", - "filters": [] - } - ] - } - ] - } - ] - }, - "error": "not enough nodes" - } - } -} diff --git a/netmap/json_tests/filter_invalid_integer.json b/netmap/json_tests/filter_invalid_integer.json deleted file mode 100644 index c2a5898..0000000 --- a/netmap/json_tests/filter_invalid_integer.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "name": "invalid integer field", - "nodes": [ - { - "attributes": [ - { - "key": "IntegerField", - "value": "true" - } - ] - }, - { - "attributes": [ - { - "key": "IntegerField", - "value": "str" - } - ] - } - ], - "tests": { - "empty string is not casted to 0": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "IntegerField", - "op": "LE", - "value": "8", - "filters": [] - } - ] - }, - "error": "not enough nodes" - }, - "non-empty string is not casted to a number": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "IntegerField", - "op": "GE", - "value": "0", - "filters": [] - } - ] - }, - "error": "not enough nodes" - } - } -} diff --git a/netmap/json_tests/filter_simple.json b/netmap/json_tests/filter_simple.json deleted file mode 100644 index f54b217..0000000 --- a/netmap/json_tests/filter_simple.json +++ /dev/null @@ -1,403 +0,0 @@ -{ - "name": "single-op filters", - "nodes": [ - { - "attributes": [ - { - "key": "Rating", - "value": "4" - }, - { - "key": "Country", - "value": "Germany" - } - ] - } - ], - "tests": { - "GE true": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Rating", - "op": "GE", - "value": "4", - "filters": [] - } - ] - }, - "result": [ - [ - 0 - ] - ] - }, - "GE false": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Rating", - "op": "GE", - "value": "5", - "filters": [] - } - ] - }, - "error": "not enough nodes" - }, - "GT true": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Rating", - "op": "GT", - "value": "3", - "filters": [] - } - ] - }, - "result": [ - [ - 0 - ] - ] - }, - "GT false": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Rating", - "op": "GT", - "value": "4", - "filters": [] - } - ] - }, - "error": "not enough nodes" - }, - "LE true": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Rating", - "op": "LE", - "value": "4", - "filters": [] - } - ] - }, - "result": [ - [ - 0 - ] - ] - }, - "LE false": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Rating", - "op": "LE", - "value": "3", - "filters": [] - } - ] - }, - "error": "not enough nodes" - }, - "LT true": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Rating", - "op": "LT", - "value": "5", - "filters": [] - } - ] - }, - "result": [ - [ - 0 - ] - ] - }, - "LT false": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Rating", - "op": "LT", - "value": "4", - "filters": [] - } - ] - }, - "error": "not enough nodes" - }, - "EQ true": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Country", - "op": "EQ", - "value": "Germany", - "filters": [] - } - ] - }, - "result": [ - [ - 0 - ] - ] - }, - "EQ false": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Country", - "op": "EQ", - "value": "China", - "filters": [] - } - ] - }, - "error": "not enough nodes" - }, - "NE true": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Country", - "op": "NE", - "value": "France", - "filters": [] - } - ] - }, - "result": [ - [ - 0 - ] - ] - }, - "NE false": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "S" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "S", - "count": 1, - "clause": "DISTINCT", - "attribute": "", - "filter": "Main" - } - ], - "filters": [ - { - "name": "Main", - "key": "Country", - "op": "NE", - "value": "Germany", - "filters": [] - } - ] - }, - "error": "not enough nodes" - } - } -} diff --git a/netmap/json_tests/hrw_sort.json b/netmap/json_tests/hrw_sort.json deleted file mode 100644 index 49bdf2d..0000000 --- a/netmap/json_tests/hrw_sort.json +++ /dev/null @@ -1,165 +0,0 @@ -{ - "name": "HRW ordering", - "nodes": [ - { - "attributes": [ - { - "key": "Country", - "value": "Germany" - }, - { - "key": "Price", - "value": "2" - }, - { - "key": "Capacity", - "value": "10000" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Germany" - }, - { - "key": "Price", - "value": "4" - }, - { - "key": "Capacity", - "value": "1" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "France" - }, - { - "key": "Price", - "value": "3" - }, - { - "key": "Capacity", - "value": "10" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Price", - "value": "2" - }, - { - "key": "Capacity", - "value": "10000" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Price", - "value": "1" - }, - { - "key": "Capacity", - "value": "10000" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Capacity", - "value": "10000" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "France" - }, - { - "key": "Price", - "value": "100" - }, - { - "key": "Capacity", - "value": "1" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "France" - }, - { - "key": "Price", - "value": "7" - }, - { - "key": "Capacity", - "value": "10000" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Price", - "value": "2" - }, - { - "key": "Capacity", - "value": "1" - } - ] - } - ], - "tests": { - "select 3 nodes in 3 distinct countries, same placement": { - "policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]}, - "pivot": "Y29udGFpbmVySUQ=", - "result": [ [ 5, 0, 7 ] ], - "placement": { - "pivot": "b2JqZWN0SUQ=", - "result": [ [ 5, 0, 7 ] ] - } - }, - "select 6 nodes in 3 distinct countries, different placement": { - "policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]}, - "pivot": "Y29udGFpbmVySUQ=", - "result": [ [ 5, 4, 0, 1, 7, 2 ] ], - "placement": { - "pivot": "b2JqZWN0SUQ=", - "result": [ [ 5, 4, 0, 7, 2, 1 ] ] - } - } - } -} diff --git a/netmap/json_tests/issue213.json b/netmap/json_tests/issue213.json deleted file mode 100644 index 0e25779..0000000 --- a/netmap/json_tests/issue213.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "name": "unnamed selector (nspcc-dev/neofs-api-go#213)", - "nodes": [ - { - "attributes": [ - { - "key": "Location", - "value": "Europe" - }, - { - "key": "Country", - "value": "Russia" - }, - { - "key": "City", - "value": "Moscow" - } - ] - }, - { - "attributes": [ - { - "key": "Location", - "value": "Europe" - }, - { - "key": "Country", - "value": "Russia" - }, - { - "key": "City", - "value": "Saint-Petersburg" - } - ] - }, - { - "attributes": [ - { - "key": "Location", - "value": "Europe" - }, - { - "key": "Country", - "value": "Sweden" - }, - { - "key": "City", - "value": "Stockholm" - } - ] - }, - { - "attributes": [ - { - "key": "Location", - "value": "Europe" - }, - { - "key": "Country", - "value": "Finalnd" - }, - { - "key": "City", - "value": "Helsinki" - } - ] - } - ], - "tests": { - "test": { - "policy": { - "replicas": [ - { - "count": 4, - "selector": "" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "", - "count": 4, - "clause": "DISTINCT", - "attribute": "", - "filter": "LOC_EU" - } - ], - "filters": [ - { - "name": "LOC_EU", - "key": "Location", - "op": "EQ", - "value": "Europe", - "filters": [] - } - ] - }, - "result": [ - [ - 0, - 1, - 2, - 3 - ] - ] - } - } -} diff --git a/netmap/json_tests/many_selects.json b/netmap/json_tests/many_selects.json deleted file mode 100644 index cbe96b1..0000000 --- a/netmap/json_tests/many_selects.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "name": "single-op filters", - "nodes": [ - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Rating", - "value": "1" - }, - { - "key": "City", - "value": "SPB" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Germany" - }, - { - "key": "Rating", - "value": "5" - }, - { - "key": "City", - "value": "Berlin" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Rating", - "value": "6" - }, - { - "key": "City", - "value": "Moscow" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "France" - }, - { - "key": "Rating", - "value": "4" - }, - { - "key": "City", - "value": "Paris" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "France" - }, - { - "key": "Rating", - "value": "1" - }, - { - "key": "City", - "value": "Lyon" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Rating", - "value": "5" - }, - { - "key": "City", - "value": "SPB" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Rating", - "value": "7" - }, - { - "key": "City", - "value": "Moscow" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Germany" - }, - { - "key": "Rating", - "value": "3" - }, - { - "key": "City", - "value": "Darmstadt" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Germany" - }, - { - "key": "Rating", - "value": "7" - }, - { - "key": "City", - "value": "Frankfurt" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Rating", - "value": "9" - }, - { - "key": "City", - "value": "SPB" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - }, - { - "key": "Rating", - "value": "9" - }, - { - "key": "City", - "value": "SPB" - } - ] - } - ], - "tests": { - "Select": { - "policy": {"replicas":[{"count":1,"selector":"SameRU"},{"count":1,"selector":"DistinctRU"},{"count":1,"selector":"Good"},{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"SameRU","count":2,"clause":"SAME","attribute":"City","filter":"FromRU"},{"name":"DistinctRU","count":2,"clause":"DISTINCT","attribute":"City","filter":"FromRU"},{"name":"Good","count":2,"clause":"DISTINCT","attribute":"Country","filter":"Good"},{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[{"name":"FromRU","key":"Country","op":"EQ","value":"Russia"},{"name":"Good","key":"Rating","op":"GE","value":"4"}]}, - "result": [ - [0, 5, 9, 10], - [2, 6, 0, 5], - [1, 8, 2, 5], - [3, 4, 1, 7, 0, 2] - ] - } - } -} diff --git a/netmap/json_tests/multiple_rep.json b/netmap/json_tests/multiple_rep.json deleted file mode 100644 index 97ae4d1..0000000 --- a/netmap/json_tests/multiple_rep.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "name": "multiple replicas (#215)", - "nodes": [ - { - "attributes": [ - { - "key": "City", - "value": "Saint-Petersburg" - } - ] - }, - { - "attributes": [ - { - "key": "City", - "value": "Moscow" - } - ] - }, - { - "attributes": [ - { - "key": "City", - "value": "Berlin" - } - ] - }, - { - "attributes": [ - { - "key": "City", - "value": "Paris" - } - ] - } - ], - "tests": { - "test": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "LOC_SPB_PLACE" - }, - { - "count": 1, - "selector": "LOC_MSK_PLACE" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "LOC_SPB_PLACE", - "count": 1, - "clause": "CLAUSE_UNSPECIFIED", - "attribute": "", - "filter": "LOC_SPB" - }, - { - "name": "LOC_MSK_PLACE", - "count": 1, - "clause": "CLAUSE_UNSPECIFIED", - "attribute": "", - "filter": "LOC_MSK" - } - ], - "filters": [ - { - "name": "LOC_SPB", - "key": "City", - "op": "EQ", - "value": "Saint-Petersburg", - "filters": [] - }, - { - "name": "LOC_MSK", - "key": "City", - "op": "EQ", - "value": "Moscow", - "filters": [] - } - ] - }, - "result": [ - [ - 0 - ], - [ - 1 - ] - ] - } - } -} diff --git a/netmap/json_tests/multiple_rep_asymmetric.json b/netmap/json_tests/multiple_rep_asymmetric.json deleted file mode 100644 index 8454ffd..0000000 --- a/netmap/json_tests/multiple_rep_asymmetric.json +++ /dev/null @@ -1,331 +0,0 @@ -{ - "name": "multiple REP, asymmetric", - "nodes": [ - { - "attributes": [ - { - "key": "ID", - "value": "1" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "St.Petersburg" - }, - { - "key": "SSD", - "value": "0" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "2" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "St.Petersburg" - }, - { - "key": "SSD", - "value": "1" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "3" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "Moscow" - }, - { - "key": "SSD", - "value": "1" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "4" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "Moscow" - }, - { - "key": "SSD", - "value": "1" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "5" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "St.Petersburg" - }, - { - "key": "SSD", - "value": "1" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "6" - }, - { - "key": "Continent", - "value": "NA" - }, - { - "key": "City", - "value": "NewYork" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "7" - }, - { - "key": "Continent", - "value": "AF" - }, - { - "key": "City", - "value": "Cairo" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "8" - }, - { - "key": "Continent", - "value": "AF" - }, - { - "key": "City", - "value": "Cairo" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "9" - }, - { - "key": "Continent", - "value": "SA" - }, - { - "key": "City", - "value": "Lima" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "10" - }, - { - "key": "Continent", - "value": "AF" - }, - { - "key": "City", - "value": "Cairo" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "11" - }, - { - "key": "Continent", - "value": "NA" - }, - { - "key": "City", - "value": "NewYork" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "12" - }, - { - "key": "Continent", - "value": "NA" - }, - { - "key": "City", - "value": "LosAngeles" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "13" - }, - { - "key": "Continent", - "value": "SA" - }, - { - "key": "City", - "value": "Lima" - } - ] - } - ], - "tests": { - "test": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "SPB" - }, - { - "count": 2, - "selector": "Americas" - } - ], - "containerBackupFactor": 2, - "selectors": [ - { - "name": "SPB", - "count": 1, - "clause": "SAME", - "attribute": "City", - "filter": "SPBSSD" - }, - { - "name": "Americas", - "count": 2, - "clause": "DISTINCT", - "attribute": "City", - "filter": "Americas" - } - ], - "filters": [ - { - "name": "SPBSSD", - "key": "", - "op": "AND", - "value": "", - "filters": [ - { - "name": "", - "key": "Country", - "op": "EQ", - "value": "RU", - "filters": [] - }, - { - "name": "", - "key": "City", - "op": "EQ", - "value": "St.Petersburg", - "filters": [] - }, - { - "name": "", - "key": "SSD", - "op": "EQ", - "value": "1", - "filters": [] - } - ] - }, - { - "name": "Americas", - "key": "", - "op": "OR", - "value": "", - "filters": [ - { - "name": "", - "key": "Continent", - "op": "EQ", - "value": "NA", - "filters": [] - }, - { - "name": "", - "key": "Continent", - "op": "EQ", - "value": "SA", - "filters": [] - } - ] - } - ] - }, - "result": [ - [ - 1, - 4 - ], - [ - 8, - 12, - 5, - 10 - ] - ] - } - } -} diff --git a/netmap/json_tests/non_strict.json b/netmap/json_tests/non_strict.json deleted file mode 100644 index 2d564d7..0000000 --- a/netmap/json_tests/non_strict.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "name": "non-strict selections", - "comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.", - "nodes": [ - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Germany" - } - ] - }, - { - "attributes": [ ] - } - ], - "tests": { - "not enough nodes (backup factor)": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "MyStore" - } - ], - "containerBackupFactor": 2, - "selectors": [ - { - "name": "MyStore", - "count": 2, - "clause": "DISTINCT", - "attribute": "Country", - "filter": "FromRU" - } - ], - "filters": [ - { - "name": "FromRU", - "key": "Country", - "op": "EQ", - "value": "Russia", - "filters": [ ] - } - ] - }, - "result": [ - [ - 0 - ] - ] - }, - "not enough nodes (buckets)": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "MyStore" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "MyStore", - "count": 2, - "clause": "DISTINCT", - "attribute": "Country", - "filter": "FromRU" - } - ], - "filters": [ - { - "name": "FromRU", - "key": "Country", - "op": "EQ", - "value": "Russia", - "filters": [ ] - } - ] - }, - "result": [ - [ - 0 - ] - ] - } - } -} diff --git a/netmap/json_tests/rep_only.json b/netmap/json_tests/rep_only.json deleted file mode 100644 index db37e62..0000000 --- a/netmap/json_tests/rep_only.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "name": "REP X", - "nodes": [ - { - "publicKey": "", - "addresses": [], - "attributes": [ - { - "key": "City", - "value": "Saint-Petersburg", - "parents": [] - } - ], - "state": "UNSPECIFIED" - }, - { - "publicKey": "", - "addresses": [], - "attributes": [ - { - "key": "City", - "value": "Moscow", - "parents": [] - } - ], - "state": "UNSPECIFIED" - }, - { - "publicKey": "", - "addresses": [], - "attributes": [ - { - "key": "City", - "value": "Berlin", - "parents": [] - } - ], - "state": "UNSPECIFIED" - }, - { - "publicKey": "", - "addresses": [], - "attributes": [ - { - "key": "City", - "value": "Paris", - "parents": [] - } - ], - "state": "UNSPECIFIED" - } - ], - "tests": { - "REP 1": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "" - } - ], - "containerBackupFactor": 0, - "selectors": [], - "filters": [] - }, - "result": [ - [ - 0, - 1, - 2 - ] - ] - }, - "REP 3": { - "policy": { - "replicas": [ - { - "count": 3, - "selector": "" - } - ], - "containerBackupFactor": 0, - "selectors": [], - "filters": [] - }, - "result": [ - [ - 0, - 3, - 1, - 2 - ] - ] - }, - "REP 5": { - "policy": { - "replicas": [ - { - "count": 5, - "selector": "" - } - ], - "containerBackupFactor": 0, - "selectors": [], - "filters": [] - }, - "result": [ - [ - 0, - 1, - 2, - 3 - ] - ] - } - } -} diff --git a/netmap/json_tests/select_no_attribute.json b/netmap/json_tests/select_no_attribute.json deleted file mode 100644 index 2f14f31..0000000 --- a/netmap/json_tests/select_no_attribute.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "name": "select with unspecified attribute", - "nodes": [ - { - "attributes": [ - { - "key": "ID", - "value": "1" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "St.Petersburg" - }, - { - "key": "SSD", - "value": "0" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "2" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "St.Petersburg" - }, - { - "key": "SSD", - "value": "1" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "3" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "Moscow" - }, - { - "key": "SSD", - "value": "1" - } - ] - }, - { - "attributes": [ - { - "key": "ID", - "value": "4" - }, - { - "key": "Country", - "value": "RU" - }, - { - "key": "City", - "value": "Moscow" - }, - { - "key": "SSD", - "value": "1" - } - ] - } - ], - "tests": { - "test": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "X" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "X", - "count": 4, - "clause": "DISTINCT", - "attribute": "", - "filter": "*" - } - ], - "filters": [] - }, - "result": [ - [ - 0, - 1, - 2, - 3 - ] - ] - } - } -} diff --git a/netmap/json_tests/selector_invalid.json b/netmap/json_tests/selector_invalid.json deleted file mode 100644 index ef307a7..0000000 --- a/netmap/json_tests/selector_invalid.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "name": "invalid selections", - "nodes": [ - { - "attributes": [ - { - "key": "Country", - "value": "Russia" - } - ] - }, - { - "attributes": [ - { - "key": "Country", - "value": "Germany" - } - ] - }, - { - "attributes": [] - } - ], - "tests": { - "missing filter": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "MyStore" - } - ], - "containerBackupFactor": 1, - "selectors": [ - { - "name": "MyStore", - "count": 1, - "clause": "DISTINCT", - "attribute": "Country", - "filter": "FromNL" - } - ], - "filters": [ - { - "name": "FromRU", - "key": "Country", - "op": "EQ", - "value": "Russia", - "filters": [] - } - ] - }, - "error": "filter not found" - }, - "not enough nodes (filter results in empty set)": { - "policy": { - "replicas": [ - { - "count": 1, - "selector": "MyStore" - } - ], - "containerBackupFactor": 2, - "selectors": [ - { - "name": "MyStore", - "count": 2, - "clause": "DISTINCT", - "attribute": "Country", - "filter": "FromMoon" - } - ], - "filters": [ - { - "name": "FromMoon", - "key": "Country", - "op": "EQ", - "value": "Moon", - "filters": [] - } - ] - }, - "error": "not enough nodes" - } - } -} diff --git a/netmap/yml_tests/cbf_default.yml b/netmap/yml_tests/cbf_default.yml new file mode 100644 index 0000000..c43a703 --- /dev/null +++ b/netmap/yml_tests/cbf_default.yml @@ -0,0 +1,48 @@ +name: default CBF is 3 +nodes: + - attributes: + - key: Location + value: Europe + - key: Country + value: RU + - key: City + value: St.Petersburg + - attributes: + - key: Location + value: Europe + - key: Country + value: RU + - key: City + value: Moscow + - attributes: + - key: Location + value: Europe + - key: Country + value: DE + - key: City + value: Berlin + - attributes: + - key: Location + value: Europe + - key: Country + value: FR + - key: City + value: Paris +tests: + set default CBF: + policy: + replicas: + - count: 1 + selector: EU + containerBackupFactor: 0 + selectors: + - name: EU + count: 1 + clause: SAME + attribute: Location + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 diff --git a/netmap/yml_tests/cbf_minimal.yml b/netmap/yml_tests/cbf_minimal.yml new file mode 100644 index 0000000..2fe2642 --- /dev/null +++ b/netmap/yml_tests/cbf_minimal.yml @@ -0,0 +1,52 @@ +name: Real node count multiplier is in range [1, specified CBF] +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: DE + - attributes: + - key: ID + value: '2' + - key: Country + value: DE + - attributes: + - key: ID + value: '3' + - key: Country + value: DE +tests: + select 2, CBF is 2: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 2 + selectors: + - name: X + count: 2 + clause: SAME + attribute: Country + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + select 3, CBF is 2: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 2 + selectors: + - name: X + count: 3 + clause: SAME + attribute: Country + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 diff --git a/netmap/yml_tests/cbf_requirements.yml b/netmap/yml_tests/cbf_requirements.yml new file mode 100644 index 0000000..ccd58d4 --- /dev/null +++ b/netmap/yml_tests/cbf_requirements.yml @@ -0,0 +1,82 @@ +name: CBF requirements +nodes: + - attributes: + - key: ID + value: '1' + - key: Attr + value: Same + - attributes: + - key: ID + value: '2' + - key: Attr + value: Same + - attributes: + - key: ID + value: '3' + - key: Attr + value: Same + - attributes: + - key: ID + value: '4' + - key: Attr + value: Same +tests: + default CBF, no selector: + policy: + replicas: + - count: 2 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + explicit CBF, no selector: + policy: + replicas: + - count: 2 + containerBackupFactor: 3 + selectors: [] + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + select distinct, weak CBF: + policy: + replicas: + - count: 2 + selector: X + containerBackupFactor: 3 + selectors: + - name: X + count: 2 + clause: DISTINCT + filter: '*' + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + select same, weak CBF: + policy: + replicas: + - count: 2 + selector: X + containerBackupFactor: 3 + selectors: + - name: X + count: 2 + clause: SAME + attribute: Attr + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/netmap/yml_tests/filter_complex.yml b/netmap/yml_tests/filter_complex.yml new file mode 100644 index 0000000..a65fa17 --- /dev/null +++ b/netmap/yml_tests/filter_complex.yml @@ -0,0 +1,210 @@ +name: compound filter +nodes: + - attributes: + - key: Storage + value: SSD + - key: Rating + value: '10' + - key: IntField + value: '100' + - key: Param + value: Value1 +tests: + good: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - op: OR + filters: + - key: Param + op: EQ + value: Value1 + filters: [] + - key: Param + op: EQ + value: Value2 + filters: [] + result: + - - 0 + bad storage type: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: HDD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value1 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] + error: not enough nodes + bad rating: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '15' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value1 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] + error: not enough nodes + bad param: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value0 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] + error: not enough nodes diff --git a/netmap/yml_tests/filter_invalid_integer.yml b/netmap/yml_tests/filter_invalid_integer.yml new file mode 100644 index 0000000..e65ab24 --- /dev/null +++ b/netmap/yml_tests/filter_invalid_integer.yml @@ -0,0 +1,45 @@ +name: invalid integer field +nodes: + - attributes: + - key: IntegerField + value: 'true' + - attributes: + - key: IntegerField + value: str +tests: + empty string is not casted to 0: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: IntegerField + op: LE + value: '8' + filters: [] + error: not enough nodes + non-empty string is not casted to a number: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: IntegerField + op: GE + value: '0' + filters: [] + error: not enough nodes diff --git a/netmap/yml_tests/filter_simple.yml b/netmap/yml_tests/filter_simple.yml new file mode 100644 index 0000000..a24814a --- /dev/null +++ b/netmap/yml_tests/filter_simple.yml @@ -0,0 +1,230 @@ +name: single-op filters +nodes: + - attributes: + - key: Rating + value: '4' + - key: Country + value: Germany +tests: + GE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GE + value: '4' + filters: [] + result: + - - 0 + GE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GE + value: '5' + filters: [] + error: not enough nodes + GT true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GT + value: '3' + filters: [] + result: + - - 0 + GT false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GT + value: '4' + filters: [] + error: not enough nodes + LE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LE + value: '4' + filters: [] + result: + - - 0 + LE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LE + value: '3' + filters: [] + error: not enough nodes + LT true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LT + value: '5' + filters: [] + result: + - - 0 + LT false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LT + value: '4' + filters: [] + error: not enough nodes + EQ true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: EQ + value: Germany + filters: [] + result: + - - 0 + EQ false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: EQ + value: China + filters: [] + error: not enough nodes + NE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: NE + value: France + filters: [] + result: + - - 0 + NE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: NE + value: Germany + filters: [] + error: not enough nodes diff --git a/netmap/yml_tests/hrw_sort.yml b/netmap/yml_tests/hrw_sort.yml new file mode 100644 index 0000000..c84f7c9 --- /dev/null +++ b/netmap/yml_tests/hrw_sort.yml @@ -0,0 +1,118 @@ +name: HRW ordering +nodes: + - attributes: + - key: Country + value: Germany + - key: Price + value: '2' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Germany + - key: Price + value: '4' + - key: Capacity + value: '1' + - attributes: + - key: Country + value: France + - key: Price + value: '3' + - key: Capacity + value: '10' + - attributes: + - key: Country + value: Russia + - key: Price + value: '2' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Price + value: '1' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: France + - key: Price + value: '100' + - key: Capacity + value: '1' + - attributes: + - key: Country + value: France + - key: Price + value: '7' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Price + value: '2' + - key: Capacity + value: '1' +tests: + select 3 nodes in 3 distinct countries, same placement: + policy: + replicas: + - count: 1 + selector: Main + containerBackupFactor: 1 + selectors: + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: [] + pivot: Y29udGFpbmVySUQ= + result: + - - 5 + - 0 + - 7 + placement: + pivot: b2JqZWN0SUQ= + result: + - - 5 + - 0 + - 7 + select 6 nodes in 3 distinct countries, different placement: + policy: + replicas: + - count: 1 + selector: Main + containerBackupFactor: 2 + selectors: + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: [] + pivot: Y29udGFpbmVySUQ= + result: + - - 5 + - 4 + - 0 + - 1 + - 7 + - 2 + placement: + pivot: b2JqZWN0SUQ= + result: + - - 5 + - 4 + - 0 + - 7 + - 2 + - 1 diff --git a/netmap/yml_tests/issue213.yml b/netmap/yml_tests/issue213.yml new file mode 100644 index 0000000..8e8aea4 --- /dev/null +++ b/netmap/yml_tests/issue213.yml @@ -0,0 +1,52 @@ +name: unnamed selector (nspcc-dev/neofs-api-go#213) +nodes: + - attributes: + - key: Location + value: Europe + - key: Country + value: Russia + - key: City + value: Moscow + - attributes: + - key: Location + value: Europe + - key: Country + value: Russia + - key: City + value: Saint-Petersburg + - attributes: + - key: Location + value: Europe + - key: Country + value: Sweden + - key: City + value: Stockholm + - attributes: + - key: Location + value: Europe + - key: Country + value: Finalnd + - key: City + value: Helsinki +tests: + test: + policy: + replicas: + - count: 4 + containerBackupFactor: 1 + selectors: + - name: '' + count: 4 + clause: DISTINCT + filter: LOC_EU + filters: + - name: LOC_EU + key: Location + op: EQ + value: Europe + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/netmap/yml_tests/many_selects.yml b/netmap/yml_tests/many_selects.yml new file mode 100644 index 0000000..29efd43 --- /dev/null +++ b/netmap/yml_tests/many_selects.yml @@ -0,0 +1,141 @@ +name: single-op filters +nodes: + - attributes: + - key: Country + value: Russia + - key: Rating + value: '1' + - key: City + value: SPB + - attributes: + - key: Country + value: Germany + - key: Rating + value: '5' + - key: City + value: Berlin + - attributes: + - key: Country + value: Russia + - key: Rating + value: '6' + - key: City + value: Moscow + - attributes: + - key: Country + value: France + - key: Rating + value: '4' + - key: City + value: Paris + - attributes: + - key: Country + value: France + - key: Rating + value: '1' + - key: City + value: Lyon + - attributes: + - key: Country + value: Russia + - key: Rating + value: '5' + - key: City + value: SPB + - attributes: + - key: Country + value: Russia + - key: Rating + value: '7' + - key: City + value: Moscow + - attributes: + - key: Country + value: Germany + - key: Rating + value: '3' + - key: City + value: Darmstadt + - attributes: + - key: Country + value: Germany + - key: Rating + value: '7' + - key: City + value: Frankfurt + - attributes: + - key: Country + value: Russia + - key: Rating + value: '9' + - key: City + value: SPB + - attributes: + - key: Country + value: Russia + - key: Rating + value: '9' + - key: City + value: SPB +tests: + Select: + policy: + replicas: + - count: 1 + selector: SameRU + - count: 1 + selector: DistinctRU + - count: 1 + selector: Good + - count: 1 + selector: Main + containerBackupFactor: 2 + selectors: + - name: SameRU + count: 2 + clause: SAME + attribute: City + filter: FromRU + - name: DistinctRU + count: 2 + clause: DISTINCT + attribute: City + filter: FromRU + - name: Good + count: 2 + clause: DISTINCT + attribute: Country + filter: Good + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + - name: Good + key: Rating + op: GE + value: '4' + result: + - - 0 + - 5 + - 9 + - 10 + - - 2 + - 6 + - 0 + - 5 + - - 1 + - 8 + - 2 + - 5 + - - 3 + - 4 + - 1 + - 7 + - 0 + - 2 diff --git a/netmap/yml_tests/multiple_rep.yml b/netmap/yml_tests/multiple_rep.yml new file mode 100644 index 0000000..448214f --- /dev/null +++ b/netmap/yml_tests/multiple_rep.yml @@ -0,0 +1,46 @@ +name: multiple replicas (#215) +nodes: + - attributes: + - key: City + value: Saint-Petersburg + - attributes: + - key: City + value: Moscow + - attributes: + - key: City + value: Berlin + - attributes: + - key: City + value: Paris +tests: + test: + policy: + replicas: + - count: 1 + selector: LOC_SPB_PLACE + - count: 1 + selector: LOC_MSK_PLACE + containerBackupFactor: 1 + selectors: + - name: LOC_SPB_PLACE + count: 1 + clause: CLAUSE_UNSPECIFIED + filter: LOC_SPB + - name: LOC_MSK_PLACE + count: 1 + clause: CLAUSE_UNSPECIFIED + filter: LOC_MSK + filters: + - name: LOC_SPB + key: City + op: EQ + value: Saint-Petersburg + filters: [] + - name: LOC_MSK + key: City + op: EQ + value: Moscow + filters: [] + result: + - - 0 + - - 1 diff --git a/netmap/yml_tests/multiple_rep_asymmetric.yml b/netmap/yml_tests/multiple_rep_asymmetric.yml new file mode 100644 index 0000000..61f8f76 --- /dev/null +++ b/netmap/yml_tests/multiple_rep_asymmetric.yml @@ -0,0 +1,162 @@ +name: multiple REP, asymmetric +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '0' + - attributes: + - key: ID + value: '2' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '3' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '4' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '5' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '6' + - key: Continent + value: NA + - key: City + value: NewYork + - attributes: + - key: ID + value: '7' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '8' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '9' + - key: Continent + value: SA + - key: City + value: Lima + - attributes: + - key: ID + value: '10' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '11' + - key: Continent + value: NA + - key: City + value: NewYork + - attributes: + - key: ID + value: '12' + - key: Continent + value: NA + - key: City + value: LosAngeles + - attributes: + - key: ID + value: '13' + - key: Continent + value: SA + - key: City + value: Lima +tests: + test: + policy: + replicas: + - count: 1 + selector: SPB + - count: 2 + selector: Americas + containerBackupFactor: 2 + selectors: + - name: SPB + count: 1 + clause: SAME + attribute: City + filter: SPBSSD + - name: Americas + count: 2 + clause: DISTINCT + attribute: City + filter: Americas + filters: + - name: SPBSSD + op: AND + filters: + - name: '' + key: Country + op: EQ + value: RU + filters: [] + - name: '' + key: City + op: EQ + value: St.Petersburg + filters: [] + - name: '' + key: SSD + op: EQ + value: '1' + filters: [] + - name: Americas + op: OR + filters: + - name: '' + key: Continent + op: EQ + value: NA + filters: [] + - name: '' + key: Continent + op: EQ + value: SA + filters: [] + result: + - - 1 + - 4 + - - 8 + - 12 + - 5 + - 10 diff --git a/netmap/yml_tests/non_strict.yml b/netmap/yml_tests/non_strict.yml new file mode 100644 index 0000000..a01986d --- /dev/null +++ b/netmap/yml_tests/non_strict.yml @@ -0,0 +1,52 @@ +name: non-strict selections +comment: These test specify loose selection behaviour, to allow fetching already PUT + objects even when there is not enough nodes to select from. +nodes: + - attributes: + - key: Country + value: Russia + - attributes: + - key: Country + value: Germany + - attributes: [] +tests: + not enough nodes (backup factor): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 2 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromRU + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + result: + - - 0 + not enough nodes (buckets): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 1 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromRU + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + result: + - - 0 diff --git a/netmap/yml_tests/rep_only.yml b/netmap/yml_tests/rep_only.yml new file mode 100644 index 0000000..80bf06b --- /dev/null +++ b/netmap/yml_tests/rep_only.yml @@ -0,0 +1,66 @@ +name: REP X +nodes: + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Saint-Petersburg + parents: [] + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Moscow + parents: [] + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Berlin + parents: [] + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Paris + parents: [] + state: UNSPECIFIED +tests: + REP 1: + policy: + replicas: + - count: 1 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 1 + - 2 + REP 3: + policy: + replicas: + - count: 3 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 3 + - 1 + - 2 + REP 5: + policy: + replicas: + - count: 5 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/netmap/yml_tests/select_no_attribute.yml b/netmap/yml_tests/select_no_attribute.yml new file mode 100644 index 0000000..02046f3 --- /dev/null +++ b/netmap/yml_tests/select_no_attribute.yml @@ -0,0 +1,56 @@ +name: select with unspecified attribute +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '0' + - attributes: + - key: ID + value: '2' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '3' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '4' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' +tests: + test: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 1 + selectors: + - name: X + count: 4 + clause: DISTINCT + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/netmap/yml_tests/selector_invalid.yml b/netmap/yml_tests/selector_invalid.yml new file mode 100644 index 0000000..d6e3b4e --- /dev/null +++ b/netmap/yml_tests/selector_invalid.yml @@ -0,0 +1,48 @@ +name: invalid selections +nodes: + - attributes: + - key: Country + value: Russia + - attributes: + - key: Country + value: Germany + - attributes: [] +tests: + missing filter: + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 1 + selectors: + - name: MyStore + count: 1 + clause: DISTINCT + attribute: Country + filter: FromNL + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + error: filter not found + not enough nodes (filter results in empty set): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 2 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromMoon + filters: + - name: FromMoon + key: Country + op: EQ + value: Moon + filters: [] + error: not enough nodes -- 2.45.2 From 03d35dd1f380701709a27eda846f61adf49d8611 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 16 Nov 2023 15:16:48 +0300 Subject: [PATCH 2/2] [#166] netmap: Add support YML tests Signed-off-by: Alexander Chuprov --- netmap/policy.go | 5 + netmap/{json_test.go => yml_test.go} | 48 ++++++--- netmap/yml_unmarshal.go | 142 +++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 15 deletions(-) rename netmap/{json_test.go => yml_test.go} (74%) create mode 100644 netmap/yml_unmarshal.go diff --git a/netmap/policy.go b/netmap/policy.go index 52cee8f..5d85897 100644 --- a/netmap/policy.go +++ b/netmap/policy.go @@ -217,6 +217,11 @@ func (s *Selector) SelectByBucketAttribute(bucket string) { s.m.SetAttribute(bucket) } +// SetClause sets the clause for the Selector. +func (s *Selector) SetClause(clause netmap.Clause) { + s.m.SetClause(clause) +} + // SelectSame makes selection algorithm to select only nodes having the same values // of the bucket attribute. // diff --git a/netmap/json_test.go b/netmap/yml_test.go similarity index 74% rename from netmap/json_test.go rename to netmap/yml_test.go index 1f63852..a744a01 100644 --- a/netmap/json_test.go +++ b/netmap/yml_test.go @@ -1,6 +1,7 @@ package netmap import ( + "encoding/base64" "encoding/json" "fmt" "os" @@ -8,24 +9,41 @@ import ( "testing" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) // TestCase represents collection of placement policy tests for a single node set. type TestCase struct { - Name string `json:"name"` - Nodes []NodeInfo `json:"nodes"` + Name string `json:"name" yaml:"name"` + Nodes []NodeInfo `json:"nodes" yaml:"nodes"` Tests map[string]struct { - Policy PlacementPolicy `json:"policy"` - Pivot []byte `json:"pivot,omitempty"` - Result [][]int `json:"result,omitempty"` - Error string `json:"error,omitempty"` + Policy PlacementPolicy + Pivot Base64 + Result [][]int `json:"result,omitempty" yaml:"result,omitempty"` + Error string `json:"error,omitempty" yaml:"error,omitempty"` Placement struct { - Pivot []byte - Result [][]int - } `json:"placement,omitempty"` + Pivot Base64 `json:"pivot" yaml:"pivot"` + Result [][]int `json:"result,omitempty" yaml:"result,omitempty"` + } `json:"placement,omitempty" yaml:"placement,omitempty"` } } +// Base64 is a type that will hold the decoded Base64 data. +type Base64 []byte + +func (b *Base64) UnmarshalYAML(unmarshal func(interface{}) error) error { + var base64Str string + if err := unmarshal(&base64Str); err != nil { + return err + } + decodedBytes, err := base64.StdEncoding.DecodeString(base64Str) + if err != nil { + return err + } + *b = decodedBytes + return nil +} + var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy) func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) { @@ -39,7 +57,7 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn } func TestPlacementPolicy_Interopability(t *testing.T) { - const testsDir = "./json_tests" + const testsDir = "./yml_tests" f, err := os.Open(testsDir) require.NoError(t, err) @@ -53,7 +71,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) { require.NoError(t, err) var tc TestCase - require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) + require.NoError(t, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) srcNodes := make([]NodeInfo, len(tc.Nodes)) copy(srcNodes, tc.Nodes) @@ -88,7 +106,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) { } func BenchmarkPlacementPolicyInteropability(b *testing.B) { - const testsDir = "./json_tests" + const testsDir = "./yml_tests" f, err := os.Open(testsDir) require.NoError(b, err) @@ -101,7 +119,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) { require.NoError(b, err) var tc TestCase - require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) + require.NoError(b, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) b.Run(tc.Name, func(b *testing.B) { var nm NetMap @@ -140,12 +158,12 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) { } func BenchmarkManySelects(b *testing.B) { - testsFile := filepath.Join("json_tests", "many_selects.json") + testsFile := filepath.Join("yml_tests", "many_selects.yml") bs, err := os.ReadFile(testsFile) require.NoError(b, err) var tc TestCase - require.NoError(b, json.Unmarshal(bs, &tc)) + require.NoError(b, yaml.Unmarshal(bs, &tc)) tt, ok := tc.Tests["Select"] require.True(b, ok) diff --git a/netmap/yml_unmarshal.go b/netmap/yml_unmarshal.go new file mode 100644 index 0000000..fa94586 --- /dev/null +++ b/netmap/yml_unmarshal.go @@ -0,0 +1,142 @@ +package netmap + +import ( + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" +) + +type tempPlacementPolicy struct { + BackupFactor uint32 `yaml:"containerBackupFactor"` + Filters []tempFilter `yaml:"filters"` + Selectors []tempSelector `yaml:"selectors"` + Replicas []tempReplica `yaml:"replicas"` + Unique bool `yaml:"unique"` +} + +type tempFilter struct { + Name string `yaml:"name"` + Key string `yaml:"key"` + Op string `yaml:"op"` + Value string `yaml:"value"` + Filters []tempFilter `yaml:"filters"` +} + +type tempSelector struct { + Name string `yaml:"name"` + Count uint32 `yaml:"count"` + Clause string `yaml:"clause"` + Attribute string `yaml:"attribute"` + Filter string `yaml:"filter"` +} + +type tempReplica struct { + Count uint32 `yaml:"count"` + Selector string `yaml:"selector"` +} + +func convertNFilters(temp []tempFilter) []netmap.Filter { + var filters []netmap.Filter + for _, tf := range temp { + filters = append(filters, convertNFilter(tf)) + } + return filters +} + +var stringToOperationMap = map[string]netmap.Operation{ + "EQ": netmap.EQ, + "NE": netmap.NE, + "GT": netmap.GT, + "GE": netmap.GE, + "LT": netmap.LT, + "LE": netmap.LE, + "OR": netmap.OR, + "AND": netmap.AND, + "NOT": netmap.NOT, +} + +func convertStringToOperation(opStr string) netmap.Operation { + opStr = strings.ToUpper(opStr) + if op, exists := stringToOperationMap[opStr]; exists { + return op + } + return netmap.UnspecifiedOperation +} + +func convertStringToClause(clauseStr string) netmap.Clause { + switch strings.ToUpper(clauseStr) { + case "DISTINCT": + return netmap.Distinct + default: + return netmap.Same + } +} + +func convertNFilter(temp tempFilter) netmap.Filter { + filter := netmap.Filter{} + filter.SetKey(temp.Key) + filter.SetName(temp.Name) + filter.SetValue(temp.Value) + filter.SetOp(convertStringToOperation(temp.Op)) + + if temp.Filters != nil { + filter.SetFilters(convertNFilters(temp.Filters)) + } + return filter +} + +func (p *PlacementPolicy) UnmarshalYAML(unmarshal func(interface{}) error) error { + var temp tempPlacementPolicy + if err := unmarshal(&temp); err != nil { + return err + } + + for _, ts := range temp.Filters { + netmapFilters := convertNFilter(ts) + p.AddFilters(Filter{m: netmapFilters}) + } + + for _, ts := range temp.Selectors { + selector := Selector{} + selector.SetName(ts.Name) + selector.SetNumberOfNodes(ts.Count) + selector.SetClause(convertStringToClause(ts.Clause)) + selector.SelectByBucketAttribute(ts.Attribute) + selector.SetFilterName(ts.Filter) + p.AddSelectors(selector) + } + + for _, tr := range temp.Replicas { + replica := ReplicaDescriptor{} + replica.SetSelectorName(tr.Selector) + replica.m.SetCount(tr.Count) + p.AddReplicas(replica) + } + + p.SetContainerBackupFactor(temp.BackupFactor) + p.SetUnique(temp.Unique) + + return nil +} + +type Attribute struct { + Key string `yaml:"key"` + Value string `yaml:"value"` +} + +type tempNode struct { + Attributes []Attribute `yaml:"attributes"` +} + +func (x *NodeInfo) UnmarshalYAML(unmarshal func(interface{}) error) error { + var temp tempNode + if err := unmarshal(&temp); err != nil { + return err + } + + for _, atr := range temp.Attributes { + x.SetAttribute(atr.Key, atr.Value) + } + + return nil +} -- 2.45.2