From 03d35dd1f380701709a27eda846f61adf49d8611 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 16 Nov 2023 15:16:48 +0300 Subject: [PATCH] [#166] netmap: Add support YML tests Signed-off-by: Alexander Chuprov --- netmap/policy.go | 5 + netmap/{json_test.go => yml_test.go} | 48 ++++++--- netmap/yml_unmarshal.go | 142 +++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 15 deletions(-) rename netmap/{json_test.go => yml_test.go} (74%) create mode 100644 netmap/yml_unmarshal.go diff --git a/netmap/policy.go b/netmap/policy.go index 52cee8f..5d85897 100644 --- a/netmap/policy.go +++ b/netmap/policy.go @@ -217,6 +217,11 @@ func (s *Selector) SelectByBucketAttribute(bucket string) { s.m.SetAttribute(bucket) } +// SetClause sets the clause for the Selector. +func (s *Selector) SetClause(clause netmap.Clause) { + s.m.SetClause(clause) +} + // SelectSame makes selection algorithm to select only nodes having the same values // of the bucket attribute. // diff --git a/netmap/json_test.go b/netmap/yml_test.go similarity index 74% rename from netmap/json_test.go rename to netmap/yml_test.go index 1f63852..a744a01 100644 --- a/netmap/json_test.go +++ b/netmap/yml_test.go @@ -1,6 +1,7 @@ package netmap import ( + "encoding/base64" "encoding/json" "fmt" "os" @@ -8,24 +9,41 @@ import ( "testing" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" ) // TestCase represents collection of placement policy tests for a single node set. type TestCase struct { - Name string `json:"name"` - Nodes []NodeInfo `json:"nodes"` + Name string `json:"name" yaml:"name"` + Nodes []NodeInfo `json:"nodes" yaml:"nodes"` Tests map[string]struct { - Policy PlacementPolicy `json:"policy"` - Pivot []byte `json:"pivot,omitempty"` - Result [][]int `json:"result,omitempty"` - Error string `json:"error,omitempty"` + Policy PlacementPolicy + Pivot Base64 + Result [][]int `json:"result,omitempty" yaml:"result,omitempty"` + Error string `json:"error,omitempty" yaml:"error,omitempty"` Placement struct { - Pivot []byte - Result [][]int - } `json:"placement,omitempty"` + Pivot Base64 `json:"pivot" yaml:"pivot"` + Result [][]int `json:"result,omitempty" yaml:"result,omitempty"` + } `json:"placement,omitempty" yaml:"placement,omitempty"` } } +// Base64 is a type that will hold the decoded Base64 data. +type Base64 []byte + +func (b *Base64) UnmarshalYAML(unmarshal func(interface{}) error) error { + var base64Str string + if err := unmarshal(&base64Str); err != nil { + return err + } + decodedBytes, err := base64.StdEncoding.DecodeString(base64Str) + if err != nil { + return err + } + *b = decodedBytes + return nil +} + var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy) func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) { @@ -39,7 +57,7 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn } func TestPlacementPolicy_Interopability(t *testing.T) { - const testsDir = "./json_tests" + const testsDir = "./yml_tests" f, err := os.Open(testsDir) require.NoError(t, err) @@ -53,7 +71,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) { require.NoError(t, err) var tc TestCase - require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) + require.NoError(t, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) srcNodes := make([]NodeInfo, len(tc.Nodes)) copy(srcNodes, tc.Nodes) @@ -88,7 +106,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) { } func BenchmarkPlacementPolicyInteropability(b *testing.B) { - const testsDir = "./json_tests" + const testsDir = "./yml_tests" f, err := os.Open(testsDir) require.NoError(b, err) @@ -101,7 +119,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) { require.NoError(b, err) var tc TestCase - require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) + require.NoError(b, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) b.Run(tc.Name, func(b *testing.B) { var nm NetMap @@ -140,12 +158,12 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) { } func BenchmarkManySelects(b *testing.B) { - testsFile := filepath.Join("json_tests", "many_selects.json") + testsFile := filepath.Join("yml_tests", "many_selects.yml") bs, err := os.ReadFile(testsFile) require.NoError(b, err) var tc TestCase - require.NoError(b, json.Unmarshal(bs, &tc)) + require.NoError(b, yaml.Unmarshal(bs, &tc)) tt, ok := tc.Tests["Select"] require.True(b, ok) diff --git a/netmap/yml_unmarshal.go b/netmap/yml_unmarshal.go new file mode 100644 index 0000000..fa94586 --- /dev/null +++ b/netmap/yml_unmarshal.go @@ -0,0 +1,142 @@ +package netmap + +import ( + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" +) + +type tempPlacementPolicy struct { + BackupFactor uint32 `yaml:"containerBackupFactor"` + Filters []tempFilter `yaml:"filters"` + Selectors []tempSelector `yaml:"selectors"` + Replicas []tempReplica `yaml:"replicas"` + Unique bool `yaml:"unique"` +} + +type tempFilter struct { + Name string `yaml:"name"` + Key string `yaml:"key"` + Op string `yaml:"op"` + Value string `yaml:"value"` + Filters []tempFilter `yaml:"filters"` +} + +type tempSelector struct { + Name string `yaml:"name"` + Count uint32 `yaml:"count"` + Clause string `yaml:"clause"` + Attribute string `yaml:"attribute"` + Filter string `yaml:"filter"` +} + +type tempReplica struct { + Count uint32 `yaml:"count"` + Selector string `yaml:"selector"` +} + +func convertNFilters(temp []tempFilter) []netmap.Filter { + var filters []netmap.Filter + for _, tf := range temp { + filters = append(filters, convertNFilter(tf)) + } + return filters +} + +var stringToOperationMap = map[string]netmap.Operation{ + "EQ": netmap.EQ, + "NE": netmap.NE, + "GT": netmap.GT, + "GE": netmap.GE, + "LT": netmap.LT, + "LE": netmap.LE, + "OR": netmap.OR, + "AND": netmap.AND, + "NOT": netmap.NOT, +} + +func convertStringToOperation(opStr string) netmap.Operation { + opStr = strings.ToUpper(opStr) + if op, exists := stringToOperationMap[opStr]; exists { + return op + } + return netmap.UnspecifiedOperation +} + +func convertStringToClause(clauseStr string) netmap.Clause { + switch strings.ToUpper(clauseStr) { + case "DISTINCT": + return netmap.Distinct + default: + return netmap.Same + } +} + +func convertNFilter(temp tempFilter) netmap.Filter { + filter := netmap.Filter{} + filter.SetKey(temp.Key) + filter.SetName(temp.Name) + filter.SetValue(temp.Value) + filter.SetOp(convertStringToOperation(temp.Op)) + + if temp.Filters != nil { + filter.SetFilters(convertNFilters(temp.Filters)) + } + return filter +} + +func (p *PlacementPolicy) UnmarshalYAML(unmarshal func(interface{}) error) error { + var temp tempPlacementPolicy + if err := unmarshal(&temp); err != nil { + return err + } + + for _, ts := range temp.Filters { + netmapFilters := convertNFilter(ts) + p.AddFilters(Filter{m: netmapFilters}) + } + + for _, ts := range temp.Selectors { + selector := Selector{} + selector.SetName(ts.Name) + selector.SetNumberOfNodes(ts.Count) + selector.SetClause(convertStringToClause(ts.Clause)) + selector.SelectByBucketAttribute(ts.Attribute) + selector.SetFilterName(ts.Filter) + p.AddSelectors(selector) + } + + for _, tr := range temp.Replicas { + replica := ReplicaDescriptor{} + replica.SetSelectorName(tr.Selector) + replica.m.SetCount(tr.Count) + p.AddReplicas(replica) + } + + p.SetContainerBackupFactor(temp.BackupFactor) + p.SetUnique(temp.Unique) + + return nil +} + +type Attribute struct { + Key string `yaml:"key"` + Value string `yaml:"value"` +} + +type tempNode struct { + Attributes []Attribute `yaml:"attributes"` +} + +func (x *NodeInfo) UnmarshalYAML(unmarshal func(interface{}) error) error { + var temp tempNode + if err := unmarshal(&temp); err != nil { + return err + } + + for _, atr := range temp.Attributes { + x.SetAttribute(atr.Key, atr.Value) + } + + return nil +}