[#166] netmap: Add support YML tests

Signed-off-by: Alexander Chuprov <a.chuprov@yadro.com>
This commit is contained in:
Alexander Chuprov 2023-11-16 15:16:48 +03:00
parent dea8759762
commit 03d35dd1f3
3 changed files with 180 additions and 15 deletions

View file

@ -217,6 +217,11 @@ func (s *Selector) SelectByBucketAttribute(bucket string) {
s.m.SetAttribute(bucket)
}
// SetClause sets the clause for the Selector.
func (s *Selector) SetClause(clause netmap.Clause) {
s.m.SetClause(clause)
}
// SelectSame makes selection algorithm to select only nodes having the same values
// of the bucket attribute.
//

View file

@ -1,6 +1,7 @@
package netmap
import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
@ -8,24 +9,41 @@ import (
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
// TestCase represents collection of placement policy tests for a single node set.
type TestCase struct {
Name string `json:"name"`
Nodes []NodeInfo `json:"nodes"`
Name string `json:"name" yaml:"name"`
Nodes []NodeInfo `json:"nodes" yaml:"nodes"`
Tests map[string]struct {
Policy PlacementPolicy `json:"policy"`
Pivot []byte `json:"pivot,omitempty"`
Result [][]int `json:"result,omitempty"`
Error string `json:"error,omitempty"`
Policy PlacementPolicy
Pivot Base64
Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
Error string `json:"error,omitempty" yaml:"error,omitempty"`
Placement struct {
Pivot []byte
Result [][]int
} `json:"placement,omitempty"`
Pivot Base64 `json:"pivot" yaml:"pivot"`
Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
} `json:"placement,omitempty" yaml:"placement,omitempty"`
}
}
// Base64 is a type that will hold the decoded Base64 data.
type Base64 []byte
func (b *Base64) UnmarshalYAML(unmarshal func(interface{}) error) error {
var base64Str string
if err := unmarshal(&base64Str); err != nil {
return err
}
decodedBytes, err := base64.StdEncoding.DecodeString(base64Str)
if err != nil {
return err
}
*b = decodedBytes
return nil
}
var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy)
func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
@ -39,7 +57,7 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn
}
func TestPlacementPolicy_Interopability(t *testing.T) {
const testsDir = "./json_tests"
const testsDir = "./yml_tests"
f, err := os.Open(testsDir)
require.NoError(t, err)
@ -53,7 +71,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
require.NoError(t, err)
var tc TestCase
require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
require.NoError(t, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
srcNodes := make([]NodeInfo, len(tc.Nodes))
copy(srcNodes, tc.Nodes)
@ -88,7 +106,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
}
func BenchmarkPlacementPolicyInteropability(b *testing.B) {
const testsDir = "./json_tests"
const testsDir = "./yml_tests"
f, err := os.Open(testsDir)
require.NoError(b, err)
@ -101,7 +119,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
require.NoError(b, err)
var tc TestCase
require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
require.NoError(b, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
b.Run(tc.Name, func(b *testing.B) {
var nm NetMap
@ -140,12 +158,12 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
}
func BenchmarkManySelects(b *testing.B) {
testsFile := filepath.Join("json_tests", "many_selects.json")
testsFile := filepath.Join("yml_tests", "many_selects.yml")
bs, err := os.ReadFile(testsFile)
require.NoError(b, err)
var tc TestCase
require.NoError(b, json.Unmarshal(bs, &tc))
require.NoError(b, yaml.Unmarshal(bs, &tc))
tt, ok := tc.Tests["Select"]
require.True(b, ok)

142
netmap/yml_unmarshal.go Normal file
View file

@ -0,0 +1,142 @@
package netmap
import (
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
)
type tempPlacementPolicy struct {
BackupFactor uint32 `yaml:"containerBackupFactor"`
Filters []tempFilter `yaml:"filters"`
Selectors []tempSelector `yaml:"selectors"`
Replicas []tempReplica `yaml:"replicas"`
Unique bool `yaml:"unique"`
}
type tempFilter struct {
Name string `yaml:"name"`
Key string `yaml:"key"`
Op string `yaml:"op"`
Value string `yaml:"value"`
Filters []tempFilter `yaml:"filters"`
}
type tempSelector struct {
Name string `yaml:"name"`
Count uint32 `yaml:"count"`
Clause string `yaml:"clause"`
Attribute string `yaml:"attribute"`
Filter string `yaml:"filter"`
}
type tempReplica struct {
Count uint32 `yaml:"count"`
Selector string `yaml:"selector"`
}
func convertNFilters(temp []tempFilter) []netmap.Filter {
var filters []netmap.Filter
for _, tf := range temp {
filters = append(filters, convertNFilter(tf))
}
return filters
}
var stringToOperationMap = map[string]netmap.Operation{
"EQ": netmap.EQ,
"NE": netmap.NE,
"GT": netmap.GT,
"GE": netmap.GE,
"LT": netmap.LT,
"LE": netmap.LE,
"OR": netmap.OR,
"AND": netmap.AND,
"NOT": netmap.NOT,
}
func convertStringToOperation(opStr string) netmap.Operation {
opStr = strings.ToUpper(opStr)
if op, exists := stringToOperationMap[opStr]; exists {
return op
}
return netmap.UnspecifiedOperation
}
func convertStringToClause(clauseStr string) netmap.Clause {
switch strings.ToUpper(clauseStr) {
case "DISTINCT":
return netmap.Distinct
default:
return netmap.Same
}
}
func convertNFilter(temp tempFilter) netmap.Filter {
filter := netmap.Filter{}
filter.SetKey(temp.Key)
filter.SetName(temp.Name)
filter.SetValue(temp.Value)
filter.SetOp(convertStringToOperation(temp.Op))
if temp.Filters != nil {
filter.SetFilters(convertNFilters(temp.Filters))
}
return filter
}
func (p *PlacementPolicy) UnmarshalYAML(unmarshal func(interface{}) error) error {
var temp tempPlacementPolicy
if err := unmarshal(&temp); err != nil {
return err
}
for _, ts := range temp.Filters {
netmapFilters := convertNFilter(ts)
p.AddFilters(Filter{m: netmapFilters})
}
for _, ts := range temp.Selectors {
selector := Selector{}
selector.SetName(ts.Name)
selector.SetNumberOfNodes(ts.Count)
selector.SetClause(convertStringToClause(ts.Clause))
selector.SelectByBucketAttribute(ts.Attribute)
selector.SetFilterName(ts.Filter)
p.AddSelectors(selector)
}
for _, tr := range temp.Replicas {
replica := ReplicaDescriptor{}
replica.SetSelectorName(tr.Selector)
replica.m.SetCount(tr.Count)
p.AddReplicas(replica)
}
p.SetContainerBackupFactor(temp.BackupFactor)
p.SetUnique(temp.Unique)
return nil
}
type Attribute struct {
Key string `yaml:"key"`
Value string `yaml:"value"`
}
type tempNode struct {
Attributes []Attribute `yaml:"attributes"`
}
func (x *NodeInfo) UnmarshalYAML(unmarshal func(interface{}) error) error {
var temp tempNode
if err := unmarshal(&temp); err != nil {
return err
}
for _, atr := range temp.Attributes {
x.SetAttribute(atr.Key, atr.Value)
}
return nil
}