forked from TrueCloudLab/frostfs-sdk-go
[#166] netmap: Add support YML tests
Signed-off-by: Alexander Chuprov <a.chuprov@yadro.com>
This commit is contained in:
parent
dea8759762
commit
03d35dd1f3
3 changed files with 180 additions and 15 deletions
|
@ -217,6 +217,11 @@ func (s *Selector) SelectByBucketAttribute(bucket string) {
|
||||||
s.m.SetAttribute(bucket)
|
s.m.SetAttribute(bucket)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetClause sets the clause for the Selector.
|
||||||
|
func (s *Selector) SetClause(clause netmap.Clause) {
|
||||||
|
s.m.SetClause(clause)
|
||||||
|
}
|
||||||
|
|
||||||
// SelectSame makes selection algorithm to select only nodes having the same values
|
// SelectSame makes selection algorithm to select only nodes having the same values
|
||||||
// of the bucket attribute.
|
// of the bucket attribute.
|
||||||
//
|
//
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package netmap
|
package netmap
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
@ -8,24 +9,41 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestCase represents collection of placement policy tests for a single node set.
|
// TestCase represents collection of placement policy tests for a single node set.
|
||||||
type TestCase struct {
|
type TestCase struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name" yaml:"name"`
|
||||||
Nodes []NodeInfo `json:"nodes"`
|
Nodes []NodeInfo `json:"nodes" yaml:"nodes"`
|
||||||
Tests map[string]struct {
|
Tests map[string]struct {
|
||||||
Policy PlacementPolicy `json:"policy"`
|
Policy PlacementPolicy
|
||||||
Pivot []byte `json:"pivot,omitempty"`
|
Pivot Base64
|
||||||
Result [][]int `json:"result,omitempty"`
|
Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
|
||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty" yaml:"error,omitempty"`
|
||||||
Placement struct {
|
Placement struct {
|
||||||
Pivot []byte
|
Pivot Base64 `json:"pivot" yaml:"pivot"`
|
||||||
Result [][]int
|
Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
|
||||||
} `json:"placement,omitempty"`
|
} `json:"placement,omitempty" yaml:"placement,omitempty"`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Base64 is a type that will hold the decoded Base64 data.
|
||||||
|
type Base64 []byte
|
||||||
|
|
||||||
|
func (b *Base64) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
var base64Str string
|
||||||
|
if err := unmarshal(&base64Str); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
decodedBytes, err := base64.StdEncoding.DecodeString(base64Str)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*b = decodedBytes
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy)
|
var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy)
|
||||||
|
|
||||||
func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
|
func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
|
||||||
|
@ -39,7 +57,7 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPlacementPolicy_Interopability(t *testing.T) {
|
func TestPlacementPolicy_Interopability(t *testing.T) {
|
||||||
const testsDir = "./json_tests"
|
const testsDir = "./yml_tests"
|
||||||
|
|
||||||
f, err := os.Open(testsDir)
|
f, err := os.Open(testsDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -53,7 +71,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
var tc TestCase
|
var tc TestCase
|
||||||
require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
|
require.NoError(t, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
|
||||||
|
|
||||||
srcNodes := make([]NodeInfo, len(tc.Nodes))
|
srcNodes := make([]NodeInfo, len(tc.Nodes))
|
||||||
copy(srcNodes, tc.Nodes)
|
copy(srcNodes, tc.Nodes)
|
||||||
|
@ -88,7 +106,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkPlacementPolicyInteropability(b *testing.B) {
|
func BenchmarkPlacementPolicyInteropability(b *testing.B) {
|
||||||
const testsDir = "./json_tests"
|
const testsDir = "./yml_tests"
|
||||||
|
|
||||||
f, err := os.Open(testsDir)
|
f, err := os.Open(testsDir)
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
|
@ -101,7 +119,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
|
|
||||||
var tc TestCase
|
var tc TestCase
|
||||||
require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
|
require.NoError(b, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
|
||||||
|
|
||||||
b.Run(tc.Name, func(b *testing.B) {
|
b.Run(tc.Name, func(b *testing.B) {
|
||||||
var nm NetMap
|
var nm NetMap
|
||||||
|
@ -140,12 +158,12 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkManySelects(b *testing.B) {
|
func BenchmarkManySelects(b *testing.B) {
|
||||||
testsFile := filepath.Join("json_tests", "many_selects.json")
|
testsFile := filepath.Join("yml_tests", "many_selects.yml")
|
||||||
bs, err := os.ReadFile(testsFile)
|
bs, err := os.ReadFile(testsFile)
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
|
|
||||||
var tc TestCase
|
var tc TestCase
|
||||||
require.NoError(b, json.Unmarshal(bs, &tc))
|
require.NoError(b, yaml.Unmarshal(bs, &tc))
|
||||||
tt, ok := tc.Tests["Select"]
|
tt, ok := tc.Tests["Select"]
|
||||||
require.True(b, ok)
|
require.True(b, ok)
|
||||||
|
|
142
netmap/yml_unmarshal.go
Normal file
142
netmap/yml_unmarshal.go
Normal file
|
@ -0,0 +1,142 @@
|
||||||
|
package netmap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type tempPlacementPolicy struct {
|
||||||
|
BackupFactor uint32 `yaml:"containerBackupFactor"`
|
||||||
|
Filters []tempFilter `yaml:"filters"`
|
||||||
|
Selectors []tempSelector `yaml:"selectors"`
|
||||||
|
Replicas []tempReplica `yaml:"replicas"`
|
||||||
|
Unique bool `yaml:"unique"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type tempFilter struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Key string `yaml:"key"`
|
||||||
|
Op string `yaml:"op"`
|
||||||
|
Value string `yaml:"value"`
|
||||||
|
Filters []tempFilter `yaml:"filters"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type tempSelector struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Count uint32 `yaml:"count"`
|
||||||
|
Clause string `yaml:"clause"`
|
||||||
|
Attribute string `yaml:"attribute"`
|
||||||
|
Filter string `yaml:"filter"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type tempReplica struct {
|
||||||
|
Count uint32 `yaml:"count"`
|
||||||
|
Selector string `yaml:"selector"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertNFilters(temp []tempFilter) []netmap.Filter {
|
||||||
|
var filters []netmap.Filter
|
||||||
|
for _, tf := range temp {
|
||||||
|
filters = append(filters, convertNFilter(tf))
|
||||||
|
}
|
||||||
|
return filters
|
||||||
|
}
|
||||||
|
|
||||||
|
var stringToOperationMap = map[string]netmap.Operation{
|
||||||
|
"EQ": netmap.EQ,
|
||||||
|
"NE": netmap.NE,
|
||||||
|
"GT": netmap.GT,
|
||||||
|
"GE": netmap.GE,
|
||||||
|
"LT": netmap.LT,
|
||||||
|
"LE": netmap.LE,
|
||||||
|
"OR": netmap.OR,
|
||||||
|
"AND": netmap.AND,
|
||||||
|
"NOT": netmap.NOT,
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertStringToOperation(opStr string) netmap.Operation {
|
||||||
|
opStr = strings.ToUpper(opStr)
|
||||||
|
if op, exists := stringToOperationMap[opStr]; exists {
|
||||||
|
return op
|
||||||
|
}
|
||||||
|
return netmap.UnspecifiedOperation
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertStringToClause(clauseStr string) netmap.Clause {
|
||||||
|
switch strings.ToUpper(clauseStr) {
|
||||||
|
case "DISTINCT":
|
||||||
|
return netmap.Distinct
|
||||||
|
default:
|
||||||
|
return netmap.Same
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertNFilter(temp tempFilter) netmap.Filter {
|
||||||
|
filter := netmap.Filter{}
|
||||||
|
filter.SetKey(temp.Key)
|
||||||
|
filter.SetName(temp.Name)
|
||||||
|
filter.SetValue(temp.Value)
|
||||||
|
filter.SetOp(convertStringToOperation(temp.Op))
|
||||||
|
|
||||||
|
if temp.Filters != nil {
|
||||||
|
filter.SetFilters(convertNFilters(temp.Filters))
|
||||||
|
}
|
||||||
|
return filter
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *PlacementPolicy) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
var temp tempPlacementPolicy
|
||||||
|
if err := unmarshal(&temp); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ts := range temp.Filters {
|
||||||
|
netmapFilters := convertNFilter(ts)
|
||||||
|
p.AddFilters(Filter{m: netmapFilters})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ts := range temp.Selectors {
|
||||||
|
selector := Selector{}
|
||||||
|
selector.SetName(ts.Name)
|
||||||
|
selector.SetNumberOfNodes(ts.Count)
|
||||||
|
selector.SetClause(convertStringToClause(ts.Clause))
|
||||||
|
selector.SelectByBucketAttribute(ts.Attribute)
|
||||||
|
selector.SetFilterName(ts.Filter)
|
||||||
|
p.AddSelectors(selector)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tr := range temp.Replicas {
|
||||||
|
replica := ReplicaDescriptor{}
|
||||||
|
replica.SetSelectorName(tr.Selector)
|
||||||
|
replica.m.SetCount(tr.Count)
|
||||||
|
p.AddReplicas(replica)
|
||||||
|
}
|
||||||
|
|
||||||
|
p.SetContainerBackupFactor(temp.BackupFactor)
|
||||||
|
p.SetUnique(temp.Unique)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type Attribute struct {
|
||||||
|
Key string `yaml:"key"`
|
||||||
|
Value string `yaml:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type tempNode struct {
|
||||||
|
Attributes []Attribute `yaml:"attributes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *NodeInfo) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
var temp tempNode
|
||||||
|
if err := unmarshal(&temp); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, atr := range temp.Attributes {
|
||||||
|
x.SetAttribute(atr.Key, atr.Value)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
Loading…
Reference in a new issue