forked from TrueCloudLab/frostfs-node
[#1513] Upgrade NeoFS SDK Go with changed netmap package
Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich
LeL
parent
24b4c1ecf4
commit
21d2f8f861
70 changed files with 878 additions and 992 deletions
|
|
@ -8,104 +8,63 @@ import (
|
|||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
)
|
||||
|
||||
const (
|
||||
pairSeparator = "/"
|
||||
keyValueSeparator = ":"
|
||||
)
|
||||
const keyValueSeparator = ":"
|
||||
|
||||
var (
|
||||
errEmptyChain = errors.New("empty attribute chain")
|
||||
errNonUniqueBucket = errors.New("attributes must contain unique keys")
|
||||
errUnexpectedKey = errors.New("attributes contain unexpected key")
|
||||
)
|
||||
|
||||
// ParseV2Attributes parses strings like "K1:V1/K2:V2/K3:V3" into netmap
|
||||
// attributes. Supports escaped symbols "\:", "\/" and "\\".
|
||||
func ParseV2Attributes(attrs []string, excl []string) ([]netmap.NodeAttribute, error) {
|
||||
restricted := make(map[string]struct{}, len(excl))
|
||||
for i := range excl {
|
||||
restricted[excl[i]] = struct{}{}
|
||||
}
|
||||
|
||||
cache := make(map[string]*netmap.NodeAttribute, len(attrs))
|
||||
result := make([]netmap.NodeAttribute, 0, len(attrs))
|
||||
// ReadNodeAttributes parses node attributes from list of string in "Key:Value" format
|
||||
// and writes them into netmap.NodeInfo instance. Supports escaped symbols
|
||||
// "\:", "\/" and "\\".
|
||||
func ReadNodeAttributes(dst *netmap.NodeInfo, attrs []string) error {
|
||||
cache := make(map[string]struct{}, len(attrs))
|
||||
|
||||
for i := range attrs {
|
||||
line := strings.Trim(attrs[i], pairSeparator)
|
||||
line = replaceEscaping(line, false) // replaced escaped symbols with non-printable symbols
|
||||
chain := strings.Split(line, pairSeparator)
|
||||
if len(chain) == 0 {
|
||||
return nil, errEmptyChain
|
||||
line := replaceEscaping(attrs[i], false) // replaced escaped symbols with non-printable symbols
|
||||
|
||||
words := strings.Split(line, keyValueSeparator)
|
||||
if len(words) != 2 {
|
||||
return errors.New("missing attribute key and/or value")
|
||||
}
|
||||
|
||||
var parentKey string // backtrack parents in next pairs
|
||||
|
||||
for j := range chain {
|
||||
pair := strings.Split(chain[j], keyValueSeparator)
|
||||
if len(pair) != 2 {
|
||||
return nil, fmt.Errorf("incorrect attribute pair %s", chain[j])
|
||||
}
|
||||
|
||||
key := pair[0]
|
||||
value := pair[1]
|
||||
|
||||
attribute, present := cache[key]
|
||||
if present && attribute.Value() != value {
|
||||
return nil, errNonUniqueBucket
|
||||
}
|
||||
|
||||
if _, ok := restricted[key]; ok {
|
||||
return nil, errUnexpectedKey
|
||||
}
|
||||
|
||||
if !present {
|
||||
result = append(result, netmap.NodeAttribute{})
|
||||
attribute = &result[len(result)-1]
|
||||
cache[key] = attribute
|
||||
|
||||
// replace non-printable symbols with escaped symbols without escape character
|
||||
key = replaceEscaping(key, true)
|
||||
value = replaceEscaping(value, true)
|
||||
|
||||
attribute.SetKey(key)
|
||||
attribute.SetValue(value)
|
||||
}
|
||||
|
||||
if parentKey != "" {
|
||||
parentKeys := attribute.ParentKeys()
|
||||
if !hasString(parentKeys, parentKey) {
|
||||
attribute.SetParentKeys(append(parentKeys, parentKey)...)
|
||||
}
|
||||
}
|
||||
|
||||
parentKey = key
|
||||
_, ok := cache[words[0]]
|
||||
if ok {
|
||||
return fmt.Errorf("duplicated keys %s", words[0])
|
||||
}
|
||||
|
||||
cache[words[0]] = struct{}{}
|
||||
|
||||
// replace non-printable symbols with escaped symbols without escape character
|
||||
words[0] = replaceEscaping(words[0], true)
|
||||
words[1] = replaceEscaping(words[1], true)
|
||||
fmt.Println(words[0], words[1])
|
||||
|
||||
if words[0] == "" {
|
||||
return errors.New("empty key")
|
||||
} else if words[1] == "" {
|
||||
return errors.New("empty value")
|
||||
}
|
||||
|
||||
dst.SetAttribute(words[0], words[1])
|
||||
}
|
||||
|
||||
return result, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func replaceEscaping(target string, rollback bool) (s string) {
|
||||
const escChar = `\`
|
||||
|
||||
var (
|
||||
oldPairSep = escChar + pairSeparator
|
||||
oldKVSep = escChar + keyValueSeparator
|
||||
oldEsc = escChar + escChar
|
||||
newPairSep = string(uint8(1))
|
||||
newKVSep = string(uint8(2))
|
||||
newEsc = string(uint8(3))
|
||||
oldKVSep = escChar + keyValueSeparator
|
||||
oldEsc = escChar + escChar
|
||||
newKVSep = string(uint8(2))
|
||||
newEsc = string(uint8(3))
|
||||
)
|
||||
|
||||
if rollback {
|
||||
oldPairSep, oldKVSep, oldEsc = newPairSep, newKVSep, newEsc
|
||||
newPairSep = pairSeparator
|
||||
oldKVSep, oldEsc = newKVSep, newEsc
|
||||
newKVSep = keyValueSeparator
|
||||
newEsc = escChar
|
||||
}
|
||||
|
||||
s = strings.ReplaceAll(target, oldEsc, newEsc)
|
||||
s = strings.ReplaceAll(s, oldPairSep, newPairSep)
|
||||
s = strings.ReplaceAll(s, oldKVSep, newKVSep)
|
||||
|
||||
return
|
||||
|
|
|
|||
|
|
@ -4,117 +4,96 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-node/pkg/util/attributes"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestParseV2Attributes(t *testing.T) {
|
||||
t.Run("empty", func(t *testing.T) {
|
||||
attrs, err := attributes.ParseV2Attributes(nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, attrs, 0)
|
||||
func testAttributeMap(t *testing.T, mSrc, mExp map[string]string) {
|
||||
var node netmap.NodeInfo
|
||||
|
||||
s := make([]string, 0, len(mSrc))
|
||||
for k, v := range mSrc {
|
||||
s = append(s, k+":"+v)
|
||||
}
|
||||
|
||||
err := attributes.ReadNodeAttributes(&node, s)
|
||||
require.NoError(t, err)
|
||||
|
||||
if mExp == nil {
|
||||
mExp = mSrc
|
||||
}
|
||||
|
||||
node.IterateAttributes(func(key, value string) {
|
||||
v, ok := mExp[key]
|
||||
require.True(t, ok)
|
||||
require.Equal(t, value, v)
|
||||
delete(mExp, key)
|
||||
})
|
||||
|
||||
t.Run("non unique bucket keys", func(t *testing.T) {
|
||||
good := []string{
|
||||
"StorageType:HDD/RPM:7200",
|
||||
"StorageType:HDD/SMR:True",
|
||||
}
|
||||
_, err := attributes.ParseV2Attributes(good, nil)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, mExp)
|
||||
}
|
||||
|
||||
bad := append(good, "StorageType:SSD/Cell:QLC")
|
||||
_, err = attributes.ParseV2Attributes(bad, nil)
|
||||
func TestParseV2Attributes(t *testing.T) {
|
||||
t.Run("empty", func(t *testing.T) {
|
||||
var node netmap.NodeInfo
|
||||
err := attributes.ReadNodeAttributes(&node, nil)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, node.NumberOfAttributes())
|
||||
})
|
||||
|
||||
t.Run("empty key and/or value", func(t *testing.T) {
|
||||
var node netmap.NodeInfo
|
||||
err := attributes.ReadNodeAttributes(&node, []string{
|
||||
":HDD",
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
err = attributes.ReadNodeAttributes(&node, []string{
|
||||
"StorageType:",
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
err = attributes.ReadNodeAttributes(&node, []string{
|
||||
":",
|
||||
})
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("non-unique keys", func(t *testing.T) {
|
||||
var node netmap.NodeInfo
|
||||
err := attributes.ReadNodeAttributes(&node, []string{
|
||||
"StorageType:HDD",
|
||||
"StorageType:HDD",
|
||||
})
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("malformed", func(t *testing.T) {
|
||||
_, err := attributes.ParseV2Attributes([]string{"..."}, nil)
|
||||
var node netmap.NodeInfo
|
||||
err := attributes.ReadNodeAttributes(&node, []string{"..."})
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = attributes.ParseV2Attributes([]string{"a:b", ""}, nil)
|
||||
err = attributes.ReadNodeAttributes(&node, []string{"a:b", ""})
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = attributes.ParseV2Attributes([]string{"//"}, nil)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("unexpected", func(t *testing.T) {
|
||||
unexpectedBucket := []string{
|
||||
"Location:Europe/City:Moscow",
|
||||
"Price:100",
|
||||
}
|
||||
_, err := attributes.ParseV2Attributes(unexpectedBucket, []string{"Price"})
|
||||
err = attributes.ReadNodeAttributes(&node, []string{"//"})
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("correct", func(t *testing.T) {
|
||||
from := []string{
|
||||
"/Location:Europe/Country:Sweden/City:Stockholm",
|
||||
"/StorageType:HDD/RPM:7200",
|
||||
}
|
||||
attrs, err := attributes.ParseV2Attributes(from, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, attrs, 5)
|
||||
testAttributeMap(t, map[string]string{
|
||||
"Location": "Europe",
|
||||
"StorageType": "HDD",
|
||||
}, nil)
|
||||
})
|
||||
|
||||
t.Run("escape characters", func(t *testing.T) {
|
||||
from := []string{
|
||||
`/K\:ey1:V\/alue\\/Ke\/y2:Va\:lue`,
|
||||
}
|
||||
attrs, err := attributes.ParseV2Attributes(from, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, `K:ey1`, attrs[0].Key())
|
||||
require.Equal(t, `V/alue\`, attrs[0].Value())
|
||||
require.Equal(t, `Ke/y2`, attrs[1].Key())
|
||||
require.Equal(t, `Va:lue`, attrs[1].Value())
|
||||
})
|
||||
|
||||
t.Run("same attributes", func(t *testing.T) {
|
||||
from := []string{"/a:b", "/a:b"}
|
||||
attrs, err := attributes.ParseV2Attributes(from, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, attrs, 1)
|
||||
|
||||
t.Run("with escape characters", func(t *testing.T) {
|
||||
from = []string{`/a\::b\/`, `/a\::b\/`}
|
||||
attrs, err := attributes.ParseV2Attributes(from, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, attrs, 1)
|
||||
testAttributeMap(t, map[string]string{
|
||||
`K\:ey1`: `V\/alue`,
|
||||
`Ke\/y2`: `Va\:lue`,
|
||||
}, map[string]string{
|
||||
`K:ey1`: `V\/alue`,
|
||||
`Ke\/y2`: `Va:lue`,
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("multiple parents", func(t *testing.T) {
|
||||
from := []string{
|
||||
"/parent1:x/child:x",
|
||||
"/parent2:x/child:x",
|
||||
"/parent2:x/child:x/subchild:x",
|
||||
}
|
||||
attrs, err := attributes.ParseV2Attributes(from, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
var flag bool
|
||||
for _, attr := range attrs {
|
||||
if attr.Key() == "child" {
|
||||
flag = true
|
||||
require.Equal(t, []string{"parent1", "parent2"}, attr.ParentKeys())
|
||||
}
|
||||
}
|
||||
require.True(t, flag)
|
||||
})
|
||||
|
||||
t.Run("consistent order in chain", func(t *testing.T) {
|
||||
from := []string{"/a:1/b:2/c:3"}
|
||||
|
||||
for i := 0; i < 10000; i++ {
|
||||
attrs, err := attributes.ParseV2Attributes(from, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, "a", attrs[0].Key())
|
||||
require.Equal(t, "1", attrs[0].Value())
|
||||
require.Equal(t, "b", attrs[1].Key())
|
||||
require.Equal(t, "2", attrs[1].Value())
|
||||
require.Equal(t, "c", attrs[2].Key())
|
||||
require.Equal(t, "3", attrs[2].Value())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue