[#134] v2/netmap: Update unified structures

Signed-off-by: Alex Vanin <alexey@nspcc.ru>
This commit is contained in:
Alex Vanin 2020-09-04 13:05:56 +03:00 committed by Stanislav Bogatyrev
parent 505ef59dd6
commit 148618b5b3
4 changed files with 824 additions and 15 deletions

View file

@ -4,13 +4,130 @@ import (
netmap "github.com/nspcc-dev/neofs-api-go/v2/netmap/grpc" netmap "github.com/nspcc-dev/neofs-api-go/v2/netmap/grpc"
) )
func FilterToGRPCMessage(f *Filter) *netmap.Filter {
if f == nil {
return nil
}
m := new(netmap.Filter)
m.SetName(f.GetName())
m.SetKey(f.GetKey())
m.SetValue(f.GetValue())
m.SetOp(OperationToGRPCMessage(f.GetOp()))
filters := make([]*netmap.Filter, 0, len(f.GetFilters()))
for _, filter := range f.GetFilters() {
filters = append(filters, FilterToGRPCMessage(filter))
}
m.SetFilters(filters)
return m
}
func FilterFromGRPCMessage(m *netmap.Filter) *Filter {
if m == nil {
return nil
}
f := new(Filter)
f.SetName(m.GetName())
f.SetKey(m.GetKey())
f.SetValue(m.GetValue())
f.SetOp(OperationFromGRPCMessage(m.GetOp()))
filters := make([]*Filter, 0, len(f.GetFilters()))
for _, filter := range m.GetFilters() {
filters = append(filters, FilterFromGRPCMessage(filter))
}
f.SetFilters(filters)
return f
}
func SelectorToGRPCMessage(s *Selector) *netmap.Selector {
if s == nil {
return nil
}
m := new(netmap.Selector)
m.SetName(s.GetName())
m.SetCount(s.GetCount())
m.SetFilter(s.GetFilter())
m.SetAttribute(s.GetAttribute())
return m
}
func SelectorFromGRPCMessage(m *netmap.Selector) *Selector {
if m == nil {
return nil
}
s := new(Selector)
s.SetName(m.GetName())
s.SetCount(m.GetCount())
s.SetFilter(m.GetFilter())
s.SetAttribute(m.GetAttribute())
return s
}
func ReplicaToGRPCMessage(r *Replica) *netmap.Replica {
if r == nil {
return nil
}
m := new(netmap.Replica)
m.SetCount(r.GetCount())
m.SetSelector(r.GetSelector())
return m
}
func ReplicaFromGRPCMessage(m *netmap.Replica) *Replica {
if m == nil {
return nil
}
r := new(Replica)
r.SetSelector(m.GetSelector())
r.SetCount(m.GetCount())
return r
}
func PlacementPolicyToGRPCMessage(p *PlacementPolicy) *netmap.PlacementPolicy { func PlacementPolicyToGRPCMessage(p *PlacementPolicy) *netmap.PlacementPolicy {
if p == nil { if p == nil {
return nil return nil
} }
// TODO: fill me filters := make([]*netmap.Filter, 0, len(p.GetFilters()))
return nil for _, filter := range p.GetFilters() {
filters = append(filters, FilterToGRPCMessage(filter))
}
selectors := make([]*netmap.Selector, 0, len(p.GetSelectors()))
for _, selector := range p.GetSelectors() {
selectors = append(selectors, SelectorToGRPCMessage(selector))
}
replicas := make([]*netmap.Replica, 0, len(p.GetReplicas()))
for _, replica := range p.GetReplicas() {
replicas = append(replicas, ReplicaToGRPCMessage(replica))
}
m := new(netmap.PlacementPolicy)
m.SetContainerBackupFactor(p.GetContainerBackupFactor())
m.SetFilters(filters)
m.SetSelectors(selectors)
m.SetReplicas(replicas)
return m
} }
func PlacementPolicyFromGRPCMessage(m *netmap.PlacementPolicy) *PlacementPolicy { func PlacementPolicyFromGRPCMessage(m *netmap.PlacementPolicy) *PlacementPolicy {
@ -18,8 +135,45 @@ func PlacementPolicyFromGRPCMessage(m *netmap.PlacementPolicy) *PlacementPolicy
return nil return nil
} }
// TODO: fill me filters := make([]*Filter, 0, len(m.GetFilters()))
return nil for _, filter := range m.GetFilters() {
filters = append(filters, FilterFromGRPCMessage(filter))
}
selectors := make([]*Selector, 0, len(m.GetSelectors()))
for _, selector := range m.GetSelectors() {
selectors = append(selectors, SelectorFromGRPCMessage(selector))
}
replicas := make([]*Replica, 0, len(m.GetReplicas()))
for _, replica := range m.GetReplicas() {
replicas = append(replicas, ReplicaFromGRPCMessage(replica))
}
p := new(PlacementPolicy)
p.SetContainerBackupFactor(m.GetContainerBackupFactor())
p.SetFilters(filters)
p.SetSelectors(selectors)
p.SetReplicas(replicas)
return p
}
func OperationToGRPCMessage(n Operation) netmap.Operation {
return netmap.Operation(n)
}
func OperationFromGRPCMessage(n netmap.Operation) Operation {
return Operation(n)
}
func NodeStateToGRPCMessage(n NodeState) netmap.NodeInfo_State {
return netmap.NodeInfo_State(n)
}
func NodeStateFromRPCMessage(n netmap.NodeInfo_State) NodeState {
return NodeState(n)
} }
func AttributeToGRPCMessage(a *Attribute) *netmap.NodeInfo_Attribute { func AttributeToGRPCMessage(a *Attribute) *netmap.NodeInfo_Attribute {
@ -31,6 +185,7 @@ func AttributeToGRPCMessage(a *Attribute) *netmap.NodeInfo_Attribute {
m.SetKey(a.GetKey()) m.SetKey(a.GetKey())
m.SetValue(a.GetValue()) m.SetValue(a.GetValue())
m.SetParents(a.GetParents())
return m return m
} }
@ -44,6 +199,7 @@ func AttributeFromGRPCMessage(m *netmap.NodeInfo_Attribute) *Attribute {
a.SetKey(m.GetKey()) a.SetKey(m.GetKey())
a.SetValue(m.GetValue()) a.SetValue(m.GetValue())
a.SetParents(m.GetParents())
return a return a
} }
@ -57,6 +213,7 @@ func NodeInfoToGRPCMessage(n *NodeInfo) *netmap.NodeInfo {
m.SetPublicKey(n.GetPublicKey()) m.SetPublicKey(n.GetPublicKey())
m.SetAddress(n.GetAddress()) m.SetAddress(n.GetAddress())
m.SetState(NodeStateToGRPCMessage(n.GetState()))
attr := n.GetAttributes() attr := n.GetAttributes()
attrMsg := make([]*netmap.NodeInfo_Attribute, 0, len(attr)) attrMsg := make([]*netmap.NodeInfo_Attribute, 0, len(attr))
@ -79,6 +236,7 @@ func NodeInfoFromGRPCMessage(m *netmap.NodeInfo) *NodeInfo {
a.SetPublicKey(m.GetPublicKey()) a.SetPublicKey(m.GetPublicKey())
a.SetAddress(m.GetAddress()) a.SetAddress(m.GetAddress())
a.SetState(NodeStateFromRPCMessage(m.GetState()))
attrMsg := m.GetAttributes() attrMsg := m.GetAttributes()
attr := make([]*Attribute, 0, len(attrMsg)) attr := make([]*Attribute, 0, len(attrMsg))

View file

@ -5,22 +5,256 @@ import (
) )
const ( const (
keyAttributeField = 1 nameFilterField = 1
valueAttributeField = 2 keyFilterField = 2
opFilterField = 3
valueFilterField = 4
filtersFilterField = 5
nameSelectorField = 1
countSelectorField = 2
attributeSelectorField = 3
filterSelectorField = 4
countReplicaField = 1
selectorReplicaField = 2
replicasPolicyField = 1
backupPolicyField = 2
selectorsPolicyField = 3
filtersPolicyField = 4
keyAttributeField = 1
valueAttributeField = 2
parentsAttributeField = 3
keyNodeInfoField = 1 keyNodeInfoField = 1
addressNodeInfoField = 2 addressNodeInfoField = 2
attributesNodeInfoField = 3 attributesNodeInfoField = 3
stateNodeInfoField = 4
) )
func (f *Filter) StableMarshal(buf []byte) ([]byte, error) {
if f == nil {
return []byte{}, nil
}
if buf == nil {
buf = make([]byte, f.StableSize())
}
var (
offset, n int
err error
)
n, err = proto.StringMarshal(nameFilterField, buf[offset:], f.name)
if err != nil {
return nil, err
}
offset += n
n, err = proto.StringMarshal(keyFilterField, buf[offset:], f.key)
if err != nil {
return nil, err
}
offset += n
n, err = proto.EnumMarshal(opFilterField, buf[offset:], int32(f.op))
if err != nil {
return nil, err
}
offset += n
n, err = proto.StringMarshal(valueFilterField, buf[offset:], f.value)
if err != nil {
return nil, err
}
offset += n
for i := range f.filters {
n, err = proto.NestedStructureMarshal(filtersFilterField, buf[offset:], f.filters[i])
if err != nil {
return nil, err
}
offset += n
}
return buf, nil
}
func (f *Filter) StableSize() (size int) {
size += proto.StringSize(nameFilterField, f.name)
size += proto.StringSize(keyFilterField, f.key)
size += proto.EnumSize(opFilterField, int32(f.op))
size += proto.StringSize(valueFilterField, f.value)
for i := range f.filters {
size += proto.NestedStructureSize(filtersFilterField, f.filters[i])
}
return size
}
func (s *Selector) StableMarshal(buf []byte) ([]byte, error) {
if s == nil {
return []byte{}, nil
}
if buf == nil {
buf = make([]byte, s.StableSize())
}
var (
offset, n int
err error
)
n, err = proto.StringMarshal(nameSelectorField, buf[offset:], s.name)
if err != nil {
return nil, err
}
offset += n
n, err = proto.UInt32Marshal(countSelectorField, buf[offset:], s.count)
if err != nil {
return nil, err
}
offset += n
n, err = proto.StringMarshal(attributeSelectorField, buf[offset:], s.attribute)
if err != nil {
return nil, err
}
offset += n
n, err = proto.StringMarshal(filterSelectorField, buf[offset:], s.filter)
if err != nil {
return nil, err
}
return buf, nil
}
func (s *Selector) StableSize() (size int) {
size += proto.StringSize(nameSelectorField, s.name)
size += proto.UInt32Size(countSelectorField, s.count)
size += proto.StringSize(attributeSelectorField, s.attribute)
size += proto.StringSize(filterSelectorField, s.filter)
return size
}
func (r *Replica) StableMarshal(buf []byte) ([]byte, error) {
if r == nil {
return []byte{}, nil
}
if buf == nil {
buf = make([]byte, r.StableSize())
}
var (
offset, n int
err error
)
n, err = proto.UInt32Marshal(countReplicaField, buf[offset:], r.count)
if err != nil {
return nil, err
}
offset += n
n, err = proto.StringMarshal(selectorReplicaField, buf[offset:], r.selector)
if err != nil {
return nil, err
}
return buf, nil
}
func (r *Replica) StableSize() (size int) {
size += proto.UInt32Size(countReplicaField, r.count)
size += proto.StringSize(selectorReplicaField, r.selector)
return size
}
func (p *PlacementPolicy) StableMarshal(buf []byte) ([]byte, error) { func (p *PlacementPolicy) StableMarshal(buf []byte) ([]byte, error) {
// todo: implement me if p == nil {
return nil, nil return []byte{}, nil
}
if buf == nil {
buf = make([]byte, p.StableSize())
}
var (
offset, n int
err error
)
for i := range p.replicas {
n, err = proto.NestedStructureMarshal(replicasPolicyField, buf[offset:], p.replicas[i])
if err != nil {
return nil, err
}
offset += n
}
n, err = proto.UInt32Marshal(backupPolicyField, buf[offset:], p.backupFactor)
if err != nil {
return nil, err
}
offset += n
for i := range p.selectors {
n, err = proto.NestedStructureMarshal(selectorsPolicyField, buf[offset:], p.selectors[i])
if err != nil {
return nil, err
}
offset += n
}
for i := range p.filters {
n, err = proto.NestedStructureMarshal(filtersPolicyField, buf[offset:], p.filters[i])
if err != nil {
return nil, err
}
offset += n
}
return buf, nil
} }
func (p *PlacementPolicy) StableSize() (size int) { func (p *PlacementPolicy) StableSize() (size int) {
// todo: implement me for i := range p.replicas {
return 0 size += proto.NestedStructureSize(replicasPolicyField, p.replicas[i])
}
size += proto.UInt32Size(backupPolicyField, p.backupFactor)
for i := range p.selectors {
size += proto.NestedStructureSize(selectorsPolicyField, p.selectors[i])
}
for i := range p.filters {
size += proto.NestedStructureSize(filtersPolicyField, p.filters[i])
}
return size
} }
func (a *Attribute) StableMarshal(buf []byte) ([]byte, error) { func (a *Attribute) StableMarshal(buf []byte) ([]byte, error) {
@ -49,6 +283,17 @@ func (a *Attribute) StableMarshal(buf []byte) ([]byte, error) {
return nil, err return nil, err
} }
offset += n
for i := range a.parents {
n, err = proto.StringMarshal(parentsAttributeField, buf[offset:], a.parents[i])
if err != nil {
return nil, err
}
offset += n
}
return buf, nil return buf, nil
} }
@ -60,6 +305,10 @@ func (a *Attribute) StableSize() (size int) {
size += proto.StringSize(keyAttributeField, a.key) size += proto.StringSize(keyAttributeField, a.key)
size += proto.StringSize(valueAttributeField, a.value) size += proto.StringSize(valueAttributeField, a.value)
for i := range a.parents {
size += proto.StringSize(parentsAttributeField, a.parents[i])
}
return size return size
} }
@ -100,6 +349,11 @@ func (ni *NodeInfo) StableMarshal(buf []byte) ([]byte, error) {
offset += n offset += n
} }
n, err = proto.EnumMarshal(stateNodeInfoField, buf[offset:], int32(ni.state))
if err != nil {
return nil, err
}
return buf, nil return buf, nil
} }
@ -114,5 +368,7 @@ func (ni *NodeInfo) StableSize() (size int) {
size += proto.NestedStructureSize(attributesNodeInfoField, ni.attributes[i]) size += proto.NestedStructureSize(attributesNodeInfoField, ni.attributes[i])
} }
size += proto.EnumSize(stateNodeInfoField, int32(ni.state))
return size return size
} }

View file

@ -25,7 +25,7 @@ func TestAttribute_StableMarshal(t *testing.T) {
}) })
} }
func TestNodeInfo(t *testing.T) { func TestNodeInfo_StableMarshal(t *testing.T) {
from := generateNodeInfo("publicKey", "/multi/addr", 10) from := generateNodeInfo("publicKey", "/multi/addr", 10)
transport := new(grpc.NodeInfo) transport := new(grpc.NodeInfo)
@ -41,10 +41,75 @@ func TestNodeInfo(t *testing.T) {
}) })
} }
func TestFilter_StableMarshal(t *testing.T) {
from := generateFilter("key", "value", false)
transport := new(grpc.Filter)
t.Run("non empty", func(t *testing.T) {
wire, err := from.StableMarshal(nil)
require.NoError(t, err)
err = transport.Unmarshal(wire)
require.NoError(t, err)
to := netmap.FilterFromGRPCMessage(transport)
require.Equal(t, from, to)
})
}
func TestSelector_StableMarshal(t *testing.T) {
from := generateSelector("name")
transport := new(grpc.Selector)
t.Run("non empty", func(t *testing.T) {
wire, err := from.StableMarshal(nil)
require.NoError(t, err)
err = transport.Unmarshal(wire)
require.NoError(t, err)
to := netmap.SelectorFromGRPCMessage(transport)
require.Equal(t, from, to)
})
}
func TestReplica_StableMarshal(t *testing.T) {
from := generateReplica("selector")
transport := new(grpc.Replica)
t.Run("non empty", func(t *testing.T) {
wire, err := from.StableMarshal(nil)
require.NoError(t, err)
err = transport.Unmarshal(wire)
require.NoError(t, err)
to := netmap.ReplicaFromGRPCMessage(transport)
require.Equal(t, from, to)
})
}
func TestPlacementPolicy_StableSize(t *testing.T) {
from := generatePolicy(3)
transport := new(grpc.PlacementPolicy)
t.Run("non empty", func(t *testing.T) {
wire, err := from.StableMarshal(nil)
require.NoError(t, err)
err = transport.Unmarshal(wire)
require.NoError(t, err)
to := netmap.PlacementPolicyFromGRPCMessage(transport)
require.Equal(t, from, to)
})
}
func generateAttribute(k, v string) *netmap.Attribute { func generateAttribute(k, v string) *netmap.Attribute {
attr := new(netmap.Attribute) attr := new(netmap.Attribute)
attr.SetKey(k) attr.SetKey(k)
attr.SetValue(v) attr.SetValue(v)
attr.SetParents([]string{k, v})
return attr return attr
} }
@ -53,6 +118,7 @@ func generateNodeInfo(key, addr string, n int) *netmap.NodeInfo {
nodeInfo := new(netmap.NodeInfo) nodeInfo := new(netmap.NodeInfo)
nodeInfo.SetPublicKey([]byte(key)) nodeInfo.SetPublicKey([]byte(key))
nodeInfo.SetAddress(addr) nodeInfo.SetAddress(addr)
nodeInfo.SetState(netmap.Online)
attrs := make([]*netmap.Attribute, n) attrs := make([]*netmap.Attribute, n)
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
@ -64,3 +130,61 @@ func generateNodeInfo(key, addr string, n int) *netmap.NodeInfo {
return nodeInfo return nodeInfo
} }
func generateFilter(key, value string, fin bool) *netmap.Filter {
f := new(netmap.Filter)
f.SetKey(key)
f.SetValue(value)
f.SetName("name")
f.SetOp(netmap.AND)
if !fin {
ff := generateFilter(key+"fin", value+"fin", true)
f.SetFilters([]*netmap.Filter{ff})
} else {
f.SetFilters([]*netmap.Filter{})
}
return f
}
func generateSelector(name string) *netmap.Selector {
s := new(netmap.Selector)
s.SetName(name)
s.SetAttribute("attribute")
s.SetCount(10)
s.SetFilter("filter")
return s
}
func generateReplica(selector string) *netmap.Replica {
r := new(netmap.Replica)
r.SetCount(10)
r.SetSelector(selector)
return r
}
func generatePolicy(n int) *netmap.PlacementPolicy {
var (
p = new(netmap.PlacementPolicy)
f = make([]*netmap.Filter, 0, n)
s = make([]*netmap.Selector, 0, n)
r = make([]*netmap.Replica, 0, n)
)
for i := 0; i < n; i++ {
ind := strconv.Itoa(i)
f = append(f, generateFilter("key"+ind, "val"+ind, false))
s = append(s, generateSelector("name"+ind))
r = append(r, generateReplica("selector"+ind))
}
p.SetFilters(f)
p.SetSelectors(s)
p.SetReplicas(r)
p.SetContainerBackupFactor(10)
return p
}

View file

@ -1,13 +1,39 @@
package netmap package netmap
type Filter struct {
name string
key string
op Operation
value string
filters []*Filter
}
type Selector struct {
name string
count uint32
attribute string
filter string
}
type Replica struct {
count uint32
selector string
}
type Operation uint32
type PlacementPolicy struct { type PlacementPolicy struct {
// TODO: fill me replicas []*Replica
backupFactor uint32
selectors []*Selector
filters []*Filter
} }
// Attribute of storage node. // Attribute of storage node.
type Attribute struct { type Attribute struct {
key string key string
value string value string
parents []string
} }
// NodeInfo of storage node. // NodeInfo of storage node.
@ -15,17 +41,234 @@ type NodeInfo struct {
publicKey []byte publicKey []byte
address string address string
attributes []*Attribute attributes []*Attribute
state NodeState
} }
// NodeState of storage node. // NodeState of storage node.
type NodeState uint32 type NodeState uint32
const ( const (
Unspecified NodeState = iota UnspecifiedState NodeState = iota
Online Online
Offline Offline
) )
const (
UnspecifiedOperation Operation = iota
EQ
NE
GT
GE
LT
LE
OR
AND
)
func (f *Filter) GetFilters() []*Filter {
if f != nil {
return f.filters
}
return nil
}
func (f *Filter) SetFilters(filters []*Filter) {
if f != nil {
f.filters = filters
}
}
func (f *Filter) GetValue() string {
if f != nil {
return f.value
}
return ""
}
func (f *Filter) SetValue(value string) {
if f != nil {
f.value = value
}
}
func (f *Filter) GetOp() Operation {
if f != nil {
return f.op
}
return UnspecifiedOperation
}
func (f *Filter) SetOp(op Operation) {
if f != nil {
f.op = op
}
}
func (f *Filter) GetKey() string {
if f != nil {
return f.key
}
return ""
}
func (f *Filter) SetKey(key string) {
if f != nil {
f.key = key
}
}
func (f *Filter) GetName() string {
if f != nil {
return f.name
}
return ""
}
func (f *Filter) SetName(name string) {
if f != nil {
f.name = name
}
}
func (s *Selector) GetFilter() string {
if s != nil {
return s.filter
}
return ""
}
func (s *Selector) SetFilter(filter string) {
if s != nil {
s.filter = filter
}
}
func (s *Selector) GetAttribute() string {
if s != nil {
return s.attribute
}
return ""
}
func (s *Selector) SetAttribute(attribute string) {
if s != nil {
s.attribute = attribute
}
}
func (s *Selector) GetCount() uint32 {
if s != nil {
return s.count
}
return 0
}
func (s *Selector) SetCount(count uint32) {
if s != nil {
s.count = count
}
}
func (s *Selector) GetName() string {
if s != nil {
return s.name
}
return ""
}
func (s *Selector) SetName(name string) {
if s != nil {
s.name = name
}
}
func (r *Replica) GetSelector() string {
if r != nil {
return r.selector
}
return ""
}
func (r *Replica) SetSelector(selector string) {
if r != nil {
r.selector = selector
}
}
func (r *Replica) GetCount() uint32 {
if r != nil {
return r.count
}
return 0
}
func (r *Replica) SetCount(count uint32) {
if r != nil {
r.count = count
}
}
func (p *PlacementPolicy) GetFilters() []*Filter {
if p != nil {
return p.filters
}
return nil
}
func (p *PlacementPolicy) SetFilters(filters []*Filter) {
if p != nil {
}
p.filters = filters
}
func (p *PlacementPolicy) GetSelectors() []*Selector {
if p != nil {
return p.selectors
}
return nil
}
func (p *PlacementPolicy) SetSelectors(selectors []*Selector) {
if p != nil {
p.selectors = selectors
}
}
func (p *PlacementPolicy) GetContainerBackupFactor() uint32 {
if p != nil {
return p.backupFactor
}
return 0
}
func (p *PlacementPolicy) SetContainerBackupFactor(backupFactor uint32) {
if p != nil {
p.backupFactor = backupFactor
}
}
func (p *PlacementPolicy) GetReplicas() []*Replica {
return p.replicas
}
func (p *PlacementPolicy) SetReplicas(replicas []*Replica) {
p.replicas = replicas
}
func (a *Attribute) GetKey() string { func (a *Attribute) GetKey() string {
if a != nil { if a != nil {
return a.key return a.key
@ -54,6 +297,20 @@ func (a *Attribute) SetValue(v string) {
} }
} }
func (a *Attribute) GetParents() []string {
if a != nil {
return a.parents
}
return nil
}
func (a *Attribute) SetParents(parent []string) {
if a != nil {
a.parents = parent
}
}
func (ni *NodeInfo) GetPublicKey() []byte { func (ni *NodeInfo) GetPublicKey() []byte {
if ni != nil { if ni != nil {
return ni.publicKey return ni.publicKey
@ -95,3 +352,17 @@ func (ni *NodeInfo) SetAttributes(v []*Attribute) {
ni.attributes = v ni.attributes = v
} }
} }
func (ni *NodeInfo) GetState() NodeState {
if ni != nil {
return ni.state
}
return UnspecifiedState
}
func (ni *NodeInfo) SetState(state NodeState) {
if ni != nil {
ni.state = state
}
}