From 148618b5b3ca8f92729251e0879712a59b377f3d Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 4 Sep 2020 13:05:56 +0300 Subject: [PATCH] [#134] v2/netmap: Update unified structures Signed-off-by: Alex Vanin --- v2/netmap/convert.go | 166 ++++++++++++++++++++++- v2/netmap/marshal.go | 268 +++++++++++++++++++++++++++++++++++- v2/netmap/marshal_test.go | 126 ++++++++++++++++- v2/netmap/types.go | 279 +++++++++++++++++++++++++++++++++++++- 4 files changed, 824 insertions(+), 15 deletions(-) diff --git a/v2/netmap/convert.go b/v2/netmap/convert.go index b8713fe..caf7e33 100644 --- a/v2/netmap/convert.go +++ b/v2/netmap/convert.go @@ -4,13 +4,130 @@ import ( netmap "github.com/nspcc-dev/neofs-api-go/v2/netmap/grpc" ) +func FilterToGRPCMessage(f *Filter) *netmap.Filter { + if f == nil { + return nil + } + + m := new(netmap.Filter) + + m.SetName(f.GetName()) + m.SetKey(f.GetKey()) + m.SetValue(f.GetValue()) + m.SetOp(OperationToGRPCMessage(f.GetOp())) + + filters := make([]*netmap.Filter, 0, len(f.GetFilters())) + for _, filter := range f.GetFilters() { + filters = append(filters, FilterToGRPCMessage(filter)) + } + m.SetFilters(filters) + + return m +} + +func FilterFromGRPCMessage(m *netmap.Filter) *Filter { + if m == nil { + return nil + } + + f := new(Filter) + f.SetName(m.GetName()) + f.SetKey(m.GetKey()) + f.SetValue(m.GetValue()) + f.SetOp(OperationFromGRPCMessage(m.GetOp())) + + filters := make([]*Filter, 0, len(f.GetFilters())) + for _, filter := range m.GetFilters() { + filters = append(filters, FilterFromGRPCMessage(filter)) + } + f.SetFilters(filters) + + return f +} + +func SelectorToGRPCMessage(s *Selector) *netmap.Selector { + if s == nil { + return nil + } + + m := new(netmap.Selector) + + m.SetName(s.GetName()) + m.SetCount(s.GetCount()) + m.SetFilter(s.GetFilter()) + m.SetAttribute(s.GetAttribute()) + + return m +} + +func SelectorFromGRPCMessage(m *netmap.Selector) *Selector { + if m == nil { + return nil + } + + s := new(Selector) + + s.SetName(m.GetName()) + s.SetCount(m.GetCount()) + s.SetFilter(m.GetFilter()) + s.SetAttribute(m.GetAttribute()) + + return s +} + +func ReplicaToGRPCMessage(r *Replica) *netmap.Replica { + if r == nil { + return nil + } + + m := new(netmap.Replica) + + m.SetCount(r.GetCount()) + m.SetSelector(r.GetSelector()) + + return m +} + +func ReplicaFromGRPCMessage(m *netmap.Replica) *Replica { + if m == nil { + return nil + } + + r := new(Replica) + r.SetSelector(m.GetSelector()) + r.SetCount(m.GetCount()) + + return r +} + func PlacementPolicyToGRPCMessage(p *PlacementPolicy) *netmap.PlacementPolicy { if p == nil { return nil } - // TODO: fill me - return nil + filters := make([]*netmap.Filter, 0, len(p.GetFilters())) + for _, filter := range p.GetFilters() { + filters = append(filters, FilterToGRPCMessage(filter)) + } + + selectors := make([]*netmap.Selector, 0, len(p.GetSelectors())) + for _, selector := range p.GetSelectors() { + selectors = append(selectors, SelectorToGRPCMessage(selector)) + } + + replicas := make([]*netmap.Replica, 0, len(p.GetReplicas())) + for _, replica := range p.GetReplicas() { + replicas = append(replicas, ReplicaToGRPCMessage(replica)) + } + + m := new(netmap.PlacementPolicy) + + m.SetContainerBackupFactor(p.GetContainerBackupFactor()) + m.SetFilters(filters) + m.SetSelectors(selectors) + m.SetReplicas(replicas) + + return m } func PlacementPolicyFromGRPCMessage(m *netmap.PlacementPolicy) *PlacementPolicy { @@ -18,8 +135,45 @@ func PlacementPolicyFromGRPCMessage(m *netmap.PlacementPolicy) *PlacementPolicy return nil } - // TODO: fill me - return nil + filters := make([]*Filter, 0, len(m.GetFilters())) + for _, filter := range m.GetFilters() { + filters = append(filters, FilterFromGRPCMessage(filter)) + } + + selectors := make([]*Selector, 0, len(m.GetSelectors())) + for _, selector := range m.GetSelectors() { + selectors = append(selectors, SelectorFromGRPCMessage(selector)) + } + + replicas := make([]*Replica, 0, len(m.GetReplicas())) + for _, replica := range m.GetReplicas() { + replicas = append(replicas, ReplicaFromGRPCMessage(replica)) + } + + p := new(PlacementPolicy) + + p.SetContainerBackupFactor(m.GetContainerBackupFactor()) + p.SetFilters(filters) + p.SetSelectors(selectors) + p.SetReplicas(replicas) + + return p +} + +func OperationToGRPCMessage(n Operation) netmap.Operation { + return netmap.Operation(n) +} + +func OperationFromGRPCMessage(n netmap.Operation) Operation { + return Operation(n) +} + +func NodeStateToGRPCMessage(n NodeState) netmap.NodeInfo_State { + return netmap.NodeInfo_State(n) +} + +func NodeStateFromRPCMessage(n netmap.NodeInfo_State) NodeState { + return NodeState(n) } func AttributeToGRPCMessage(a *Attribute) *netmap.NodeInfo_Attribute { @@ -31,6 +185,7 @@ func AttributeToGRPCMessage(a *Attribute) *netmap.NodeInfo_Attribute { m.SetKey(a.GetKey()) m.SetValue(a.GetValue()) + m.SetParents(a.GetParents()) return m } @@ -44,6 +199,7 @@ func AttributeFromGRPCMessage(m *netmap.NodeInfo_Attribute) *Attribute { a.SetKey(m.GetKey()) a.SetValue(m.GetValue()) + a.SetParents(m.GetParents()) return a } @@ -57,6 +213,7 @@ func NodeInfoToGRPCMessage(n *NodeInfo) *netmap.NodeInfo { m.SetPublicKey(n.GetPublicKey()) m.SetAddress(n.GetAddress()) + m.SetState(NodeStateToGRPCMessage(n.GetState())) attr := n.GetAttributes() attrMsg := make([]*netmap.NodeInfo_Attribute, 0, len(attr)) @@ -79,6 +236,7 @@ func NodeInfoFromGRPCMessage(m *netmap.NodeInfo) *NodeInfo { a.SetPublicKey(m.GetPublicKey()) a.SetAddress(m.GetAddress()) + a.SetState(NodeStateFromRPCMessage(m.GetState())) attrMsg := m.GetAttributes() attr := make([]*Attribute, 0, len(attrMsg)) diff --git a/v2/netmap/marshal.go b/v2/netmap/marshal.go index b5025f5..a38d8a2 100644 --- a/v2/netmap/marshal.go +++ b/v2/netmap/marshal.go @@ -5,22 +5,256 @@ import ( ) const ( - keyAttributeField = 1 - valueAttributeField = 2 + nameFilterField = 1 + keyFilterField = 2 + opFilterField = 3 + valueFilterField = 4 + filtersFilterField = 5 + + nameSelectorField = 1 + countSelectorField = 2 + attributeSelectorField = 3 + filterSelectorField = 4 + + countReplicaField = 1 + selectorReplicaField = 2 + + replicasPolicyField = 1 + backupPolicyField = 2 + selectorsPolicyField = 3 + filtersPolicyField = 4 + + keyAttributeField = 1 + valueAttributeField = 2 + parentsAttributeField = 3 keyNodeInfoField = 1 addressNodeInfoField = 2 attributesNodeInfoField = 3 + stateNodeInfoField = 4 ) +func (f *Filter) StableMarshal(buf []byte) ([]byte, error) { + if f == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, f.StableSize()) + } + + var ( + offset, n int + err error + ) + + n, err = proto.StringMarshal(nameFilterField, buf[offset:], f.name) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.StringMarshal(keyFilterField, buf[offset:], f.key) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.EnumMarshal(opFilterField, buf[offset:], int32(f.op)) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.StringMarshal(valueFilterField, buf[offset:], f.value) + if err != nil { + return nil, err + } + + offset += n + + for i := range f.filters { + n, err = proto.NestedStructureMarshal(filtersFilterField, buf[offset:], f.filters[i]) + if err != nil { + return nil, err + } + + offset += n + } + + return buf, nil +} + +func (f *Filter) StableSize() (size int) { + size += proto.StringSize(nameFilterField, f.name) + size += proto.StringSize(keyFilterField, f.key) + size += proto.EnumSize(opFilterField, int32(f.op)) + size += proto.StringSize(valueFilterField, f.value) + for i := range f.filters { + size += proto.NestedStructureSize(filtersFilterField, f.filters[i]) + } + + return size +} + +func (s *Selector) StableMarshal(buf []byte) ([]byte, error) { + if s == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, s.StableSize()) + } + + var ( + offset, n int + err error + ) + + n, err = proto.StringMarshal(nameSelectorField, buf[offset:], s.name) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.UInt32Marshal(countSelectorField, buf[offset:], s.count) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.StringMarshal(attributeSelectorField, buf[offset:], s.attribute) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.StringMarshal(filterSelectorField, buf[offset:], s.filter) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (s *Selector) StableSize() (size int) { + size += proto.StringSize(nameSelectorField, s.name) + size += proto.UInt32Size(countSelectorField, s.count) + size += proto.StringSize(attributeSelectorField, s.attribute) + size += proto.StringSize(filterSelectorField, s.filter) + + return size +} + +func (r *Replica) StableMarshal(buf []byte) ([]byte, error) { + if r == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, r.StableSize()) + } + + var ( + offset, n int + err error + ) + + n, err = proto.UInt32Marshal(countReplicaField, buf[offset:], r.count) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.StringMarshal(selectorReplicaField, buf[offset:], r.selector) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (r *Replica) StableSize() (size int) { + size += proto.UInt32Size(countReplicaField, r.count) + size += proto.StringSize(selectorReplicaField, r.selector) + + return size +} + func (p *PlacementPolicy) StableMarshal(buf []byte) ([]byte, error) { - // todo: implement me - return nil, nil + if p == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, p.StableSize()) + } + + var ( + offset, n int + err error + ) + + for i := range p.replicas { + n, err = proto.NestedStructureMarshal(replicasPolicyField, buf[offset:], p.replicas[i]) + if err != nil { + return nil, err + } + + offset += n + } + + n, err = proto.UInt32Marshal(backupPolicyField, buf[offset:], p.backupFactor) + if err != nil { + return nil, err + } + + offset += n + + for i := range p.selectors { + n, err = proto.NestedStructureMarshal(selectorsPolicyField, buf[offset:], p.selectors[i]) + if err != nil { + return nil, err + } + + offset += n + } + + for i := range p.filters { + n, err = proto.NestedStructureMarshal(filtersPolicyField, buf[offset:], p.filters[i]) + if err != nil { + return nil, err + } + + offset += n + } + + return buf, nil } func (p *PlacementPolicy) StableSize() (size int) { - // todo: implement me - return 0 + for i := range p.replicas { + size += proto.NestedStructureSize(replicasPolicyField, p.replicas[i]) + } + + size += proto.UInt32Size(backupPolicyField, p.backupFactor) + + for i := range p.selectors { + size += proto.NestedStructureSize(selectorsPolicyField, p.selectors[i]) + } + + for i := range p.filters { + size += proto.NestedStructureSize(filtersPolicyField, p.filters[i]) + } + + return size } func (a *Attribute) StableMarshal(buf []byte) ([]byte, error) { @@ -49,6 +283,17 @@ func (a *Attribute) StableMarshal(buf []byte) ([]byte, error) { return nil, err } + offset += n + + for i := range a.parents { + n, err = proto.StringMarshal(parentsAttributeField, buf[offset:], a.parents[i]) + if err != nil { + return nil, err + } + + offset += n + } + return buf, nil } @@ -60,6 +305,10 @@ func (a *Attribute) StableSize() (size int) { size += proto.StringSize(keyAttributeField, a.key) size += proto.StringSize(valueAttributeField, a.value) + for i := range a.parents { + size += proto.StringSize(parentsAttributeField, a.parents[i]) + } + return size } @@ -100,6 +349,11 @@ func (ni *NodeInfo) StableMarshal(buf []byte) ([]byte, error) { offset += n } + n, err = proto.EnumMarshal(stateNodeInfoField, buf[offset:], int32(ni.state)) + if err != nil { + return nil, err + } + return buf, nil } @@ -114,5 +368,7 @@ func (ni *NodeInfo) StableSize() (size int) { size += proto.NestedStructureSize(attributesNodeInfoField, ni.attributes[i]) } + size += proto.EnumSize(stateNodeInfoField, int32(ni.state)) + return size } diff --git a/v2/netmap/marshal_test.go b/v2/netmap/marshal_test.go index 1292c40..7178a20 100644 --- a/v2/netmap/marshal_test.go +++ b/v2/netmap/marshal_test.go @@ -25,7 +25,7 @@ func TestAttribute_StableMarshal(t *testing.T) { }) } -func TestNodeInfo(t *testing.T) { +func TestNodeInfo_StableMarshal(t *testing.T) { from := generateNodeInfo("publicKey", "/multi/addr", 10) transport := new(grpc.NodeInfo) @@ -41,10 +41,75 @@ func TestNodeInfo(t *testing.T) { }) } +func TestFilter_StableMarshal(t *testing.T) { + from := generateFilter("key", "value", false) + transport := new(grpc.Filter) + + t.Run("non empty", func(t *testing.T) { + wire, err := from.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + to := netmap.FilterFromGRPCMessage(transport) + require.Equal(t, from, to) + }) +} + +func TestSelector_StableMarshal(t *testing.T) { + from := generateSelector("name") + transport := new(grpc.Selector) + + t.Run("non empty", func(t *testing.T) { + wire, err := from.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + to := netmap.SelectorFromGRPCMessage(transport) + require.Equal(t, from, to) + }) +} + +func TestReplica_StableMarshal(t *testing.T) { + from := generateReplica("selector") + transport := new(grpc.Replica) + + t.Run("non empty", func(t *testing.T) { + wire, err := from.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + to := netmap.ReplicaFromGRPCMessage(transport) + require.Equal(t, from, to) + }) +} + +func TestPlacementPolicy_StableSize(t *testing.T) { + from := generatePolicy(3) + transport := new(grpc.PlacementPolicy) + + t.Run("non empty", func(t *testing.T) { + wire, err := from.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + to := netmap.PlacementPolicyFromGRPCMessage(transport) + require.Equal(t, from, to) + }) +} + func generateAttribute(k, v string) *netmap.Attribute { attr := new(netmap.Attribute) attr.SetKey(k) attr.SetValue(v) + attr.SetParents([]string{k, v}) return attr } @@ -53,6 +118,7 @@ func generateNodeInfo(key, addr string, n int) *netmap.NodeInfo { nodeInfo := new(netmap.NodeInfo) nodeInfo.SetPublicKey([]byte(key)) nodeInfo.SetAddress(addr) + nodeInfo.SetState(netmap.Online) attrs := make([]*netmap.Attribute, n) for i := 0; i < n; i++ { @@ -64,3 +130,61 @@ func generateNodeInfo(key, addr string, n int) *netmap.NodeInfo { return nodeInfo } + +func generateFilter(key, value string, fin bool) *netmap.Filter { + f := new(netmap.Filter) + f.SetKey(key) + f.SetValue(value) + f.SetName("name") + f.SetOp(netmap.AND) + if !fin { + ff := generateFilter(key+"fin", value+"fin", true) + f.SetFilters([]*netmap.Filter{ff}) + } else { + f.SetFilters([]*netmap.Filter{}) + } + + return f +} + +func generateSelector(name string) *netmap.Selector { + s := new(netmap.Selector) + s.SetName(name) + s.SetAttribute("attribute") + s.SetCount(10) + s.SetFilter("filter") + + return s +} + +func generateReplica(selector string) *netmap.Replica { + r := new(netmap.Replica) + r.SetCount(10) + r.SetSelector(selector) + + return r +} + +func generatePolicy(n int) *netmap.PlacementPolicy { + var ( + p = new(netmap.PlacementPolicy) + f = make([]*netmap.Filter, 0, n) + s = make([]*netmap.Selector, 0, n) + r = make([]*netmap.Replica, 0, n) + ) + + for i := 0; i < n; i++ { + ind := strconv.Itoa(i) + + f = append(f, generateFilter("key"+ind, "val"+ind, false)) + s = append(s, generateSelector("name"+ind)) + r = append(r, generateReplica("selector"+ind)) + } + + p.SetFilters(f) + p.SetSelectors(s) + p.SetReplicas(r) + p.SetContainerBackupFactor(10) + + return p +} diff --git a/v2/netmap/types.go b/v2/netmap/types.go index 7df923c..c16fa50 100644 --- a/v2/netmap/types.go +++ b/v2/netmap/types.go @@ -1,13 +1,39 @@ package netmap +type Filter struct { + name string + key string + op Operation + value string + filters []*Filter +} + +type Selector struct { + name string + count uint32 + attribute string + filter string +} + +type Replica struct { + count uint32 + selector string +} + +type Operation uint32 + type PlacementPolicy struct { - // TODO: fill me + replicas []*Replica + backupFactor uint32 + selectors []*Selector + filters []*Filter } // Attribute of storage node. type Attribute struct { - key string - value string + key string + value string + parents []string } // NodeInfo of storage node. @@ -15,17 +41,234 @@ type NodeInfo struct { publicKey []byte address string attributes []*Attribute + state NodeState } // NodeState of storage node. type NodeState uint32 const ( - Unspecified NodeState = iota + UnspecifiedState NodeState = iota Online Offline ) +const ( + UnspecifiedOperation Operation = iota + EQ + NE + GT + GE + LT + LE + OR + AND +) + +func (f *Filter) GetFilters() []*Filter { + if f != nil { + return f.filters + } + + return nil +} + +func (f *Filter) SetFilters(filters []*Filter) { + if f != nil { + f.filters = filters + } +} + +func (f *Filter) GetValue() string { + if f != nil { + return f.value + } + + return "" +} + +func (f *Filter) SetValue(value string) { + if f != nil { + f.value = value + } +} + +func (f *Filter) GetOp() Operation { + if f != nil { + return f.op + } + return UnspecifiedOperation +} + +func (f *Filter) SetOp(op Operation) { + if f != nil { + f.op = op + } +} + +func (f *Filter) GetKey() string { + if f != nil { + return f.key + } + + return "" +} + +func (f *Filter) SetKey(key string) { + if f != nil { + f.key = key + } +} + +func (f *Filter) GetName() string { + if f != nil { + return f.name + } + + return "" +} + +func (f *Filter) SetName(name string) { + if f != nil { + f.name = name + } +} + +func (s *Selector) GetFilter() string { + if s != nil { + return s.filter + } + + return "" +} + +func (s *Selector) SetFilter(filter string) { + if s != nil { + s.filter = filter + } +} + +func (s *Selector) GetAttribute() string { + if s != nil { + return s.attribute + } + + return "" +} + +func (s *Selector) SetAttribute(attribute string) { + if s != nil { + s.attribute = attribute + } +} + +func (s *Selector) GetCount() uint32 { + if s != nil { + return s.count + } + + return 0 +} + +func (s *Selector) SetCount(count uint32) { + if s != nil { + s.count = count + } +} + +func (s *Selector) GetName() string { + if s != nil { + return s.name + } + + return "" +} + +func (s *Selector) SetName(name string) { + if s != nil { + s.name = name + } +} + +func (r *Replica) GetSelector() string { + if r != nil { + return r.selector + } + + return "" +} + +func (r *Replica) SetSelector(selector string) { + if r != nil { + r.selector = selector + } +} + +func (r *Replica) GetCount() uint32 { + if r != nil { + return r.count + } + + return 0 +} + +func (r *Replica) SetCount(count uint32) { + if r != nil { + r.count = count + } +} + +func (p *PlacementPolicy) GetFilters() []*Filter { + if p != nil { + return p.filters + } + + return nil +} + +func (p *PlacementPolicy) SetFilters(filters []*Filter) { + if p != nil { + + } + p.filters = filters +} + +func (p *PlacementPolicy) GetSelectors() []*Selector { + if p != nil { + return p.selectors + } + + return nil +} + +func (p *PlacementPolicy) SetSelectors(selectors []*Selector) { + if p != nil { + p.selectors = selectors + } +} + +func (p *PlacementPolicy) GetContainerBackupFactor() uint32 { + if p != nil { + return p.backupFactor + } + + return 0 +} + +func (p *PlacementPolicy) SetContainerBackupFactor(backupFactor uint32) { + if p != nil { + p.backupFactor = backupFactor + } +} + +func (p *PlacementPolicy) GetReplicas() []*Replica { + return p.replicas +} + +func (p *PlacementPolicy) SetReplicas(replicas []*Replica) { + p.replicas = replicas +} + func (a *Attribute) GetKey() string { if a != nil { return a.key @@ -54,6 +297,20 @@ func (a *Attribute) SetValue(v string) { } } +func (a *Attribute) GetParents() []string { + if a != nil { + return a.parents + } + + return nil +} + +func (a *Attribute) SetParents(parent []string) { + if a != nil { + a.parents = parent + } +} + func (ni *NodeInfo) GetPublicKey() []byte { if ni != nil { return ni.publicKey @@ -95,3 +352,17 @@ func (ni *NodeInfo) SetAttributes(v []*Attribute) { ni.attributes = v } } + +func (ni *NodeInfo) GetState() NodeState { + if ni != nil { + return ni.state + } + + return UnspecifiedState +} + +func (ni *NodeInfo) SetState(state NodeState) { + if ni != nil { + ni.state = state + } +}