carpawell/frostfs-sdk-go
1
0
Fork 0
forked from TrueCloudLab/frostfs-sdk-go
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use CID to prevent incorrect usage in netmap (#396)

Browse source 
close #216
This commit is contained in:
Roman Khimov 2023-04-21 15:25:53 +03:00 committed by GitHub
commit dbbb22ca28
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 61 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
netmap/json_test.go
View file

@ -6,6 +6,8 @@ import (
"path/filepath"
"testing"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/stretchr/testify/require"
)
@ -37,6 +39,20 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn
}
}
func compareNodesIgnoreOrder(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
require.Equal(t, len(expected), len(actual))
for i := range expected {
require.Equal(t, len(expected[i]), len(actual[i]))
var expectedNodes []NodeInfo
for _, index := range expected[i] {
expectedNodes = append(expectedNodes, nodes[index])
}
require.ElementsMatch(t, expectedNodes, actual[i])
}
}
func TestPlacementPolicy_Interopability(t *testing.T) {
const testsDir = "./json_tests"
@ -62,7 +78,10 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
for name, tt := range tc.Tests {
t.Run(name, func(t *testing.T) {
v, err := nm.ContainerNodes(tt.Policy, tt.Pivot)
var pivot cid.ID
copy(pivot[:], tt.Pivot)
v, err := nm.ContainerNodes(tt.Policy, pivot)
if tt.Result == nil {
require.Error(t, err)
require.Contains(t, err.Error(), tt.Error)
@ -70,10 +89,13 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
require.NoError(t, err)
require.Equal(t, srcNodes, tc.Nodes)
compareNodes(t, tt.Result, tc.Nodes, v)
compareNodesIgnoreOrder(t, tt.Result, tc.Nodes, v)
if tt.Placement.Result != nil {
res, err := nm.PlacementVectors(v, tt.Placement.Pivot)
var placementPivot oid.ID
copy(placementPivot[:], tt.Placement.Pivot)
res, err := nm.PlacementVectors(v, placementPivot)
require.NoError(t, err)
compareNodes(t, tt.Placement.Result, tc.Nodes, res)
require.Equal(t, srcNodes, tc.Nodes)
@ -108,11 +130,14 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
for name, tt := range tc.Tests {
b.Run(name, func(b *testing.B) {
var pivot cid.ID
copy(pivot[:], tt.Pivot)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
v, err := nm.ContainerNodes(tt.Policy, tt.Pivot)
v, err := nm.ContainerNodes(tt.Policy, pivot)
b.StopTimer()
if tt.Result == nil {
require.Error(b, err)
@ -120,11 +145,14 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
} else {
require.NoError(b, err)
compareNodes(b, tt.Result, tc.Nodes, v)
compareNodesIgnoreOrder(b, tt.Result, tc.Nodes, v)
if tt.Placement.Result != nil {
var placementPivot oid.ID
copy(placementPivot[:], tt.Placement.Pivot)
b.StartTimer()
res, err := nm.PlacementVectors(v, tt.Placement.Pivot)
res, err := nm.PlacementVectors(v, placementPivot)
b.StopTimer()
require.NoError(b, err)
compareNodes(b, tt.Placement.Result, tc.Nodes, res)
@ -150,11 +178,14 @@ func BenchmarkManySelects(b *testing.B) {
var nm NetMap
nm.SetNodes(tc.Nodes)
var pivot cid.ID
copy(pivot[:], tt.Pivot)
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_, err = nm.ContainerNodes(tt.Policy, tt.Pivot)
_, err = nm.ContainerNodes(tt.Policy, pivot)
if err != nil {
b.FailNow()
}

4
netmap/json_tests/many_selects.json
View file

@ -183,9 +183,9 @@
"policy": {"replicas":[{"count":1,"selector":"SameRU"},{"count":1,"selector":"DistinctRU"},{"count":1,"selector":"Good"},{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"SameRU","count":2,"clause":"SAME","attribute":"City","filter":"FromRU"},{"name":"DistinctRU","count":2,"clause":"DISTINCT","attribute":"City","filter":"FromRU"},{"name":"Good","count":2,"clause":"DISTINCT","attribute":"Country","filter":"Good"},{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[{"name":"FromRU","key":"Country","op":"EQ","value":"Russia"},{"name":"Good","key":"Rating","op":"GE","value":"4"}]},
"result": [
[0, 5, 9, 10],
[2, 6, 0, 5],
[0, 5, 2, 6],
[1, 8, 2, 5],
[3, 4, 1, 7, 0, 2]
[0, 2, 1, 7, 3, 4]
]
}
}

6
netmap/json_tests/multiple_rep_asymmetric.json
View file

@ -321,10 +321,10 @@
4
],
[
8,
12,
5,
10
10,
8,
12
]
]
}

21
netmap/netmap.go
View file

@ -1,10 +1,13 @@
package netmap
import (
"crypto/sha256"
"fmt"
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
// NetMap represents NeoFS network map. It includes information about all
@ -140,11 +143,14 @@ func flattenNodes(ns []nodes) nodes {
}
// PlacementVectors sorts container nodes returned by ContainerNodes method
// and returns placement vectors for the entity identified by the given pivot.
// and returns placement vectors for the entity identified by the given object id.
// For example, in order to build node list to store the object, binary-encoded
// object identifier can be used as pivot. Result is deterministic for
// the fixed NetMap and parameters.
func (m NetMap) PlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeInfo, error) {
func (m NetMap) PlacementVectors(vectors [][]NodeInfo, objectID oid.ID) ([][]NodeInfo, error) {
pivot := make([]byte, sha256.Size)
objectID.Encode(pivot)
h := hrw.Hash(pivot)
wf := defaultWeightFunc(m.nodes)
result := make([][]NodeInfo, len(vectors))
@ -162,15 +168,18 @@ func (m NetMap) PlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeIn
// given PlacementPolicy to the NetMap. Each line of the list corresponds to a
// replica descriptor. Line order corresponds to order of ReplicaDescriptor list
// in the policy. Nodes are pre-filtered according to the Filter list from
// the policy, and then selected by Selector list. Result is deterministic for
// the fixed NetMap and parameters.
// the policy, and then selected by Selector list. Result is not deterministic and
// node order in each vector may vary for call.
//
// Result can be used in PlacementVectors.
func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, error) {
func (m NetMap) ContainerNodes(p PlacementPolicy, containerID cid.ID) ([][]NodeInfo, error) {
c := newContext(m)
c.setPivot(pivot)
c.setCBF(p.backupFactor)
pivot := make([]byte, sha256.Size)
containerID.Encode(pivot)
c.setPivot(pivot)
if err := c.processFilters(p); err != nil {
return nil, err
}

5
netmap/selector_test.go
View file

@ -9,6 +9,7 @@ import (
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
"github.com/stretchr/testify/require"
)
@ -129,7 +130,7 @@ func BenchmarkPolicyHRWType(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := nm.ContainerNodes(p, []byte{1})
_, err := nm.ContainerNodes(p, cid.ID{1})
if err != nil {
b.Fatal()
}
@ -173,7 +174,7 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
nm.SetNodes(nodeList)
getIndices := func(t *testing.T) (uint64, uint64) {
v, err := nm.ContainerNodes(p, []byte{1})
v, err := nm.ContainerNodes(p, cid.ID{1})
require.NoError(t, err)
nss := make([]nodes, len(v))