2021-10-27 10:00:35 +00:00
|
|
|
package netmap
|
|
|
|
|
|
|
|
import (
|
2024-07-12 07:22:35 +00:00
|
|
|
"cmp"
|
2023-09-08 14:15:08 +00:00
|
|
|
"crypto/rand"
|
2023-06-05 10:38:15 +00:00
|
|
|
"encoding/binary"
|
2021-10-27 10:00:35 +00:00
|
|
|
"fmt"
|
2023-09-08 14:15:08 +00:00
|
|
|
mrand "math/rand"
|
2024-09-17 14:32:56 +00:00
|
|
|
"reflect"
|
2024-07-12 07:22:35 +00:00
|
|
|
"slices"
|
2021-12-24 11:31:34 +00:00
|
|
|
"strconv"
|
2024-06-24 11:45:59 +00:00
|
|
|
"strings"
|
2021-10-27 10:00:35 +00:00
|
|
|
"testing"
|
|
|
|
|
2023-03-07 11:20:03 +00:00
|
|
|
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
|
|
|
|
"git.frostfs.info/TrueCloudLab/hrw"
|
2024-09-17 14:32:56 +00:00
|
|
|
"github.com/stretchr/testify/assert"
|
2021-10-27 10:00:35 +00:00
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
)
|
|
|
|
|
2021-12-24 11:31:34 +00:00
|
|
|
func BenchmarkHRWSort(b *testing.B) {
|
|
|
|
const netmapSize = 1000
|
|
|
|
|
2022-06-07 02:12:39 +00:00
|
|
|
vectors := make([]nodes, netmapSize)
|
2021-12-24 11:31:34 +00:00
|
|
|
weights := make([]float64, netmapSize)
|
2022-06-07 02:12:39 +00:00
|
|
|
for i := range vectors {
|
|
|
|
key := make([]byte, 33)
|
|
|
|
rand.Read(key)
|
|
|
|
|
|
|
|
var node NodeInfo
|
2022-06-07 08:25:34 +00:00
|
|
|
node.SetPrice(1)
|
|
|
|
node.SetCapacity(100)
|
2022-06-07 02:12:39 +00:00
|
|
|
node.SetPublicKey(key)
|
|
|
|
|
|
|
|
vectors[i] = nodes{node}
|
2023-09-08 14:15:08 +00:00
|
|
|
weights[i] = float64(mrand.Uint32()%10) / 10.0
|
2021-12-24 11:31:34 +00:00
|
|
|
}
|
|
|
|
|
2023-09-08 14:15:08 +00:00
|
|
|
pivot := mrand.Uint64()
|
2021-12-24 11:31:34 +00:00
|
|
|
b.Run("sort by index, no weight", func(b *testing.B) {
|
2022-06-07 02:12:39 +00:00
|
|
|
realNodes := make([]nodes, netmapSize)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.ResetTimer()
|
2024-09-03 10:20:30 +00:00
|
|
|
for range b.N {
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StopTimer()
|
2022-06-07 02:12:39 +00:00
|
|
|
copy(realNodes, vectors)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StartTimer()
|
|
|
|
|
|
|
|
hrw.SortSliceByIndex(realNodes, pivot)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
b.Run("sort by value, no weight", func(b *testing.B) {
|
2022-06-07 02:12:39 +00:00
|
|
|
realNodes := make([]nodes, netmapSize)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.ResetTimer()
|
2024-09-03 10:20:30 +00:00
|
|
|
for range b.N {
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StopTimer()
|
2022-06-07 02:12:39 +00:00
|
|
|
copy(realNodes, vectors)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StartTimer()
|
|
|
|
|
2023-02-28 11:04:02 +00:00
|
|
|
hrw.SortHasherSliceByValue(realNodes, pivot)
|
2021-12-24 11:31:34 +00:00
|
|
|
}
|
|
|
|
})
|
|
|
|
b.Run("only sort by index", func(b *testing.B) {
|
2022-06-07 02:12:39 +00:00
|
|
|
realNodes := make([]nodes, netmapSize)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.ResetTimer()
|
2024-09-03 10:20:30 +00:00
|
|
|
for range b.N {
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StopTimer()
|
2022-06-07 02:12:39 +00:00
|
|
|
copy(realNodes, vectors)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StartTimer()
|
|
|
|
|
|
|
|
hrw.SortSliceByWeightIndex(realNodes, weights, pivot)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
b.Run("sort by value", func(b *testing.B) {
|
2022-06-07 02:12:39 +00:00
|
|
|
realNodes := make([]nodes, netmapSize)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.ResetTimer()
|
2024-09-03 10:20:30 +00:00
|
|
|
for range b.N {
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StopTimer()
|
2022-06-07 02:12:39 +00:00
|
|
|
copy(realNodes, vectors)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StartTimer()
|
|
|
|
|
2023-02-28 11:04:02 +00:00
|
|
|
hrw.SortHasherSliceByWeightValue(realNodes, weights, pivot)
|
2021-12-24 11:31:34 +00:00
|
|
|
}
|
|
|
|
})
|
|
|
|
b.Run("sort by ID, then by index (deterministic)", func(b *testing.B) {
|
2022-06-07 02:12:39 +00:00
|
|
|
realNodes := make([]nodes, netmapSize)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.ResetTimer()
|
2024-09-03 10:20:30 +00:00
|
|
|
for range b.N {
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StopTimer()
|
2022-06-07 02:12:39 +00:00
|
|
|
copy(realNodes, vectors)
|
2021-12-24 11:31:34 +00:00
|
|
|
b.StartTimer()
|
|
|
|
|
2024-07-12 07:22:35 +00:00
|
|
|
slices.SortFunc(vectors, func(vi, vj nodes) int {
|
|
|
|
return cmp.Compare(vi[0].Hash(), vj[0].Hash())
|
2021-12-24 11:31:34 +00:00
|
|
|
})
|
|
|
|
hrw.SortSliceByWeightIndex(realNodes, weights, pivot)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func BenchmarkPolicyHRWType(b *testing.B) {
|
|
|
|
const netmapSize = 100
|
|
|
|
|
|
|
|
p := newPlacementPolicy(1,
|
2022-06-07 08:25:34 +00:00
|
|
|
[]ReplicaDescriptor{
|
2021-12-24 11:31:34 +00:00
|
|
|
newReplica(1, "loc1"),
|
2023-11-21 09:57:35 +00:00
|
|
|
newReplica(1, "loc2"),
|
|
|
|
},
|
2022-03-11 09:13:08 +00:00
|
|
|
[]Selector{
|
2022-06-07 08:25:34 +00:00
|
|
|
newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame),
|
2023-11-21 09:57:35 +00:00
|
|
|
newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame),
|
|
|
|
},
|
2022-03-11 09:13:08 +00:00
|
|
|
[]Filter{
|
2022-06-07 08:25:34 +00:00
|
|
|
newFilter("loc1", "Location", "Shanghai", netmap.EQ),
|
|
|
|
newFilter("loc2", "Location", "Shanghai", netmap.NE),
|
2021-12-24 11:31:34 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
nodes := make([]NodeInfo, netmapSize)
|
|
|
|
for i := range nodes {
|
|
|
|
var loc string
|
|
|
|
switch i % 20 {
|
|
|
|
case 0:
|
|
|
|
loc = "Shanghai"
|
|
|
|
default:
|
|
|
|
loc = strconv.Itoa(i % 20)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Having the same price and capacity ensures equal weights for all nodes.
|
|
|
|
// This way placement is more dependent on the initial order.
|
|
|
|
nodes[i] = nodeInfoFromAttributes("Location", loc, "Price", "1", "Capacity", "10")
|
|
|
|
pub := make([]byte, 33)
|
|
|
|
pub[0] = byte(i)
|
|
|
|
nodes[i].SetPublicKey(pub)
|
|
|
|
}
|
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
var nm NetMap
|
2022-06-07 02:12:39 +00:00
|
|
|
nm.SetNodes(nodes)
|
2021-12-24 11:31:34 +00:00
|
|
|
|
|
|
|
b.ResetTimer()
|
2024-09-03 10:20:30 +00:00
|
|
|
for range b.N {
|
2022-06-07 08:25:34 +00:00
|
|
|
_, err := nm.ContainerNodes(p, []byte{1})
|
2021-12-24 11:31:34 +00:00
|
|
|
if err != nil {
|
|
|
|
b.Fatal()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
|
|
|
|
const netmapSize = 100
|
|
|
|
|
|
|
|
p := newPlacementPolicy(1,
|
2022-06-07 08:25:34 +00:00
|
|
|
[]ReplicaDescriptor{
|
2021-12-24 11:31:34 +00:00
|
|
|
newReplica(1, "loc1"),
|
2023-11-21 09:57:35 +00:00
|
|
|
newReplica(1, "loc2"),
|
|
|
|
},
|
2022-03-11 09:13:08 +00:00
|
|
|
[]Selector{
|
2022-06-07 08:25:34 +00:00
|
|
|
newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame),
|
2023-11-21 09:57:35 +00:00
|
|
|
newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame),
|
|
|
|
},
|
2022-03-11 09:13:08 +00:00
|
|
|
[]Filter{
|
2022-06-07 08:25:34 +00:00
|
|
|
newFilter("loc1", "Location", "Shanghai", netmap.EQ),
|
|
|
|
newFilter("loc2", "Location", "Shanghai", netmap.NE),
|
2021-12-24 11:31:34 +00:00
|
|
|
})
|
|
|
|
|
2022-06-07 02:12:39 +00:00
|
|
|
nodeList := make([]NodeInfo, netmapSize)
|
|
|
|
for i := range nodeList {
|
2021-12-24 11:31:34 +00:00
|
|
|
var loc string
|
|
|
|
switch i % 20 {
|
|
|
|
case 0:
|
|
|
|
loc = "Shanghai"
|
|
|
|
default:
|
|
|
|
loc = strconv.Itoa(i % 20)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Having the same price and capacity ensures equal weights for all nodes.
|
|
|
|
// This way placement is more dependent on the initial order.
|
2022-06-07 02:12:39 +00:00
|
|
|
nodeList[i] = nodeInfoFromAttributes("Location", loc, "Price", "1", "Capacity", "10")
|
2021-12-24 11:31:34 +00:00
|
|
|
pub := make([]byte, 33)
|
|
|
|
pub[0] = byte(i)
|
2022-06-07 02:12:39 +00:00
|
|
|
nodeList[i].SetPublicKey(pub)
|
2021-12-24 11:31:34 +00:00
|
|
|
}
|
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
var nm NetMap
|
2022-06-07 02:12:39 +00:00
|
|
|
nm.SetNodes(nodeList)
|
|
|
|
|
|
|
|
getIndices := func(t *testing.T) (uint64, uint64) {
|
2022-06-07 08:25:34 +00:00
|
|
|
v, err := nm.ContainerNodes(p, []byte{1})
|
2021-12-24 11:31:34 +00:00
|
|
|
require.NoError(t, err)
|
2022-06-07 02:12:39 +00:00
|
|
|
|
|
|
|
nss := make([]nodes, len(v))
|
|
|
|
for i := range v {
|
|
|
|
nss[i] = v[i]
|
|
|
|
}
|
|
|
|
|
|
|
|
ns := flattenNodes(nss)
|
2021-12-24 11:31:34 +00:00
|
|
|
require.Equal(t, 2, len(ns))
|
2022-06-07 02:12:39 +00:00
|
|
|
return ns[0].Hash(), ns[1].Hash()
|
2021-12-24 11:31:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
a, b := getIndices(t)
|
2024-09-03 10:20:30 +00:00
|
|
|
for range 10 {
|
2021-12-24 11:31:34 +00:00
|
|
|
x, y := getIndices(t)
|
|
|
|
require.Equal(t, a, x)
|
|
|
|
require.Equal(t, b, y)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-27 10:00:35 +00:00
|
|
|
func TestPlacementPolicy_ProcessSelectors(t *testing.T) {
|
|
|
|
p := newPlacementPolicy(2, nil,
|
2022-03-11 09:13:08 +00:00
|
|
|
[]Selector{
|
2022-06-07 08:25:34 +00:00
|
|
|
newSelector("SameRU", "City", 2, "FromRU", (*Selector).SelectSame),
|
|
|
|
newSelector("DistinctRU", "City", 2, "FromRU", (*Selector).SelectDistinct),
|
|
|
|
newSelector("Good", "Country", 2, "Good", (*Selector).SelectDistinct),
|
|
|
|
newSelector("Main", "Country", 3, "*", (*Selector).SelectDistinct),
|
2021-10-27 10:00:35 +00:00
|
|
|
},
|
2022-03-11 09:13:08 +00:00
|
|
|
[]Filter{
|
2022-06-07 08:25:34 +00:00
|
|
|
newFilter("FromRU", "Country", "Russia", netmap.EQ),
|
|
|
|
newFilter("Good", "Rating", "4", netmap.GE),
|
2021-10-27 10:00:35 +00:00
|
|
|
})
|
|
|
|
nodes := []NodeInfo{
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "1", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("Country", "Germany", "Rating", "5", "City", "Berlin"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "6", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("Country", "France", "Rating", "4", "City", "Paris"),
|
|
|
|
nodeInfoFromAttributes("Country", "France", "Rating", "1", "City", "Lyon"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "5", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "7", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("Country", "Germany", "Rating", "3", "City", "Darmstadt"),
|
2023-05-24 09:50:13 +00:00
|
|
|
nodeInfoFromAttributes("Country", "Germany", "Rating", "7", "City", "Frankfurt"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
|
|
|
|
}
|
|
|
|
|
|
|
|
var nm NetMap
|
|
|
|
nm.SetNodes(nodes)
|
|
|
|
c := newContext(nm)
|
|
|
|
c.setCBF(p.backupFactor)
|
|
|
|
require.NoError(t, c.processFilters(p))
|
|
|
|
require.NoError(t, c.processSelectors(p))
|
|
|
|
|
|
|
|
for _, s := range p.selectors {
|
|
|
|
sel := c.selections[s.GetName()]
|
|
|
|
s := c.processedSelectors[s.GetName()]
|
|
|
|
bucketCount, nodesInBucket := calcNodesCount(*s)
|
|
|
|
nodesInBucket *= int(c.cbf)
|
|
|
|
targ := fmt.Sprintf("selector '%s'", s.GetName())
|
|
|
|
require.Equal(t, bucketCount, len(sel), targ)
|
|
|
|
fName := s.GetFilter()
|
|
|
|
for _, res := range sel {
|
|
|
|
require.Equal(t, nodesInBucket, len(res), targ)
|
|
|
|
for j := range res {
|
|
|
|
require.True(t, fName == mainFilterName || c.match(c.processedFilters[fName], res[j]), targ)
|
|
|
|
}
|
2023-06-05 10:38:15 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-06-24 11:45:59 +00:00
|
|
|
func TestPlacementPolicy_Like(t *testing.T) {
|
|
|
|
nodes := []NodeInfo{
|
|
|
|
nodeInfoFromAttributes("Country", "Russia"),
|
|
|
|
nodeInfoFromAttributes("Country", "Germany"),
|
|
|
|
nodeInfoFromAttributes("Country", "Belarus"),
|
|
|
|
}
|
|
|
|
|
|
|
|
var nm NetMap
|
|
|
|
nm.SetNodes(nodes)
|
|
|
|
|
|
|
|
t.Run("LIKE all", func(t *testing.T) {
|
|
|
|
ssNamed := []Selector{newSelector("X", "Country", 4, "FromRU", (*Selector).SelectDistinct)}
|
|
|
|
fsNamed := []Filter{newFilter("FromRU", "Country", "*", netmap.LIKE)}
|
|
|
|
rsNamed := []ReplicaDescriptor{newReplica(4, "X")}
|
|
|
|
pNamed := newPlacementPolicy(0, rsNamed, ssNamed, fsNamed)
|
|
|
|
|
|
|
|
n, err := nm.ContainerNodes(pNamed, []byte{1})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, 3, len(n[0]))
|
|
|
|
for _, n := range n[0] {
|
|
|
|
require.True(t, strings.Contains("GermanyRussiaBelarus", n.Attribute("Country")))
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("LIKE no wildcard", func(t *testing.T) {
|
|
|
|
ssNamed := []Selector{newSelector("X", "Country", 1, "FromRU", (*Selector).SelectDistinct)}
|
|
|
|
fsNamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.LIKE)}
|
|
|
|
rsNamed := []ReplicaDescriptor{newReplica(1, "X")}
|
|
|
|
pNamed := newPlacementPolicy(0, rsNamed, ssNamed, fsNamed)
|
|
|
|
|
|
|
|
n, err := nm.ContainerNodes(pNamed, []byte{1})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, 1, len(n[0]))
|
|
|
|
for _, n := range n[0] {
|
|
|
|
require.True(t, n.Attribute("Country") == "Russia")
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("LIKE prefix", func(t *testing.T) {
|
|
|
|
ssNamed := []Selector{newSelector("X", "Country", 1, "FromRU", (*Selector).SelectDistinct)}
|
|
|
|
fsNamed := []Filter{newFilter("FromRU", "Country", "Ge*", netmap.LIKE)}
|
|
|
|
rsNamed := []ReplicaDescriptor{newReplica(1, "X")}
|
|
|
|
pNamed := newPlacementPolicy(0, rsNamed, ssNamed, fsNamed)
|
|
|
|
|
|
|
|
n, err := nm.ContainerNodes(pNamed, []byte{1})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, 1, len(n[0]))
|
|
|
|
for _, n := range n[0] {
|
|
|
|
require.True(t, n.Attribute("Country") == "Germany")
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("LIKE suffix", func(t *testing.T) {
|
|
|
|
ssNamed := []Selector{newSelector("X", "Country", 1, "FromRU", (*Selector).SelectDistinct)}
|
|
|
|
fsNamed := []Filter{newFilter("FromRU", "Country", "*sia", netmap.LIKE)}
|
|
|
|
rsNamed := []ReplicaDescriptor{newReplica(1, "X")}
|
|
|
|
pNamed := newPlacementPolicy(0, rsNamed, ssNamed, fsNamed)
|
|
|
|
|
|
|
|
n, err := nm.ContainerNodes(pNamed, []byte{1})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, 1, len(n[0]))
|
|
|
|
for _, n := range n[0] {
|
|
|
|
require.True(t, n.Attribute("Country") == "Russia")
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("LIKE middle", func(t *testing.T) {
|
|
|
|
ssNamed := []Selector{newSelector("X", "Country", 2, "FromRU", (*Selector).SelectDistinct)}
|
|
|
|
fsNamed := []Filter{newFilter("FromRU", "Country", "*us*", netmap.LIKE)}
|
|
|
|
rsNamed := []ReplicaDescriptor{newReplica(2, "X")}
|
|
|
|
pNamed := newPlacementPolicy(0, rsNamed, ssNamed, fsNamed)
|
|
|
|
|
|
|
|
n, err := nm.ContainerNodes(pNamed, []byte{1})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, 2, len(n[0]))
|
|
|
|
for _, n := range n[0] {
|
|
|
|
require.True(t, strings.Contains("RussiaBelarus", n.Attribute("Country")))
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2023-06-05 10:38:15 +00:00
|
|
|
func TestPlacementPolicy_Unique(t *testing.T) {
|
|
|
|
p := newPlacementPolicy(2,
|
|
|
|
[]ReplicaDescriptor{
|
|
|
|
newReplica(1, "S"),
|
|
|
|
newReplica(1, "S"),
|
|
|
|
},
|
|
|
|
[]Selector{
|
|
|
|
newSelector("S", "City", 1, "*", (*Selector).SelectSame),
|
|
|
|
},
|
|
|
|
[]Filter{})
|
|
|
|
p.unique = true
|
|
|
|
|
|
|
|
var nodes []NodeInfo
|
|
|
|
for i, city := range []string{"Moscow", "Berlin", "Shenzhen"} {
|
2024-09-03 10:20:30 +00:00
|
|
|
for j := range 3 {
|
2023-06-05 10:38:15 +00:00
|
|
|
node := nodeInfoFromAttributes("City", city)
|
|
|
|
node.SetPublicKey(binary.BigEndian.AppendUint16(nil, uint16(i*4+j)))
|
|
|
|
nodes = append(nodes, node)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
var nm NetMap
|
|
|
|
nm.SetNodes(nodes)
|
|
|
|
|
|
|
|
v, err := nm.ContainerNodes(p, nil)
|
|
|
|
require.NoError(t, err)
|
|
|
|
for i, vi := range v {
|
|
|
|
for _, ni := range vi {
|
2024-09-03 10:20:30 +00:00
|
|
|
for j := range i {
|
2023-06-05 10:38:15 +00:00
|
|
|
for _, nj := range v[j] {
|
|
|
|
require.NotEqual(t, ni.hash, nj.hash)
|
|
|
|
}
|
|
|
|
}
|
2023-05-24 09:50:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-08 13:57:41 +00:00
|
|
|
func TestPlacementPolicy_SingleOmitNames(t *testing.T) {
|
|
|
|
nodes := []NodeInfo{
|
|
|
|
nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("ID", "2", "Country", "Germany", "City", "Berlin"),
|
|
|
|
nodeInfoFromAttributes("ID", "3", "Country", "Russia", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("ID", "4", "Country", "France", "City", "Paris"),
|
|
|
|
nodeInfoFromAttributes("ID", "5", "Country", "France", "City", "Lyon"),
|
|
|
|
nodeInfoFromAttributes("ID", "6", "Country", "Russia", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("ID", "7", "Country", "Russia", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("ID", "8", "Country", "Germany", "City", "Darmstadt"),
|
|
|
|
nodeInfoFromAttributes("ID", "9", "Country", "Germany", "City", "Frankfurt"),
|
|
|
|
nodeInfoFromAttributes("ID", "10", "Country", "Russia", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("ID", "11", "Country", "Russia", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("ID", "12", "Country", "Germany", "City", "London"),
|
|
|
|
}
|
|
|
|
for i := range nodes {
|
|
|
|
pub := make([]byte, 33)
|
|
|
|
rand.Read(pub)
|
|
|
|
nodes[i].SetPublicKey(pub)
|
|
|
|
}
|
|
|
|
|
|
|
|
var nm NetMap
|
|
|
|
nm.SetNodes(nodes)
|
|
|
|
|
|
|
|
for _, unique := range []bool{false, true} {
|
|
|
|
t.Run(fmt.Sprintf("unique=%t", unique), func(t *testing.T) {
|
|
|
|
ssNamed := []Selector{newSelector("X", "City", 2, "FromRU", (*Selector).SelectDistinct)}
|
|
|
|
fsNamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
|
|
|
|
rsNamed := []ReplicaDescriptor{newReplica(1, "X")}
|
|
|
|
pNamed := newPlacementPolicy(3, rsNamed, ssNamed, fsNamed)
|
|
|
|
pNamed.unique = unique
|
|
|
|
|
|
|
|
vNamed, err := nm.ContainerNodes(pNamed, []byte{1})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
ssUnnamed := []Selector{newSelector("", "City", 2, "FromRU", (*Selector).SelectDistinct)}
|
|
|
|
fsUnnamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
|
|
|
|
rsUnnamed := []ReplicaDescriptor{newReplica(1, "")}
|
|
|
|
pUnnamed := newPlacementPolicy(3, rsUnnamed, ssUnnamed, fsUnnamed)
|
|
|
|
pUnnamed.unique = unique
|
|
|
|
|
|
|
|
vUnnamed, err := nm.ContainerNodes(pUnnamed, []byte{1})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Equal(t, vNamed, vUnnamed)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-07-10 11:49:40 +00:00
|
|
|
func TestPlacementPolicy_MultiREP(t *testing.T) {
|
|
|
|
nodes := []NodeInfo{
|
|
|
|
nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("ID", "2", "Country", "Germany", "City", "Berlin"),
|
|
|
|
nodeInfoFromAttributes("ID", "3", "Country", "Russia", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("ID", "4", "Country", "France", "City", "Paris"),
|
|
|
|
nodeInfoFromAttributes("ID", "5", "Country", "France", "City", "Lyon"),
|
|
|
|
nodeInfoFromAttributes("ID", "6", "Country", "Russia", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("ID", "7", "Country", "Russia", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("ID", "8", "Country", "Germany", "City", "Darmstadt"),
|
|
|
|
nodeInfoFromAttributes("ID", "9", "Country", "Germany", "City", "Frankfurt"),
|
|
|
|
nodeInfoFromAttributes("ID", "10", "Country", "Russia", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("ID", "11", "Country", "Russia", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("ID", "12", "Country", "Germany", "City", "London"),
|
|
|
|
}
|
|
|
|
for i := range nodes {
|
|
|
|
pub := make([]byte, 33)
|
|
|
|
rand.Read(pub)
|
|
|
|
nodes[i].SetPublicKey(pub)
|
|
|
|
}
|
|
|
|
|
|
|
|
var nm NetMap
|
|
|
|
nm.SetNodes(nodes)
|
|
|
|
|
|
|
|
ss := []Selector{newSelector("SameRU", "City", 2, "FromRU", (*Selector).SelectDistinct)}
|
|
|
|
fs := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
|
|
|
|
|
|
|
|
for _, unique := range []bool{false, true} {
|
|
|
|
for _, additional := range []int{0, 1, 2} {
|
|
|
|
t.Run(fmt.Sprintf("unique=%t, additional=%d", unique, additional), func(t *testing.T) {
|
|
|
|
rs := []ReplicaDescriptor{newReplica(1, "SameRU")}
|
2024-09-03 10:20:30 +00:00
|
|
|
for range additional {
|
2023-07-10 11:49:40 +00:00
|
|
|
rs = append(rs, newReplica(1, ""))
|
|
|
|
}
|
|
|
|
|
|
|
|
p := newPlacementPolicy(3, rs, ss, fs)
|
|
|
|
p.unique = unique
|
|
|
|
|
|
|
|
v, err := nm.ContainerNodes(p, []byte{1})
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, 1+additional, len(v))
|
|
|
|
require.Equal(t, 6, len(v[0]))
|
|
|
|
|
|
|
|
for i := 1; i < additional; i++ {
|
|
|
|
require.Equal(t, 3, len(v[i]))
|
|
|
|
if !unique {
|
|
|
|
require.Equal(t, v[1], v[i])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if unique {
|
|
|
|
seen := make(map[string]bool)
|
|
|
|
for i := range v {
|
|
|
|
for j := range v[i] {
|
|
|
|
attr := v[i][j].Attribute("ID")
|
|
|
|
require.NotEmpty(t, attr)
|
|
|
|
require.False(t, seen[attr])
|
|
|
|
|
|
|
|
seen[attr] = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-24 09:50:13 +00:00
|
|
|
func TestPlacementPolicy_ProcessSelectorsExceptForNodes(t *testing.T) {
|
|
|
|
p := newPlacementPolicy(1, nil,
|
|
|
|
[]Selector{
|
|
|
|
newSelector("ExceptRU", "City", 2, "ExceptRU", (*Selector).SelectSame),
|
|
|
|
},
|
|
|
|
[]Filter{
|
|
|
|
newFilter("ExceptRU", "", "", netmap.NOT,
|
|
|
|
newFilter("", "", "", netmap.AND,
|
|
|
|
newFilter("", "City", "Lyon", netmap.EQ),
|
|
|
|
newFilter("", "Rating", "10", netmap.LE),
|
|
|
|
),
|
|
|
|
),
|
|
|
|
})
|
|
|
|
nodes := []NodeInfo{
|
|
|
|
nodeInfoFromAttributes("Country", "Germany", "Rating", "1", "City", "Berlin"),
|
|
|
|
nodeInfoFromAttributes("Country", "Germany", "Rating", "5", "City", "Berlin"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "6", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("Country", "France", "Rating", "4", "City", "Paris"),
|
|
|
|
nodeInfoFromAttributes("Country", "France", "Rating", "1", "City", "Lyon"),
|
|
|
|
nodeInfoFromAttributes("Country", "France", "Rating", "5", "City", "Lyon"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "7", "City", "Moscow"),
|
|
|
|
nodeInfoFromAttributes("Country", "Germany", "Rating", "3", "City", "Darmstadt"),
|
2021-10-27 10:00:35 +00:00
|
|
|
nodeInfoFromAttributes("Country", "Germany", "Rating", "7", "City", "Frankfurt"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
|
|
|
|
nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
|
|
|
|
}
|
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
var nm NetMap
|
2022-06-07 02:12:39 +00:00
|
|
|
nm.SetNodes(nodes)
|
2022-06-07 08:25:34 +00:00
|
|
|
c := newContext(nm)
|
|
|
|
c.setCBF(p.backupFactor)
|
2021-10-27 10:00:35 +00:00
|
|
|
require.NoError(t, c.processFilters(p))
|
|
|
|
require.NoError(t, c.processSelectors(p))
|
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
for _, s := range p.selectors {
|
|
|
|
sel := c.selections[s.GetName()]
|
|
|
|
s := c.processedSelectors[s.GetName()]
|
|
|
|
bucketCount, nodesInBucket := calcNodesCount(*s)
|
2021-10-27 10:00:35 +00:00
|
|
|
nodesInBucket *= int(c.cbf)
|
2022-06-07 08:25:34 +00:00
|
|
|
targ := fmt.Sprintf("selector '%s'", s.GetName())
|
2021-10-27 10:00:35 +00:00
|
|
|
require.Equal(t, bucketCount, len(sel), targ)
|
2022-06-07 08:25:34 +00:00
|
|
|
fName := s.GetFilter()
|
2021-10-27 10:00:35 +00:00
|
|
|
for _, res := range sel {
|
|
|
|
require.Equal(t, nodesInBucket, len(res), targ)
|
|
|
|
for j := range res {
|
2022-06-07 08:25:34 +00:00
|
|
|
require.True(t, fName == mainFilterName || c.match(c.processedFilters[fName], res[j]), targ)
|
2021-10-27 10:00:35 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-09-17 14:32:56 +00:00
|
|
|
func TestPlacementPolicy_NonAsciiAttributes(t *testing.T) {
|
|
|
|
p := newPlacementPolicy(
|
|
|
|
1,
|
|
|
|
[]ReplicaDescriptor{
|
|
|
|
newReplica(2, "Nodes"),
|
|
|
|
newReplica(2, "Nodes"),
|
|
|
|
},
|
|
|
|
[]Selector{
|
|
|
|
newSelector("Nodes", "Цвет", 2, "Colorful", (*Selector).SelectSame),
|
|
|
|
},
|
|
|
|
[]Filter{
|
|
|
|
newFilter("Colorful", "", "", netmap.OR,
|
|
|
|
newFilter("", "Цвет", "Красный", netmap.EQ),
|
|
|
|
newFilter("", "Цвет", "Синий", netmap.EQ),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
)
|
|
|
|
p.SetUnique(true)
|
|
|
|
|
|
|
|
nodes := []NodeInfo{
|
|
|
|
nodeInfoFromAttributes("Цвет", "Красный", "Форма", "Треугольник"),
|
|
|
|
nodeInfoFromAttributes("Цвет", "Красный", "Форма", "Круг"),
|
|
|
|
nodeInfoFromAttributes("Цвет", "Синий", "Форма", "Треугольник"),
|
|
|
|
nodeInfoFromAttributes("Цвет", "Синий", "Форма", "Круг"),
|
|
|
|
nodeInfoFromAttributes("Свойство", "Мягкий", "Форма", "Треугольник"),
|
|
|
|
nodeInfoFromAttributes("Свойство", "Теплый", "Форма", "Круг"),
|
|
|
|
}
|
|
|
|
for i := range nodes {
|
|
|
|
nodes[i].SetPublicKey([]byte{byte(i)})
|
|
|
|
}
|
|
|
|
|
|
|
|
redNodes := nodes[:2]
|
|
|
|
blueNodes := nodes[2:4]
|
|
|
|
|
|
|
|
var nm NetMap
|
|
|
|
nm.SetNodes(nodes)
|
|
|
|
|
|
|
|
pivot := make([]byte, 42)
|
|
|
|
_, _ = rand.Read(pivot)
|
|
|
|
|
|
|
|
nodesPerReplica, err := nm.ContainerNodes(p, pivot)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, nodesPerReplica, 2)
|
|
|
|
|
|
|
|
for i := range nodesPerReplica {
|
|
|
|
slices.SortFunc(nodesPerReplica[i], func(n1, n2 NodeInfo) int {
|
|
|
|
pk1, pk2 := string(n1.PublicKey()), string(n2.PublicKey())
|
|
|
|
return cmp.Compare(pk1, pk2)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
redMatchFirst := reflect.DeepEqual(redNodes, nodesPerReplica[0])
|
|
|
|
blueMatchFirst := reflect.DeepEqual(blueNodes, nodesPerReplica[0])
|
|
|
|
|
|
|
|
redMatchSecond := reflect.DeepEqual(redNodes, nodesPerReplica[1])
|
|
|
|
blueMatchSecond := reflect.DeepEqual(blueNodes, nodesPerReplica[1])
|
|
|
|
|
|
|
|
assert.True(t, redMatchFirst && blueMatchSecond || blueMatchFirst && redMatchSecond)
|
|
|
|
}
|
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
func TestSelector_SetName(t *testing.T) {
|
|
|
|
const name = "some name"
|
|
|
|
var s Selector
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
require.Zero(t, s.m.GetName())
|
2021-10-27 10:00:35 +00:00
|
|
|
|
|
|
|
s.SetName(name)
|
2022-06-07 08:25:34 +00:00
|
|
|
require.Equal(t, name, s.m.GetName())
|
2021-10-27 10:00:35 +00:00
|
|
|
}
|
|
|
|
|
2022-06-14 14:27:51 +00:00
|
|
|
func TestSelector_SetNumberOfNodes(t *testing.T) {
|
|
|
|
const num = 3
|
2022-06-07 08:25:34 +00:00
|
|
|
var s Selector
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
require.Zero(t, s.m.GetCount())
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-14 14:27:51 +00:00
|
|
|
s.SetNumberOfNodes(num)
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-14 14:27:51 +00:00
|
|
|
require.EqualValues(t, num, s.m.GetCount())
|
2021-10-27 10:00:35 +00:00
|
|
|
}
|
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
func TestSelectorClauses(t *testing.T) {
|
|
|
|
var s Selector
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
require.Equal(t, netmap.UnspecifiedClause, s.m.GetClause())
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
s.SelectDistinct()
|
|
|
|
require.Equal(t, netmap.Distinct, s.m.GetClause())
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
s.SelectSame()
|
|
|
|
require.Equal(t, netmap.Same, s.m.GetClause())
|
2021-10-27 10:00:35 +00:00
|
|
|
}
|
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
func TestSelector_SelectByBucketAttribute(t *testing.T) {
|
|
|
|
const attr = "some attribute"
|
|
|
|
var s Selector
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
require.Zero(t, s.m.GetAttribute())
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
s.SelectByBucketAttribute(attr)
|
|
|
|
require.Equal(t, attr, s.m.GetAttribute())
|
2021-10-27 10:00:35 +00:00
|
|
|
}
|
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
func TestSelector_SetFilterName(t *testing.T) {
|
|
|
|
const fName = "some filter"
|
|
|
|
var s Selector
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
require.Zero(t, s.m.GetFilter())
|
2021-10-27 10:00:35 +00:00
|
|
|
|
2022-06-07 08:25:34 +00:00
|
|
|
s.SetFilterName(fName)
|
|
|
|
require.Equal(t, fName, s.m.GetFilter())
|
2021-10-27 10:00:35 +00:00
|
|
|
}
|