frostfs-sdk-go/netmap/selector_test.go

package netmap

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	mrand "math/rand"
	"sort"
	"strconv"
	"testing"

	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
	"git.frostfs.info/TrueCloudLab/hrw"
	"github.com/stretchr/testify/require"
)

func BenchmarkHRWSort(b *testing.B) {
	const netmapSize = 1000

	vectors := make([]nodes, netmapSize)
	weights := make([]float64, netmapSize)
	for i := range vectors {
		key := make([]byte, 33)
		rand.Read(key)

		var node NodeInfo
		node.SetPrice(1)
		node.SetCapacity(100)
		node.SetPublicKey(key)

		vectors[i] = nodes{node}
		weights[i] = float64(mrand.Uint32()%10) / 10.0
	}

	pivot := mrand.Uint64()
	b.Run("sort by index, no weight", func(b *testing.B) {
		realNodes := make([]nodes, netmapSize)
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			b.StopTimer()
			copy(realNodes, vectors)
			b.StartTimer()

			hrw.SortSliceByIndex(realNodes, pivot)
		}
	})
	b.Run("sort by value, no weight", func(b *testing.B) {
		realNodes := make([]nodes, netmapSize)
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			b.StopTimer()
			copy(realNodes, vectors)
			b.StartTimer()

			hrw.SortHasherSliceByValue(realNodes, pivot)
		}
	})
	b.Run("only sort by index", func(b *testing.B) {
		realNodes := make([]nodes, netmapSize)
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			b.StopTimer()
			copy(realNodes, vectors)
			b.StartTimer()

			hrw.SortSliceByWeightIndex(realNodes, weights, pivot)
		}
	})
	b.Run("sort by value", func(b *testing.B) {
		realNodes := make([]nodes, netmapSize)
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			b.StopTimer()
			copy(realNodes, vectors)
			b.StartTimer()

			hrw.SortHasherSliceByWeightValue(realNodes, weights, pivot)
		}
	})
	b.Run("sort by ID, then by index (deterministic)", func(b *testing.B) {
		realNodes := make([]nodes, netmapSize)
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			b.StopTimer()
			copy(realNodes, vectors)
			b.StartTimer()

			sort.Slice(vectors, func(i, j int) bool {
				return less(vectors[i][0], vectors[j][0])
			})
			hrw.SortSliceByWeightIndex(realNodes, weights, pivot)
		}
	})
}

func BenchmarkPolicyHRWType(b *testing.B) {
	const netmapSize = 100

	p := newPlacementPolicy(1,
		[]ReplicaDescriptor{
			newReplica(1, "loc1"),
			newReplica(1, "loc2"),
		},
		[]Selector{
			newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame),
			newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame),
		},
		[]Filter{
			newFilter("loc1", "Location", "Shanghai", netmap.EQ),
			newFilter("loc2", "Location", "Shanghai", netmap.NE),
		})

	nodes := make([]NodeInfo, netmapSize)
	for i := range nodes {
		var loc string
		switch i % 20 {
		case 0:
			loc = "Shanghai"
		default:
			loc = strconv.Itoa(i % 20)
		}

		// Having the same price and capacity ensures equal weights for all nodes.
		// This way placement is more dependent on the initial order.
		nodes[i] = nodeInfoFromAttributes("Location", loc, "Price", "1", "Capacity", "10")
		pub := make([]byte, 33)
		pub[0] = byte(i)
		nodes[i].SetPublicKey(pub)
	}

	var nm NetMap
	nm.SetNodes(nodes)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err := nm.ContainerNodes(p, []byte{1})
		if err != nil {
			b.Fatal()
		}
	}
}

func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
	const netmapSize = 100

	p := newPlacementPolicy(1,
		[]ReplicaDescriptor{
			newReplica(1, "loc1"),
			newReplica(1, "loc2"),
		},
		[]Selector{
			newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame),
			newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame),
		},
		[]Filter{
			newFilter("loc1", "Location", "Shanghai", netmap.EQ),
			newFilter("loc2", "Location", "Shanghai", netmap.NE),
		})

	nodeList := make([]NodeInfo, netmapSize)
	for i := range nodeList {
		var loc string
		switch i % 20 {
		case 0:
			loc = "Shanghai"
		default:
			loc = strconv.Itoa(i % 20)
		}

		// Having the same price and capacity ensures equal weights for all nodes.
		// This way placement is more dependent on the initial order.
		nodeList[i] = nodeInfoFromAttributes("Location", loc, "Price", "1", "Capacity", "10")
		pub := make([]byte, 33)
		pub[0] = byte(i)
		nodeList[i].SetPublicKey(pub)
	}

	var nm NetMap
	nm.SetNodes(nodeList)

	getIndices := func(t *testing.T) (uint64, uint64) {
		v, err := nm.ContainerNodes(p, []byte{1})
		require.NoError(t, err)

		nss := make([]nodes, len(v))
		for i := range v {
			nss[i] = v[i]
		}

		ns := flattenNodes(nss)
		require.Equal(t, 2, len(ns))
		return ns[0].Hash(), ns[1].Hash()
	}

	a, b := getIndices(t)
	for i := 0; i < 10; i++ {
		x, y := getIndices(t)
		require.Equal(t, a, x)
		require.Equal(t, b, y)
	}
}

func TestPlacementPolicy_ProcessSelectors(t *testing.T) {
	p := newPlacementPolicy(2, nil,
		[]Selector{
			newSelector("SameRU", "City", 2, "FromRU", (*Selector).SelectSame),
			newSelector("DistinctRU", "City", 2, "FromRU", (*Selector).SelectDistinct),
			newSelector("Good", "Country", 2, "Good", (*Selector).SelectDistinct),
			newSelector("Main", "Country", 3, "*", (*Selector).SelectDistinct),
		},
		[]Filter{
			newFilter("FromRU", "Country", "Russia", netmap.EQ),
			newFilter("Good", "Rating", "4", netmap.GE),
		})
	nodes := []NodeInfo{
		nodeInfoFromAttributes("Country", "Russia", "Rating", "1", "City", "SPB"),
		nodeInfoFromAttributes("Country", "Germany", "Rating", "5", "City", "Berlin"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "6", "City", "Moscow"),
		nodeInfoFromAttributes("Country", "France", "Rating", "4", "City", "Paris"),
		nodeInfoFromAttributes("Country", "France", "Rating", "1", "City", "Lyon"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "5", "City", "SPB"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "7", "City", "Moscow"),
		nodeInfoFromAttributes("Country", "Germany", "Rating", "3", "City", "Darmstadt"),
		nodeInfoFromAttributes("Country", "Germany", "Rating", "7", "City", "Frankfurt"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
	}

	var nm NetMap
	nm.SetNodes(nodes)
	c := newContext(nm)
	c.setCBF(p.backupFactor)
	require.NoError(t, c.processFilters(p))
	require.NoError(t, c.processSelectors(p))

	for _, s := range p.selectors {
		sel := c.selections[s.GetName()]
		s := c.processedSelectors[s.GetName()]
		bucketCount, nodesInBucket := calcNodesCount(*s)
		nodesInBucket *= int(c.cbf)
		targ := fmt.Sprintf("selector '%s'", s.GetName())
		require.Equal(t, bucketCount, len(sel), targ)
		fName := s.GetFilter()
		for _, res := range sel {
			require.Equal(t, nodesInBucket, len(res), targ)
			for j := range res {
				require.True(t, fName == mainFilterName || c.match(c.processedFilters[fName], res[j]), targ)
			}
		}
	}
}

func TestPlacementPolicy_Unique(t *testing.T) {
	p := newPlacementPolicy(2,
		[]ReplicaDescriptor{
			newReplica(1, "S"),
			newReplica(1, "S"),
		},
		[]Selector{
			newSelector("S", "City", 1, "*", (*Selector).SelectSame),
		},
		[]Filter{})
	p.unique = true

	var nodes []NodeInfo
	for i, city := range []string{"Moscow", "Berlin", "Shenzhen"} {
		for j := 0; j < 3; j++ {
			node := nodeInfoFromAttributes("City", city)
			node.SetPublicKey(binary.BigEndian.AppendUint16(nil, uint16(i*4+j)))
			nodes = append(nodes, node)
		}
	}

	var nm NetMap
	nm.SetNodes(nodes)

	v, err := nm.ContainerNodes(p, nil)
	require.NoError(t, err)
	for i, vi := range v {
		for _, ni := range vi {
			for j := 0; j < i; j++ {
				for _, nj := range v[j] {
					require.NotEqual(t, ni.hash, nj.hash)
				}
			}
		}
	}
}

func TestPlacementPolicy_SingleOmitNames(t *testing.T) {
	nodes := []NodeInfo{
		nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"),
		nodeInfoFromAttributes("ID", "2", "Country", "Germany", "City", "Berlin"),
		nodeInfoFromAttributes("ID", "3", "Country", "Russia", "City", "Moscow"),
		nodeInfoFromAttributes("ID", "4", "Country", "France", "City", "Paris"),
		nodeInfoFromAttributes("ID", "5", "Country", "France", "City", "Lyon"),
		nodeInfoFromAttributes("ID", "6", "Country", "Russia", "City", "SPB"),
		nodeInfoFromAttributes("ID", "7", "Country", "Russia", "City", "Moscow"),
		nodeInfoFromAttributes("ID", "8", "Country", "Germany", "City", "Darmstadt"),
		nodeInfoFromAttributes("ID", "9", "Country", "Germany", "City", "Frankfurt"),
		nodeInfoFromAttributes("ID", "10", "Country", "Russia", "City", "SPB"),
		nodeInfoFromAttributes("ID", "11", "Country", "Russia", "City", "Moscow"),
		nodeInfoFromAttributes("ID", "12", "Country", "Germany", "City", "London"),
	}
	for i := range nodes {
		pub := make([]byte, 33)
		rand.Read(pub)
		nodes[i].SetPublicKey(pub)
	}

	var nm NetMap
	nm.SetNodes(nodes)

	for _, unique := range []bool{false, true} {
		t.Run(fmt.Sprintf("unique=%t", unique), func(t *testing.T) {
			ssNamed := []Selector{newSelector("X", "City", 2, "FromRU", (*Selector).SelectDistinct)}
			fsNamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
			rsNamed := []ReplicaDescriptor{newReplica(1, "X")}
			pNamed := newPlacementPolicy(3, rsNamed, ssNamed, fsNamed)
			pNamed.unique = unique

			vNamed, err := nm.ContainerNodes(pNamed, []byte{1})
			require.NoError(t, err)

			ssUnnamed := []Selector{newSelector("", "City", 2, "FromRU", (*Selector).SelectDistinct)}
			fsUnnamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
			rsUnnamed := []ReplicaDescriptor{newReplica(1, "")}
			pUnnamed := newPlacementPolicy(3, rsUnnamed, ssUnnamed, fsUnnamed)
			pUnnamed.unique = unique

			vUnnamed, err := nm.ContainerNodes(pUnnamed, []byte{1})
			require.NoError(t, err)

			require.Equal(t, vNamed, vUnnamed)
		})
	}
}

func TestPlacementPolicy_MultiREP(t *testing.T) {
	nodes := []NodeInfo{
		nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"),
		nodeInfoFromAttributes("ID", "2", "Country", "Germany", "City", "Berlin"),
		nodeInfoFromAttributes("ID", "3", "Country", "Russia", "City", "Moscow"),
		nodeInfoFromAttributes("ID", "4", "Country", "France", "City", "Paris"),
		nodeInfoFromAttributes("ID", "5", "Country", "France", "City", "Lyon"),
		nodeInfoFromAttributes("ID", "6", "Country", "Russia", "City", "SPB"),
		nodeInfoFromAttributes("ID", "7", "Country", "Russia", "City", "Moscow"),
		nodeInfoFromAttributes("ID", "8", "Country", "Germany", "City", "Darmstadt"),
		nodeInfoFromAttributes("ID", "9", "Country", "Germany", "City", "Frankfurt"),
		nodeInfoFromAttributes("ID", "10", "Country", "Russia", "City", "SPB"),
		nodeInfoFromAttributes("ID", "11", "Country", "Russia", "City", "Moscow"),
		nodeInfoFromAttributes("ID", "12", "Country", "Germany", "City", "London"),
	}
	for i := range nodes {
		pub := make([]byte, 33)
		rand.Read(pub)
		nodes[i].SetPublicKey(pub)
	}

	var nm NetMap
	nm.SetNodes(nodes)

	ss := []Selector{newSelector("SameRU", "City", 2, "FromRU", (*Selector).SelectDistinct)}
	fs := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}

	for _, unique := range []bool{false, true} {
		for _, additional := range []int{0, 1, 2} {
			t.Run(fmt.Sprintf("unique=%t, additional=%d", unique, additional), func(t *testing.T) {
				rs := []ReplicaDescriptor{newReplica(1, "SameRU")}
				for i := 0; i < additional; i++ {
					rs = append(rs, newReplica(1, ""))
				}

				p := newPlacementPolicy(3, rs, ss, fs)
				p.unique = unique

				v, err := nm.ContainerNodes(p, []byte{1})
				require.NoError(t, err)
				require.Equal(t, 1+additional, len(v))
				require.Equal(t, 6, len(v[0]))

				for i := 1; i < additional; i++ {
					require.Equal(t, 3, len(v[i]))
					if !unique {
						require.Equal(t, v[1], v[i])
					}
				}

				if unique {
					seen := make(map[string]bool)
					for i := range v {
						for j := range v[i] {
							attr := v[i][j].Attribute("ID")
							require.NotEmpty(t, attr)
							require.False(t, seen[attr])

							seen[attr] = true
						}
					}
				}
			})
		}
	}
}

func TestPlacementPolicy_ProcessSelectorsExceptForNodes(t *testing.T) {
	p := newPlacementPolicy(1, nil,
		[]Selector{
			newSelector("ExceptRU", "City", 2, "ExceptRU", (*Selector).SelectSame),
		},
		[]Filter{
			newFilter("ExceptRU", "", "", netmap.NOT,
				newFilter("", "", "", netmap.AND,
					newFilter("", "City", "Lyon", netmap.EQ),
					newFilter("", "Rating", "10", netmap.LE),
				),
			),
		})
	nodes := []NodeInfo{
		nodeInfoFromAttributes("Country", "Germany", "Rating", "1", "City", "Berlin"),
		nodeInfoFromAttributes("Country", "Germany", "Rating", "5", "City", "Berlin"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "6", "City", "Moscow"),
		nodeInfoFromAttributes("Country", "France", "Rating", "4", "City", "Paris"),
		nodeInfoFromAttributes("Country", "France", "Rating", "1", "City", "Lyon"),
		nodeInfoFromAttributes("Country", "France", "Rating", "5", "City", "Lyon"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "7", "City", "Moscow"),
		nodeInfoFromAttributes("Country", "Germany", "Rating", "3", "City", "Darmstadt"),
		nodeInfoFromAttributes("Country", "Germany", "Rating", "7", "City", "Frankfurt"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
		nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
	}

	var nm NetMap
	nm.SetNodes(nodes)
	c := newContext(nm)
	c.setCBF(p.backupFactor)
	require.NoError(t, c.processFilters(p))
	require.NoError(t, c.processSelectors(p))

	for _, s := range p.selectors {
		sel := c.selections[s.GetName()]
		s := c.processedSelectors[s.GetName()]
		bucketCount, nodesInBucket := calcNodesCount(*s)
		nodesInBucket *= int(c.cbf)
		targ := fmt.Sprintf("selector '%s'", s.GetName())
		require.Equal(t, bucketCount, len(sel), targ)
		fName := s.GetFilter()
		for _, res := range sel {
			require.Equal(t, nodesInBucket, len(res), targ)
			for j := range res {
				require.True(t, fName == mainFilterName || c.match(c.processedFilters[fName], res[j]), targ)
			}
		}
	}
}

func TestSelector_SetName(t *testing.T) {
	const name = "some name"
	var s Selector

	require.Zero(t, s.m.GetName())

	s.SetName(name)
	require.Equal(t, name, s.m.GetName())
}

func TestSelector_SetNumberOfNodes(t *testing.T) {
	const num = 3
	var s Selector

	require.Zero(t, s.m.GetCount())

	s.SetNumberOfNodes(num)

	require.EqualValues(t, num, s.m.GetCount())
}

func TestSelectorClauses(t *testing.T) {
	var s Selector

	require.Equal(t, netmap.UnspecifiedClause, s.m.GetClause())

	s.SelectDistinct()
	require.Equal(t, netmap.Distinct, s.m.GetClause())

	s.SelectSame()
	require.Equal(t, netmap.Same, s.m.GetClause())
}

func TestSelector_SelectByBucketAttribute(t *testing.T) {
	const attr = "some attribute"
	var s Selector

	require.Zero(t, s.m.GetAttribute())

	s.SelectByBucketAttribute(attr)
	require.Equal(t, attr, s.m.GetAttribute())
}

func TestSelector_SetFilterName(t *testing.T) {
	const fName = "some filter"
	var s Selector

	require.Zero(t, s.m.GetFilter())

	s.SetFilterName(fName)
	require.Equal(t, fName, s.m.GetFilter())
}