forked from TrueCloudLab/frostfs-node
Anton Nikiforov
f12f04199e
Placement vector may contain fewer nodes count than it required by policy due to the outage of the one of the node. Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
603 lines
15 KiB
Go
603 lines
15 KiB
Go
package placement
|
|
|
|
import (
|
|
"strconv"
|
|
"testing"
|
|
|
|
netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
type testBuilder struct {
|
|
vectors [][]netmap.NodeInfo
|
|
}
|
|
|
|
func (b testBuilder) BuildPlacement(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) {
|
|
return b.vectors, nil
|
|
}
|
|
|
|
func testNode(v uint32) (n netmap.NodeInfo) {
|
|
ip := "/ip4/0.0.0.0/tcp/" + strconv.Itoa(int(v))
|
|
n.SetNetworkEndpoints(ip)
|
|
n.SetPublicKey([]byte(ip))
|
|
|
|
return n
|
|
}
|
|
|
|
func copyVectors(v [][]netmap.NodeInfo) [][]netmap.NodeInfo {
|
|
vc := make([][]netmap.NodeInfo, 0, len(v))
|
|
|
|
for i := range v {
|
|
ns := make([]netmap.NodeInfo, len(v[i]))
|
|
copy(ns, v[i])
|
|
|
|
vc = append(vc, ns)
|
|
}
|
|
|
|
return vc
|
|
}
|
|
|
|
func testPlacement(ss []int, rs []int) ([][]netmap.NodeInfo, container.Container) {
|
|
return placement(ss, rs, nil)
|
|
}
|
|
|
|
func testECPlacement(ss []int, ec [][]int) ([][]netmap.NodeInfo, container.Container) {
|
|
return placement(ss, nil, ec)
|
|
}
|
|
|
|
func placement(ss []int, rs []int, ec [][]int) ([][]netmap.NodeInfo, container.Container) {
|
|
nodes := make([][]netmap.NodeInfo, 0, len(rs))
|
|
replicas := make([]netmap.ReplicaDescriptor, 0, len(rs))
|
|
num := uint32(0)
|
|
|
|
for i := range ss {
|
|
ns := make([]netmap.NodeInfo, 0, ss[i])
|
|
|
|
for range ss[i] {
|
|
ns = append(ns, testNode(num))
|
|
num++
|
|
}
|
|
|
|
nodes = append(nodes, ns)
|
|
|
|
var rd netmap.ReplicaDescriptor
|
|
if len(rs) > 0 {
|
|
rd.SetNumberOfObjects(uint32(rs[i]))
|
|
} else {
|
|
rd.SetECDataCount(uint32(ec[i][0]))
|
|
rd.SetECParityCount(uint32(ec[i][1]))
|
|
}
|
|
|
|
replicas = append(replicas, rd)
|
|
}
|
|
|
|
var policy netmap.PlacementPolicy
|
|
policy.AddReplicas(replicas...)
|
|
|
|
var cnr container.Container
|
|
cnr.SetPlacementPolicy(policy)
|
|
|
|
return nodes, cnr
|
|
}
|
|
|
|
func assertSameAddress(t *testing.T, ni netmap.NodeInfo, addr network.AddressGroup) {
|
|
var netAddr network.AddressGroup
|
|
|
|
err := netAddr.FromIterator(netmapcore.Node(ni))
|
|
require.NoError(t, err)
|
|
require.True(t, netAddr.Intersects(addr))
|
|
}
|
|
|
|
func TestTraverserObjectScenarios(t *testing.T) {
|
|
t.Run("search scenario", func(t *testing.T) {
|
|
selectors := []int{2, 3}
|
|
replicas := []int{1, 2}
|
|
|
|
nodes, cnr := testPlacement(selectors, replicas)
|
|
|
|
nodesCopy := copyVectors(nodes)
|
|
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{vectors: nodesCopy}),
|
|
WithoutSuccessTracking(),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
for i := range selectors {
|
|
addrs := tr.Next()
|
|
|
|
require.Len(t, addrs, len(nodes[i]))
|
|
|
|
for j, n := range nodes[i] {
|
|
assertSameAddress(t, n, addrs[j].Addresses())
|
|
}
|
|
}
|
|
|
|
require.Empty(t, tr.Next())
|
|
require.True(t, tr.Success())
|
|
})
|
|
|
|
t.Run("read scenario", func(t *testing.T) {
|
|
selectors := []int{5, 3}
|
|
replicas := []int{2, 2}
|
|
|
|
nodes, cnr := testPlacement(selectors, replicas)
|
|
|
|
nodesCopy := copyVectors(nodes)
|
|
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{
|
|
vectors: nodesCopy,
|
|
}),
|
|
SuccessAfter(1),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
for range len(nodes[0]) {
|
|
require.NotNil(t, tr.Next())
|
|
}
|
|
|
|
var n network.AddressGroup
|
|
|
|
err = n.FromIterator(netmapcore.Node(nodes[1][0]))
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, []Node{{addresses: n, key: []byte("/ip4/0.0.0.0/tcp/5")}}, tr.Next())
|
|
})
|
|
|
|
t.Run("put scenario", func(t *testing.T) {
|
|
selectors := []int{5, 3}
|
|
replicas := []int{2, 2}
|
|
|
|
nodes, cnr := testPlacement(selectors, replicas)
|
|
|
|
nodesCopy := copyVectors(nodes)
|
|
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{vectors: nodesCopy}),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
fn := func(curVector int) {
|
|
for i := 0; i+replicas[curVector] < selectors[curVector]; i += replicas[curVector] {
|
|
addrs := tr.Next()
|
|
require.Len(t, addrs, replicas[curVector])
|
|
|
|
for j := range addrs {
|
|
assertSameAddress(t, nodes[curVector][i+j], addrs[j].Addresses())
|
|
}
|
|
}
|
|
|
|
require.Empty(t, tr.Next())
|
|
require.False(t, tr.Success())
|
|
|
|
for range replicas[curVector] {
|
|
tr.SubmitSuccess()
|
|
}
|
|
}
|
|
|
|
for i := range selectors {
|
|
fn(i)
|
|
|
|
if i < len(selectors)-1 {
|
|
require.False(t, tr.Success())
|
|
} else {
|
|
require.True(t, tr.Success())
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("local operation scenario", func(t *testing.T) {
|
|
selectors := []int{2, 3}
|
|
replicas := []int{1, 2}
|
|
|
|
nodes, cnr := testPlacement(selectors, replicas)
|
|
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{
|
|
vectors: [][]netmap.NodeInfo{{nodes[1][1]}}, // single node (local)
|
|
}),
|
|
SuccessAfter(1),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
require.NotEmpty(t, tr.Next())
|
|
require.False(t, tr.Success())
|
|
|
|
// add 1 OK
|
|
tr.SubmitSuccess()
|
|
|
|
// nothing more to do
|
|
require.Empty(t, tr.Next())
|
|
|
|
// common success
|
|
require.True(t, tr.Success())
|
|
})
|
|
}
|
|
|
|
func TestTraverserRemValues(t *testing.T) {
|
|
selectors := []int{3, 4, 5}
|
|
replicas := []int{2, 3, 4}
|
|
|
|
nodes, cnr := testPlacement(selectors, replicas)
|
|
nodesCopy := copyVectors(nodes)
|
|
|
|
testCases := [...]struct {
|
|
name string
|
|
copyNumbers []uint32
|
|
expectedRem []int
|
|
expectedErr error
|
|
}{
|
|
{
|
|
name: "zero copy numbers",
|
|
copyNumbers: []uint32{},
|
|
expectedRem: replicas,
|
|
},
|
|
{
|
|
name: "compatible zero copy numbers, len 1",
|
|
copyNumbers: []uint32{0},
|
|
expectedRem: replicas,
|
|
},
|
|
{
|
|
name: "compatible zero copy numbers, len 3",
|
|
copyNumbers: []uint32{0, 0, 0},
|
|
expectedRem: replicas,
|
|
},
|
|
{
|
|
name: "copy numbers for all replicas",
|
|
copyNumbers: []uint32{1, 1, 1},
|
|
expectedRem: []int{1, 1, 1},
|
|
},
|
|
{
|
|
name: "single copy numbers for multiple replicas",
|
|
copyNumbers: []uint32{1},
|
|
expectedRem: []int{1}, // may be a bit unexpected
|
|
},
|
|
{
|
|
name: "multiple copy numbers for multiple replicas",
|
|
copyNumbers: []uint32{1, 1, 4},
|
|
expectedRem: []int{1, 1, 4},
|
|
},
|
|
{
|
|
name: "incompatible copies number vector",
|
|
copyNumbers: []uint32{1, 1},
|
|
expectedErr: errCopiesNumberLen,
|
|
},
|
|
}
|
|
|
|
for _, testCase := range testCases {
|
|
t.Run(testCase.name, func(t *testing.T) {
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{vectors: nodesCopy}),
|
|
WithCopyNumbers(testCase.copyNumbers),
|
|
)
|
|
if testCase.expectedErr == nil {
|
|
require.NoError(t, err, testCase.name)
|
|
require.Equal(t, testCase.expectedRem, tr.rem, testCase.name)
|
|
} else {
|
|
require.Error(t, err, testCase.expectedErr, testCase.name)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type nodeState struct {
|
|
node *netmap.NodeInfo
|
|
}
|
|
|
|
func (n *nodeState) LocalNodeInfo() *netmap.NodeInfo {
|
|
return n.node
|
|
}
|
|
|
|
func TestTraverserPriorityMetrics(t *testing.T) {
|
|
t.Run("one rep one metric", func(t *testing.T) {
|
|
selectors := []int{4}
|
|
replicas := []int{3}
|
|
|
|
nodes, cnr := testPlacement(selectors, replicas)
|
|
|
|
// Node_0, PK - ip4/0.0.0.0/tcp/0
|
|
nodes[0][0].SetAttribute("ClusterName", "A")
|
|
// Node_1, PK - ip4/0.0.0.0/tcp/1
|
|
nodes[0][1].SetAttribute("ClusterName", "A")
|
|
// Node_2, PK - ip4/0.0.0.0/tcp/2
|
|
nodes[0][2].SetAttribute("ClusterName", "B")
|
|
// Node_3, PK - ip4/0.0.0.0/tcp/3
|
|
nodes[0][3].SetAttribute("ClusterName", "B")
|
|
|
|
sdkNode := testNode(5)
|
|
sdkNode.SetAttribute("ClusterName", "B")
|
|
|
|
nodesCopy := copyVectors(nodes)
|
|
|
|
m := []Metric{NewAttributeMetric("ClusterName")}
|
|
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{
|
|
vectors: nodesCopy,
|
|
}),
|
|
WithoutSuccessTracking(),
|
|
WithPriorityMetrics(m),
|
|
WithNodeState(&nodeState{
|
|
node: &sdkNode,
|
|
}),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
// Without priority metric `ClusterName` the order will be:
|
|
// [ {Node_0 A}, {Node_1 A}, {Node_2 B}, {Node_3 B}]
|
|
// With priority metric `ClusterName` and current node in cluster B
|
|
// the order should be:
|
|
// [ {Node_2 B}, {Node_0 A}, {Node_1 A}, {Node_3 B}]
|
|
next := tr.Next()
|
|
require.NotNil(t, next)
|
|
require.Equal(t, 3, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[2].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
// The last node is
|
|
require.Equal(t, 1, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[0].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Nil(t, next)
|
|
})
|
|
|
|
t.Run("one rep one metric fewer nodes", func(t *testing.T) {
|
|
selectors := []int{2}
|
|
replicas := []int{3}
|
|
|
|
nodes, cnr := testPlacement(selectors, replicas)
|
|
|
|
// Node_0, PK - ip4/0.0.0.0/tcp/0
|
|
nodes[0][0].SetAttribute("ClusterName", "A")
|
|
// Node_1, PK - ip4/0.0.0.0/tcp/1
|
|
nodes[0][1].SetAttribute("ClusterName", "B")
|
|
|
|
sdkNode := testNode(5)
|
|
sdkNode.SetAttribute("ClusterName", "B")
|
|
|
|
nodesCopy := copyVectors(nodes)
|
|
|
|
m := []Metric{NewAttributeMetric("ClusterName")}
|
|
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{
|
|
vectors: nodesCopy,
|
|
}),
|
|
WithoutSuccessTracking(),
|
|
WithPriorityMetrics(m),
|
|
WithNodeState(&nodeState{
|
|
node: &sdkNode,
|
|
}),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
// Without priority metric `ClusterName` the order will be:
|
|
// [ {Node_0 A}, {Node_1 A} ]
|
|
// With priority metric `ClusterName` and current node in cluster B
|
|
// the order should be:
|
|
// [ {Node_1 B}, {Node_0 A} ]
|
|
next := tr.Next()
|
|
require.NotNil(t, next)
|
|
require.Equal(t, 2, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Nil(t, next)
|
|
})
|
|
|
|
t.Run("two reps two metrics", func(t *testing.T) {
|
|
selectors := []int{3, 3}
|
|
replicas := []int{2, 2}
|
|
|
|
nodes, cnr := testPlacement(selectors, replicas)
|
|
|
|
// REPLICA #1
|
|
// Node_0, PK - ip4/0.0.0.0/tcp/0
|
|
nodes[0][0].SetAttribute("ClusterName", "A")
|
|
nodes[0][0].SetAttribute("UN-LOCODE", "RU LED")
|
|
|
|
// Node_1, PK - ip4/0.0.0.0/tcp/1
|
|
nodes[0][1].SetAttribute("ClusterName", "A")
|
|
nodes[0][1].SetAttribute("UN-LOCODE", "FI HEL")
|
|
|
|
// Node_2, PK - ip4/0.0.0.0/tcp/2
|
|
nodes[0][2].SetAttribute("ClusterName", "A")
|
|
nodes[0][2].SetAttribute("UN-LOCODE", "RU LED")
|
|
|
|
// REPLICA #2
|
|
// Node_3 ip4/0.0.0.0/tcp/3
|
|
nodes[1][0].SetAttribute("ClusterName", "B")
|
|
nodes[1][0].SetAttribute("UN-LOCODE", "RU MOW")
|
|
|
|
// Node_4, PK - ip4/0.0.0.0/tcp/4
|
|
nodes[1][1].SetAttribute("ClusterName", "B")
|
|
nodes[1][1].SetAttribute("UN-LOCODE", "RU DME")
|
|
|
|
// Node_5, PK - ip4/0.0.0.0/tcp/5
|
|
nodes[1][2].SetAttribute("ClusterName", "B")
|
|
nodes[1][2].SetAttribute("UN-LOCODE", "RU MOW")
|
|
|
|
sdkNode := testNode(9)
|
|
sdkNode.SetAttribute("ClusterName", "B")
|
|
sdkNode.SetAttribute("UN-LOCODE", "RU DME")
|
|
|
|
nodesCopy := copyVectors(nodes)
|
|
|
|
m := []Metric{
|
|
NewAttributeMetric("ClusterName"),
|
|
NewAttributeMetric("UN-LOCODE"),
|
|
}
|
|
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{
|
|
vectors: nodesCopy,
|
|
}),
|
|
WithoutSuccessTracking(),
|
|
WithPriorityMetrics(m),
|
|
WithNodeState(&nodeState{
|
|
node: &sdkNode,
|
|
}),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
// Check that nodes in the same cluster and
|
|
// in the same location should be the first in slice.
|
|
// Nodes which are follow criteria but stay outside the replica
|
|
// should be in the next slice.
|
|
|
|
next := tr.Next()
|
|
require.Equal(t, 4, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[1].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[2].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[3].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Equal(t, 2, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Nil(t, next)
|
|
|
|
sdkNode.SetAttribute("ClusterName", "B")
|
|
sdkNode.SetAttribute("UN-LOCODE", "RU MOW")
|
|
|
|
nodesCopy = copyVectors(nodes)
|
|
|
|
tr, err = NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{
|
|
vectors: nodesCopy,
|
|
}),
|
|
WithoutSuccessTracking(),
|
|
WithPriorityMetrics(m),
|
|
WithNodeState(&nodeState{
|
|
node: &sdkNode,
|
|
}),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
next = tr.Next()
|
|
require.Equal(t, 4, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[1].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[2].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[3].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Equal(t, 2, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Nil(t, next)
|
|
|
|
sdkNode.SetAttribute("ClusterName", "A")
|
|
sdkNode.SetAttribute("UN-LOCODE", "RU LED")
|
|
|
|
nodesCopy = copyVectors(nodes)
|
|
|
|
tr, err = NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{
|
|
vectors: nodesCopy,
|
|
}),
|
|
WithoutSuccessTracking(),
|
|
WithPriorityMetrics(m),
|
|
WithNodeState(&nodeState{
|
|
node: &sdkNode,
|
|
}),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
next = tr.Next()
|
|
require.Equal(t, 4, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[1].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[2].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[3].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Equal(t, 2, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Nil(t, next)
|
|
})
|
|
|
|
t.Run("ec container", func(t *testing.T) {
|
|
selectors := []int{4}
|
|
ec := [][]int{{2, 1}}
|
|
|
|
nodes, cnr := testECPlacement(selectors, ec)
|
|
|
|
// Node_0, PK - ip4/0.0.0.0/tcp/0
|
|
nodes[0][0].SetAttribute("ClusterName", "A")
|
|
// Node_1, PK - ip4/0.0.0.0/tcp/1
|
|
nodes[0][1].SetAttribute("ClusterName", "A")
|
|
// Node_2, PK - ip4/0.0.0.0/tcp/2
|
|
nodes[0][2].SetAttribute("ClusterName", "B")
|
|
// Node_3, PK - ip4/0.0.0.0/tcp/3
|
|
nodes[0][3].SetAttribute("ClusterName", "B")
|
|
|
|
sdkNode := testNode(5)
|
|
sdkNode.SetAttribute("ClusterName", "B")
|
|
|
|
nodesCopy := copyVectors(nodes)
|
|
|
|
m := []Metric{NewAttributeMetric("ClusterName")}
|
|
|
|
tr, err := NewTraverser(
|
|
ForContainer(cnr),
|
|
UseBuilder(&testBuilder{
|
|
vectors: nodesCopy,
|
|
}),
|
|
WithoutSuccessTracking(),
|
|
WithPriorityMetrics(m),
|
|
WithNodeState(&nodeState{
|
|
node: &sdkNode,
|
|
}),
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
// Without priority metric `ClusterName` the order will be:
|
|
// [ {Node_0 A}, {Node_1 A}, {Node_2 B}, {Node_3 B}]
|
|
// With priority metric `ClusterName` and current node in cluster B
|
|
// the order should be:
|
|
// [ {Node_2 B}, {Node_0 A}, {Node_1 A}, {Node_3 B}]
|
|
next := tr.Next()
|
|
require.NotNil(t, next)
|
|
require.Equal(t, 3, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey()))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[2].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
// The last node is
|
|
require.Equal(t, 1, len(next))
|
|
require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[0].PublicKey()))
|
|
|
|
next = tr.Next()
|
|
require.Nil(t, next)
|
|
})
|
|
}
|