forked from TrueCloudLab/frostfs-node
[#1129] policer: Pull required EC chunks
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
d45d086acd
commit
cbe9757490
13 changed files with 744 additions and 71 deletions
402
pkg/services/policer/ec_test.go
Normal file
402
pkg/services/policer/ec_test.go
Normal file
|
|
@ -0,0 +1,402 @@
|
|||
package policer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
|
||||
objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestECChunkHasValidPlacement(t *testing.T) {
|
||||
t.Parallel()
|
||||
chunkAddress := oidtest.Address()
|
||||
parentID := oidtest.ID()
|
||||
|
||||
var policy netmapSDK.PlacementPolicy
|
||||
require.NoError(t, policy.DecodeString("EC 2.1"))
|
||||
|
||||
cnr := &container.Container{}
|
||||
cnr.Value.Init()
|
||||
cnr.Value.SetPlacementPolicy(policy)
|
||||
containerSrc := containerSrc{
|
||||
get: func(id cid.ID) (*container.Container, error) {
|
||||
if id.Equals(chunkAddress.Container()) {
|
||||
return cnr, nil
|
||||
}
|
||||
return nil, new(apistatus.ContainerNotFound)
|
||||
},
|
||||
}
|
||||
|
||||
nodes := make([]netmapSDK.NodeInfo, 4)
|
||||
for i := range nodes {
|
||||
nodes[i].SetPublicKey([]byte{byte(i)})
|
||||
}
|
||||
|
||||
placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) {
|
||||
if cnr.Equals(chunkAddress.Container()) && obj.Equals(parentID) {
|
||||
return [][]netmapSDK.NodeInfo{nodes}, nil
|
||||
}
|
||||
return nil, errors.New("unexpected placement build")
|
||||
}
|
||||
|
||||
headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) {
|
||||
require.True(t, raw, "remote header for parent object must be called with raw flag")
|
||||
index := int(ni.PublicKey()[0])
|
||||
require.True(t, index == 1 || index == 2, "invalid node to get parent header")
|
||||
require.True(t, a.Container() == chunkAddress.Container() && a.Object() == parentID, "invalid address to get remote header")
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = uint32(index)
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
|
||||
p := New(
|
||||
WithContainerSource(containerSrc),
|
||||
WithPlacementBuilder(placementBuilderFunc(placementBuilder)),
|
||||
WithNetmapKeys(announcedKeysFunc(func(k []byte) bool {
|
||||
return bytes.Equal(k, nodes[0].PublicKey())
|
||||
})),
|
||||
WithRemoteObjectHeaderFunc(headFn),
|
||||
WithPool(testPool(t)),
|
||||
)
|
||||
|
||||
objInfo := objectcore.Info{
|
||||
Address: chunkAddress,
|
||||
Type: objectSDK.TypeRegular,
|
||||
ECInfo: &objectcore.ECInfo{
|
||||
ParentID: parentID,
|
||||
Index: 0,
|
||||
Total: 3,
|
||||
},
|
||||
}
|
||||
err := p.processObject(context.Background(), objInfo)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestECChunkHasInvalidPlacement(t *testing.T) {
|
||||
t.Parallel()
|
||||
chunkAddress := oidtest.Address()
|
||||
parentID := oidtest.ID()
|
||||
chunkObject := objectSDK.New()
|
||||
chunkObject.SetContainerID(chunkAddress.Container())
|
||||
chunkObject.SetID(chunkAddress.Object())
|
||||
chunkObject.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
|
||||
chunkObject.SetPayloadSize(uint64(10))
|
||||
chunkObject.SetECHeader(objectSDK.NewECHeader(parentID, 1, 3, []byte{}, 0))
|
||||
|
||||
var policy netmapSDK.PlacementPolicy
|
||||
require.NoError(t, policy.DecodeString("EC 2.1"))
|
||||
|
||||
cnr := &container.Container{}
|
||||
cnr.Value.Init()
|
||||
cnr.Value.SetPlacementPolicy(policy)
|
||||
containerSrc := containerSrc{
|
||||
get: func(id cid.ID) (*container.Container, error) {
|
||||
if id.Equals(chunkAddress.Container()) {
|
||||
return cnr, nil
|
||||
}
|
||||
return nil, new(apistatus.ContainerNotFound)
|
||||
},
|
||||
}
|
||||
|
||||
nodes := make([]netmapSDK.NodeInfo, 4)
|
||||
for i := range nodes {
|
||||
nodes[i].SetPublicKey([]byte{byte(i)})
|
||||
}
|
||||
|
||||
placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) {
|
||||
if cnr.Equals(chunkAddress.Container()) && obj.Equals(parentID) {
|
||||
return [][]netmapSDK.NodeInfo{nodes}, nil
|
||||
}
|
||||
return nil, errors.New("unexpected placement build")
|
||||
}
|
||||
|
||||
objInfo := objectcore.Info{
|
||||
Address: chunkAddress,
|
||||
Type: objectSDK.TypeRegular,
|
||||
ECInfo: &objectcore.ECInfo{
|
||||
ParentID: parentID,
|
||||
Index: 1,
|
||||
Total: 3,
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("node0 has chunk1, node1 has chunk0 and chunk1", func(t *testing.T) {
|
||||
// policer should pull chunk0 on first run and drop chunk1 on second run
|
||||
var allowDrop bool
|
||||
requiredChunkID := oidtest.ID()
|
||||
headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) {
|
||||
if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a == chunkAddress && !raw {
|
||||
return chunkObject, nil
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = 1
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
ch.Index = 0
|
||||
ch.SetID(requiredChunkID)
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[2].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = 2
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[3].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
return nil, new(apistatus.ObjectNotFound)
|
||||
}
|
||||
require.Fail(t, "unexpected remote HEAD")
|
||||
return nil, fmt.Errorf("unexpected remote HEAD")
|
||||
}
|
||||
|
||||
localHeadF := func(_ context.Context, addr oid.Address) (*objectSDK.Object, error) {
|
||||
require.True(t, addr.Container() == chunkAddress.Container() && addr.Object() == parentID, "unexpected local HEAD")
|
||||
if allowDrop {
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = 1
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
ch.SetID(requiredChunkID)
|
||||
ch.Index = 0
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = 1
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
|
||||
var pullCounter atomic.Int64
|
||||
var dropped []oid.Address
|
||||
p := New(
|
||||
WithContainerSource(containerSrc),
|
||||
WithPlacementBuilder(placementBuilderFunc(placementBuilder)),
|
||||
WithNetmapKeys(announcedKeysFunc(func(k []byte) bool {
|
||||
return bytes.Equal(k, nodes[0].PublicKey())
|
||||
})),
|
||||
WithRemoteObjectHeaderFunc(headFn),
|
||||
WithLocalObjectHeaderFunc(localHeadF),
|
||||
WithReplicator(pullFunc(func(ctx context.Context, r replicator.Task) {
|
||||
require.True(t, r.Addr.Container() == chunkAddress.Container() && r.Addr.Object() == requiredChunkID &&
|
||||
len(r.Nodes) == 1 && bytes.Equal(r.Nodes[0].PublicKey(), nodes[1].PublicKey()), "invalid pull task")
|
||||
pullCounter.Add(1)
|
||||
})),
|
||||
WithRedundantCopyCallback(func(ctx context.Context, a oid.Address) {
|
||||
require.True(t, allowDrop, "invalid redundent copy call")
|
||||
dropped = append(dropped, a)
|
||||
}),
|
||||
WithPool(testPool(t)),
|
||||
)
|
||||
|
||||
err := p.processObject(context.Background(), objInfo)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(1), pullCounter.Load(), "invalid pull count")
|
||||
require.Equal(t, 0, len(dropped), "invalid dropped count")
|
||||
allowDrop = true
|
||||
err = p.processObject(context.Background(), objInfo)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(1), pullCounter.Load(), "invalid pull count")
|
||||
require.Equal(t, 1, len(dropped), "invalid dropped count")
|
||||
require.True(t, chunkAddress.Equals(dropped[0]), "invalid dropped object")
|
||||
})
|
||||
|
||||
t.Run("node0 has chunk0 and chunk1, node1 has chunk1", func(t *testing.T) {
|
||||
// policer should drop chunk1
|
||||
headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) {
|
||||
if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a == chunkAddress && !raw {
|
||||
return chunkObject, nil
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(chunkAddress.Object())
|
||||
ch.Index = 1
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[2].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = 2
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[3].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
return nil, new(apistatus.ObjectNotFound)
|
||||
}
|
||||
require.Fail(t, "unexpected remote HEAD")
|
||||
return nil, fmt.Errorf("unexpected remote HEAD")
|
||||
}
|
||||
|
||||
localHeadF := func(_ context.Context, addr oid.Address) (*objectSDK.Object, error) {
|
||||
require.True(t, addr.Container() == chunkAddress.Container() && addr.Object() == parentID, "unexpected local HEAD")
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(chunkAddress.Object())
|
||||
ch.Index = 1
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = 0
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
|
||||
var dropped []oid.Address
|
||||
p := New(
|
||||
WithContainerSource(containerSrc),
|
||||
WithPlacementBuilder(placementBuilderFunc(placementBuilder)),
|
||||
WithNetmapKeys(announcedKeysFunc(func(k []byte) bool {
|
||||
return bytes.Equal(k, nodes[0].PublicKey())
|
||||
})),
|
||||
WithRemoteObjectHeaderFunc(headFn),
|
||||
WithLocalObjectHeaderFunc(localHeadF),
|
||||
WithRedundantCopyCallback(func(ctx context.Context, a oid.Address) {
|
||||
dropped = append(dropped, a)
|
||||
}),
|
||||
WithPool(testPool(t)),
|
||||
)
|
||||
|
||||
err := p.processObject(context.Background(), objInfo)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, len(dropped), "invalid dropped count")
|
||||
require.True(t, chunkAddress.Equals(dropped[0]), "invalid dropped object")
|
||||
})
|
||||
|
||||
t.Run("node0 has chunk0 and chunk1, node1 has no chunks", func(t *testing.T) {
|
||||
// policer should replicate chunk1 to node1 on first run and drop chunk1 on node0 on second run
|
||||
var secondRun bool
|
||||
headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) {
|
||||
if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a == chunkAddress && !raw {
|
||||
if !secondRun {
|
||||
return nil, new(apistatus.ObjectNotFound)
|
||||
}
|
||||
return chunkObject, nil
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(chunkAddress.Object())
|
||||
ch.Index = 1
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[2].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = 2
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
if bytes.Equal(ni.PublicKey(), nodes[3].PublicKey()) && a.Container() == chunkAddress.Container() &&
|
||||
a.Object() == parentID && raw {
|
||||
return nil, new(apistatus.ObjectNotFound)
|
||||
}
|
||||
require.Fail(t, "unexpected remote HEAD")
|
||||
return nil, fmt.Errorf("unexpected remote HEAD")
|
||||
}
|
||||
|
||||
localHeadF := func(_ context.Context, addr oid.Address) (*objectSDK.Object, error) {
|
||||
require.True(t, addr.Container() == chunkAddress.Container() && addr.Object() == parentID, "unexpected local HEAD")
|
||||
ei := objectSDK.NewECInfo()
|
||||
var ch objectSDK.ECChunk
|
||||
ch.SetID(chunkAddress.Object())
|
||||
ch.Index = 1
|
||||
ch.Total = 3
|
||||
ei.AddChunk(ch)
|
||||
ch.SetID(oidtest.ID())
|
||||
ch.Index = 0
|
||||
ei.AddChunk(ch)
|
||||
return nil, objectSDK.NewECInfoError(ei)
|
||||
}
|
||||
|
||||
var dropped []oid.Address
|
||||
var replicated []replicator.Task
|
||||
p := New(
|
||||
WithContainerSource(containerSrc),
|
||||
WithPlacementBuilder(placementBuilderFunc(placementBuilder)),
|
||||
WithNetmapKeys(announcedKeysFunc(func(k []byte) bool {
|
||||
return bytes.Equal(k, nodes[0].PublicKey())
|
||||
})),
|
||||
WithRemoteObjectHeaderFunc(headFn),
|
||||
WithLocalObjectHeaderFunc(localHeadF),
|
||||
WithRedundantCopyCallback(func(ctx context.Context, a oid.Address) {
|
||||
dropped = append(dropped, a)
|
||||
}),
|
||||
WithReplicator(replicatorFunc(func(ctx context.Context, t replicator.Task, tr replicator.TaskResult) {
|
||||
replicated = append(replicated, t)
|
||||
})),
|
||||
WithPool(testPool(t)),
|
||||
)
|
||||
|
||||
err := p.processObject(context.Background(), objInfo)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(dropped), "invalid dropped count")
|
||||
require.Equal(t, 1, len(replicated), "invalid replicated count")
|
||||
require.Equal(t, chunkAddress, replicated[0].Addr, "invalid replicated object")
|
||||
require.True(t, bytes.Equal(replicated[0].Nodes[0].PublicKey(), nodes[1].PublicKey()), "invalid replicate target")
|
||||
|
||||
secondRun = true
|
||||
err = p.processObject(context.Background(), objInfo)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, len(replicated), "invalid replicated count")
|
||||
require.Equal(t, chunkAddress, replicated[0].Addr, "invalid replicated object")
|
||||
require.True(t, bytes.Equal(replicated[0].Nodes[0].PublicKey(), nodes[1].PublicKey()), "invalid replicate target")
|
||||
require.Equal(t, 1, len(dropped), "invalid dropped count")
|
||||
require.True(t, chunkAddress.Equals(dropped[0]), "invalid dropped object")
|
||||
})
|
||||
}
|
||||
|
||||
type pullFunc func(context.Context, replicator.Task)
|
||||
|
||||
func (f pullFunc) HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func (f pullFunc) HandlePullTask(ctx context.Context, task replicator.Task) {
|
||||
f(ctx, task)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue