forked from TrueCloudLab/frostfs-sdk-go
[#223] object: Introduce new fields for ECHeader
* Introduce `parentSplitID`, `parentSplitParentID` fields for `ECHeader`; * Fix ECHeader's constructor; * Fix `Split` and `Reconstruct`; * Add unit-tests. Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
This commit is contained in:
parent
09b79d13f3
commit
3de256d05e
4 changed files with 152 additions and 13 deletions
|
@ -11,6 +11,8 @@ import (
|
||||||
// ECHeader represents erasure coding header.
|
// ECHeader represents erasure coding header.
|
||||||
type ECHeader struct {
|
type ECHeader struct {
|
||||||
parent oid.ID
|
parent oid.ID
|
||||||
|
parentSplitID *SplitID
|
||||||
|
parentSplitParentID *oid.ID
|
||||||
index uint32
|
index uint32
|
||||||
total uint32
|
total uint32
|
||||||
header []byte
|
header []byte
|
||||||
|
@ -18,9 +20,11 @@ type ECHeader struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewECHeader constructs new erasure coding header.
|
// NewECHeader constructs new erasure coding header.
|
||||||
func NewECHeader(parent oid.ID, index, total uint32, header []byte, headerLength uint32) *ECHeader {
|
func NewECHeader(parent oid.ID, parentSplitID *SplitID, parentSplitParentID *oid.ID, index, total uint32, header []byte, headerLength uint32) *ECHeader {
|
||||||
return &ECHeader{
|
return &ECHeader{
|
||||||
parent: parent,
|
parent: parent,
|
||||||
|
parentSplitID: parentSplitID,
|
||||||
|
parentSplitParentID: parentSplitParentID,
|
||||||
index: index,
|
index: index,
|
||||||
total: total,
|
total: total,
|
||||||
header: header,
|
header: header,
|
||||||
|
@ -32,6 +36,14 @@ func NewECHeader(parent oid.ID, index, total uint32, header []byte, headerLength
|
||||||
func (e *ECHeader) WriteToV2(h *object.ECHeader) {
|
func (e *ECHeader) WriteToV2(h *object.ECHeader) {
|
||||||
var parent refs.ObjectID
|
var parent refs.ObjectID
|
||||||
e.parent.WriteToV2(&parent)
|
e.parent.WriteToV2(&parent)
|
||||||
|
h.ParentSplitID = e.parentSplitID.ToV2()
|
||||||
|
|
||||||
|
if e.parentSplitParentID != nil {
|
||||||
|
parentSplitParentID := new(refs.ObjectID)
|
||||||
|
e.parentSplitParentID.WriteToV2(parentSplitParentID)
|
||||||
|
h.ParentSplitParentID = parentSplitParentID
|
||||||
|
}
|
||||||
|
|
||||||
h.Parent = &parent
|
h.Parent = &parent
|
||||||
h.Index = e.index
|
h.Index = e.index
|
||||||
h.Total = e.total
|
h.Total = e.total
|
||||||
|
@ -49,6 +61,13 @@ func (e *ECHeader) ReadFromV2(h *object.ECHeader) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = e.parent.ReadFromV2(*h.Parent)
|
_ = e.parent.ReadFromV2(*h.Parent)
|
||||||
|
e.parentSplitID = NewSplitIDFromV2(h.ParentSplitID)
|
||||||
|
if h.ParentSplitParentID != nil {
|
||||||
|
if e.parentSplitParentID == nil {
|
||||||
|
e.parentSplitParentID = new(oid.ID)
|
||||||
|
}
|
||||||
|
_ = e.parentSplitParentID.ReadFromV2(*h.ParentSplitParentID)
|
||||||
|
}
|
||||||
e.index = h.Index
|
e.index = h.Index
|
||||||
e.total = h.Total
|
e.total = h.Total
|
||||||
e.header = h.Header
|
e.header = h.Header
|
||||||
|
@ -88,6 +107,22 @@ func (e *ECHeader) SetParent(id oid.ID) {
|
||||||
e.parent = id
|
e.parent = id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e *ECHeader) ParentSplitID() *SplitID {
|
||||||
|
return e.parentSplitID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ECHeader) SetParentSplitID(parentSplitID *SplitID) {
|
||||||
|
e.parentSplitID = parentSplitID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ECHeader) ParentSplitParentID() *oid.ID {
|
||||||
|
return e.parentSplitParentID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ECHeader) SetParentSplitParentID(parentSplitParentID *oid.ID) {
|
||||||
|
e.parentSplitParentID = parentSplitParentID
|
||||||
|
}
|
||||||
|
|
||||||
func (e *ECHeader) Index() uint32 {
|
func (e *ECHeader) Index() uint32 {
|
||||||
return e.index
|
return e.index
|
||||||
}
|
}
|
||||||
|
|
|
@ -87,6 +87,8 @@ func (c *Constructor) ReconstructParts(parts []*objectSDK.Object, required []boo
|
||||||
ec := parts[nonNilPart].GetECHeader()
|
ec := parts[nonNilPart].GetECHeader()
|
||||||
parent := ec.Parent()
|
parent := ec.Parent()
|
||||||
total := ec.Total()
|
total := ec.Total()
|
||||||
|
splitID := ec.ParentSplitID()
|
||||||
|
parSplitParID := ec.ParentSplitParentID()
|
||||||
|
|
||||||
for i := range required {
|
for i := range required {
|
||||||
if parts[i] != nil || !required[i] {
|
if parts[i] != nil || !required[i] {
|
||||||
|
@ -97,7 +99,7 @@ func (c *Constructor) ReconstructParts(parts []*objectSDK.Object, required []boo
|
||||||
copyRequiredFields(part, parts[nonNilPart])
|
copyRequiredFields(part, parts[nonNilPart])
|
||||||
part.SetPayload(c.payloadShards[i])
|
part.SetPayload(c.payloadShards[i])
|
||||||
part.SetPayloadSize(uint64(len(c.payloadShards[i])))
|
part.SetPayloadSize(uint64(len(c.payloadShards[i])))
|
||||||
part.SetECHeader(objectSDK.NewECHeader(parent, uint32(i), total,
|
part.SetECHeader(objectSDK.NewECHeader(parent, splitID, parSplitParID, uint32(i), total,
|
||||||
c.headerShards[i], c.headerLength))
|
c.headerShards[i], c.headerLength))
|
||||||
|
|
||||||
if err := setIDWithSignature(part, key); err != nil {
|
if err := setIDWithSignature(part, key); err != nil {
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"crypto/ecdsa"
|
"crypto/ecdsa"
|
||||||
|
|
||||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Split splits fully formed object into multiple chunks.
|
// Split splits fully formed object into multiple chunks.
|
||||||
|
@ -32,7 +33,15 @@ func (c *Constructor) Split(obj *objectSDK.Object, key *ecdsa.PrivateKey) ([]*ob
|
||||||
chunk.SetPayload(payloadShards[i])
|
chunk.SetPayload(payloadShards[i])
|
||||||
chunk.SetPayloadSize(uint64(len(payloadShards[i])))
|
chunk.SetPayloadSize(uint64(len(payloadShards[i])))
|
||||||
|
|
||||||
ec := objectSDK.NewECHeader(parent, uint32(i), uint32(len(payloadShards)), headerShards[i], uint32(len(header)))
|
var parentSplitParentID *oid.ID
|
||||||
|
if obj.HasParent() {
|
||||||
|
splitParentID, ok := obj.Parent().ID()
|
||||||
|
if ok {
|
||||||
|
parentSplitParentID = &splitParentID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ec := objectSDK.NewECHeader(parent, obj.SplitID(), parentSplitParentID, uint32(i), uint32(len(payloadShards)), headerShards[i], uint32(len(header)))
|
||||||
chunk.SetECHeader(ec)
|
chunk.SetECHeader(ec)
|
||||||
if err := setIDWithSignature(chunk, key); err != nil {
|
if err := setIDWithSignature(chunk, key); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
|
|
||||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode"
|
||||||
|
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -41,5 +42,97 @@ func TestSplitMaxShardCount(t *testing.T) {
|
||||||
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
t.Run("ec parents are children of last Split part", func(t *testing.T) {
|
||||||
|
c, err := erasurecode.NewConstructor(1, erasurecode.MaxShardCount-1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
splitted1 := newObject(t, 1024, pk)
|
||||||
|
splitted2 := newObject(t, 1024, pk)
|
||||||
|
|
||||||
|
splittedId1, _ := splitted1.ID()
|
||||||
|
splittedId2, _ := splitted2.ID()
|
||||||
|
|
||||||
|
splitID := objectSDK.NewSplitID()
|
||||||
|
parent := objectSDK.New()
|
||||||
|
parent.SetID(oidtest.ID())
|
||||||
|
parent.SetChildren(splittedId1, splittedId2)
|
||||||
|
parent.SetSplitID(splitID)
|
||||||
|
|
||||||
|
splitted1.SetSplitID(splitID)
|
||||||
|
splitted1.SetParent(parent)
|
||||||
|
splitted2.SetSplitID(splitID)
|
||||||
|
splitted2.SetParent(parent)
|
||||||
|
|
||||||
|
parts1, err := c.Split(splitted1, &pk.PrivateKey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, parts1, erasurecode.MaxShardCount)
|
||||||
|
|
||||||
|
for _, part := range parts1 {
|
||||||
|
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||||
|
|
||||||
|
require.NotNil(t, part.ECHeader().ParentSplitID())
|
||||||
|
require.Equal(t, *splitID, *part.ECHeader().ParentSplitID())
|
||||||
|
require.NotNil(t, part.ECHeader().ParentSplitParentID())
|
||||||
|
}
|
||||||
|
|
||||||
|
parts2, err := c.Split(splitted2, &pk.PrivateKey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, parts1, erasurecode.MaxShardCount)
|
||||||
|
|
||||||
|
for _, part := range parts2 {
|
||||||
|
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||||
|
|
||||||
|
require.NotNil(t, part.ECHeader().ParentSplitID())
|
||||||
|
require.Equal(t, *splitID, *part.ECHeader().ParentSplitID())
|
||||||
|
require.NotNil(t, part.ECHeader().ParentSplitParentID())
|
||||||
|
}
|
||||||
|
|
||||||
|
})
|
||||||
|
t.Run("ec parents are children of non-last Split part", func(t *testing.T) {
|
||||||
|
c, err := erasurecode.NewConstructor(1, erasurecode.MaxShardCount-1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
splitted1 := newObject(t, 1024, pk)
|
||||||
|
splitted2 := newObject(t, 1024, pk)
|
||||||
|
|
||||||
|
splittedId1, _ := splitted1.ID()
|
||||||
|
splittedId2, _ := splitted2.ID()
|
||||||
|
|
||||||
|
splitID := objectSDK.NewSplitID()
|
||||||
|
parent := objectSDK.New()
|
||||||
|
// Such parent has got no ID.
|
||||||
|
parent.SetChildren(splittedId1, splittedId2)
|
||||||
|
parent.SetSplitID(splitID)
|
||||||
|
|
||||||
|
splitted1.SetSplitID(splitID)
|
||||||
|
splitted1.SetParent(parent)
|
||||||
|
splitted2.SetSplitID(splitID)
|
||||||
|
splitted2.SetParent(parent)
|
||||||
|
|
||||||
|
parts1, err := c.Split(splitted1, &pk.PrivateKey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, parts1, erasurecode.MaxShardCount)
|
||||||
|
|
||||||
|
for _, part := range parts1 {
|
||||||
|
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||||
|
|
||||||
|
require.NotNil(t, part.ECHeader().ParentSplitID())
|
||||||
|
require.Equal(t, *splitID, *part.ECHeader().ParentSplitID())
|
||||||
|
require.Nil(t, part.ECHeader().ParentSplitParentID())
|
||||||
|
}
|
||||||
|
|
||||||
|
parts2, err := c.Split(splitted2, &pk.PrivateKey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, parts1, erasurecode.MaxShardCount)
|
||||||
|
|
||||||
|
for _, part := range parts2 {
|
||||||
|
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||||
|
|
||||||
|
require.NotNil(t, part.ECHeader().ParentSplitID())
|
||||||
|
require.Equal(t, *splitID, *part.ECHeader().ParentSplitID())
|
||||||
|
require.Nil(t, part.ECHeader().ParentSplitParentID())
|
||||||
|
}
|
||||||
|
|
||||||
|
})
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue