[#223] object: Introduce new fields for `ECHeader`
* Add `parentSplitID`, `parentSplitIndex` and setters/getters for them; * Fix `NewECHeader` constructor; * Add unit-test for Split with ec-parents case. Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
parent
4367817039
commit
facf1f2a1e
4
go.mod
4
go.mod
|
@ -2,8 +2,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-sdk-go
|
|||
|
||||
go 1.20
|
||||
|
||||
replace git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240513140355-4fe42ac4add1 => git.frostfs.info/aarifullin/frostfs-api-go/v2 v2.15.1-0.20240516090342-053675de7228
|
||||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240506114654-b171364079c3
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240513140355-4fe42ac4add1
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1
|
||||
|
|
4
go.sum
4
go.sum
|
@ -31,8 +31,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
|
|||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240506114654-b171364079c3 h1:Fhkq+F6AiYhkJlqq0IXnA5eUuTPbhRqAwlcHTPzUF2g=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240506114654-b171364079c3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
|
||||
|
@ -43,6 +41,8 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9m
|
|||
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc=
|
||||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA=
|
||||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8=
|
||||
git.frostfs.info/aarifullin/frostfs-api-go/v2 v2.15.1-0.20240516090342-053675de7228 h1:xgzdq+obsk9Rjs2bSdv5lT7FT78gzScuh/RIFdfCTnM=
|
||||
git.frostfs.info/aarifullin/frostfs-api-go/v2 v2.15.1-0.20240516090342-053675de7228/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0=
|
||||
|
|
|
@ -10,21 +10,25 @@ import (
|
|||
|
||||
// ECHeader represents erasure coding header.
|
||||
type ECHeader struct {
|
||||
parent oid.ID
|
||||
index uint32
|
||||
total uint32
|
||||
header []byte
|
||||
headerLength uint32
|
||||
parent oid.ID
|
||||
parentSplitID *SplitID
|
||||
parentSplitParentID *oid.ID
|
||||
index uint32
|
||||
total uint32
|
||||
header []byte
|
||||
headerLength uint32
|
||||
}
|
||||
|
||||
// NewECHeader constructs new erasure coding header.
|
||||
func NewECHeader(parent oid.ID, index, total uint32, header []byte, headerLength uint32) *ECHeader {
|
||||
func NewECHeader(parent oid.ID, parentSplitID *SplitID, parentSplitParentID *oid.ID, index, total uint32, header []byte, headerLength uint32) *ECHeader {
|
||||
return &ECHeader{
|
||||
parent: parent,
|
||||
index: index,
|
||||
total: total,
|
||||
header: header,
|
||||
headerLength: headerLength,
|
||||
parent: parent,
|
||||
parentSplitID: parentSplitID,
|
||||
parentSplitParentID: parentSplitParentID,
|
||||
index: index,
|
||||
total: total,
|
||||
header: header,
|
||||
headerLength: headerLength,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -32,6 +36,14 @@ func NewECHeader(parent oid.ID, index, total uint32, header []byte, headerLength
|
|||
func (e *ECHeader) WriteToV2(h *object.ECHeader) {
|
||||
var parent refs.ObjectID
|
||||
e.parent.WriteToV2(&parent)
|
||||
h.ParentSplitID = e.parentSplitID.ToV2()
|
||||
|
||||
if e.parentSplitParentID != nil {
|
||||
parentSplitParentID := new(refs.ObjectID)
|
||||
e.parentSplitParentID.WriteToV2(parentSplitParentID)
|
||||
h.ParentSplitParentID = parentSplitParentID
|
||||
}
|
||||
|
||||
h.Parent = &parent
|
||||
h.Index = e.index
|
||||
h.Total = e.total
|
||||
|
@ -49,10 +61,17 @@ func (e *ECHeader) ReadFromV2(h *object.ECHeader) error {
|
|||
}
|
||||
|
||||
_ = e.parent.ReadFromV2(*h.Parent)
|
||||
e.index = h.Index
|
||||
e.total = h.Total
|
||||
e.header = h.Header
|
||||
e.headerLength = h.HeaderLength
|
||||
e.SetParentSplitID(NewSplitIDFromV2(h.ParentSplitID))
|
||||
if h.ParentSplitParentID != nil {
|
||||
if e.parentSplitParentID == nil {
|
||||
e.parentSplitParentID = new(oid.ID)
|
||||
}
|
||||
_ = e.parentSplitParentID.ReadFromV2(*h.ParentSplitParentID)
|
||||
}
|
||||
e.SetIndex(h.Index)
|
||||
e.SetTotal(h.Total)
|
||||
e.SetHeader(h.Header)
|
||||
e.SetHeaderLength(h.HeaderLength)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -88,6 +107,22 @@ func (e *ECHeader) SetParent(id oid.ID) {
|
|||
e.parent = id
|
||||
}
|
||||
|
||||
func (e *ECHeader) ParentSplitID() *SplitID {
|
||||
return e.parentSplitID
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetParentSplitID(parentSplitID *SplitID) {
|
||||
e.parentSplitID = parentSplitID
|
||||
}
|
||||
|
||||
func (e *ECHeader) ParentSplitParentID() *oid.ID {
|
||||
return e.parentSplitParentID
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetParentSplitParentID(parentSplitParentID *oid.ID) {
|
||||
e.parentSplitParentID = parentSplitParentID
|
||||
}
|
||||
|
||||
func (e *ECHeader) Index() uint32 {
|
||||
return e.index
|
||||
}
|
||||
|
|
|
@ -87,6 +87,8 @@ func (c *Constructor) ReconstructParts(parts []*objectSDK.Object, required []boo
|
|||
ec := parts[nonNilPart].GetECHeader()
|
||||
parent := ec.Parent()
|
||||
total := ec.Total()
|
||||
splitID := ec.ParentSplitID()
|
||||
parSplitParID := ec.ParentSplitParentID()
|
||||
|
||||
for i := range required {
|
||||
if parts[i] != nil || !required[i] {
|
||||
|
@ -97,7 +99,7 @@ func (c *Constructor) ReconstructParts(parts []*objectSDK.Object, required []boo
|
|||
copyRequiredFields(part, parts[nonNilPart])
|
||||
part.SetPayload(c.payloadShards[i])
|
||||
part.SetPayloadSize(uint64(len(c.payloadShards[i])))
|
||||
part.SetECHeader(objectSDK.NewECHeader(parent, uint32(i), total,
|
||||
part.SetECHeader(objectSDK.NewECHeader(parent, splitID, parSplitParID, uint32(i), total,
|
||||
c.headerShards[i], c.headerLength))
|
||||
|
||||
if err := setIDWithSignature(part, key); err != nil {
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
"crypto/ecdsa"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
// Split splits fully formed object into multiple chunks.
|
||||
|
@ -32,7 +33,13 @@ func (c *Constructor) Split(obj *objectSDK.Object, key *ecdsa.PrivateKey) ([]*ob
|
|||
chunk.SetPayload(payloadShards[i])
|
||||
chunk.SetPayloadSize(uint64(len(payloadShards[i])))
|
||||
|
||||
ec := objectSDK.NewECHeader(parent, uint32(i), uint32(len(payloadShards)), headerShards[i], uint32(len(header)))
|
||||
var parentSplitParentID *oid.ID
|
||||
if obj.HasParent() {
|
||||
splitParentID, _ := obj.Parent().ID()
|
||||
parentSplitParentID = &splitParentID
|
||||
}
|
||||
|
||||
ec := objectSDK.NewECHeader(parent, obj.SplitID(), parentSplitParentID, uint32(i), uint32(len(payloadShards)), headerShards[i], uint32(len(header)))
|
||||
chunk.SetECHeader(ec)
|
||||
if err := setIDWithSignature(chunk, key); err != nil {
|
||||
return nil, err
|
||||
|
|
|
@ -42,4 +42,50 @@ func TestSplitMaxShardCount(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("ec parents are split parts", func(t *testing.T) {
|
||||
c, err := erasurecode.NewConstructor(1, erasurecode.MaxShardCount-1)
|
||||
require.NoError(t, err)
|
||||
|
||||
splitted1 := newObject(t, 1024, pk)
|
||||
splitted2 := newObject(t, 1024, pk)
|
||||
|
||||
splittedId1, _ := splitted1.ID()
|
||||
splittedId2, _ := splitted2.ID()
|
||||
|
||||
splitID := objectSDK.NewSplitID()
|
||||
parent := objectSDK.New()
|
||||
parent.SetChildren(splittedId1, splittedId2)
|
||||
parent.SetSplitID(splitID)
|
||||
|
||||
splitted1.SetSplitID(splitID)
|
||||
splitted1.SetParent(parent)
|
||||
splitted2.SetSplitID(splitID)
|
||||
splitted2.SetParent(parent)
|
||||
|
||||
parts1, err := c.Split(splitted1, &pk.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parts1, erasurecode.MaxShardCount)
|
||||
|
||||
for _, part := range parts1 {
|
||||
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||
|
||||
require.NotNil(t, part.ECHeader().ParentSplitID())
|
||||
require.Equal(t, *splitID, *part.ECHeader().ParentSplitID())
|
||||
require.NotNil(t, part.ECHeader().ParentSplitParentID())
|
||||
}
|
||||
|
||||
parts2, err := c.Split(splitted2, &pk.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parts1, erasurecode.MaxShardCount)
|
||||
|
||||
for _, part := range parts2 {
|
||||
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||
|
||||
require.NotNil(t, part.ECHeader().ParentSplitID())
|
||||
require.Equal(t, *splitID, *part.ECHeader().ParentSplitID())
|
||||
require.NotNil(t, part.ECHeader().ParentSplitParentID())
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue