forked from TrueCloudLab/frostfs-sdk-go
[#205] object: Initial EC implementation
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
parent
e9be3e6d94
commit
bd2d350b09
12 changed files with 933 additions and 0 deletions
87
object/erasurecode/constructor.go
Normal file
87
object/erasurecode/constructor.go
Normal file
|
@ -0,0 +1,87 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/klauspost/reedsolomon"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrMalformedSlice is returned when a slice of EC chunks is inconsistent.
|
||||
ErrMalformedSlice = errors.New("inconsistent EC headers")
|
||||
// ErrInvShardNum is returned from NewConstructor when the number of shards is invalid.
|
||||
ErrInvShardNum = reedsolomon.ErrInvShardNum
|
||||
// ErrMaxShardNum is returned from NewConstructor when the number of shards is too big.
|
||||
ErrMaxShardNum = reedsolomon.ErrMaxShardNum
|
||||
)
|
||||
|
||||
// MaxShardCount is the maximum number of shards.
|
||||
const MaxShardCount = 256
|
||||
|
||||
// Constructor is a wrapper around encoder allowing to reconstruct objects.
|
||||
// It's methods are not thread-safe.
|
||||
type Constructor struct {
|
||||
enc reedsolomon.Encoder
|
||||
headerLength uint32
|
||||
payloadShards [][]byte
|
||||
headerShards [][]byte
|
||||
}
|
||||
|
||||
// NewConstructor returns new constructor instance.
|
||||
func NewConstructor(dataCount int, parityCount int) (*Constructor, error) {
|
||||
// The library supports up to 65536 shards with some restrictions.
|
||||
// This can easily result in OOM or panic, thus SDK declares it's own restriction.
|
||||
if dataCount+parityCount > MaxShardCount {
|
||||
return nil, ErrMaxShardNum
|
||||
}
|
||||
|
||||
enc, err := reedsolomon.New(dataCount, parityCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Constructor{enc: enc}, nil
|
||||
}
|
||||
|
||||
// clear clears internal state of the constructor, so it can be reused.
|
||||
func (c *Constructor) clear() {
|
||||
c.headerLength = 0
|
||||
c.payloadShards = nil
|
||||
c.headerShards = nil
|
||||
}
|
||||
|
||||
func (c *Constructor) fillHeader(parts []*objectSDK.Object) error {
|
||||
shards := make([][]byte, len(parts))
|
||||
headerLength := 0
|
||||
for i := range parts {
|
||||
if parts[i] == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var err error
|
||||
headerLength, err = validatePart(parts, i, headerLength)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
shards[i] = parts[i].GetECHeader().Header()
|
||||
}
|
||||
|
||||
c.headerLength = uint32(headerLength)
|
||||
c.headerShards = shards
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillPayload fills the payload shards.
|
||||
// Currently there is no case when it can be called without reconstructing header,
|
||||
// thus fillHeader() must be called before and this function performs no validation.
|
||||
func (c *Constructor) fillPayload(parts []*objectSDK.Object) {
|
||||
shards := make([][]byte, len(parts))
|
||||
for i := range parts {
|
||||
if parts[i] == nil {
|
||||
continue
|
||||
}
|
||||
shards[i] = parts[i].Payload()
|
||||
}
|
||||
c.payloadShards = shards
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue