object: Initial EC implementation #69
30 changed files with 250 additions and 3 deletions
BIN
accounting/grpc/service.pb.go
generated
BIN
accounting/grpc/service.pb.go
generated
Binary file not shown.
BIN
accounting/grpc/service_grpc.pb.go
generated
BIN
accounting/grpc/service_grpc.pb.go
generated
Binary file not shown.
BIN
accounting/grpc/types.pb.go
generated
BIN
accounting/grpc/types.pb.go
generated
Binary file not shown.
BIN
acl/grpc/types.pb.go
generated
BIN
acl/grpc/types.pb.go
generated
Binary file not shown.
BIN
audit/grpc/types.pb.go
generated
BIN
audit/grpc/types.pb.go
generated
Binary file not shown.
BIN
container/grpc/service.pb.go
generated
BIN
container/grpc/service.pb.go
generated
Binary file not shown.
BIN
container/grpc/service_grpc.pb.go
generated
BIN
container/grpc/service_grpc.pb.go
generated
Binary file not shown.
BIN
container/grpc/types.pb.go
generated
BIN
container/grpc/types.pb.go
generated
Binary file not shown.
BIN
lock/grpc/types.pb.go
generated
BIN
lock/grpc/types.pb.go
generated
Binary file not shown.
|
@ -142,6 +142,8 @@ func (r *Replica) ToGRPCMessage() grpc.Message {
|
|||
|
||||
m.SetSelector(r.selector)
|
||||
m.SetCount(r.count)
|
||||
m.EcDataCount = r.ecDataCount
|
||||
m.EcParityCount = r.ecParityCount
|
||||
}
|
||||
|
||||
return m
|
||||
|
@ -155,6 +157,8 @@ func (r *Replica) FromGRPCMessage(m grpc.Message) error {
|
|||
|
||||
r.selector = v.GetSelector()
|
||||
r.count = v.GetCount()
|
||||
r.ecDataCount = v.GetEcDataCount()
|
||||
r.ecParityCount = v.GetEcParityCount()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
BIN
netmap/grpc/service.pb.go
generated
BIN
netmap/grpc/service.pb.go
generated
Binary file not shown.
BIN
netmap/grpc/service_grpc.pb.go
generated
BIN
netmap/grpc/service_grpc.pb.go
generated
Binary file not shown.
BIN
netmap/grpc/types.pb.go
generated
BIN
netmap/grpc/types.pb.go
generated
Binary file not shown.
|
@ -21,6 +21,8 @@ const (
|
|||
|
||||
countReplicaField = 1
|
||||
selectorReplicaField = 2
|
||||
ecDataCountReplicaField = 3
|
||||
ecParityCountReplicaField = 4
|
||||
|
||||
replicasPolicyField = 1
|
||||
backupPolicyField = 2
|
||||
|
@ -134,7 +136,9 @@ func (r *Replica) StableMarshal(buf []byte) []byte {
|
|||
var offset int
|
||||
|
||||
offset += protoutil.UInt32Marshal(countReplicaField, buf[offset:], r.count)
|
||||
protoutil.StringMarshal(selectorReplicaField, buf[offset:], r.selector)
|
||||
offset += protoutil.StringMarshal(selectorReplicaField, buf[offset:], r.selector)
|
||||
offset += protoutil.UInt32Marshal(ecDataCountReplicaField, buf[offset:], r.ecDataCount)
|
||||
protoutil.UInt32Marshal(ecParityCountReplicaField, buf[offset:], r.ecParityCount)
|
||||
|
||||
return buf
|
||||
}
|
||||
|
@ -146,6 +150,8 @@ func (r *Replica) StableSize() (size int) {
|
|||
|
||||
size += protoutil.UInt32Size(countReplicaField, r.count)
|
||||
size += protoutil.StringSize(selectorReplicaField, r.selector)
|
||||
size += protoutil.UInt32Size(ecDataCountReplicaField, r.ecDataCount)
|
||||
size += protoutil.UInt32Size(ecParityCountReplicaField, r.ecParityCount)
|
||||
|
||||
return size
|
||||
}
|
||||
|
|
|
@ -50,6 +50,9 @@ type Selector struct {
|
|||
type Replica struct {
|
||||
count uint32
|
||||
selector string
|
||||
|
||||
ecDataCount uint32
|
||||
ecParityCount uint32
|
||||
}
|
||||
|
||||
type Operation uint32
|
||||
|
@ -259,6 +262,30 @@ func (r *Replica) SetCount(count uint32) {
|
|||
r.count = count
|
||||
}
|
||||
|
||||
func (r *Replica) GetECDataCount() uint32 {
|
||||
if r != nil {
|
||||
return r.ecDataCount
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func (r *Replica) SetECDataCount(count uint32) {
|
||||
r.ecDataCount = count
|
||||
}
|
||||
|
||||
func (r *Replica) GetECParityCount() uint32 {
|
||||
if r != nil {
|
||||
return r.ecParityCount
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func (r *Replica) SetECParityCount(count uint32) {
|
||||
r.ecParityCount = count
|
||||
}
|
||||
|
||||
func (p *PlacementPolicy) GetUnique() bool {
|
||||
if p != nil {
|
||||
return p.unique
|
||||
|
|
|
@ -262,6 +262,51 @@ func (h *SplitHeader) FromGRPCMessage(m grpc.Message) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (h *ECHeader) ToGRPCMessage() grpc.Message {
|
||||
var m *object.Header_EC
|
||||
|
||||
if h != nil {
|
||||
m = new(object.Header_EC)
|
||||
|
||||
m.Parent = h.Parent.ToGRPCMessage().(*refsGRPC.ObjectID)
|
||||
m.Index = h.Index
|
||||
m.Total = h.Total
|
||||
m.Header = h.Header
|
||||
m.HeaderLength = h.HeaderLength
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
func (h *ECHeader) FromGRPCMessage(m grpc.Message) error {
|
||||
v, ok := m.(*object.Header_EC)
|
||||
if !ok {
|
||||
return message.NewUnexpectedMessageType(m, v)
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
par := v.GetParent()
|
||||
if par == nil {
|
||||
h.Parent = nil
|
||||
} else {
|
||||
if h.Parent == nil {
|
||||
h.Parent = new(refs.ObjectID)
|
||||
}
|
||||
|
||||
err = h.Parent.FromGRPCMessage(par)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
h.Index = v.GetIndex()
|
||||
h.Total = v.GetTotal()
|
||||
h.Header = v.GetHeader()
|
||||
h.HeaderLength = v.GetHeaderLength()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Header) ToGRPCMessage() grpc.Message {
|
||||
var m *object.Header
|
||||
|
||||
|
@ -275,6 +320,7 @@ func (h *Header) ToGRPCMessage() grpc.Message {
|
|||
m.SetContainerId(h.cid.ToGRPCMessage().(*refsGRPC.ContainerID))
|
||||
m.SetSessionToken(h.sessionToken.ToGRPCMessage().(*sessionGRPC.SessionToken))
|
||||
m.SetSplit(h.split.ToGRPCMessage().(*object.Header_Split))
|
||||
m.Ec = h.ec.ToGRPCMessage().(*object.Header_EC)
|
||||
m.SetAttributes(AttributesToGRPC(h.attr))
|
||||
m.SetPayloadLength(h.payloadLen)
|
||||
m.SetCreationEpoch(h.creatEpoch)
|
||||
|
@ -313,6 +359,9 @@ func (h *Header) FromGRPCMessage(m grpc.Message) error {
|
|||
if err := h.fillSplitHeader(v); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.fillECHeader(v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
h.attr, err = AttributesFromGRPC(v.GetAttributes())
|
||||
if err != nil {
|
||||
|
@ -417,6 +466,19 @@ func (h *Header) fillSplitHeader(v *object.Header) error {
|
|||
return h.split.FromGRPCMessage(split)
|
||||
}
|
||||
|
||||
func (h *Header) fillECHeader(v *object.Header) error {
|
||||
ec := v.GetEc()
|
||||
if ec == nil {
|
||||
h.ec = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.ec == nil {
|
||||
h.ec = new(ECHeader)
|
||||
}
|
||||
return h.ec.FromGRPCMessage(ec)
|
||||
}
|
||||
|
||||
func (h *HeaderWithSignature) ToGRPCMessage() grpc.Message {
|
||||
var m *object.HeaderWithSignature
|
||||
|
||||
|
@ -591,6 +653,50 @@ func (s *SplitInfo) FromGRPCMessage(m grpc.Message) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *ECInfo) ToGRPCMessage() grpc.Message {
|
||||
var m *object.ECInfo
|
||||
|
||||
if s != nil {
|
||||
m = new(object.ECInfo)
|
||||
|
||||
if s.Chunks != nil {
|
||||
chunks := make([]*object.ECInfo_Chunk, len(s.Chunks))
|
||||
for i := range chunks {
|
||||
chunks[i] = &object.ECInfo_Chunk{
|
||||
Id: s.Chunks[i].ID.ToGRPCMessage().(*refsGRPC.ObjectID),
|
||||
Index: s.Chunks[i].Index,
|
||||
Total: s.Chunks[i].Total,
|
||||
}
|
||||
}
|
||||
m.Chunks = chunks
|
||||
}
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
func (s *ECInfo) FromGRPCMessage(m grpc.Message) error {
|
||||
v, ok := m.(*object.ECInfo)
|
||||
if !ok {
|
||||
return message.NewUnexpectedMessageType(m, v)
|
||||
}
|
||||
|
||||
chunks := v.GetChunks()
|
||||
if chunks == nil {
|
||||
s.Chunks = nil
|
||||
} else {
|
||||
s.Chunks = make([]ECChunk, len(chunks))
|
||||
for i := range chunks {
|
||||
if err := s.Chunks[i].ID.FromGRPCMessage(chunks[i].Id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.Chunks[i].Index = chunks[i].Index
|
||||
s.Chunks[i].Total = chunks[i].Total
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *GetRequestBody) ToGRPCMessage() grpc.Message {
|
||||
var m *object.GetRequest_Body
|
||||
|
||||
|
|
BIN
object/grpc/service.pb.go
generated
BIN
object/grpc/service.pb.go
generated
Binary file not shown.
BIN
object/grpc/service_grpc.pb.go
generated
BIN
object/grpc/service_grpc.pb.go
generated
Binary file not shown.
BIN
object/grpc/types.pb.go
generated
BIN
object/grpc/types.pb.go
generated
Binary file not shown.
|
@ -26,6 +26,12 @@ const (
|
|||
splitHdrChildrenField = 5
|
||||
splitHdrSplitIDField = 6
|
||||
|
||||
ecHdrParentField = 1
|
||||
ecHdrIndexField = 2
|
||||
ecHdrTotalField = 3
|
||||
ecHdrHeaderLengthField = 4
|
||||
ecHdrHeaderField = 5
|
||||
|
||||
hdrVersionField = 1
|
||||
hdrContainerIDField = 2
|
||||
hdrOwnerIDField = 3
|
||||
|
@ -37,6 +43,7 @@ const (
|
|||
hdrSessionTokenField = 9
|
||||
hdrAttributesField = 10
|
||||
hdrSplitField = 11
|
||||
hdrECField = 12
|
||||
|
||||
hdrWithSigHeaderField = 1
|
||||
hdrWithSigSignatureField = 2
|
||||
|
@ -229,6 +236,43 @@ func (h *SplitHeader) Unmarshal(data []byte) error {
|
|||
return message.Unmarshal(h, data, new(object.Header_Split))
|
||||
}
|
||||
|
||||
func (h *ECHeader) StableMarshal(buf []byte) []byte {
|
||||
if h == nil {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
if buf == nil {
|
||||
buf = make([]byte, h.StableSize())
|
||||
}
|
||||
|
||||
var offset int
|
||||
|
||||
offset += proto.NestedStructureMarshal(ecHdrParentField, buf[offset:], h.Parent)
|
||||
offset += proto.UInt32Marshal(ecHdrIndexField, buf[offset:], h.Index)
|
||||
offset += proto.UInt32Marshal(ecHdrTotalField, buf[offset:], h.Total)
|
||||
offset += proto.UInt32Marshal(ecHdrHeaderLengthField, buf[offset:], h.HeaderLength)
|
||||
proto.BytesMarshal(ecHdrHeaderField, buf[offset:], h.Header)
|
||||
return buf
|
||||
}
|
||||
|
||||
func (h *ECHeader) StableSize() (size int) {
|
||||
if h == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
size += proto.NestedStructureSize(ecHdrParentField, h.Parent)
|
||||
size += proto.UInt32Size(ecHdrIndexField, h.Index)
|
||||
size += proto.UInt32Size(ecHdrTotalField, h.Total)
|
||||
size += proto.UInt32Size(ecHdrHeaderLengthField, h.HeaderLength)
|
||||
size += proto.BytesSize(ecHdrHeaderField, h.Header)
|
||||
|
||||
return size
|
||||
}
|
||||
|
||||
func (h *ECHeader) Unmarshal(data []byte) error {
|
||||
return message.Unmarshal(h, data, new(object.Header_EC))
|
||||
}
|
||||
|
||||
func (h *Header) StableMarshal(buf []byte) []byte {
|
||||
if h == nil {
|
||||
return []byte{}
|
||||
|
@ -255,6 +299,7 @@ func (h *Header) StableMarshal(buf []byte) []byte {
|
|||
}
|
||||
|
||||
proto.NestedStructureMarshal(hdrSplitField, buf[offset:], h.split)
|
||||
proto.NestedStructureMarshal(hdrECField, buf[offset:], h.ec)
|
||||
|
||||
return buf
|
||||
}
|
||||
|
@ -277,6 +322,7 @@ func (h *Header) StableSize() (size int) {
|
|||
size += proto.NestedStructureSize(hdrAttributesField, &h.attr[i])
|
||||
}
|
||||
size += proto.NestedStructureSize(hdrSplitField, h.split)
|
||||
size += proto.NestedStructureSize(hdrECField, h.ec)
|
||||
|
||||
return size
|
||||
}
|
||||
|
|
|
@ -16,6 +16,7 @@ func TestMessageConvert(t *testing.T) {
|
|||
func(empty bool) message.Message { return objecttest.GenerateHeader(empty) },
|
||||
func(empty bool) message.Message { return objecttest.GenerateObject(empty) },
|
||||
func(empty bool) message.Message { return objecttest.GenerateSplitInfo(empty) },
|
||||
func(empty bool) message.Message { return objecttest.GenerateECInfo(empty) },
|
||||
func(empty bool) message.Message { return objecttest.GenerateGetRequestBody(empty) },
|
||||
func(empty bool) message.Message { return objecttest.GenerateGetRequest(empty) },
|
||||
func(empty bool) message.Message { return objecttest.GenerateGetObjectPartInit(empty) },
|
||||
|
|
|
@ -137,6 +137,31 @@ func GenerateSplitInfo(empty bool) *object.SplitInfo {
|
|||
return m
|
||||
}
|
||||
|
||||
func GenerateECInfo(empty bool) *object.ECInfo {
|
||||
m := new(object.ECInfo)
|
||||
|
||||
if !empty {
|
||||
m.Chunks = make([]object.ECChunk, 2)
|
||||
for i := range m.Chunks {
|
||||
m.Chunks[i] = *GenerateECChunk(false)
|
||||
}
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
func GenerateECChunk(empty bool) *object.ECChunk {
|
||||
m := new(object.ECChunk)
|
||||
|
||||
if !empty {
|
||||
m.ID = *refstest.GenerateObjectID(false)
|
||||
m.Index = 4
|
||||
m.Total = 7
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
func GenerateGetRequestBody(empty bool) *object.GetRequestBody {
|
||||
m := new(object.GetRequestBody)
|
||||
|
||||
|
|
|
@ -39,6 +39,15 @@ type SplitHeader struct {
|
|||
splitID []byte
|
||||
}
|
||||
|
||||
type ECHeader struct {
|
||||
Parent *refs.ObjectID
|
||||
Index uint32
|
||||
Total uint32
|
||||
Header []byte
|
||||
HeaderLength uint32
|
||||
Signature []byte
|
||||
}
|
||||
|
||||
type Header struct {
|
||||
version *refs.Version
|
||||
|
||||
|
@ -59,6 +68,8 @@ type Header struct {
|
|||
attr []Attribute
|
||||
|
||||
split *SplitHeader
|
||||
|
||||
ec *ECHeader
|
||||
}
|
||||
|
||||
type HeaderWithSignature struct {
|
||||
|
@ -88,6 +99,16 @@ type SplitInfo struct {
|
|||
link *refs.ObjectID
|
||||
}
|
||||
|
||||
type ECChunk struct {
|
||||
ID refs.ObjectID
|
||||
Index uint32
|
||||
Total uint32
|
||||
}
|
||||
|
||||
type ECInfo struct {
|
||||
Chunks []ECChunk
|
||||
}
|
||||
|
||||
type GetRequestBody struct {
|
||||
addr *refs.Address
|
||||
|
||||
|
@ -655,6 +676,17 @@ func (h *Header) SetSplit(v *SplitHeader) {
|
|||
h.split = v
|
||||
}
|
||||
|
||||
func (h *Header) GetEC() *ECHeader {
|
||||
if h != nil {
|
||||
return h.ec
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Header) SetEC(v *ECHeader) {
|
||||
h.ec = v
|
||||
}
|
||||
|
||||
func (h *HeaderWithSignature) GetHeader() *Header {
|
||||
if h != nil {
|
||||
return h.header
|
||||
|
|
BIN
refs/grpc/types.pb.go
generated
BIN
refs/grpc/types.pb.go
generated
Binary file not shown.
BIN
session/grpc/service.pb.go
generated
BIN
session/grpc/service.pb.go
generated
Binary file not shown.
BIN
session/grpc/service_grpc.pb.go
generated
BIN
session/grpc/service_grpc.pb.go
generated
Binary file not shown.
BIN
session/grpc/types.pb.go
generated
BIN
session/grpc/types.pb.go
generated
Binary file not shown.
BIN
status/grpc/types.pb.go
generated
BIN
status/grpc/types.pb.go
generated
Binary file not shown.
BIN
tombstone/grpc/types.pb.go
generated
BIN
tombstone/grpc/types.pb.go
generated
Binary file not shown.
BIN
util/proto/test/test.pb.go
generated
BIN
util/proto/test/test.pb.go
generated
Binary file not shown.
Loading…
Reference in a new issue