From 93d2d129846ab732b5318b63f1cd1f1a755c8889 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 18 Aug 2020 18:28:02 +0300 Subject: [PATCH] Add stable marshal for object structure Signed-off-by: Alex Vanin --- v2/object/marshal.go | 450 ++++++++++++++++++++++++++++++++++++++ v2/object/marshal_test.go | 233 ++++++++++++++++++++ 2 files changed, 683 insertions(+) create mode 100644 v2/object/marshal.go create mode 100644 v2/object/marshal_test.go diff --git a/v2/object/marshal.go b/v2/object/marshal.go new file mode 100644 index 00000000..76938466 --- /dev/null +++ b/v2/object/marshal.go @@ -0,0 +1,450 @@ +package object + +import ( + "github.com/nspcc-dev/neofs-api-go/util/proto" +) + +const ( + shortHdrVersionField = 1 + shortHdrEpochField = 2 + shortHdrOwnerField = 3 + shortHdrObjectTypeField = 4 + shortHdrPayloadLength = 5 + + attributeKeyField = 1 + attributeValueField = 2 + + splitHdrParentField = 1 + splitHdrPreviousField = 2 + splitHdrParentSignatureField = 3 + splitHdrParentHeaderField = 4 + splitHdrChildrenField = 5 + + hdrVersionField = 1 + hdrContainerIDField = 2 + hdrOwnerIDField = 3 + hdrEpochField = 4 + hdrPayloadLengthField = 5 + hdrPayloadHashField = 6 + hdrObjectTypeField = 7 + hdrHomomorphicHashField = 8 + hdrSessionTokenField = 9 + hdrAttributesField = 10 + hdrSplitField = 11 + + objIDField = 1 + objSignatureField = 2 + objHeaderField = 3 + objPayloadField = 4 +) + +func (h *ShortHeader) StableMarshal(buf []byte) ([]byte, error) { + if h == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, h.StableSize()) + } + + var ( + offset, n int + err error + ) + + n, err = proto.NestedStructureMarshal(shortHdrVersionField, buf[offset:], h.version) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.UInt64Marshal(shortHdrEpochField, buf[offset:], h.creatEpoch) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(shortHdrOwnerField, buf[offset:], h.ownerID) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.EnumMarshal(shortHdrObjectTypeField, buf[offset:], int32(h.typ)) + if err != nil { + return nil, err + } + + offset += n + + _, err = proto.UInt64Marshal(shortHdrPayloadLength, buf[offset:], h.payloadLen) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (h *ShortHeader) StableSize() (size int) { + if h == nil { + return 0 + } + + size += proto.NestedStructureSize(shortHdrVersionField, h.version) + size += proto.UInt64Size(shortHdrEpochField, h.creatEpoch) + size += proto.NestedStructureSize(shortHdrOwnerField, h.ownerID) + size += proto.EnumSize(shortHdrObjectTypeField, int32(h.typ)) + size += proto.UInt64Size(shortHdrPayloadLength, h.payloadLen) + + return size +} + +func (a *Attribute) StableMarshal(buf []byte) ([]byte, error) { + if a == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, a.StableSize()) + } + + var ( + offset, n int + err error + ) + + n, err = proto.StringMarshal(attributeKeyField, buf[offset:], a.key) + if err != nil { + return nil, err + } + + offset += n + + _, err = proto.StringMarshal(attributeValueField, buf[offset:], a.val) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (a *Attribute) StableSize() (size int) { + if a == nil { + return 0 + } + + size += proto.StringSize(shortHdrVersionField, a.key) + size += proto.StringSize(shortHdrEpochField, a.val) + + return size +} + +func (h *SplitHeader) StableMarshal(buf []byte) ([]byte, error) { + if h == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, h.StableSize()) + } + + var ( + offset, n int + err error + ) + + n, err = proto.NestedStructureMarshal(splitHdrParentField, buf[offset:], h.par) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(splitHdrPreviousField, buf[offset:], h.prev) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(splitHdrParentSignatureField, buf[offset:], h.parSig) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(splitHdrParentHeaderField, buf[offset:], h.parHdr) + if err != nil { + return nil, err + } + + offset += n + + for i := range h.children { + n, err = proto.NestedStructureMarshal(splitHdrChildrenField, buf[offset:], h.children[i]) + if err != nil { + return nil, err + } + + offset += n + } + + return buf, nil +} + +func (h *SplitHeader) StableSize() (size int) { + if h == nil { + return 0 + } + + size += proto.NestedStructureSize(splitHdrParentField, h.par) + size += proto.NestedStructureSize(splitHdrPreviousField, h.prev) + size += proto.NestedStructureSize(splitHdrParentSignatureField, h.parSig) + size += proto.NestedStructureSize(splitHdrParentHeaderField, h.parHdr) + + for i := range h.children { + size += proto.NestedStructureSize(splitHdrChildrenField, h.children[i]) + } + + return size +} + +func (h *Header) StableMarshal(buf []byte) ([]byte, error) { + if h == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, h.StableSize()) + } + + var ( + offset, n int + err error + ) + + n, err = proto.NestedStructureMarshal(hdrVersionField, buf[offset:], h.version) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(hdrContainerIDField, buf[offset:], h.cid) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(hdrOwnerIDField, buf[offset:], h.ownerID) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.UInt64Marshal(hdrEpochField, buf[offset:], h.creatEpoch) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.UInt64Marshal(hdrPayloadLengthField, buf[offset:], h.payloadLen) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.BytesMarshal(hdrPayloadHashField, buf[offset:], h.payloadHash) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.EnumMarshal(hdrObjectTypeField, buf[offset:], int32(h.typ)) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.BytesMarshal(hdrHomomorphicHashField, buf[offset:], h.homoHash) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(hdrSessionTokenField, buf[offset:], h.sessionToken) + if err != nil { + return nil, err + } + + offset += n + + for i := range h.attr { + n, err = proto.NestedStructureMarshal(hdrAttributesField, buf[offset:], h.attr[i]) + if err != nil { + return nil, err + } + + offset += n + } + + _, err = proto.NestedStructureMarshal(hdrSplitField, buf[offset:], h.split) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (h *Header) StableSize() (size int) { + if h == nil { + return 0 + } + + size += proto.NestedStructureSize(hdrVersionField, h.version) + size += proto.NestedStructureSize(hdrContainerIDField, h.cid) + size += proto.NestedStructureSize(hdrOwnerIDField, h.ownerID) + size += proto.UInt64Size(hdrEpochField, h.creatEpoch) + size += proto.UInt64Size(hdrPayloadLengthField, h.payloadLen) + size += proto.BytesSize(hdrPayloadHashField, h.payloadHash) + size += proto.EnumSize(hdrObjectTypeField, int32(h.typ)) + size += proto.BytesSize(hdrHomomorphicHashField, h.homoHash) + size += proto.NestedStructureSize(hdrSessionTokenField, h.sessionToken) + for i := range h.attr { + size += proto.NestedStructureSize(hdrAttributesField, h.attr[i]) + } + size += proto.NestedStructureSize(hdrSplitField, h.split) + + return size +} + +func (o *Object) StableMarshal(buf []byte) ([]byte, error) { + if o == nil { + return []byte{}, nil + } + + if buf == nil { + buf = make([]byte, o.StableSize()) + } + + var ( + offset, n int + err error + ) + + n, err = proto.NestedStructureMarshal(objIDField, buf[offset:], o.objectID) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(objSignatureField, buf[offset:], o.idSig) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.NestedStructureMarshal(objHeaderField, buf[offset:], o.header) + if err != nil { + return nil, err + } + + offset += n + + _, err = proto.BytesMarshal(objPayloadField, buf[offset:], o.payload) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (o *Object) StableSize() (size int) { + if o == nil { + return 0 + } + + size += proto.NestedStructureSize(objIDField, o.objectID) + size += proto.NestedStructureSize(objSignatureField, o.idSig) + size += proto.NestedStructureSize(objHeaderField, o.header) + size += proto.BytesSize(objPayloadField, o.payload) + + return size +} + +func (r *PutRequestBody) StableMarshal(buf []byte) ([]byte, error) { + panic("not implemented") +} + +func (r *PutRequestBody) StableSize() (size int) { + panic("not implemented") +} + +func (r *GetResponseBody) StableMarshal(buf []byte) ([]byte, error) { + panic("not implemented") +} + +func (r *GetResponseBody) StableSize() (size int) { + panic("not implemented") +} + +func (r *PutResponseBody) StableMarshal(buf []byte) ([]byte, error) { + panic("not implemented") +} + +func (r *PutResponseBody) StableSize() (size int) { + panic("not implemented") +} + +func (r *DeleteResponseBody) StableMarshal(buf []byte) ([]byte, error) { + panic("not implemented") +} + +func (r *DeleteResponseBody) StableSize() (size int) { + panic("not implemented") +} + +func (r *HeadResponseBody) StableMarshal(buf []byte) ([]byte, error) { + panic("not implemented") +} + +func (r *HeadResponseBody) StableSize() (size int) { + panic("not implemented") +} + +func (r *SearchResponseBody) StableMarshal(buf []byte) ([]byte, error) { + panic("not implemented") +} + +func (r *SearchResponseBody) StableSize() (size int) { + panic("not implemented") +} + +func (r *GetRangeResponseBody) StableMarshal(buf []byte) ([]byte, error) { + panic("not implemented") +} + +func (r *GetRangeResponseBody) StableSize() (size int) { + panic("not implemented") +} + +func (r *GetRangeHashResponseBody) StableMarshal(buf []byte) ([]byte, error) { + panic("not implemented") +} + +func (r *GetRangeHashResponseBody) StableSize() (size int) { + panic("not implemented") +} diff --git a/v2/object/marshal_test.go b/v2/object/marshal_test.go new file mode 100644 index 00000000..a2a37acf --- /dev/null +++ b/v2/object/marshal_test.go @@ -0,0 +1,233 @@ +package object_test + +import ( + "testing" + + "github.com/nspcc-dev/neofs-api-go/v2/object" + grpc "github.com/nspcc-dev/neofs-api-go/v2/object/grpc" + "github.com/nspcc-dev/neofs-api-go/v2/refs" + "github.com/nspcc-dev/neofs-api-go/v2/service" + "github.com/stretchr/testify/require" +) + +func TestShortHeader_StableMarshal(t *testing.T) { + hdrFrom := generateShortHeader("Owner ID") + transport := new(grpc.ShortHeader) + + t.Run("non empty", func(t *testing.T) { + wire, err := hdrFrom.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + hdrTo := object.ShortHeaderFromGRPCMessage(transport) + require.Equal(t, hdrFrom, hdrTo) + }) +} + +func TestAttribute_StableMarshal(t *testing.T) { + from := generateAttribute("Key", "Value") + transport := new(grpc.Header_Attribute) + + t.Run("non empty", func(t *testing.T) { + wire, err := from.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + to := object.AttributeFromGRPCMessage(transport) + require.Equal(t, from, to) + }) +} + +func TestSplitHeader_StableMarshal(t *testing.T) { + from := generateSplit("Split Outside") + hdr := generateHeader(123) + from.SetParentHeader(hdr) + + transport := new(grpc.Header_Split) + + t.Run("non empty", func(t *testing.T) { + wire, err := from.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + to := object.SplitHeaderFromGRPCMessage(transport) + require.Equal(t, from, to) + }) +} + +func TestHeader_StableMarshal(t *testing.T) { + insideHeader := generateHeader(100) + split := generateSplit("Split") + split.SetParentHeader(insideHeader) + + from := generateHeader(500) + from.SetSplit(split) + + transport := new(grpc.Header) + + t.Run("non empty", func(t *testing.T) { + wire, err := from.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + to := object.HeaderFromGRPCMessage(transport) + require.Equal(t, from, to) + }) +} + +func TestObject_StableMarshal(t *testing.T) { + from := generateObject("Payload") + transport := new(grpc.Object) + + t.Run("non empty", func(t *testing.T) { + wire, err := from.StableMarshal(nil) + require.NoError(t, err) + + err = transport.Unmarshal(wire) + require.NoError(t, err) + + to := object.ObjectFromGRPCMessage(transport) + require.Equal(t, from, to) + }) +} + +func generateOwner(id string) *refs.OwnerID { + owner := new(refs.OwnerID) + owner.SetValue([]byte(id)) + + return owner +} + +func generateObjectID(id string) *refs.ObjectID { + oid := new(refs.ObjectID) + oid.SetValue([]byte(id)) + + return oid +} + +func generateContainerID(id string) *refs.ContainerID { + cid := new(refs.ContainerID) + cid.SetValue([]byte(id)) + + return cid +} + +func generateSignature(k, v string) *service.Signature { + sig := new(service.Signature) + sig.SetKey([]byte(k)) + sig.SetSign([]byte(v)) + + return sig +} + +func generateVersion(maj, min uint32) *service.Version { + version := new(service.Version) + version.SetMajor(maj) + version.SetMinor(min) + + return version +} + +func generateSessionToken(id string) *service.SessionToken { + lifetime := new(service.TokenLifetime) + lifetime.SetExp(1) + lifetime.SetNbf(2) + lifetime.SetIat(3) + + addr := new(refs.Address) + addr.SetContainerID(generateContainerID("Container ID")) + addr.SetObjectID(generateObjectID("Object ID")) + + objectCtx := new(service.ObjectSessionContext) + objectCtx.SetVerb(service.ObjectVerbPut) + objectCtx.SetAddress(addr) + + tokenBody := new(service.SessionTokenBody) + tokenBody.SetID([]byte(id)) + tokenBody.SetOwnerID(generateOwner("Owner ID")) + tokenBody.SetSessionKey([]byte(id)) + tokenBody.SetLifetime(lifetime) + tokenBody.SetContext(objectCtx) + + sessionToken := new(service.SessionToken) + sessionToken.SetBody(tokenBody) + sessionToken.SetSignature(generateSignature("public key", id)) + + return sessionToken +} + +func generateShortHeader(id string) *object.ShortHeader { + hdr := new(object.ShortHeader) + hdr.SetOwnerID(generateOwner(id)) + hdr.SetVersion(generateVersion(2, 0)) + hdr.SetCreationEpoch(200) + hdr.SetObjectType(object.TypeRegular) + hdr.SetPayloadLength(10) + + return hdr +} + +func generateAttribute(k, v string) *object.Attribute { + attr := new(object.Attribute) + attr.SetValue(v) + attr.SetKey(k) + + return attr +} + +func generateSplit(sig string) *object.SplitHeader { + split := new(object.SplitHeader) + split.SetChildren([]*refs.ObjectID{ + generateObjectID("Child 1"), + generateObjectID("Child 2"), + }) + split.SetParent(generateObjectID("Parent")) + split.SetParentSignature(generateSignature("Key", sig)) + split.SetPrevious(generateObjectID("Previous")) + + return split +} + +func generateHeader(ln uint64) *object.Header { + hdr := new(object.Header) + hdr.SetPayloadLength(ln) + hdr.SetCreationEpoch(ln / 2) + hdr.SetVersion(generateVersion(2, 0)) + hdr.SetOwnerID(generateOwner("Owner ID")) + hdr.SetContainerID(generateContainerID("Contanier ID")) + hdr.SetAttributes([]*object.Attribute{ + generateAttribute("One", "Two"), + generateAttribute("Three", "Four"), + }) + hdr.SetHomomorphicHash([]byte("Homomorphic Hash")) + hdr.SetObjectType(object.TypeRegular) + hdr.SetPayloadHash([]byte("Payload Hash")) + hdr.SetSessionToken(generateSessionToken(string(ln))) + + return hdr +} + +func generateObject(data string) *object.Object { + insideHeader := generateHeader(100) + split := generateSplit("Split") + split.SetParentHeader(insideHeader) + + outsideHeader := generateHeader(500) + outsideHeader.SetSplit(split) + + obj := new(object.Object) + obj.SetSignature(generateSignature("Public Key", "Signature")) + obj.SetObjectID(generateObjectID("Object ID")) + obj.SetPayload([]byte(data)) + obj.SetHeader(outsideHeader) + + return obj +}