From ded06674ac1b028377d72f74eb61745efdc867cf Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 7 Dec 2020 18:59:01 +0300 Subject: [PATCH] [#225] v2/object: Support more SplitInfo structures - SplitInfo structure in object.HeadResponse - SplitInfo structure in object.GetRangeResponse - Raw flag in object.GetRangeRequest Signed-off-by: Alex Vanin --- v2/object/convert.go | 64 ++++++++++++++++++++++++++++++++-- v2/object/marshal.go | 63 ++++++++++++++++++++++++++++++---- v2/object/marshal_test.go | 72 ++++++++++++++++++++++++++++++--------- v2/object/types.go | 52 +++++++++++++++++++++++++--- 4 files changed, 223 insertions(+), 28 deletions(-) diff --git a/v2/object/convert.go b/v2/object/convert.go index 54075784..24d4f47a 100644 --- a/v2/object/convert.go +++ b/v2/object/convert.go @@ -1040,6 +1040,10 @@ func HeadResponseBodyToGRPCMessage(r *HeadResponseBody) *object.HeadResponse_Bod m.SetShortHeader( ShortHeaderToGRPCMessage(t), ) + case *SplitInfo: + m.SetSplitInfo( + SplitInfoToGRPCMessage(t), + ) default: panic(fmt.Sprintf("unknown header part %T", t)) } @@ -1064,6 +1068,10 @@ func HeadResponseBodyFromGRPCMessage(m *object.HeadResponse_Body) *HeadResponseB r.SetHeaderPart( ShortHeaderFromGRPCMessage(v.ShortHeader), ) + case *object.HeadResponse_Body_SplitInfo: + r.SetHeaderPart( + SplitInfoFromGRPCMessage(v.SplitInfo), + ) default: panic(fmt.Sprintf("unknown header part %T", v)) } @@ -1332,6 +1340,8 @@ func GetRangeRequestBodyToGRPCMessage(r *GetRangeRequestBody) *object.GetRangeRe RangeToGRPCMessage(r.GetRange()), ) + m.SetRaw(r.GetRaw()) + return m } @@ -1350,6 +1360,8 @@ func GetRangeRequestBodyFromGRPCMessage(m *object.GetRangeRequest_Body) *GetRang RangeFromGRPCMessage(m.GetRange()), ) + r.SetRaw(m.GetRaw()) + return r } @@ -1385,6 +1397,30 @@ func GetRangeRequestFromGRPCMessage(m *object.GetRangeRequest) *GetRangeRequest return r } +func GetRangePartChunkToGRPCMessage(r *GetRangePartChunk) *object.GetRangeResponse_Body_Chunk { + if r == nil { + return nil + } + + m := new(object.GetRangeResponse_Body_Chunk) + + m.SetChunk(r.GetChunk()) + + return m +} + +func GetRangePartChunkFromGRPCMessage(m *object.GetRangeResponse_Body_Chunk) *GetRangePartChunk { + if m == nil { + return nil + } + + r := new(GetRangePartChunk) + + r.SetChunk(m.GetChunk()) + + return r +} + func GetRangeResponseBodyToGRPCMessage(r *GetRangeResponseBody) *object.GetRangeResponse_Body { if r == nil { return nil @@ -1392,7 +1428,19 @@ func GetRangeResponseBodyToGRPCMessage(r *GetRangeResponseBody) *object.GetRange m := new(object.GetRangeResponse_Body) - m.SetChunk(r.GetChunk()) + switch v := r.GetRangePart(); t := v.(type) { + case nil: + case *GetRangePartChunk: + m.SetChunk( + GetRangePartChunkToGRPCMessage(t), + ) + case *SplitInfo: + m.SetSplitInfo( + SplitInfoToGRPCMessage(t), + ) + default: + panic(fmt.Sprintf("unknown get range part %T", t)) + } return m } @@ -1404,7 +1452,19 @@ func GetRangeResponseBodyFromGRPCMessage(m *object.GetRangeResponse_Body) *GetRa r := new(GetRangeResponseBody) - r.SetChunk(m.GetChunk()) + switch v := m.GetRangePart().(type) { + case nil: + case *object.GetRangeResponse_Body_Chunk: + r.SetRangePart( + GetRangePartChunkFromGRPCMessage(v), + ) + case *object.GetRangeResponse_Body_SplitInfo: + r.SetRangePart( + SplitInfoFromGRPCMessage(v.SplitInfo), + ) + default: + panic(fmt.Sprintf("unknown get range part %T", v)) + } return r } diff --git a/v2/object/marshal.go b/v2/object/marshal.go index fc71f562..04737b60 100644 --- a/v2/object/marshal.go +++ b/v2/object/marshal.go @@ -76,6 +76,7 @@ const ( headRespBodyHeaderField = 1 headRespBodyShortHeaderField = 2 + headRespBodySplitInfoField = 3 searchFilterMatchField = 1 searchFilterNameField = 2 @@ -92,8 +93,10 @@ const ( getRangeReqBodyAddressField = 1 getRangeReqBodyRangeField = 2 + getRangeReqBodyRawField = 3 - getRangeRespChunkField = 1 + getRangeRespChunkField = 1 + getRangeRespSplitInfoField = 2 getRangeHashReqBodyAddressField = 1 getRangeHashReqBodyRangesField = 2 @@ -1030,6 +1033,13 @@ func (r *HeadResponseBody) StableMarshal(buf []byte) ([]byte, error) { return nil, err } } + case *SplitInfo: + if v != nil { + _, err := proto.NestedStructureMarshal(headRespBodySplitInfoField, buf, v) + if err != nil { + return nil, err + } + } default: panic("unknown one of object put request body type") } @@ -1053,6 +1063,10 @@ func (r *HeadResponseBody) StableSize() (size int) { if v != nil { size += proto.NestedStructureSize(headRespBodyShortHeaderField, v) } + case *SplitInfo: + if v != nil { + size += proto.NestedStructureSize(headRespBodySplitInfoField, v) + } default: panic("unknown one of object put request body type") } @@ -1263,7 +1277,14 @@ func (r *GetRangeRequestBody) StableMarshal(buf []byte) ([]byte, error) { offset += n - _, err = proto.NestedStructureMarshal(getRangeReqBodyRangeField, buf[offset:], r.rng) + n, err = proto.NestedStructureMarshal(getRangeReqBodyRangeField, buf[offset:], r.rng) + if err != nil { + return nil, err + } + + offset += n + + _, err = proto.BoolMarshal(getRangeReqBodyRawField, buf[offset:], r.raw) if err != nil { return nil, err } @@ -1278,6 +1299,7 @@ func (r *GetRangeRequestBody) StableSize() (size int) { size += proto.NestedStructureSize(getRangeReqBodyAddressField, r.addr) size += proto.NestedStructureSize(getRangeReqBodyRangeField, r.rng) + size += proto.BoolSize(getRangeReqBodyRawField, r.raw) return size } @@ -1291,9 +1313,25 @@ func (r *GetRangeResponseBody) StableMarshal(buf []byte) ([]byte, error) { buf = make([]byte, r.StableSize()) } - _, err := proto.BytesMarshal(getRangeRespChunkField, buf, r.chunk) - if err != nil { - return nil, err + if r.rngPart != nil { + switch v := r.rngPart.(type) { + case *GetRangePartChunk: + if v != nil { + _, err := proto.BytesMarshal(getRangeRespChunkField, buf, v.chunk) + if err != nil { + return nil, err + } + } + case *SplitInfo: + if v != nil { + _, err := proto.NestedStructureMarshal(getRangeRespSplitInfoField, buf, v) + if err != nil { + return nil, err + } + } + default: + panic("unknown one of object get range request body type") + } } return buf, nil @@ -1304,7 +1342,20 @@ func (r *GetRangeResponseBody) StableSize() (size int) { return 0 } - size += proto.BytesSize(getRangeRespChunkField, r.chunk) + if r.rngPart != nil { + switch v := r.rngPart.(type) { + case *GetRangePartChunk: + if v != nil { + size += proto.BytesSize(getRangeRespChunkField, v.chunk) + } + case *SplitInfo: + if v != nil { + size = proto.NestedStructureSize(getRangeRespSplitInfoField, v) + } + default: + panic("unknown one of object get range request body type") + } + } return size } diff --git a/v2/object/marshal_test.go b/v2/object/marshal_test.go index ee79a3f6..cc702afc 100644 --- a/v2/object/marshal_test.go +++ b/v2/object/marshal_test.go @@ -239,8 +239,9 @@ func TestSplitHeaderFromGRPCMessage(t *testing.T) { } func TestHeadResponseBody_StableMarshal(t *testing.T) { - shortFrom := generateHeadResponseBody(true) - fullFrom := generateHeadResponseBody(false) + shortFrom := generateHeadResponseBody(0) + fullFrom := generateHeadResponseBody(1) + splitInfoFrom := generateHeadResponseBody(2) transport := new(grpc.HeadResponse_Body) t.Run("short header non empty", func(t *testing.T) { @@ -264,6 +265,17 @@ func TestHeadResponseBody_StableMarshal(t *testing.T) { to := object.HeadResponseBodyFromGRPCMessage(transport) require.Equal(t, fullFrom, to) }) + + t.Run("split info non empty", func(t *testing.T) { + wire, err := splitInfoFrom.StableMarshal(nil) + require.NoError(t, err) + + err = goproto.Unmarshal(wire, transport) + require.NoError(t, err) + + to := object.HeadResponseBodyFromGRPCMessage(transport) + require.Equal(t, splitInfoFrom, to) + }) } func TestSearchRequestBody_StableMarshal(t *testing.T) { @@ -315,18 +327,30 @@ func TestGetRangeRequestBody_StableMarshal(t *testing.T) { } func TestGetRangeResponseBody_StableMarshal(t *testing.T) { - from := generateRangeResponseBody("some data") + dataFrom := generateRangeResponseBody("some data", true) + splitInfoFrom := generateRangeResponseBody("some data", false) transport := new(grpc.GetRangeResponse_Body) - t.Run("non empty", func(t *testing.T) { - wire, err := from.StableMarshal(nil) + t.Run("data non empty", func(t *testing.T) { + wire, err := dataFrom.StableMarshal(nil) require.NoError(t, err) err = goproto.Unmarshal(wire, transport) require.NoError(t, err) to := object.GetRangeResponseBodyFromGRPCMessage(transport) - require.Equal(t, from, to) + require.Equal(t, dataFrom, to) + }) + + t.Run("split info non empty", func(t *testing.T) { + wire, err := splitInfoFrom.StableMarshal(nil) + require.NoError(t, err) + + err = goproto.Unmarshal(wire, transport) + require.NoError(t, err) + + to := object.GetRangeResponseBodyFromGRPCMessage(transport) + require.Equal(t, splitInfoFrom, to) }) } @@ -546,11 +570,7 @@ func generateGetResponseBody(i int) *object.GetResponseBody { chunk.SetChunk([]byte("Some data chunk")) part = chunk default: - splitInfo := new(object.SplitInfo) - splitInfo.SetSplitID([]byte("splitID")) - splitInfo.SetLastPart(generateObjectID("Right ID")) - splitInfo.SetLink(generateObjectID("Link ID")) - part = splitInfo + part = generateSplitInfo() } resp.SetObjectPart(part) @@ -606,14 +626,17 @@ func generateHeadRequestBody(cid, oid string) *object.HeadRequestBody { return req } -func generateHeadResponseBody(flag bool) *object.HeadResponseBody { +func generateHeadResponseBody(flag int) *object.HeadResponseBody { req := new(object.HeadResponseBody) var part object.GetHeaderPart - if flag { + switch flag { + case 0: part = generateShortHeader("short id") - } else { + case 1: part = generateHeaderWithSignature() + default: + part = generateSplitInfo() } req.SetHeaderPart(part) @@ -677,13 +700,21 @@ func generateRangeRequestBody(cid, oid string) *object.GetRangeRequestBody { req := new(object.GetRangeRequestBody) req.SetAddress(generateAddress(cid, oid)) req.SetRange(generateRange(10, 20)) + req.SetRaw(true) return req } -func generateRangeResponseBody(data string) *object.GetRangeResponseBody { +func generateRangeResponseBody(data string, flag bool) *object.GetRangeResponseBody { resp := new(object.GetRangeResponseBody) - resp.SetChunk([]byte(data)) + + if flag { + p := new(object.GetRangePartChunk) + p.SetChunk([]byte(data)) + resp.SetRangePart(p) + } else { + resp.SetRangePart(generateSplitInfo()) + } return resp } @@ -729,3 +760,12 @@ func TestObject_StableUnmarshal(t *testing.T) { require.Equal(t, obj, obj2) } + +func generateSplitInfo() *object.SplitInfo { + splitInfo := new(object.SplitInfo) + splitInfo.SetSplitID([]byte("splitID")) + splitInfo.SetLastPart(generateObjectID("Right ID")) + splitInfo.SetLink(generateObjectID("Link ID")) + + return splitInfo +} diff --git a/v2/object/types.go b/v2/object/types.go index 7a94c5de..2ab3ff5f 100644 --- a/v2/object/types.go +++ b/v2/object/types.go @@ -181,10 +181,20 @@ type GetRangeRequestBody struct { addr *refs.Address rng *Range + + raw bool +} + +type GetRangePart interface { + getRangePart() +} + +type GetRangePartChunk struct { + chunk []byte } type GetRangeResponseBody struct { - chunk []byte + rngPart GetRangePart } type GetRangeHashRequestBody struct { @@ -680,7 +690,11 @@ func (s *SplitInfo) SetLink(v *refs.ObjectID) { } } -func (s *SplitInfo) getObjectPart() {} // implement interface to be one of response body +func (s *SplitInfo) getObjectPart() {} + +func (s *SplitInfo) getHeaderPart() {} + +func (s *SplitInfo) getRangePart() {} func (r *GetRequestBody) GetAddress() *refs.Address { if r != nil { @@ -1530,6 +1544,20 @@ func (r *GetRangeRequestBody) SetRange(v *Range) { } } +func (r *GetRangeRequestBody) GetRaw() bool { + if r != nil { + return r.raw + } + + return false +} + +func (r *GetRangeRequestBody) SetRaw(v bool) { + if r != nil { + r.raw = v + } +} + func (r *GetRangeRequest) GetBody() *GetRangeRequestBody { if r != nil { return r.body @@ -1572,7 +1600,7 @@ func (r *GetRangeRequest) SetVerificationHeader(v *session.RequestVerificationHe } } -func (r *GetRangeResponseBody) GetChunk() []byte { +func (r *GetRangePartChunk) GetChunk() []byte { if r != nil { return r.chunk } @@ -1580,12 +1608,28 @@ func (r *GetRangeResponseBody) GetChunk() []byte { return nil } -func (r *GetRangeResponseBody) SetChunk(v []byte) { +func (r *GetRangePartChunk) SetChunk(v []byte) { if r != nil { r.chunk = v } } +func (r *GetRangePartChunk) getRangePart() {} + +func (r *GetRangeResponseBody) GetRangePart() GetRangePart { + if r != nil { + return r.rngPart + } + + return nil +} + +func (r *GetRangeResponseBody) SetRangePart(v GetRangePart) { + if r != nil { + r.rngPart = v + } +} + func (r *GetRangeResponse) GetBody() *GetRangeResponseBody { if r != nil { return r.body