From 9b63c07c59eb87667c2d9dafd4bda9b5c497842c Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Mon, 22 Mar 2021 18:04:16 +0300 Subject: [PATCH] [#170] storagegroup: Refactor and document package functionality Signed-off-by: Pavel Karpy --- storagegroup/doc.go | 36 +++++++++ storagegroup/storagegroup.go | 118 +++++++++++++++++++-------- storagegroup/storagegroup_test.go | 128 +++++++++++++++++++++++++----- storagegroup/test/doc.go | 13 +++ storagegroup/test/generate.go | 4 +- 5 files changed, 244 insertions(+), 55 deletions(-) create mode 100644 storagegroup/doc.go create mode 100644 storagegroup/test/doc.go diff --git a/storagegroup/doc.go b/storagegroup/doc.go new file mode 100644 index 00000000..207a9a7f --- /dev/null +++ b/storagegroup/doc.go @@ -0,0 +1,36 @@ +/* +Package storagegroup provides features to work with information that is +used for proof of storage in NeoFS system. + +StorageGroup type groups verification values for Data Audit sessions: + // receive sg info + + sg.ExpirationEpoch() // expiration of the storage group + sg.Members() // objects in the group + sg.ValidationDataHash() // hash for objects validation + sg.ValidationDataSize() // total objects' payload size + +Instances can be also used to process NeoFS API V2 protocol messages +(see neo.fs.v2.storagegroup package in https://github.com/nspcc-dev/neofs-api). + +On client side: + import "github.com/nspcc-dev/neofs-api-go/v2/storagegroup" + + var msg storagegroup.StorageGroup + sg.WriteToV2(&msg) + + // send msg + +On server side: + // recv msg + + var sg StorageGroupDecimal + sg.ReadFromV2(msg) + + // process sg + +Using package types in an application is recommended to potentially work with +different protocol versions with which these types are compatible. + +*/ +package storagegroup diff --git a/storagegroup/storagegroup.go b/storagegroup/storagegroup.go index fae8dd0c..5d358225 100644 --- a/storagegroup/storagegroup.go +++ b/storagegroup/storagegroup.go @@ -7,35 +7,47 @@ import ( oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) -// StorageGroup represents v2-compatible storage group. +// StorageGroup represents storage group of the NeoFS objects. +// +// StorageGroup is mutually compatible with github.com/nspcc-dev/neofs-api-go/v2/storagegroup.StorageGroup +// message. See ReadFromMessageV2 / WriteToMessageV2 methods. +// +// Instances can be created using built-in var declaration. +// +// Note that direct typecast is not safe and may result in loss of compatibility: +// _ = StorageGroup(storagegroup.StorageGroup) // not recommended type StorageGroup storagegroup.StorageGroup -// NewFromV2 wraps v2 StorageGroup message to StorageGroup. +// ReadFromV2 reads StorageGroup from the storagegroup.StorageGroup message. // -// Nil storagegroup.StorageGroup converts to nil. -func NewFromV2(aV2 *storagegroup.StorageGroup) *StorageGroup { - return (*StorageGroup)(aV2) +// See also WriteToV2. +func (sg *StorageGroup) ReadFromV2(m storagegroup.StorageGroup) { + *sg = StorageGroup(m) } -// New creates and initializes blank StorageGroup. +// WriteToV2 writes StorageGroup to the storagegroup.StorageGroup message. +// The message must not be nil. // -// Defaults: -// - size: 0; -// - exp: 0; -// - members: nil; -// - hash: nil. -func New() *StorageGroup { - return NewFromV2(new(storagegroup.StorageGroup)) +// See also ReadFromV2. +func (sg StorageGroup) WriteToV2(m *storagegroup.StorageGroup) { + *m = (storagegroup.StorageGroup)(sg) } // ValidationDataSize returns total size of the payloads // of objects in the storage group. -func (sg *StorageGroup) ValidationDataSize() uint64 { - return (*storagegroup.StorageGroup)(sg).GetValidationDataSize() +// +// Zero StorageGroup has 0 data size. +// +// See also SetValidationDataSize. +func (sg StorageGroup) ValidationDataSize() uint64 { + v2 := (storagegroup.StorageGroup)(sg) + return v2.GetValidationDataSize() } // SetValidationDataSize sets total size of the payloads // of objects in the storage group. +// +// See also ValidationDataSize. func (sg *StorageGroup) SetValidationDataSize(epoch uint64) { (*storagegroup.StorageGroup)(sg).SetValidationDataSize(epoch) } @@ -71,20 +83,32 @@ func (sg *StorageGroup) SetValidationDataHash(hash checksum.Checksum) { // ExpirationEpoch returns last NeoFS epoch number // of the storage group lifetime. -func (sg *StorageGroup) ExpirationEpoch() uint64 { - return (*storagegroup.StorageGroup)(sg).GetExpirationEpoch() +// +// Zero StorageGroup has 0 expiration epoch. +// +// See also SetExpirationEpoch. +func (sg StorageGroup) ExpirationEpoch() uint64 { + v2 := (storagegroup.StorageGroup)(sg) + return v2.GetExpirationEpoch() } // SetExpirationEpoch sets last NeoFS epoch number // of the storage group lifetime. +// +// See also ExpirationEpoch. func (sg *StorageGroup) SetExpirationEpoch(epoch uint64) { (*storagegroup.StorageGroup)(sg).SetExpirationEpoch(epoch) } // Members returns strictly ordered list of // storage group member objects. -func (sg *StorageGroup) Members() []oid.ID { - mV2 := (*storagegroup.StorageGroup)(sg).GetMembers() +// +// Zero StorageGroup has nil members value. +// +// See also SetMembers. +func (sg StorageGroup) Members() []oid.ID { + v2 := (storagegroup.StorageGroup)(sg) + mV2 := v2.GetMembers() if mV2 == nil { return nil @@ -101,6 +125,8 @@ func (sg *StorageGroup) Members() []oid.ID { // SetMembers sets strictly ordered list of // storage group member objects. +// +// See also Members. func (sg *StorageGroup) SetMembers(members []oid.ID) { mV2 := (*storagegroup.StorageGroup)(sg).GetMembers() @@ -126,29 +152,57 @@ func (sg *StorageGroup) SetMembers(members []oid.ID) { (*storagegroup.StorageGroup)(sg).SetMembers(mV2) } -// ToV2 converts StorageGroup to v2 StorageGroup message. -// -// Nil StorageGroup converts to nil. -func (sg *StorageGroup) ToV2() *storagegroup.StorageGroup { - return (*storagegroup.StorageGroup)(sg) -} - // Marshal marshals StorageGroup into a protobuf binary form. -func (sg *StorageGroup) Marshal() ([]byte, error) { - return (*storagegroup.StorageGroup)(sg).StableMarshal(nil) +// +// See also Unmarshal. +func (sg StorageGroup) Marshal() ([]byte, error) { + v2 := (storagegroup.StorageGroup)(sg) + return v2.StableMarshal(nil) } // Unmarshal unmarshals protobuf binary representation of StorageGroup. +// +// See also Marshal. func (sg *StorageGroup) Unmarshal(data []byte) error { - return (*storagegroup.StorageGroup)(sg).Unmarshal(data) + v2 := (*storagegroup.StorageGroup)(sg) + err := v2.Unmarshal(data) + if err != nil { + return err + } + + return formatCheck(v2) } // MarshalJSON encodes StorageGroup to protobuf JSON format. -func (sg *StorageGroup) MarshalJSON() ([]byte, error) { - return (*storagegroup.StorageGroup)(sg).MarshalJSON() +// +// See also UnmarshalJSON. +func (sg StorageGroup) MarshalJSON() ([]byte, error) { + v2 := (storagegroup.StorageGroup)(sg) + return v2.MarshalJSON() } // UnmarshalJSON decodes StorageGroup from protobuf JSON format. +// +// See also MarshalJSON. func (sg *StorageGroup) UnmarshalJSON(data []byte) error { - return (*storagegroup.StorageGroup)(sg).UnmarshalJSON(data) + v2 := (*storagegroup.StorageGroup)(sg) + err := v2.UnmarshalJSON(data) + if err != nil { + return err + } + + return formatCheck(v2) +} + +func formatCheck(v2 *storagegroup.StorageGroup) error { + var oID oid.ID + + for _, m := range v2.GetMembers() { + err := oID.ReadFromV2(m) + if err != nil { + return err + } + } + + return nil } diff --git a/storagegroup/storagegroup_test.go b/storagegroup/storagegroup_test.go index 6a197d58..ef74c186 100644 --- a/storagegroup/storagegroup_test.go +++ b/storagegroup/storagegroup_test.go @@ -1,9 +1,13 @@ package storagegroup_test import ( + "crypto/sha256" "testing" + "github.com/nspcc-dev/neofs-api-go/v2/refs" storagegroupV2 "github.com/nspcc-dev/neofs-api-go/v2/storagegroup" + storagegroupV2test "github.com/nspcc-dev/neofs-api-go/v2/storagegroup/test" + "github.com/nspcc-dev/neofs-sdk-go/checksum" checksumtest "github.com/nspcc-dev/neofs-sdk-go/checksum/test" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" @@ -13,7 +17,7 @@ import ( ) func TestStorageGroup(t *testing.T) { - sg := storagegroup.New() + var sg storagegroup.StorageGroup sz := uint64(13) sg.SetValidationDataSize(sz) @@ -35,6 +39,56 @@ func TestStorageGroup(t *testing.T) { require.Equal(t, members, sg.Members()) } +func TestStorageGroup_ReadFromV2(t *testing.T) { + t.Run("from zero", func(t *testing.T) { + var ( + x storagegroup.StorageGroup + v2 storagegroupV2.StorageGroup + ) + + x.ReadFromV2(v2) + + require.Zero(t, x.ExpirationEpoch()) + require.Zero(t, x.ValidationDataSize()) + _, set := x.ValidationDataHash() + require.False(t, set) + require.Zero(t, x.Members()) + }) + + t.Run("from non-zero", func(t *testing.T) { + var ( + x storagegroup.StorageGroup + v2 = storagegroupV2test.GenerateStorageGroup(false) + ) + + // https://github.com/nspcc-dev/neofs-api-go/issues/394 + v2.SetMembers(generateOIDList()) + + size := v2.GetValidationDataSize() + epoch := v2.GetExpirationEpoch() + mm := v2.GetMembers() + hashV2 := v2.GetValidationHash() + + x.ReadFromV2(*v2) + + require.Equal(t, epoch, x.ExpirationEpoch()) + require.Equal(t, size, x.ValidationDataSize()) + + var hash checksum.Checksum + hash.ReadFromV2(*hashV2) + h, set := x.ValidationDataHash() + require.True(t, set) + require.Equal(t, hash, h) + + var oidV2 refs.ObjectID + + for i, m := range mm { + x.Members()[i].WriteToV2(&oidV2) + require.Equal(t, m, oidV2) + } + }) +} + func TestStorageGroupEncoding(t *testing.T) { sg := storagegrouptest.StorageGroup() @@ -42,7 +96,7 @@ func TestStorageGroupEncoding(t *testing.T) { data, err := sg.Marshal() require.NoError(t, err) - sg2 := storagegroup.New() + var sg2 storagegroup.StorageGroup require.NoError(t, sg2.Unmarshal(data)) require.Equal(t, sg, sg2) @@ -52,32 +106,58 @@ func TestStorageGroupEncoding(t *testing.T) { data, err := sg.MarshalJSON() require.NoError(t, err) - sg2 := storagegroup.New() + var sg2 storagegroup.StorageGroup require.NoError(t, sg2.UnmarshalJSON(data)) require.Equal(t, sg, sg2) }) } -func TestNewFromV2(t *testing.T) { - t.Run("from nil", func(t *testing.T) { - var x *storagegroupV2.StorageGroup +func TestStorageGroup_WriteToV2(t *testing.T) { + t.Run("zero to v2", func(t *testing.T) { + var ( + x storagegroup.StorageGroup + v2 storagegroupV2.StorageGroup + ) - require.Nil(t, storagegroup.NewFromV2(x)) + x.WriteToV2(&v2) + + require.Nil(t, v2.GetValidationHash()) + require.Nil(t, v2.GetMembers()) + require.Zero(t, v2.GetValidationDataSize()) + require.Zero(t, v2.GetExpirationEpoch()) }) -} -func TestStorageGroup_ToV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *storagegroup.StorageGroup + t.Run("non-zero to v2", func(t *testing.T) { + var ( + x = storagegrouptest.StorageGroup() + v2 storagegroupV2.StorageGroup + ) - require.Nil(t, x.ToV2()) + x.WriteToV2(&v2) + + require.Equal(t, x.ExpirationEpoch(), v2.GetExpirationEpoch()) + require.Equal(t, x.ValidationDataSize(), v2.GetValidationDataSize()) + + var hash checksum.Checksum + hash.ReadFromV2(*v2.GetValidationHash()) + + h, set := x.ValidationDataHash() + require.True(t, set) + require.Equal(t, h, hash) + + var oidV2 refs.ObjectID + + for i, m := range x.Members() { + m.WriteToV2(&oidV2) + require.Equal(t, oidV2, v2.GetMembers()[i]) + } }) } func TestNew(t *testing.T) { t.Run("default values", func(t *testing.T) { - sg := storagegroup.New() + var sg storagegroup.StorageGroup // check initial values require.Nil(t, sg.Members()) @@ -85,13 +165,19 @@ func TestNew(t *testing.T) { require.False(t, set) require.Zero(t, sg.ExpirationEpoch()) require.Zero(t, sg.ValidationDataSize()) - - // convert to v2 message - sgV2 := sg.ToV2() - - require.Nil(t, sgV2.GetMembers()) - require.Nil(t, sgV2.GetValidationHash()) - require.Zero(t, sgV2.GetExpirationEpoch()) - require.Zero(t, sgV2.GetValidationDataSize()) }) } + +func generateOIDList() []refs.ObjectID { + const size = 3 + + mmV2 := make([]refs.ObjectID, size) + for i := 0; i < size; i++ { + oidV2 := make([]byte, sha256.Size) + oidV2[i] = byte(i) + + mmV2[i].SetValue(oidV2) + } + + return mmV2 +} diff --git a/storagegroup/test/doc.go b/storagegroup/test/doc.go new file mode 100644 index 00000000..5cfb6cba --- /dev/null +++ b/storagegroup/test/doc.go @@ -0,0 +1,13 @@ +/* +Package storagegrouptest provides functions for convenient testing of storagegroup package API. + +Note that importing the package into source files is highly discouraged. + +Random instance generation functions can be useful when testing expects any value, e.g.: + import storagegrouptest "github.com/nspcc-dev/neofs-sdk-go/storagegroup/test" + + val := storagegrouptest.StorageGroup() + // test the value + +*/ +package storagegrouptest diff --git a/storagegroup/test/generate.go b/storagegroup/test/generate.go index a5c1756b..ef119554 100644 --- a/storagegroup/test/generate.go +++ b/storagegroup/test/generate.go @@ -8,8 +8,8 @@ import ( ) // StorageGroup returns random storagegroup.StorageGroup. -func StorageGroup() *storagegroup.StorageGroup { - x := storagegroup.New() +func StorageGroup() storagegroup.StorageGroup { + var x storagegroup.StorageGroup x.SetExpirationEpoch(66) x.SetValidationDataSize(322)