diff --git a/CHANGELOG.md b/CHANGELOG.md index a3f3c8ca..a039d13e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # Changelog This is the changelog for NeoFS Proto +## [0.2.4] - 2019-12-03 + +### Added +- StorageGroup library + +### Changed +- Storage group part of object library moved into separate package +- Updated proto documentation + ## [0.2.3] - 2019-11-28 ### Removed @@ -50,3 +59,4 @@ Initial public release [0.2.1]: https://github.com/nspcc-dev/neofs-proto/compare/v0.2.0...v0.2.1 [0.2.2]: https://github.com/nspcc-dev/neofs-proto/compare/v0.2.1...v0.2.2 [0.2.3]: https://github.com/nspcc-dev/neofs-proto/compare/v0.2.2...v0.2.3 +[0.2.4]: https://github.com/nspcc-dev/neofs-proto/compare/v0.2.3...v0.2.4 diff --git a/docs/object.md b/docs/object.md index 84077332..9e7efe61 100644 --- a/docs/object.md +++ b/docs/object.md @@ -34,8 +34,6 @@ - [Link](#object.Link) - [Object](#object.Object) - [Range](#object.Range) - - [StorageGroup](#object.StorageGroup) - - [StorageGroup.Lifetime](#object.StorageGroup.Lifetime) - [SystemHeader](#object.SystemHeader) - [Tombstone](#object.Tombstone) - [Transform](#object.Transform) @@ -369,7 +367,7 @@ in distributed system. | HomoHash | [bytes](#bytes) | | HomoHash is a homomorphic hash of original object payload | | PayloadChecksum | [bytes](#bytes) | | PayloadChecksum of actual object's payload | | Integrity | [IntegrityHeader](#object.IntegrityHeader) | | Integrity header with checksum of all above headers in the object | -| StorageGroup | [StorageGroup](#object.StorageGroup) | | StorageGroup contains meta information for the data audit | +| StorageGroup | [storagegroup.StorageGroup](#storagegroup.StorageGroup) | | StorageGroup contains meta information for the data audit | @@ -421,31 +419,6 @@ in distributed system. | Length | [uint64](#uint64) | | Length of the data range | - - -### Message StorageGroup - - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| ValidationDataSize | [uint64](#uint64) | | ValidationDataSize is size of the all object's payloads included into storage group | -| ValidationHash | [bytes](#bytes) | | ValidationHash is homomorphic hash of all object's payloads included into storage group | -| lifetime | [StorageGroup.Lifetime](#object.StorageGroup.Lifetime) | | Lifetime is time until storage group is valid | - - - - -### Message StorageGroup.Lifetime - - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| unit | [StorageGroup.Lifetime.Unit](#object.StorageGroup.Lifetime.Unit) | | Unit is lifetime type | -| Value | [int64](#int64) | | Value for lifetime | - - ### Message SystemHeader @@ -514,19 +487,6 @@ in distributed system. - - -### StorageGroup.Lifetime.Unit - - -| Name | Number | Description | -| ---- | ------ | ----------- | -| Unlimited | 0 | Unlimited set if storage group always valid | -| NeoFSEpoch | 1 | NeoFSEpoch set if storage group is valid until lifetime NeoFS epoch | -| UnixTime | 2 | UnixTime set if storage group is valid until lifetime unix timestamp | - - - ### Transform.Type diff --git a/docs/storagegroup.md b/docs/storagegroup.md new file mode 100644 index 00000000..7522902c --- /dev/null +++ b/docs/storagegroup.md @@ -0,0 +1,88 @@ +# Protocol Documentation + + +## Table of Contents + +- [storagegroup/types.proto](#storagegroup/types.proto) + + - Messages + - [StorageGroup](#storagegroup.StorageGroup) + - [StorageGroup.Lifetime](#storagegroup.StorageGroup.Lifetime) + + +- [Scalar Value Types](#scalar-value-types) + + + + +

Top

+ +## storagegroup/types.proto + + + + + + + +### Message StorageGroup + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| ValidationDataSize | [uint64](#uint64) | | ValidationDataSize is size of the all object's payloads included into storage group | +| ValidationHash | [bytes](#bytes) | | ValidationHash is homomorphic hash of all object's payloads included into storage group | +| lifetime | [StorageGroup.Lifetime](#storagegroup.StorageGroup.Lifetime) | | Lifetime is time until storage group is valid | + + + + +### Message StorageGroup.Lifetime + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| unit | [StorageGroup.Lifetime.Unit](#storagegroup.StorageGroup.Lifetime.Unit) | | Unit is lifetime type | +| Value | [int64](#int64) | | Value for lifetime | + + + + + + +### StorageGroup.Lifetime.Unit + + +| Name | Number | Description | +| ---- | ------ | ----------- | +| Unlimited | 0 | Unlimited set if storage group always valid | +| NeoFSEpoch | 1 | NeoFSEpoch set if storage group is valid until lifetime NeoFS epoch | +| UnixTime | 2 | UnixTime set if storage group is valid until lifetime unix timestamp | + + + + + + +## Scalar Value Types + +| .proto Type | Notes | C++ Type | Java Type | Python Type | +| ----------- | ----- | -------- | --------- | ----------- | +| double | | double | double | float | +| float | | float | float | float | +| int32 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint32 instead. | int32 | int | int | +| int64 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint64 instead. | int64 | long | int/long | +| uint32 | Uses variable-length encoding. | uint32 | int | int/long | +| uint64 | Uses variable-length encoding. | uint64 | long | int/long | +| sint32 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int32s. | int32 | int | int | +| sint64 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int64s. | int64 | long | int/long | +| fixed32 | Always four bytes. More efficient than uint32 if values are often greater than 2^28. | uint32 | int | int | +| fixed64 | Always eight bytes. More efficient than uint64 if values are often greater than 2^56. | uint64 | long | int/long | +| sfixed32 | Always four bytes. | int32 | int | int | +| sfixed64 | Always eight bytes. | int64 | long | int/long | +| bool | | bool | boolean | boolean | +| string | A string must always contain UTF-8 encoded or 7-bit ASCII text. | string | String | str/unicode | +| bytes | May contain any arbitrary sequence of bytes. | string | ByteString | str | + diff --git a/object/extensions.go b/object/extensions.go index 1d896833..7427079b 100644 --- a/object/extensions.go +++ b/object/extensions.go @@ -1,8 +1,6 @@ package object -import ( - "github.com/nspcc-dev/neofs-proto/hash" -) +import "github.com/nspcc-dev/neofs-proto/storagegroup" // IsLinking checks if object has children links to another objects. // We have to check payload size because zero-object must have zero @@ -64,7 +62,7 @@ func (m Object) IsTombstone() bool { } // StorageGroup returns storage group structure if it is presented in extended headers. -func (m Object) StorageGroup() (*StorageGroup, error) { +func (m Object) StorageGroup() (*storagegroup.StorageGroup, error) { _, sgHdr := m.LastHeader(HeaderType(StorageGroupHdr)) if sgHdr == nil { return nil, ErrHeaderNotFound @@ -74,11 +72,6 @@ func (m Object) StorageGroup() (*StorageGroup, error) { // SetStorageGroup sets storage group header in the object. // It will replace existing storage group header or add a new one. -func (m *Object) SetStorageGroup(sg *StorageGroup) { - m.SetHeader(&Header{Value: &Header_StorageGroup{StorageGroup: sg}}) -} - -// Empty checks if storage group has some data for validation. -func (m StorageGroup) Empty() bool { - return m.ValidationDataSize == 0 && m.ValidationHash.Equal(hash.Hash{}) +func (m *Object) SetStorageGroup(group *storagegroup.StorageGroup) { + m.SetHeader(&Header{Value: &Header_StorageGroup{StorageGroup: group}}) } diff --git a/object/sg.go b/object/sg.go index 1b48b3ee..620d24dc 100644 --- a/object/sg.go +++ b/object/sg.go @@ -1,54 +1,31 @@ package object import ( - "bytes" "sort" + + "github.com/nspcc-dev/neofs-proto/refs" + "github.com/nspcc-dev/neofs-proto/storagegroup" ) // Here are defined getter functions for objects that contain storage group // information. -type ( - // IDList is a slice of object ids, that can be sorted. - IDList []ID - - // ZoneInfo provides validation info of storage group. - ZoneInfo struct { - Hash - Size uint64 - } - - // IdentificationInfo provides meta information about storage group. - IdentificationInfo struct { - SGID - CID - OwnerID - } -) - -// Len returns amount of object ids in IDList. -func (s IDList) Len() int { return len(s) } - -// Less returns byte comparision between IDList[i] and IDList[j]. -func (s IDList) Less(i, j int) bool { return bytes.Compare(s[i].Bytes(), s[j].Bytes()) == -1 } - -// Swap swaps element with i and j index in IDList. -func (s IDList) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +var _ storagegroup.Provider = (*Object)(nil) // Group returns slice of object ids that are part of a storage group. -func (m *Object) Group() []ID { +func (m *Object) Group() []refs.ObjectID { sgLinks := m.Links(Link_StorageGroup) - sort.Sort(IDList(sgLinks)) + sort.Sort(storagegroup.IDList(sgLinks)) return sgLinks } // Zones returns validation zones of storage group. -func (m *Object) Zones() []ZoneInfo { +func (m *Object) Zones() []storagegroup.ZoneInfo { sgInfo, err := m.StorageGroup() if err != nil { return nil } - return []ZoneInfo{ + return []storagegroup.ZoneInfo{ { Hash: sgInfo.ValidationHash, Size: sgInfo.ValidationDataSize, @@ -57,8 +34,8 @@ func (m *Object) Zones() []ZoneInfo { } // IDInfo returns meta information about storage group. -func (m *Object) IDInfo() *IdentificationInfo { - return &IdentificationInfo{ +func (m *Object) IDInfo() *storagegroup.IdentificationInfo { + return &storagegroup.IdentificationInfo{ SGID: m.SystemHeader.ID, CID: m.SystemHeader.CID, OwnerID: m.SystemHeader.OwnerID, diff --git a/object/sg_test.go b/object/sg_test.go index 58e20427..15cc21bf 100644 --- a/object/sg_test.go +++ b/object/sg_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/nspcc-dev/neofs-proto/hash" + "github.com/nspcc-dev/neofs-proto/storagegroup" "github.com/stretchr/testify/require" ) @@ -28,7 +29,7 @@ func TestObject_StorageGroup(t *testing.T) { } rand.Shuffle(len(obj.Headers), func(i, j int) { obj.Headers[i], obj.Headers[j] = obj.Headers[j], obj.Headers[i] }) - sort.Sort(IDList(idList)) + sort.Sort(storagegroup.IDList(idList)) require.Equal(t, idList, obj.Group()) }) t.Run("identification method", func(t *testing.T) { @@ -58,7 +59,7 @@ func TestObject_StorageGroup(t *testing.T) { Headers: []Header{ { Value: &Header_StorageGroup{ - StorageGroup: &StorageGroup{ + StorageGroup: &storagegroup.StorageGroup{ ValidationDataSize: sgSize, ValidationHash: sgHash, }, diff --git a/object/types.pb.go b/object/types.pb.go index 217ef099..459e124b 100644 Binary files a/object/types.pb.go and b/object/types.pb.go differ diff --git a/object/types.proto b/object/types.proto index f3313a3b..d2bbaab3 100644 --- a/object/types.proto +++ b/object/types.proto @@ -4,6 +4,7 @@ option go_package = "github.com/nspcc-dev/neofs-proto/object"; import "refs/types.proto"; import "session/types.proto"; +import "storagegroup/types.proto"; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; option (gogoproto.stable_marshaler_all) = true; @@ -25,25 +26,25 @@ message UserHeader { message Header { oneof Value { // Link to other objects - Link Link = 1; + Link Link = 1; // Redirect not used yet - refs.Address Redirect = 2; + refs.Address Redirect = 2; // UserHeader is a set of KV headers defined by user - UserHeader UserHeader = 3; + UserHeader UserHeader = 3; // Transform defines transform operation (e.g. payload split) - Transform Transform = 4; + Transform Transform = 4; // Tombstone header that set up in deleted objects - Tombstone Tombstone = 5; + Tombstone Tombstone = 5; // Verify header that contains session public key and user's signature - session.VerificationHeader Verify = 6; + session.VerificationHeader Verify = 6; // HomoHash is a homomorphic hash of original object payload - bytes HomoHash = 7 [(gogoproto.customtype) = "Hash"]; + bytes HomoHash = 7 [(gogoproto.customtype) = "Hash"]; // PayloadChecksum of actual object's payload - bytes PayloadChecksum = 8; + bytes PayloadChecksum = 8; // Integrity header with checksum of all above headers in the object - IntegrityHeader Integrity = 9; + IntegrityHeader Integrity = 9; // StorageGroup contains meta information for the data audit - StorageGroup StorageGroup = 10; + storagegroup.StorageGroup StorageGroup = 10; } } @@ -124,29 +125,3 @@ message Object { // Payload is an object's payload bytes Payload = 3; } - -message StorageGroup { - // ValidationDataSize is size of the all object's payloads included into storage group - uint64 ValidationDataSize = 1; - // ValidationHash is homomorphic hash of all object's payloads included into storage group - bytes ValidationHash = 2 [(gogoproto.customtype) = "Hash", (gogoproto.nullable) = false]; - - message Lifetime { - enum Unit { - // Unlimited set if storage group always valid - Unlimited = 0; - // NeoFSEpoch set if storage group is valid until lifetime NeoFS epoch - NeoFSEpoch = 1; - // UnixTime set if storage group is valid until lifetime unix timestamp - UnixTime = 2; - } - - // Unit is lifetime type - Unit unit = 1 [(gogoproto.customname) = "Unit"]; - // Value for lifetime - int64 Value = 2; - } - - // Lifetime is time until storage group is valid - Lifetime lifetime = 3 [(gogoproto.customname) = "Lifetime"]; -} diff --git a/storagegroup/storage.go b/storagegroup/storage.go new file mode 100644 index 00000000..a23ef744 --- /dev/null +++ b/storagegroup/storage.go @@ -0,0 +1,39 @@ +package storagegroup + +import ( + "context" + + "github.com/nspcc-dev/neofs-proto/refs" +) + +type ( + // Store is a interface for storing users storage group + Store interface { + Lister + Receiver + Receptacle + } + + // Lister defines list function that returns all storage groups + // created for the passed container + Lister interface { + List(ctx context.Context, cid refs.CID) ([]refs.SGID, error) + } + + // Receiver defines get function that returns asked storage group + Receiver interface { + Get(ctx context.Context, cid refs.CID, sgid refs.SGID) (Provider, error) + } + + // Receptacle defines put function that places storage group in the + // store. + Receptacle interface { + Put(ctx context.Context, sg Provider) error + } + + // InfoReceiver defines GetSGInfo function that returns storage group + // that contains passed object ids. + InfoReceiver interface { + GetSGInfo(ctx context.Context, cid refs.CID, group []refs.ObjectID) (*StorageGroup, error) + } +) diff --git a/storagegroup/types.go b/storagegroup/types.go new file mode 100644 index 00000000..6c6f775f --- /dev/null +++ b/storagegroup/types.go @@ -0,0 +1,97 @@ +package storagegroup + +import ( + "bytes" + "strconv" + "strings" + + "github.com/gogo/protobuf/proto" + "github.com/mr-tron/base58" + "github.com/nspcc-dev/neofs-proto/hash" + "github.com/nspcc-dev/neofs-proto/refs" +) + +type ( + // Hash is alias of hash.Hash for proto definition. + Hash = hash.Hash + + // Provider is an interface that defines storage group instance. + // There was different storage group implementation. Right now it + // is implemented as extended header in the object. + Provider interface { + // Group returns list of object ids of the storage group. + // This list **should be** sorted. + Group() []refs.ObjectID + + IDInfo() *IdentificationInfo + Zones() []ZoneInfo + } + + // ZoneInfo provides validation information of storage group. + ZoneInfo struct { + hash.Hash + Size uint64 + } + + // IdentificationInfo provides meta information about storage group. + IdentificationInfo struct { + CID refs.CID + SGID refs.SGID + OwnerID refs.OwnerID + } + + // IDList is a slice of object ids, that can be sorted. + IDList []refs.ObjectID +) + +var _ proto.Message = (*StorageGroup)(nil) + +// String returns string representation of StorageGroup. +func (m *StorageGroup) String() string { + b := new(strings.Builder) + b.WriteString("') + return b.String() +} + +// Empty checks if storage group has some data for validation. +func (m StorageGroup) Empty() bool { + return m.ValidationDataSize == 0 && m.ValidationHash.Equal(hash.Hash{}) +} + +// Len returns amount of object ids in IDList. +func (s IDList) Len() int { return len(s) } + +// Less returns byte comparision between IDList[i] and IDList[j]. +func (s IDList) Less(i, j int) bool { return bytes.Compare(s[i].Bytes(), s[j].Bytes()) == -1 } + +// Swap swaps element with i and j index in IDList. +func (s IDList) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// CalculateSize combines length of all zones in storage group. +func CalculateSize(sg Provider) (size uint64) { + zoneList := sg.Zones() + for i := range zoneList { + size += zoneList[i].Size + } + return +} + +// CalculateHash returns homomorphic sum of hashes +// fromm all zones in storage group. +func CalculateHash(sg Provider) (hash.Hash, error) { + var ( + zones = sg.Zones() + hashes = make([]hash.Hash, len(zones)) + ) + for i := range zones { + hashes[i] = zones[i].Hash + } + return hash.Concat(hashes) +} diff --git a/storagegroup/types.pb.go b/storagegroup/types.pb.go new file mode 100644 index 00000000..f0528853 Binary files /dev/null and b/storagegroup/types.pb.go differ diff --git a/storagegroup/types.proto b/storagegroup/types.proto new file mode 100644 index 00000000..5014b962 --- /dev/null +++ b/storagegroup/types.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; +package storagegroup; +option go_package = "github.com/nspcc-dev/neofs-proto/storagegroup"; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + +option (gogoproto.stable_marshaler_all) = true; + +message StorageGroup { + option (gogoproto.goproto_stringer) = false; + + // ValidationDataSize is size of the all object's payloads included into storage group + uint64 ValidationDataSize = 1; + // ValidationHash is homomorphic hash of all object's payloads included into storage group + bytes ValidationHash = 2 [(gogoproto.customtype) = "Hash", (gogoproto.nullable) = false]; + + message Lifetime { + enum Unit { + // Unlimited set if storage group always valid + Unlimited = 0; + // NeoFSEpoch set if storage group is valid until lifetime NeoFS epoch + NeoFSEpoch = 1; + // UnixTime set if storage group is valid until lifetime unix timestamp + UnixTime = 2; + } + + // Unit is lifetime type + Unit unit = 1 [(gogoproto.customname) = "Unit"]; + // Value for lifetime + int64 Value = 2; + } + + // Lifetime is time until storage group is valid + Lifetime lifetime = 3 [(gogoproto.customname) = "Lifetime"]; +}