Merge branch 'release/0.2.4'

CHANGELOG.md

@@ -1,6 +1,15 @@
 # Changelog
 This is the changelog for NeoFS Proto
 
+## [0.2.4] - 2019-12-03
+
+### Added
+- StorageGroup library
+
+### Changed
+- Storage group part of object library moved into separate package
+- Updated proto documentation
+
 ## [0.2.3] - 2019-11-28
 
 ### Removed
@@ -50,3 +59,4 @@ Initial public release
 [0.2.1]: https://github.com/nspcc-dev/neofs-proto/compare/v0.2.0...v0.2.1
 [0.2.2]: https://github.com/nspcc-dev/neofs-proto/compare/v0.2.1...v0.2.2
 [0.2.3]: https://github.com/nspcc-dev/neofs-proto/compare/v0.2.2...v0.2.3
+[0.2.4]: https://github.com/nspcc-dev/neofs-proto/compare/v0.2.3...v0.2.4

docs/object.md

@@ -34,8 +34,6 @@
     - [Link](#object.Link)
     - [Object](#object.Object)
     - [Range](#object.Range)
-    - [StorageGroup](#object.StorageGroup)
-    - [StorageGroup.Lifetime](#object.StorageGroup.Lifetime)
     - [SystemHeader](#object.SystemHeader)
     - [Tombstone](#object.Tombstone)
     - [Transform](#object.Transform)
@@ -369,7 +367,7 @@ in distributed system.
 | HomoHash | [bytes](#bytes) |  | HomoHash is a homomorphic hash of original object payload |
 | PayloadChecksum | [bytes](#bytes) |  | PayloadChecksum of actual object's payload |
 | Integrity | [IntegrityHeader](#object.IntegrityHeader) |  | Integrity header with checksum of all above headers in the object |
-| StorageGroup | [StorageGroup](#object.StorageGroup) |  | StorageGroup contains meta information for the data audit |
+| StorageGroup | [storagegroup.StorageGroup](#storagegroup.StorageGroup) |  | StorageGroup contains meta information for the data audit |
 
 
 <a name="object.IntegrityHeader"></a>
@@ -421,31 +419,6 @@ in distributed system.
 | Length | [uint64](#uint64) |  | Length of the data range |
 
 
-<a name="object.StorageGroup"></a>
-
-### Message StorageGroup
-
-
-
-| Field | Type | Label | Description |
-| ----- | ---- | ----- | ----------- |
-| ValidationDataSize | [uint64](#uint64) |  | ValidationDataSize is size of the all object's payloads included into storage group |
-| ValidationHash | [bytes](#bytes) |  | ValidationHash is homomorphic hash of all object's payloads included into storage group |
-| lifetime | [StorageGroup.Lifetime](#object.StorageGroup.Lifetime) |  | Lifetime is time until storage group is valid |
-
-
-<a name="object.StorageGroup.Lifetime"></a>
-
-### Message StorageGroup.Lifetime
-
-
-
-| Field | Type | Label | Description |
-| ----- | ---- | ----- | ----------- |
-| unit | [StorageGroup.Lifetime.Unit](#object.StorageGroup.Lifetime.Unit) |  | Unit is lifetime type |
-| Value | [int64](#int64) |  | Value for lifetime |
-
-
 <a name="object.SystemHeader"></a>
 
 ### Message SystemHeader
@@ -514,19 +487,6 @@ in distributed system.
 
 
 
-<a name="object.StorageGroup.Lifetime.Unit"></a>
-
-### StorageGroup.Lifetime.Unit
-
-
-| Name | Number | Description |
-| ---- | ------ | ----------- |
-| Unlimited | 0 | Unlimited set if storage group always valid |
-| NeoFSEpoch | 1 | NeoFSEpoch set if storage group is valid until lifetime NeoFS epoch |
-| UnixTime | 2 | UnixTime set if storage group is valid until lifetime unix timestamp |
-
-
-
 <a name="object.Transform.Type"></a>
 
 ### Transform.Type

docs/storagegroup.md

@@ -0,0 +1,88 @@
+# Protocol Documentation
+<a name="top"></a>
+
+## Table of Contents
+
+- [storagegroup/types.proto](#storagegroup/types.proto)
+
+  - Messages
+    - [StorageGroup](#storagegroup.StorageGroup)
+    - [StorageGroup.Lifetime](#storagegroup.StorageGroup.Lifetime)
+    
+
+- [Scalar Value Types](#scalar-value-types)
+
+
+
+<a name="storagegroup/types.proto"></a>
+<p align="right"><a href="#top">Top</a></p>
+
+## storagegroup/types.proto
+
+
+ <!-- end services -->
+
+
+<a name="storagegroup.StorageGroup"></a>
+
+### Message StorageGroup
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| ValidationDataSize | [uint64](#uint64) |  | ValidationDataSize is size of the all object's payloads included into storage group |
+| ValidationHash | [bytes](#bytes) |  | ValidationHash is homomorphic hash of all object's payloads included into storage group |
+| lifetime | [StorageGroup.Lifetime](#storagegroup.StorageGroup.Lifetime) |  | Lifetime is time until storage group is valid |
+
+
+<a name="storagegroup.StorageGroup.Lifetime"></a>
+
+### Message StorageGroup.Lifetime
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| unit | [StorageGroup.Lifetime.Unit](#storagegroup.StorageGroup.Lifetime.Unit) |  | Unit is lifetime type |
+| Value | [int64](#int64) |  | Value for lifetime |
+
+ <!-- end messages -->
+
+
+<a name="storagegroup.StorageGroup.Lifetime.Unit"></a>
+
+### StorageGroup.Lifetime.Unit
+
+
+| Name | Number | Description |
+| ---- | ------ | ----------- |
+| Unlimited | 0 | Unlimited set if storage group always valid |
+| NeoFSEpoch | 1 | NeoFSEpoch set if storage group is valid until lifetime NeoFS epoch |
+| UnixTime | 2 | UnixTime set if storage group is valid until lifetime unix timestamp |
+
+
+ <!-- end enums -->
+
+
+
+## Scalar Value Types
+
+| .proto Type | Notes | C++ Type | Java Type | Python Type |
+| ----------- | ----- | -------- | --------- | ----------- |
+| <a name="double" /> double |  | double | double | float |
+| <a name="float" /> float |  | float | float | float |
+| <a name="int32" /> int32 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint32 instead. | int32 | int | int |
+| <a name="int64" /> int64 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint64 instead. | int64 | long | int/long |
+| <a name="uint32" /> uint32 | Uses variable-length encoding. | uint32 | int | int/long |
+| <a name="uint64" /> uint64 | Uses variable-length encoding. | uint64 | long | int/long |
+| <a name="sint32" /> sint32 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int32s. | int32 | int | int |
+| <a name="sint64" /> sint64 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int64s. | int64 | long | int/long |
+| <a name="fixed32" /> fixed32 | Always four bytes. More efficient than uint32 if values are often greater than 2^28. | uint32 | int | int |
+| <a name="fixed64" /> fixed64 | Always eight bytes. More efficient than uint64 if values are often greater than 2^56. | uint64 | long | int/long |
+| <a name="sfixed32" /> sfixed32 | Always four bytes. | int32 | int | int |
+| <a name="sfixed64" /> sfixed64 | Always eight bytes. | int64 | long | int/long |
+| <a name="bool" /> bool |  | bool | boolean | boolean |
+| <a name="string" /> string | A string must always contain UTF-8 encoded or 7-bit ASCII text. | string | String | str/unicode |
+| <a name="bytes" /> bytes | May contain any arbitrary sequence of bytes. | string | ByteString | str |
+

object/extensions.go

@@ -1,8 +1,6 @@
 package object
 
-import (
+import "github.com/nspcc-dev/neofs-proto/storagegroup"
-	"github.com/nspcc-dev/neofs-proto/hash"
-)
 
 // IsLinking checks if object has children links to another objects.
 // We have to check payload size because zero-object must have zero
@@ -64,7 +62,7 @@ func (m Object) IsTombstone() bool {
 }
 
 // StorageGroup returns storage group structure if it is presented in extended headers.
-func (m Object) StorageGroup() (*StorageGroup, error) {
+func (m Object) StorageGroup() (*storagegroup.StorageGroup, error) {
 	_, sgHdr := m.LastHeader(HeaderType(StorageGroupHdr))
 	if sgHdr == nil {
 		return nil, ErrHeaderNotFound
@@ -74,11 +72,6 @@ func (m Object) StorageGroup() (*StorageGroup, error) {
 
 // SetStorageGroup sets storage group header in the object.
 // It will replace existing storage group header or add a new one.
-func (m *Object) SetStorageGroup(sg *StorageGroup) {
+func (m *Object) SetStorageGroup(group *storagegroup.StorageGroup) {
-	m.SetHeader(&Header{Value: &Header_StorageGroup{StorageGroup: sg}})
+	m.SetHeader(&Header{Value: &Header_StorageGroup{StorageGroup: group}})
-}
-
-// Empty checks if storage group has some data for validation.
-func (m StorageGroup) Empty() bool {
-	return m.ValidationDataSize == 0 && m.ValidationHash.Equal(hash.Hash{})
 }

object/sg.go

@@ -1,54 +1,31 @@
 package object
 
 import (
-	"bytes"
 	"sort"
+
+	"github.com/nspcc-dev/neofs-proto/refs"
+	"github.com/nspcc-dev/neofs-proto/storagegroup"
 )
 
 // Here are defined getter functions for objects that contain storage group
 // information.
 
-type (
+var _ storagegroup.Provider = (*Object)(nil)
-	// IDList is a slice of object ids, that can be sorted.
-	IDList []ID
-
-	// ZoneInfo provides validation info of storage group.
-	ZoneInfo struct {
-		Hash
-		Size uint64
-	}
-
-	// IdentificationInfo provides meta information about storage group.
-	IdentificationInfo struct {
-		SGID
-		CID
-		OwnerID
-	}
-)
-
-// Len returns amount of object ids in IDList.
-func (s IDList) Len() int { return len(s) }
-
-// Less returns byte comparision between IDList[i] and IDList[j].
-func (s IDList) Less(i, j int) bool { return bytes.Compare(s[i].Bytes(), s[j].Bytes()) == -1 }
-
-// Swap swaps element with i and j index in IDList.
-func (s IDList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
 
 // Group returns slice of object ids that are part of a storage group.
-func (m *Object) Group() []ID {
+func (m *Object) Group() []refs.ObjectID {
 	sgLinks := m.Links(Link_StorageGroup)
-	sort.Sort(IDList(sgLinks))
+	sort.Sort(storagegroup.IDList(sgLinks))
 	return sgLinks
 }
 
 // Zones returns validation zones of storage group.
-func (m *Object) Zones() []ZoneInfo {
+func (m *Object) Zones() []storagegroup.ZoneInfo {
 	sgInfo, err := m.StorageGroup()
 	if err != nil {
 		return nil
 	}
-	return []ZoneInfo{
+	return []storagegroup.ZoneInfo{
 		{
 			Hash: sgInfo.ValidationHash,
 			Size: sgInfo.ValidationDataSize,
@@ -57,8 +34,8 @@ func (m *Object) Zones() []ZoneInfo {
 }
 
 // IDInfo returns meta information about storage group.
-func (m *Object) IDInfo() *IdentificationInfo {
+func (m *Object) IDInfo() *storagegroup.IdentificationInfo {
-	return &IdentificationInfo{
+	return &storagegroup.IdentificationInfo{
 		SGID:    m.SystemHeader.ID,
 		CID:     m.SystemHeader.CID,
 		OwnerID: m.SystemHeader.OwnerID,

object/sg_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/nspcc-dev/neofs-proto/hash"
+	"github.com/nspcc-dev/neofs-proto/storagegroup"
 	"github.com/stretchr/testify/require"
 )
 
@@ -28,7 +29,7 @@ func TestObject_StorageGroup(t *testing.T) {
 		}
 
 		rand.Shuffle(len(obj.Headers), func(i, j int) { obj.Headers[i], obj.Headers[j] = obj.Headers[j], obj.Headers[i] })
-		sort.Sort(IDList(idList))
+		sort.Sort(storagegroup.IDList(idList))
 		require.Equal(t, idList, obj.Group())
 	})
 	t.Run("identification method", func(t *testing.T) {
@@ -58,7 +59,7 @@ func TestObject_StorageGroup(t *testing.T) {
 			Headers: []Header{
 				{
 					Value: &Header_StorageGroup{
-						StorageGroup: &StorageGroup{
+						StorageGroup: &storagegroup.StorageGroup{
 							ValidationDataSize: sgSize,
 							ValidationHash:     sgHash,
 						},

object/types.proto

@@ -4,6 +4,7 @@ option go_package = "github.com/nspcc-dev/neofs-proto/object";
 
 import "refs/types.proto";
 import "session/types.proto";
+import "storagegroup/types.proto";
 import "github.com/gogo/protobuf/gogoproto/gogo.proto";
 
 option (gogoproto.stable_marshaler_all) = true;
@@ -25,25 +26,25 @@ message UserHeader {
 message Header {
     oneof Value {
         // Link to other objects
-        Link Link                         = 1;
+        Link Link                              = 1;
         // Redirect not used yet
-        refs.Address Redirect             = 2;
+        refs.Address Redirect                  = 2;
         // UserHeader is a set of KV headers defined by user
-        UserHeader UserHeader             = 3;
+        UserHeader UserHeader                  = 3;
         // Transform defines transform operation (e.g. payload split)
-        Transform Transform               = 4;
+        Transform Transform                    = 4;
         // Tombstone header that set up in deleted objects
-        Tombstone Tombstone               = 5;
+        Tombstone Tombstone                    = 5;
         // Verify header that contains session public key and user's signature
-        session.VerificationHeader Verify = 6;
+        session.VerificationHeader Verify      = 6;
         // HomoHash is a homomorphic hash of original object payload
-        bytes HomoHash                    = 7 [(gogoproto.customtype) = "Hash"];
+        bytes HomoHash                         = 7 [(gogoproto.customtype) = "Hash"];
         // PayloadChecksum of actual object's payload
-        bytes PayloadChecksum             = 8;
+        bytes PayloadChecksum                  = 8;
         // Integrity header with checksum of all above headers in the object
-        IntegrityHeader Integrity         = 9;
+        IntegrityHeader Integrity              = 9;
         // StorageGroup contains meta information for the data audit
-        StorageGroup StorageGroup         = 10;
+        storagegroup.StorageGroup StorageGroup = 10;
     }
 }
 
@@ -124,29 +125,3 @@ message Object {
     // Payload is an object's payload
     bytes Payload             = 3;
 }
-
-message StorageGroup {
-    // ValidationDataSize is size of the all object's payloads included into storage group
-    uint64 ValidationDataSize = 1;
-    // ValidationHash is homomorphic hash of all object's payloads included into storage group
-    bytes ValidationHash = 2 [(gogoproto.customtype) = "Hash", (gogoproto.nullable) = false];
-
-    message Lifetime {
-        enum Unit {
-            // Unlimited set if storage group always valid
-            Unlimited = 0;
-            // NeoFSEpoch set if storage group is valid until lifetime NeoFS epoch
-            NeoFSEpoch = 1;
-            // UnixTime set if storage group is valid until lifetime unix timestamp
-            UnixTime = 2;
-        }
-
-        // Unit is lifetime type
-        Unit unit = 1 [(gogoproto.customname) = "Unit"];
-        // Value for lifetime
-        int64 Value = 2;
-    }
-
-    // Lifetime is time until storage group is valid
-    Lifetime lifetime = 3 [(gogoproto.customname) = "Lifetime"];
-}

storagegroup/storage.go

@@ -0,0 +1,39 @@
+package storagegroup
+
+import (
+	"context"
+
+	"github.com/nspcc-dev/neofs-proto/refs"
+)
+
+type (
+	// Store is a interface for storing users storage group
+	Store interface {
+		Lister
+		Receiver
+		Receptacle
+	}
+
+	// Lister defines list function that returns all storage groups
+	// created for the passed container
+	Lister interface {
+		List(ctx context.Context, cid refs.CID) ([]refs.SGID, error)
+	}
+
+	// Receiver defines get function that returns asked storage group
+	Receiver interface {
+		Get(ctx context.Context, cid refs.CID, sgid refs.SGID) (Provider, error)
+	}
+
+	// Receptacle defines put function that places storage group in the
+	// store.
+	Receptacle interface {
+		Put(ctx context.Context, sg Provider) error
+	}
+
+	// InfoReceiver defines GetSGInfo function that returns storage group
+	// that contains passed object ids.
+	InfoReceiver interface {
+		GetSGInfo(ctx context.Context, cid refs.CID, group []refs.ObjectID) (*StorageGroup, error)
+	}
+)

storagegroup/types.go

@@ -0,0 +1,97 @@
+package storagegroup
+
+import (
+	"bytes"
+	"strconv"
+	"strings"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/mr-tron/base58"
+	"github.com/nspcc-dev/neofs-proto/hash"
+	"github.com/nspcc-dev/neofs-proto/refs"
+)
+
+type (
+	// Hash is alias of hash.Hash for proto definition.
+	Hash = hash.Hash
+
+	// Provider is an interface that defines storage group instance.
+	// There was different storage group implementation. Right now it
+	// is implemented as extended header in the object.
+	Provider interface {
+		// Group returns list of object ids of the storage group.
+		// This list **should be** sorted.
+		Group() []refs.ObjectID
+
+		IDInfo() *IdentificationInfo
+		Zones() []ZoneInfo
+	}
+
+	// ZoneInfo provides validation information of storage group.
+	ZoneInfo struct {
+		hash.Hash
+		Size uint64
+	}
+
+	// IdentificationInfo provides meta information about storage group.
+	IdentificationInfo struct {
+		CID     refs.CID
+		SGID    refs.SGID
+		OwnerID refs.OwnerID
+	}
+
+	// IDList is a slice of object ids, that can be sorted.
+	IDList []refs.ObjectID
+)
+
+var _ proto.Message = (*StorageGroup)(nil)
+
+// String returns string representation of StorageGroup.
+func (m *StorageGroup) String() string {
+	b := new(strings.Builder)
+	b.WriteString("<SG")
+	b.WriteString(" VDS=" + strconv.FormatUint(m.ValidationDataSize, 10))
+	data := base58.Encode(m.ValidationHash.Bytes())
+	b.WriteString(" Hash=" + data[:3] + "..." + data[len(data)-3:])
+	if m.Lifetime != nil {
+		b.WriteString(" Lifetime=(" + m.Lifetime.Unit.String() + " " + strconv.FormatInt(m.Lifetime.Value, 10) + ")")
+	}
+	b.WriteByte('>')
+	return b.String()
+}
+
+// Empty checks if storage group has some data for validation.
+func (m StorageGroup) Empty() bool {
+	return m.ValidationDataSize == 0 && m.ValidationHash.Equal(hash.Hash{})
+}
+
+// Len returns amount of object ids in IDList.
+func (s IDList) Len() int { return len(s) }
+
+// Less returns byte comparision between IDList[i] and IDList[j].
+func (s IDList) Less(i, j int) bool { return bytes.Compare(s[i].Bytes(), s[j].Bytes()) == -1 }
+
+// Swap swaps element with i and j index in IDList.
+func (s IDList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+// CalculateSize combines length of all zones in storage group.
+func CalculateSize(sg Provider) (size uint64) {
+	zoneList := sg.Zones()
+	for i := range zoneList {
+		size += zoneList[i].Size
+	}
+	return
+}
+
+// CalculateHash returns homomorphic sum of hashes
+// fromm all zones in storage group.
+func CalculateHash(sg Provider) (hash.Hash, error) {
+	var (
+		zones  = sg.Zones()
+		hashes = make([]hash.Hash, len(zones))
+	)
+	for i := range zones {
+		hashes[i] = zones[i].Hash
+	}
+	return hash.Concat(hashes)
+}

storagegroup/types.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+package storagegroup;
+option go_package = "github.com/nspcc-dev/neofs-proto/storagegroup";
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.stable_marshaler_all) = true;
+
+message StorageGroup {
+    option (gogoproto.goproto_stringer) = false;
+
+    // ValidationDataSize is size of the all object's payloads included into storage group
+    uint64 ValidationDataSize = 1;
+    // ValidationHash is homomorphic hash of all object's payloads included into storage group
+    bytes ValidationHash = 2 [(gogoproto.customtype) = "Hash", (gogoproto.nullable) = false];
+
+    message Lifetime {
+        enum Unit {
+            // Unlimited set if storage group always valid
+            Unlimited = 0;
+            // NeoFSEpoch set if storage group is valid until lifetime NeoFS epoch
+            NeoFSEpoch = 1;
+            // UnixTime set if storage group is valid until lifetime unix timestamp
+            UnixTime = 2;
+        }
+
+        // Unit is lifetime type
+        Unit unit = 1 [(gogoproto.customname) = "Unit"];
+        // Value for lifetime
+        int64 Value = 2;
+    }
+
+    // Lifetime is time until storage group is valid
+    Lifetime lifetime = 3 [(gogoproto.customname) = "Lifetime"];
+}