From 818ec7f0dc9436c3c689ce59df8e5cb0491c4536 Mon Sep 17 00:00:00 2001 From: Stanislav Bogatyrev Date: Tue, 11 Aug 2020 13:54:58 +0300 Subject: [PATCH] [#38] Change ObjectID format from UUID to Hash NeoFS Object are now Content-addressed. It means the Object's address depends on it's content. ObjectID is now calculated as hash of Header, which contains a hash of payload. If either if Object's payload of Headers change, the ID will also change. Signed-off-by: Stanislav Bogatyrev --- acl/types.proto | 3 +- container/types.proto | 2 +- object/service.proto | 72 ++++++++++------- object/types.proto | 182 ++++++++++++++++-------------------------- refs/types.proto | 5 +- service/meta.proto | 8 ++ service/verify.proto | 15 ++-- 7 files changed, 130 insertions(+), 157 deletions(-) diff --git a/acl/types.proto b/acl/types.proto index cd673b4..31731fd 100644 --- a/acl/types.proto +++ b/acl/types.proto @@ -56,8 +56,7 @@ message EACLRecord { enum Header { HEADER_UNKNOWN = 0; REQUEST = 1; - OBJECT_SYSTEM = 2; - OBJECT_USER = 3; + OBJECT = 2; } // Header carries type of header. diff --git a/container/types.proto b/container/types.proto index 1b6041d..572c141 100644 --- a/container/types.proto +++ b/container/types.proto @@ -5,8 +5,8 @@ package container; option go_package = "github.com/nspcc-dev/neofs-api-go/container"; option csharp_namespace = "NeoFS.API.Container"; -import "refs/types.proto"; import "netmap/types.proto"; +import "refs/types.proto"; // Container is a structure that defines object placement behaviour. Objects // can be stored only within containers. They define placement rule, attributes diff --git a/object/service.proto b/object/service.proto index 613b61d..88c3805 100644 --- a/object/service.proto +++ b/object/service.proto @@ -54,7 +54,7 @@ service Service { message GetRequest { message Body { - // Carries the address of the requested object. + // Address of the requested object. refs.Address address = 1; // Carries the raw option flag of the request. @@ -62,7 +62,6 @@ message GetRequest { // that are physically stored on the server. bool raw = 2; } - // Body of get object request message. Body body = 1; @@ -77,16 +76,23 @@ message GetRequest { message GetResponse { message Body { + // Initialization parameters of the object got from NeoFS. + message Init { + // Object ID + refs.ObjectID object_id = 1; + // Object signature + service.Signature signature =2; + // Object header. + Header header = 3; + } // Carries the single message of the response stream. oneof ObjectPart { - // Carries the object header. - Header header = 1; - - // Carries part of the object payload. + // Initialization parameters of the object stream. + Init init =1; + // Part of the object payload. bytes chunk = 2; } } - // Body of get object response message. Body body = 1; @@ -104,25 +110,26 @@ message PutRequest { message Body { // Groups initialization parameters of object placement in NeoFS. message Init { - // Carries the header of the object to save in the system. - Header header = 1; - - // Carries the number of the object copies to store - // within the RPC call. Default zero value is processed according - // to the container placement rules. - uint32 copies_number = 2; + // Object ID, where available + refs.ObjectID object_id = 1; + // Object signature, were available + service.Signature signature =2; + // Header of the object to save in the system. + Header header = 3; + // Number of the object copies to store within the RPC call. + // Default zero value is processed according to the + // container placement rules. + uint32 copies_number = 4; } // Carries the single part of the query stream. - oneof Part { + oneof ObjectPart { // Carries the initialization parameters of the object stream. Init init = 1; - // Carries part of the object payload. bytes chunk = 2; } } - // Body of put object request message. Body body = 1; @@ -141,7 +148,6 @@ message PutResponse { // It is used to access an object in the container. refs.ObjectID object_id = 1; } - // Body of put object response message. Body body = 1; @@ -159,11 +165,9 @@ message DeleteRequest { message Body { // Carries the address of the object to be deleted. refs.Address address = 1; - // Carries identifier the object owner. refs.OwnerID owner_id = 2; } - // Body of delete object request message. Body body = 1; @@ -196,18 +200,15 @@ message DeleteResponse { message HeadRequest { message Body { - // Carries the address of the object with the requested header. + // Address of the object with the requested header. refs.Address address = 1; - - // Carries the option to crop header to main part. + // Return only minimal header subset bool main_only = 2; - // Carries the raw option flag of the request. // Raw request is sent to receive only the headers of the objects // that are physically stored on the server. bool raw = 3; } - // Body of head object request message. Body body = 1; @@ -222,10 +223,25 @@ message HeadRequest { message HeadResponse { message Body { - // Carries the requested object header. - Header header = 1; + message ShortHeader { + // Object format version. + service.Version version = 1; + // Epoch when the object was created + uint64 creation_epoch = 2; + // Object's owner + refs.OwnerID owner_id = 3; + // Type of the object payload content + ObjectType object_type = 4; + // Size of payload in bytes. + // 0xFFFFFFFFFFFFFFFF means `payload_length` is unknown + uint64 payload_length = 5; + } + // Carries the requested object header or it's part + oneof Head{ + Header header = 1; + ShortHeader short_header = 2; + } } - // Body of head object response message. Body body = 1; diff --git a/object/types.proto b/object/types.proto index 45d10b3..3ec02cb 100644 --- a/object/types.proto +++ b/object/types.proto @@ -6,128 +6,80 @@ option go_package = "github.com/nspcc-dev/neofs-api-go/object"; option csharp_namespace = "NeoFS.API.Object"; import "refs/types.proto"; +import "service/meta.proto"; import "service/verify.proto"; -// Header groups the information about the NeoFS object. +// Type of the object payload content +enum ObjectType { + // Just a normal object + REGULAR = 0; + // Used internally to identify deleted objects + TOMBSTONE = 1; + // Identifies that the object holds StorageGroup information + STORAGE_GROUP = 2; +} + message Header { - // Main groups mandatory information about the object. - // Message fields are presented in all NeoFS objects. - message Main { - // payload_length carries length of the object payload. - // Each object has a fixed payload length since it's immutable. - uint64 payload_length = 1; - // address carries object address in the NeoFS system. - // It encapsulates the object and the container identifiers. - refs.Address address = 2; - // owner_id carries identifier of the object owner. - refs.OwnerID owner_id = 3; + // Object's container + refs.ContainerID container_id = 1; + // Object's owner + refs.OwnerID owner_id = 2; + // Epoch when the object was created + uint64 creation_epoch = 3; + // Object format version. + // Effectively the version of API library used to create particular object + service.Version version = 4; + // Size of payload in bytes. + // 0xFFFFFFFFFFFFFFFF means `payload_length` is unknown + uint64 payload_length = 5; + // Hash of payload bytes + bytes payload_hash = 6; + ObjectType object_type = 7; + // Homomorphic hash of the object payload. + bytes homomorphic_hash = 8; + // Session token, if it was used during Object creation. + // Need it to verify integrity and authenticity out of Request scope. + service.SessionToken session_token = 9; + + // Attribute groups the user-defined Key-Value pairs attached to the object + message Attribute { + // string key to the object attribute + string key = 1; + // string value of the object attribute + string value = 2; } + repeated Attribute attributes = 10; - // Main carries the main part of the header. - // Main MUST NOT be NULL. - Main main = 1; - - // Extended groups additional information about the object. - // It encapsulates both user and system attributes needed to regulate - // the NeoFS sub-systems. - message Extended { - // Integrity groups evidence of the integrity of an object's structure. - message Integrity { - // payload_checksum carries the checksum of object payload bytes. - // Changing any byte of the payload changes the checksum. - // It is calculated as a SHA-256 hash over payload bytes. - bytes payload_checksum = 1; - - // header_checksum carries checksum of the object header structure. - // It covers all object attributes. Changing any field of the object except - // CreatorKey and ChecksumSignature changes the checksum. - // payload_checksum and header_checksum cannot be merged due to the need - // to verify the header in the absence of a payload (e.g. in object.Head rpc). - // It is calculated as a SHA-256 hash over marshaled object header - // with cut creator_key and checksum_signature. - bytes header_checksum = 2; - - // session_token carries token of the session within which the object was created. - // If session token is presented in object, it acts as the user's proof of the - // correctness of the creator_key. - service.SessionToken session_token = 3; - - // creator_key carries public key of the object creator in a binary format. - bytes creator_key = 4; - - // checksum_signature carries signature of the structure checksum by the object creator. - bytes checksum_signature = 5; - } - - // integrity carries object integrity evidence. - Integrity integrity = 1; - - // Attribute groups the parameters of the object attributes. - message Attribute { - // key carries the string key to the object attribute. - string key = 1; - - // value carries the string value of the object attribute. - string value = 2; - } - - // attributes carries list of the object attributes in a string key-value format. - repeated Attribute attributes = 2; - - // creation_epoch carries number of NeoFS epoch on which the object was created. - uint64 creation_epoch = 3; - - // Tombstone groups the options for deleting an object. - message Tombstone { - } - - // Tombstone marks the object to be deleted. - Tombstone tombstone = 4; - - // homomorphic_hash carries homomorphic hash of the object payload. - bytes homomorphic_hash = 5; - - // StorageGroup groups meta information about a storage group. - message StorageGroup { - } - - // StorageGroup marks an object containing information about a storage group. - StorageGroup storage_group = 6; - - // Split groups information about spawning the object through a payload - // splitting. - message Split { - // Parent carries identifier of the origin object. - refs.ObjectID parent = 1; - - // Previous carries identifier of the left split neighbor. - refs.ObjectID previous = 2; - - // Next carries identifier of the right split neighbor. - refs.ObjectID next = 3; - - // Children carries list of identifiers of the objects generated by - // splitting the current. - repeated refs.ObjectID children = 4; - - // Origin carries the header of the origin object. - Header origin = 5; - } - - // Split carries the position of the object in the split hierarchy. - Split split = 7; + // Information about spawning the objects through a payload splitting. + message Split { + // Identifier of the origin object. + // Parent and children objects must be within the same container. + // Parent object_id is known only to the minor child. + refs.ObjectID parent = 1; + // Previous carries identifier of the left split neighbor. + refs.ObjectID previous = 2; + // `signature` field of the parent object. Used to reconstruct parent. + service.Signature parent_signature = 3; + // `header` field of the parent object. Used to reconstruct parent. + Header parent_header = 4; + // Children carries list of identifiers of the objects generated by splitting the current. + repeated refs.ObjectID children = 5; } - - // Extended carries the additional part of the header. - Extended extended = 2; + // Position of the object in the split hierarchy. + Split split = 11; } -// Object groups the information about the NeoFS object. -// It consists of payload data with additional service information. +// Object structure. message Object { - // Header carries the object header. - Header header = 1; - - // Payload carries the object payload bytes. - bytes payload = 2; + // Object's unique identifier. + // Object is content-addressed. It means id will change if header or payload + // changes. It's calculated as a hash of header field, which contains hash of + // object's payload + refs.ObjectID object_id = 1; + // Signed object_id + service.Signature signature = 2; + // Object metadata headers + Header header = 3; + // Payload bytes. + bytes payload = 4; } diff --git a/refs/types.proto b/refs/types.proto index e23aa4e..0b5fcff 100644 --- a/refs/types.proto +++ b/refs/types.proto @@ -9,18 +9,17 @@ option csharp_namespace = "NeoFS.API.Refs"; message Address { // container_id carries container identifier. ContainerID container_id = 1; - // object_id carries object identifier. ObjectID object_id = 2; } -// ObjectID groups information about the NeoFS object identifier. +// NeoFS object identifier. message ObjectID { // value carries the object identifier in a binary format. bytes value = 1; } -// ContainerID groups information about the NeoFS container identifier. +// NeoFS container identifier. message ContainerID { // value carries the container identifier in a binary format. bytes value = 1; diff --git a/service/meta.proto b/service/meta.proto index 4c18181..ff5ceac 100644 --- a/service/meta.proto +++ b/service/meta.proto @@ -5,6 +5,14 @@ package service; option go_package = "github.com/nspcc-dev/neofs-api-go/service"; option csharp_namespace = "NeoFS.API.Service"; +// Represents API version used by node. +message Version { + // Major API version. + uint32 major = 1; + // Minor API version. + uint32 minor = 2; +} + // RequestMetaHeader contains information about request meta headers. message RequestMetaHeader { // Carries maximum number of nodes in the request route. diff --git a/service/verify.proto b/service/verify.proto index ba11abb..4ae10a9 100644 --- a/service/verify.proto +++ b/service/verify.proto @@ -8,17 +8,16 @@ option csharp_namespace = "NeoFS.API.Service"; import "acl/types.proto"; import "refs/types.proto"; +message Signature { + // Public key used for signing. + bytes key = 1; + // Signature + bytes sign = 2; +} + // RequestVerificationHeader is a set of signatures of every NeoFS Node that // processed request. message RequestVerificationHeader { - message Signature { - // Key is compressed public key used for signature. - bytes key = 1; - - // Sign is signature of the request or session key. - bytes sign = 2; - } - // Signatures is a set of signatures of every passed NeoFS Node repeated Signature signatures = 1;