From c8054c5daa5c3cde462792264f4e960b27ea9415 Mon Sep 17 00:00:00 2001 From: Stanislav Bogatyrev Date: Sun, 23 Aug 2020 14:42:01 +0300 Subject: [PATCH] [#58] netmap: Simplify Placement Policy Proposed protobuf format for Placement Policy simplifies direct editing in visual editors and import/export from other formats like JSON. Signed-off-by: Stanislav Bogatyrev --- netmap/types.proto | 168 +++++++++++++++++++++++-------------------- object/service.proto | 6 +- proto-docs/netmap.md | 93 ++++++++++-------------- 3 files changed, 129 insertions(+), 138 deletions(-) diff --git a/netmap/types.proto b/netmap/types.proto index ae424c5..40bad0b 100644 --- a/netmap/types.proto +++ b/netmap/types.proto @@ -5,89 +5,95 @@ package neo.fs.v2.netmap; option go_package = "github.com/nspcc-dev/neofs-api-go/v2/netmap/grpc;netmap"; option csharp_namespace = "NeoFS.API.v2.Netmap"; +// Operations on filters +enum Operation { + // No Operation defined + OPERATION_UNSPECIFIED = 0; + + // Equal + EQ = 1; + + // Not Equal + NE = 2; + + // Greater then + GT = 3; + + // Greater or equal + GE = 4; + + // Less then + LT = 5; + + // Less or equal + LE = 6; + + // Logical OR + OR = 7; + + // Logical AND + AND = 8; +} + +// Filter +message Filter { + // Name of the filter or a reference to the named filter. + // '*' means application to the whole unfiltered NetworkMap + // At top level it's used as a filter name. At lower levels it's considered to + // be a reference to another named filter + string name = 1; + + // Key to filter + string key = 2; + + // Filtering operation + Operation op = 3; + + // Value to match + string value = 4; + + // List of inner filters. Top level operation will be applied to the whole list. + repeated Filter filters = 5; +} + +// Selector +message Selector { + // Selector name to reference in object placement section + string name = 1; + + // How many nodes to select from bucket + uint32 count = 2; + + // Attribute bucket to select from + string attribute = 3; + + // Filter reference to select from + string filter = 4; +} + +// Exact bucket for each replica +message Replica { + // How many object replicas to put + uint32 count = 1; + + // Named selector bucket to put in + string selector = 2; +} + // Set of rules to select a subset of nodes able to store container's objects message PlacementPolicy { - // Replication factor - uint32 repl_factor = 1; + // Rules to set number of object replicas and place each one into a particular bucket + repeated Replica replicas = 1; - // Filters to apply to Network Map - message FilterGroup { - // Filter definition - message Filter { - // Filter identifier - string key = 1; + // Container backup factor controls how deep NeoFS will search for nodes + // alternatives to include into container. + uint32 container_backup_factor = 2; - // Minimal simple filter - message SimpleFilter { - // Filtering operation - enum Operation { - // No Operation defined - OPERATION_UNSPECIFIED= 0; + // Set of Selectors to form the container's nodes subset + repeated Selector selectors = 3; - // Equal - EQ = 1; - - // Not Equal - NE = 2; - - // Greater then - GT = 3; - - // Greater or equal - GE = 4; - - // Less then - LT = 5; - - // Less or equal - LE = 6; - - // Logical OR - OR = 7; - - // Logical AND - AND = 8; - } - // Filtering operation - Operation op = 1; - - // List of filters - message SimpleFilters { - // List of filters - repeated SimpleFilter filters = 1; - } - - // Filtering operation argument - oneof args { - // Value - string value = 2; - // Result of other filter application - SimpleFilters f_args = 3; - } - } - // The rest of filter - SimpleFilter f = 2; - } - - // Resulting filter list - repeated Filter filters = 1; - - // Selector - message Selector { - // How many to select - uint32 count = 1; - // Key to select - string key = 2; - } - - // List of selectors - repeated Selector selectors = 2; - - // Parts of graph to exclude. Internal use. - repeated uint32 exclude = 3; - } - // List of filter groups - repeated FilterGroup filter_groups = 2; + // List of named filters to reference in selectors + repeated Filter filters = 4; } // NeoFS node description @@ -105,6 +111,10 @@ message NodeInfo { // Value of the node attribute. string value = 2; + + // Parent keys, if any + // Example: For City it can be Region or Country + repeated string parents = 3; } // Carries list of the NeoFS node attributes in a string key-value format. repeated Attribute attributes = 3; @@ -112,7 +122,7 @@ message NodeInfo { // Represents the enumeration of various states of the NeoFS node. enum State { // Unknown state. - UNSPECIFIED = 0; + UNSPECIFIED = 0; // Active state in the network. ONLINE = 1; diff --git a/object/service.proto b/object/service.proto index 15ce27f..e14b0c8 100644 --- a/object/service.proto +++ b/object/service.proto @@ -85,7 +85,7 @@ message GetResponse { neo.fs.v2.refs.ObjectID object_id = 1; // Object signature - neo.fs.v2.refs.Signature signature =2; + neo.fs.v2.refs.Signature signature = 2; // Object header. Header header = 3; @@ -93,7 +93,7 @@ message GetResponse { // Carries the single message of the response stream. oneof object_part { // Initialization parameters of the object stream. - Init init =1; + Init init = 1; // Part of the object payload. bytes chunk = 2; @@ -122,7 +122,7 @@ message PutRequest { neo.fs.v2.refs.ObjectID object_id = 1; // Object signature, were available - neo.fs.v2.refs.Signature signature =2; + neo.fs.v2.refs.Signature signature = 2; // Header of the object to save in the system. Header header = 3; diff --git a/proto-docs/netmap.md b/proto-docs/netmap.md index 152e507..5d1059d 100644 --- a/proto-docs/netmap.md +++ b/proto-docs/netmap.md @@ -6,14 +6,12 @@ - [netmap/types.proto](#netmap/types.proto) - Messages + - [Filter](#neo.fs.v2.netmap.Filter) - [NodeInfo](#neo.fs.v2.netmap.NodeInfo) - [NodeInfo.Attribute](#neo.fs.v2.netmap.NodeInfo.Attribute) - [PlacementPolicy](#neo.fs.v2.netmap.PlacementPolicy) - - [PlacementPolicy.FilterGroup](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup) - - [PlacementPolicy.FilterGroup.Filter](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Filter) - - [PlacementPolicy.FilterGroup.Filter.SimpleFilter](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Filter.SimpleFilter) - - [PlacementPolicy.FilterGroup.Filter.SimpleFilter.SimpleFilters](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Filter.SimpleFilter.SimpleFilters) - - [PlacementPolicy.FilterGroup.Selector](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Selector) + - [Replica](#neo.fs.v2.netmap.Replica) + - [Selector](#neo.fs.v2.netmap.Selector) - [Scalar Value Types](#scalar-value-types) @@ -29,6 +27,21 @@ + + +### Message Filter +Filter + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| name | [string](#string) | | Name of the filter or a reference to the named filter. '*' means application to the whole unfiltered NetworkMap At top level it's used as a filter name. At lower levels it's considered to be a reference to another named filter | +| key | [string](#string) | | Key to filter | +| op | [Operation](#neo.fs.v2.netmap.Operation) | | Filtering operation | +| value | [string](#string) | | Value to match | +| filters | [Filter](#neo.fs.v2.netmap.Filter) | repeated | List of inner filters. Top level operation will be applied to the whole list. | + + ### Message NodeInfo @@ -53,6 +66,7 @@ Attributes of the NeoFS node. | ----- | ---- | ----- | ----------- | | key | [string](#string) | | Key of the node attribute. | | value | [string](#string) | | Value of the node attribute. | +| parents | [string](#string) | repeated | Parent keys, if any Example: For City it can be Region or Country | @@ -63,69 +77,36 @@ Set of rules to select a subset of nodes able to store container's objects | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| repl_factor | [uint32](#uint32) | | Replication factor | -| filter_groups | [PlacementPolicy.FilterGroup](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup) | repeated | List of filter groups | +| replicas | [Replica](#neo.fs.v2.netmap.Replica) | repeated | Rules to set number of object replicas and place each one into a particular bucket | +| container_backup_factor | [uint32](#uint32) | | Container backup factor controls how deep NeoFS will search for nodes alternatives to include into container. | +| selectors | [Selector](#neo.fs.v2.netmap.Selector) | repeated | Set of Selectors to form the container's nodes subset | +| filters | [Filter](#neo.fs.v2.netmap.Filter) | repeated | List of named filters to reference in selectors | - + -### Message PlacementPolicy.FilterGroup -Filters to apply to Network Map +### Message Replica +Exact bucket for each replica | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| filters | [PlacementPolicy.FilterGroup.Filter](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Filter) | repeated | Resulting filter list | -| selectors | [PlacementPolicy.FilterGroup.Selector](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Selector) | repeated | List of selectors | -| exclude | [uint32](#uint32) | repeated | Parts of graph to exclude. Internal use. | +| count | [uint32](#uint32) | | How many object replicas to put | +| selector | [string](#string) | | Named selector bucket to put in | - + -### Message PlacementPolicy.FilterGroup.Filter -Filter definition - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| key | [string](#string) | | Filter identifier | -| f | [PlacementPolicy.FilterGroup.Filter.SimpleFilter](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Filter.SimpleFilter) | | The rest of filter | - - - - -### Message PlacementPolicy.FilterGroup.Filter.SimpleFilter -Minimal simple filter - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| op | [PlacementPolicy.FilterGroup.Filter.SimpleFilter.Operation](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Filter.SimpleFilter.Operation) | | Filtering operation | -| value | [string](#string) | | Value | -| f_args | [PlacementPolicy.FilterGroup.Filter.SimpleFilter.SimpleFilters](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Filter.SimpleFilter.SimpleFilters) | | Result of other filter application | - - - - -### Message PlacementPolicy.FilterGroup.Filter.SimpleFilter.SimpleFilters -List of filters - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| filters | [PlacementPolicy.FilterGroup.Filter.SimpleFilter](#neo.fs.v2.netmap.PlacementPolicy.FilterGroup.Filter.SimpleFilter) | repeated | List of filters | - - - - -### Message PlacementPolicy.FilterGroup.Selector +### Message Selector Selector | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| count | [uint32](#uint32) | | How many to select | -| key | [string](#string) | | Key to select | +| name | [string](#string) | | Selector name to reference in object placement section | +| count | [uint32](#uint32) | | How many nodes to select from bucket | +| attribute | [string](#string) | | Attribute bucket to select from | +| filter | [string](#string) | | Filter reference to select from | @@ -143,10 +124,10 @@ Represents the enumeration of various states of the NeoFS node. - + -### PlacementPolicy.FilterGroup.Filter.SimpleFilter.Operation -Filtering operation +### Operation +Operations on filters | Name | Number | Description | | ---- | ------ | ----------- |