realloc/frostfs-sdk-go
1
0
Fork 0
forked from TrueCloudLab/frostfs-sdk-go
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#53] sdk-go: Drop subnet

Browse source 
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-04-14 16:31:08 +03:00
parent 591dd1247d
commit c8e620ad24
30 changed files with 38 additions and 1062 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
container/container.go
View file

@ -16,7 +16,6 @@ import (
frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto"
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
"github.com/google/uuid" "github.com/google/uuid"
@ -128,10 +127,7 @@ func checkAttributes(m container.Container) error {
} }
var err error var err error
switch key { if key == attributeTimestamp {
case container.SysAttributeSubnet:
err = new(subnetid.ID).DecodeString(val)
case attributeTimestamp:
_, err = strconv.ParseInt(val, 10, 64) _, err = strconv.ParseInt(val, 10, 64)
} }
@ -391,28 +387,6 @@ func CreatedAt(cnr Container) time.Time {
return time.Unix(sec, 0) return time.Unix(sec, 0)
} }
// SetSubnet places the Container on the specified FrostFS subnet. If called,
// container nodes will only be selected from the given subnet, otherwise from
// the entire network.
func SetSubnet(cnr *Container, subNet subnetid.ID) {
cnr.SetAttribute(container.SysAttributeSubnet, subNet.EncodeToString())
}
// Subnet return container subnet set using SetSubnet.
//
// Zero Container is bound to zero subnet.
func Subnet(cnr Container) (res subnetid.ID) {
val := cnr.Attribute(container.SysAttributeSubnet)
if val != "" {
err := res.DecodeString(val)
if err != nil {
panic(fmt.Sprintf("invalid subnet attribute: %s (%v)", val, err))
}
}
return
}
const attributeHomoHashEnabled = "true" const attributeHomoHashEnabled = "true"
// DisableHomomorphicHashing sets flag to disable homomorphic hashing of the // DisableHomomorphicHashing sets flag to disable homomorphic hashing of the

24
container/container_test.go
View file

@ -15,8 +15,6 @@ import (
containertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/test" containertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/test"
frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto"
netmaptest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap/test" netmaptest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap/test"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
subnetidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
"github.com/google/uuid" "github.com/google/uuid"
@ -233,28 +231,6 @@ func TestSetCreationTime(t *testing.T) {
require.Equal(t, creat.Unix(), container.CreatedAt(val2).Unix()) require.Equal(t, creat.Unix(), container.CreatedAt(val2).Unix())
} }
func TestSetSubnet(t *testing.T) {
var val container.Container
require.True(t, subnetid.IsZero(container.Subnet(val)))
val = containertest.Container()
sub := subnetidtest.ID()
container.SetSubnet(&val, sub)
var msg v2container.Container
val.WriteToV2(&msg)
assertContainsAttribute(t, msg, v2container.SysAttributeSubnet, sub.EncodeToString())
var val2 container.Container
require.NoError(t, val2.ReadFromV2(msg))
require.Equal(t, sub, container.Subnet(val))
}
func TestDisableHomomorphicHashing(t *testing.T) { func TestDisableHomomorphicHashing(t *testing.T) {
var val container.Container var val container.Container

3
netmap/json_tests/cbf_default.json
View file

@ -85,8 +85,7 @@
"filter": "*" "filter": "*"
} }
], ],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [

6
netmap/json_tests/cbf_minimal.json
View file

@ -57,8 +57,7 @@
"filter": "*" "filter": "*"
} }
], ],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -86,8 +85,7 @@
"filter": "*" "filter": "*"
} }
], ],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [

12
netmap/json_tests/cbf_requirements.json
View file

@ -61,8 +61,7 @@
], ],
"containerBackupFactor": 0, "containerBackupFactor": 0,
"selectors": [], "selectors": [],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -83,8 +82,7 @@
], ],
"containerBackupFactor": 3, "containerBackupFactor": 3,
"selectors": [], "selectors": [],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -113,8 +111,7 @@
"filter": "*" "filter": "*"
} }
], ],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -143,8 +140,7 @@
"filter": "*" "filter": "*"
} }
], ],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [

12
netmap/json_tests/filter_complex.json
View file

@ -107,8 +107,7 @@
} }
] ]
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -200,8 +199,7 @@
} }
] ]
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
}, },
@ -289,8 +287,7 @@
} }
] ]
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
}, },
@ -378,8 +375,7 @@
} }
] ]
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
} }

36
netmap/json_tests/filter_simple.json
View file

@ -41,8 +41,7 @@
"value": "4", "value": "4",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -76,8 +75,7 @@
"value": "5", "value": "5",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
}, },
@ -107,8 +105,7 @@
"value": "3", "value": "3",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -142,8 +139,7 @@
"value": "4", "value": "4",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
}, },
@ -173,8 +169,7 @@
"value": "4", "value": "4",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -208,8 +203,7 @@
"value": "3", "value": "3",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
}, },
@ -239,8 +233,7 @@
"value": "5", "value": "5",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -274,8 +267,7 @@
"value": "4", "value": "4",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
}, },
@ -305,8 +297,7 @@
"value": "Germany", "value": "Germany",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -340,8 +331,7 @@
"value": "China", "value": "China",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
}, },
@ -371,8 +361,7 @@
"value": "France", "value": "France",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -406,8 +395,7 @@
"value": "Germany", "value": "Germany",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
} }

4
netmap/json_tests/hrw_sort.json
View file

@ -144,7 +144,7 @@
], ],
"tests": { "tests": {
"select 3 nodes in 3 distinct countries, same placement": { "select 3 nodes in 3 distinct countries, same placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[],"subnetId":null}, "policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
"pivot": "Y29udGFpbmVySUQ=", "pivot": "Y29udGFpbmVySUQ=",
"result": [[4, 0, 7]], "result": [[4, 0, 7]],
"placement": { "placement": {
@ -153,7 +153,7 @@
} }
}, },
"select 6 nodes in 3 distinct countries, different placement": { "select 6 nodes in 3 distinct countries, different placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[],"subnetId":null}, "policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
"pivot": "Y29udGFpbmVySUQ=", "pivot": "Y29udGFpbmVySUQ=",
"result": [[4, 3, 0, 1, 7, 2]], "result": [[4, 3, 0, 1, 7, 2]],
"placement": { "placement": {

3
netmap/json_tests/issue213.json
View file

@ -93,8 +93,7 @@
"value": "Europe", "value": "Europe",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [

3
netmap/json_tests/multiple_rep.json
View file

@ -79,8 +79,7 @@
"value": "Moscow", "value": "Moscow",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [

3
netmap/json_tests/multiple_rep_asymmetric.json
View file

@ -312,8 +312,7 @@
} }
] ]
} }
], ]
"subnetId": null
}, },
"result": [ "result": [
[ [

9
netmap/json_tests/rep_only.json
View file

@ -61,8 +61,7 @@
], ],
"containerBackupFactor": 0, "containerBackupFactor": 0,
"selectors": [], "selectors": [],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -82,8 +81,7 @@
], ],
"containerBackupFactor": 0, "containerBackupFactor": 0,
"selectors": [], "selectors": [],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [
@ -104,8 +102,7 @@
], ],
"containerBackupFactor": 0, "containerBackupFactor": 0,
"selectors": [], "selectors": [],
"filters": [], "filters": []
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
} }

3
netmap/json_tests/select_no_attribute.json
View file

@ -101,8 +101,7 @@
"filter": "*" "filter": "*"
} }
], ],
"filters": [], "filters": []
"subnetId": null
}, },
"result": [ "result": [
[ [

9
netmap/json_tests/selector_invalid.json
View file

@ -43,8 +43,7 @@
"value": "Russia", "value": "Russia",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "filter not found" "error": "filter not found"
}, },
@ -69,8 +68,7 @@
"value": "Russia", "value": "Russia",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
}, },
@ -95,8 +93,7 @@
"value": "Russia", "value": "Russia",
"filters": [] "filters": []
} }
], ]
"subnetId": null
}, },
"error": "not enough nodes" "error": "not enough nodes"
} }

254
netmap/json_tests/subnet.json
View file

@ -1,254 +0,0 @@
{
"name": "subnet tests",
"nodes": [
{
"attributes": [
{
"key": "ID",
"value": "0"
},
{
"key": "City",
"value": "Paris"
},
{
"key": "__SYSTEM__SUBNET_0",
"value": "False"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "1"
},
{
"key": "City",
"value": "Paris"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "2"
},
{
"key": "City",
"value": "London"
},
{
"key": "__SYSTEM__SUBNET_1",
"value": "True"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "3"
},
{
"key": "City",
"value": "London"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "4"
},
{
"key": "City",
"value": "Toronto"
},
{
"key": "__SYSTEM__SUBNET_1",
"value": "True"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "5"
},
{
"key": "City",
"value": "Toronto"
},
{
"key": "__SYSTEM__SUBNET_2",
"value": "True"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "6"
},
{
"key": "City",
"value": "Tokyo"
},
{
"key": "__SYSTEM__SUBNET_2",
"value": "True"
}
],
"state": "UNSPECIFIED"
},
{
"attributes": [
{
"key": "ID",
"value": "7"
},
{
"key": "City",
"value": "Tokyo"
},
{
"key": "__SYSTEM__SUBNET_2",
"value": "True"
}
],
"state": "UNSPECIFIED"
}
],
"tests": {
"select from default subnet, fail": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 0,
"selectors": [
{
"name": "S",
"count": 2,
"clause": "SAME",
"attribute": "City",
"filter": "F"
}
],
"filters": [
{
"name": "F",
"key": "City",
"op": "EQ",
"value": "Paris",
"filters": []
}
],
"subnetId": null
},
"error": "not enough nodes"
},
"select from default subnet, success": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 0,
"selectors": [
{
"name": "S",
"count": 2,
"clause": "SAME",
"attribute": "City",
"filter": "F"
}
],
"filters": [
{
"name": "F",
"key": "City",
"op": "EQ",
"value": "Toronto",
"filters": []
}
],
"subnetId": null
},
"result": [
[
4,
5
]
]
},
"select from non-default subnet, success": {
"policy": {
"replicas": [
{
"count": 3,
"selector": ""
}
],
"containerBackupFactor": 0,
"selectors": [],
"filters": [],
"subnetId": {
"value": 2
}
},
"result": [
[
5,
6,
7
]
]
},
"select subnet via filters": {
"policy": {
"replicas": [
{
"count": 1,
"selector": "S"
}
],
"containerBackupFactor": 1,
"selectors": [
{
"name": "S",
"count": 1,
"clause": "SAME",
"attribute": "City",
"filter": "F"
}
],
"filters": [
{
"name": "F",
"key": "__SYSTEM__SUBNET.2.ENABLED",
"op": "EQ",
"value": "True"
}
]
},
"error": "not enough nodes"
}
}
}

90
netmap/node_info.go
View file

@ -8,9 +8,7 @@ import (
"strings" "strings"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
"git.frostfs.info/TrueCloudLab/hrw" "git.frostfs.info/TrueCloudLab/hrw"
) )
@ -54,8 +52,6 @@ func (x *NodeInfo) readFromV2(m netmap.NodeInfo, checkFieldPresence bool) error
return fmt.Errorf("duplicated attbiuted %s", key) return fmt.Errorf("duplicated attbiuted %s", key)
} }
const subnetPrefix = "__SYSTEM__SUBNET_"
switch { switch {
case key == attrCapacity: case key == attrCapacity:
_, err = strconv.ParseUint(attributes[i].GetValue(), 10, 64) _, err = strconv.ParseUint(attributes[i].GetValue(), 10, 64)
@ -68,17 +64,6 @@ func (x *NodeInfo) readFromV2(m netmap.NodeInfo, checkFieldPresence bool) error
if err != nil { if err != nil {
return fmt.Errorf("invalid %s attribute: %w", attrPrice, err) return fmt.Errorf("invalid %s attribute: %w", attrPrice, err)
} }
case strings.HasPrefix(key, subnetPrefix):
var id subnetid.ID
err = id.DecodeString(strings.TrimPrefix(key, subnetPrefix))
if err != nil {
return fmt.Errorf("invalid key to the subnet attribute %s: %w", key, err)
}
if val := attributes[i].GetValue(); val != "True" && val != "False" {
return fmt.Errorf("invalid value of the subnet attribute %s: %w", val, err)
}
default: default:
if attributes[i].GetValue() == "" { if attributes[i].GetValue() == "" {
return fmt.Errorf("empty value of the attribute %s", key) return fmt.Errorf("empty value of the attribute %s", key)
@ -484,81 +469,6 @@ func (x *NodeInfo) SortAttributes() {
x.m.SetAttributes(as) x.m.SetAttributes(as)
} }
// EnterSubnet writes storage node's intention to enter the given subnet.
//
// Zero NodeInfo belongs to zero subnet.
func (x *NodeInfo) EnterSubnet(id subnetid.ID) {
x.changeSubnet(id, true)
}
// ExitSubnet writes storage node's intention to exit the given subnet.
func (x *NodeInfo) ExitSubnet(id subnetid.ID) {
x.changeSubnet(id, false)
}
func (x *NodeInfo) changeSubnet(id subnetid.ID, isMember bool) {
var (
idv2 refs.SubnetID
info netmap.NodeSubnetInfo
)
id.WriteToV2(&idv2)
info.SetID(&idv2)
info.SetEntryFlag(isMember)
netmap.WriteSubnetInfo(&x.m, info)
}
// ErrRemoveSubnet is returned when a node needs to leave the subnet.
var ErrRemoveSubnet = netmap.ErrRemoveSubnet
// IterateSubnets iterates over all subnets the node belongs to and passes the IDs to f.
// Handler MUST NOT be nil.
//
// If f returns ErrRemoveSubnet, then removes subnet entry. Note that this leads to an
// instant mutation of NodeInfo. Breaks on any other non-nil error and returns it.
//
// Returns an error if subnet incorrectly enabled/disabled.
// Returns an error if the node is not included to any subnet by the end of the loop.
//
// See also EnterSubnet, ExitSubnet.
func (x NodeInfo) IterateSubnets(f func(subnetid.ID) error) error {
var id subnetid.ID
return netmap.IterateSubnets(&x.m, func(idv2 refs.SubnetID) error {
err := id.ReadFromV2(idv2)
if err != nil {
return fmt.Errorf("invalid subnet: %w", err)
}
err = f(id)
if errors.Is(err, ErrRemoveSubnet) {
return netmap.ErrRemoveSubnet
}
return err
})
}
var errAbortSubnetIter = errors.New("abort subnet iterator")
// BelongsToSubnet is a helper function over the IterateSubnets method which
// checks whether a node belongs to a subnet.
//
// Zero NodeInfo belongs to zero subnet only.
func BelongsToSubnet(node NodeInfo, id subnetid.ID) bool {
err := node.IterateSubnets(func(id_ subnetid.ID) error {
if id.Equals(id_) {
return errAbortSubnetIter
}
return nil
})
return errors.Is(err, errAbortSubnetIter)
}
// SetOffline sets the state of the node to "offline". When a node updates // SetOffline sets the state of the node to "offline". When a node updates
// information about itself in the network map, this action is interpreted as // information about itself in the network map, this action is interpreted as
// an intention to leave the network. // an intention to leave the network.

31
netmap/policy.go
View file

@ -8,9 +8,7 @@ import (
"strings" "strings"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap/parser" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap/parser"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
"github.com/antlr/antlr4/runtime/Go/antlr/v4" "github.com/antlr/antlr4/runtime/Go/antlr/v4"
) )
@ -25,8 +23,6 @@ import (
type PlacementPolicy struct { type PlacementPolicy struct {
backupFactor uint32 backupFactor uint32
subnet subnetid.ID
filters []netmap.Filter filters []netmap.Filter
selectors []netmap.Selector selectors []netmap.Selector
@ -40,16 +36,6 @@ func (p *PlacementPolicy) readFromV2(m netmap.PlacementPolicy, checkFieldPresenc
return errors.New("missing replicas") return errors.New("missing replicas")
} }
subnetV2 := m.GetSubnetID()
if subnetV2 != nil {
err := p.subnet.ReadFromV2(*subnetV2)
if err != nil {
return fmt.Errorf("invalid subnet: %w", err)
}
} else {
p.subnet = subnetid.ID{}
}
p.backupFactor = m.GetContainerBackupFactor() p.backupFactor = m.GetContainerBackupFactor()
p.selectors = m.GetSelectors() p.selectors = m.GetSelectors()
p.filters = m.GetFilters() p.filters = m.GetFilters()
@ -123,29 +109,12 @@ func (p *PlacementPolicy) ReadFromV2(m netmap.PlacementPolicy) error {
// //
// See also ReadFromV2. // See also ReadFromV2.
func (p PlacementPolicy) WriteToV2(m *netmap.PlacementPolicy) { func (p PlacementPolicy) WriteToV2(m *netmap.PlacementPolicy) {
var subnetV2 refs.SubnetID
p.subnet.WriteToV2(&subnetV2)
m.SetContainerBackupFactor(p.backupFactor) m.SetContainerBackupFactor(p.backupFactor)
m.SetSubnetID(&subnetV2)
m.SetFilters(p.filters) m.SetFilters(p.filters)
m.SetSelectors(p.selectors) m.SetSelectors(p.selectors)
m.SetReplicas(p.replicas) m.SetReplicas(p.replicas)
} }
// RestrictSubnet sets a rule to select nodes from the given subnet only.
// By default, nodes from zero subnet are selected (whole network map).
func (p *PlacementPolicy) RestrictSubnet(subnet subnetid.ID) {
p.subnet = subnet
}
// Subnet returns subnet set using RestrictSubnet.
//
// Zero PlacementPolicy returns zero subnet meaning unlimited.
func (p PlacementPolicy) Subnet() subnetid.ID {
return p.subnet
}
// ReplicaDescriptor replica descriptor characterizes replicas of objects from // ReplicaDescriptor replica descriptor characterizes replicas of objects from
// the subset selected by a particular Selector. // the subset selected by a particular Selector.
type ReplicaDescriptor struct { type ReplicaDescriptor struct {

9
netmap/selector.go
View file

@ -5,7 +5,6 @@ import (
"sort" "sort"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
"git.frostfs.info/TrueCloudLab/hrw" "git.frostfs.info/TrueCloudLab/hrw"
) )
@ -59,7 +58,7 @@ func calcBucketWeight(ns nodes, a aggregator, wf weightFunc) float64 {
// Last argument specifies if more buckets can be used to fulfill CBF. // Last argument specifies if more buckets can be used to fulfill CBF.
func (c *context) getSelection(p PlacementPolicy, s netmap.Selector) ([]nodes, error) { func (c *context) getSelection(p PlacementPolicy, s netmap.Selector) ([]nodes, error) {
bucketCount, nodesInBucket := calcNodesCount(s) bucketCount, nodesInBucket := calcNodesCount(s)
buckets := c.getSelectionBase(p.subnet, s) buckets := c.getSelectionBase(s)
if len(buckets) < bucketCount { if len(buckets) < bucketCount {
return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName()) return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName())
@ -132,7 +131,7 @@ type nodeAttrPair struct {
// getSelectionBase returns nodes grouped by selector attribute. // getSelectionBase returns nodes grouped by selector attribute.
// It it guaranteed that each pair will contain at least one node. // It it guaranteed that each pair will contain at least one node.
func (c *context) getSelectionBase(subnetID subnetid.ID, s netmap.Selector) []nodeAttrPair { func (c *context) getSelectionBase(s netmap.Selector) []nodeAttrPair {
fName := s.GetFilter() fName := s.GetFilter()
f := c.processedFilters[fName] f := c.processedFilters[fName]
isMain := fName == mainFilterName isMain := fName == mainFilterName
@ -141,10 +140,6 @@ func (c *context) getSelectionBase(subnetID subnetid.ID, s netmap.Selector) []no
attr := s.GetAttribute() attr := s.GetAttribute()
for i := range c.netMap.nodes { for i := range c.netMap.nodes {
// TODO(fyrchik): make `BelongsToSubnet` to accept pointer
if !BelongsToSubnet(c.netMap.nodes[i], subnetID) {
continue
}
if isMain || c.match(f, c.netMap.nodes[i]) { if isMain || c.match(f, c.netMap.nodes[i]) {
if attr == "" { if attr == "" {
// Default attribute is transparent identifier which is different for every node. // Default attribute is transparent identifier which is different for every node.

126
netmap/subnet_test.go
View file

@ -1,126 +0,0 @@
package netmap_test
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
"github.com/stretchr/testify/require"
)
func TestNodeInfoSubnets(t *testing.T) {
t.Run("enter subnet", func(t *testing.T) {
var id subnetid.ID
id.SetNumeric(13)
var node netmap.NodeInfo
node.EnterSubnet(id)
mIDs := make(map[string]struct{})
err := node.IterateSubnets(func(id subnetid.ID) error {
mIDs[id.String()] = struct{}{}
return nil
})
require.NoError(t, err)
_, ok := mIDs[id.String()]
require.True(t, ok)
})
t.Run("iterate with removal", func(t *testing.T) {
t.Run("not last", func(t *testing.T) {
var id, idrm subnetid.ID
id.SetNumeric(13)
idrm.SetNumeric(23)
var node netmap.NodeInfo
node.EnterSubnet(id)
node.EnterSubnet(idrm)
err := node.IterateSubnets(func(id subnetid.ID) error {
if subnetid.IsZero(id) || id.Equals(idrm) {
return netmap.ErrRemoveSubnet
}
return nil
})
require.NoError(t, err)
mIDs := make(map[string]struct{})
err = node.IterateSubnets(func(id subnetid.ID) error {
mIDs[id.String()] = struct{}{}
return nil
})
require.NoError(t, err)
var zeroID subnetid.ID
_, ok := mIDs[zeroID.String()]
require.False(t, ok)
_, ok = mIDs[idrm.String()]
require.False(t, ok)
_, ok = mIDs[id.String()]
require.True(t, ok)
})
t.Run("last", func(t *testing.T) {
var node netmap.NodeInfo
err := node.IterateSubnets(func(id subnetid.ID) error {
return netmap.ErrRemoveSubnet
})
require.Error(t, err)
})
})
}
func TestEnterSubnet(t *testing.T) {
var (
id subnetid.ID
node netmap.NodeInfo
)
require.True(t, netmap.BelongsToSubnet(node, id))
node.EnterSubnet(id)
require.True(t, netmap.BelongsToSubnet(node, id))
node.ExitSubnet(id)
require.False(t, netmap.BelongsToSubnet(node, id))
id.SetNumeric(10)
node.EnterSubnet(id)
require.True(t, netmap.BelongsToSubnet(node, id))
require.False(t, netmap.BelongsToSubnet(node, subnetid.ID{}))
node.ExitSubnet(id)
require.False(t, netmap.BelongsToSubnet(node, id))
require.False(t, netmap.BelongsToSubnet(node, subnetid.ID{}))
}
func TestBelongsToSubnet(t *testing.T) {
var id, idMiss, idZero subnetid.ID
id.SetNumeric(13)
idMiss.SetNumeric(23)
var node netmap.NodeInfo
node.EnterSubnet(id)
require.True(t, netmap.BelongsToSubnet(node, idZero))
require.True(t, netmap.BelongsToSubnet(node, id))
require.False(t, netmap.BelongsToSubnet(node, idMiss))
}

2
netmap/test/generate.go
View file

@ -4,7 +4,6 @@ import (
"math/rand" "math/rand"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
subnetidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id/test"
) )
func filter(withInner bool) (x netmap.Filter) { func filter(withInner bool) (x netmap.Filter) {
@ -48,7 +47,6 @@ func PlacementPolicy() (p netmap.PlacementPolicy) {
p.AddFilters(Filter(), Filter()) p.AddFilters(Filter(), Filter())
p.AddReplicas(Replica(), Replica()) p.AddReplicas(Replica(), Replica())
p.AddSelectors(Selector(), Selector()) p.AddSelectors(Selector(), Selector())
p.RestrictSubnet(subnetidtest.ID())
return return
} }

9
subnet/doc.go
View file

@ -1,9 +0,0 @@
/*
Package subnet collects functionality related to the FrostFS subnets.
Subnet of a particular FrostFS network consists of a subset of the storage nodes
of that network. Subnet of the whole network is called zero. Info type acts as
a subnet descriptor. Each subnet is owned by the user who created it. Information
about all subnets is stored in the Subnet contract of the FrostFS Sidechain.
*/
package subnet

9
subnet/id/doc.go
View file

@ -1,9 +0,0 @@
/*
Package subnetid provides primitives to work with subnet identification in FrostFS.
ID type is used for global subnet identity inside the FrostFS network.
Using package types in an application is recommended to potentially work with
different protocol versions with which these types are compatible.
*/
package subnetid

105
subnet/id/id.go
View file

@ -1,105 +0,0 @@
package subnetid
import (
"fmt"
"strconv"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
)
// ID represents unique identifier of the subnet in the FrostFS network.
//
// ID is mutually compatible with git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs.SubnetID
// message. See ReadFromV2 / WriteToV2 methods.
//
// Instances can be created using built-in var declaration. Zero value is
// equivalent to identifier of the zero subnet (whole FrostFS network).
type ID struct {
m refs.SubnetID
}
// ReadFromV2 reads ID from the refs.SubnetID message. Checks if the
// message conforms to FrostFS API V2 protocol.
//
// See also WriteToV2.
func (x *ID) ReadFromV2(msg refs.SubnetID) error {
x.m = msg
return nil
}
// WriteToV2 writes ID to refs.SubnetID message structure. The message MUST NOT
// be nil.
//
// See also ReadFromV2.
func (x ID) WriteToV2(msg *refs.SubnetID) {
*msg = x.m
}
// Equals defines a comparison relation between two ID instances.
//
// Note that comparison using '==' operator is not recommended since it MAY result
// in loss of compatibility.
func (x ID) Equals(x2 ID) bool {
return x.m.GetValue() == x2.m.GetValue()
}
// EncodeToString encodes ID into FrostFS API protocol string (base10 encoding).
//
// See also DecodeString.
func (x ID) EncodeToString() string {
return strconv.FormatUint(uint64(x.m.GetValue()), 10)
}
// DecodeString decodes string calculated using EncodeToString. Returns
// an error if s is malformed.
func (x *ID) DecodeString(s string) error {
num, err := strconv.ParseUint(s, 10, 32)
if err != nil {
return fmt.Errorf("invalid numeric value: %w", err)
}
x.m.SetValue(uint32(num))
return nil
}
// String implements fmt.Stringer.
//
// String is designed to be human-readable, and its format MAY differ between
// SDK versions. String MAY return same result as EncodeToString. String MUST NOT
// be used to encode ID into FrostFS protocol string.
func (x ID) String() string {
return "#" + strconv.FormatUint(uint64(x.m.GetValue()), 10)
}
// Marshal encodes ID into a binary format of the FrostFS API protocol
// (Protocol Buffers with direct field order).
//
// See also Unmarshal.
func (x ID) Marshal() []byte {
return x.m.StableMarshal(nil)
}
// Unmarshal decodes binary ID calculated using Marshal. Returns an error
// describing a format violation.
func (x *ID) Unmarshal(data []byte) error {
return x.m.Unmarshal(data)
}
// SetNumeric sets ID value in numeric format. By default, number is 0 which
// refers to the zero subnet.
func (x *ID) SetNumeric(num uint32) {
x.m.SetValue(num)
}
// IsZero compares id with zero subnet ID.
func IsZero(id ID) bool {
return id.Equals(ID{})
}
// MakeZero makes ID to refer to zero subnet.
//
// Makes no sense to call on zero value (e.g. declared using var).
func MakeZero(id *ID) {
id.SetNumeric(0)
}

100
subnet/id/id_test.go
View file

@ -1,100 +0,0 @@
package subnetid_test
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
subnetidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id/test"
"github.com/stretchr/testify/require"
)
func TestIsZero(t *testing.T) {
var id subnetid.ID
require.True(t, subnetid.IsZero(id))
id.SetNumeric(13)
require.False(t, subnetid.IsZero(id))
id.SetNumeric(0)
require.True(t, subnetid.IsZero(id))
}
func TestID_ReadFromV2(t *testing.T) {
const num = 13
var id1 subnetid.ID
id1.SetNumeric(num)
var idv2 refs.SubnetID
idv2.SetValue(num)
var id2 subnetid.ID
require.NoError(t, id2.ReadFromV2(idv2))
require.True(t, id1.Equals(id2))
}
func TestID_WriteToV2(t *testing.T) {
const num = 13
var (
id subnetid.ID
idv2 refs.SubnetID
)
id.WriteToV2(&idv2)
require.Zero(t, idv2.GetValue())
id.SetNumeric(num)
id.WriteToV2(&idv2)
require.EqualValues(t, num, idv2.GetValue())
}
func TestID_Equals(t *testing.T) {
const num = 13
var id1, id2, idOther, id0 subnetid.ID
id0.Equals(subnetid.ID{})
id1.SetNumeric(num)
id2.SetNumeric(num)
idOther.SetNumeric(num + 1)
require.True(t, id1.Equals(id2))
require.False(t, id1.Equals(idOther))
require.False(t, id2.Equals(idOther))
}
func TestSubnetIDEncoding(t *testing.T) {
id := subnetidtest.ID()
t.Run("binary", func(t *testing.T) {
var id2 subnetid.ID
require.NoError(t, id2.Unmarshal(id.Marshal()))
require.True(t, id2.Equals(id))
})
t.Run("text", func(t *testing.T) {
var id2 subnetid.ID
require.NoError(t, id2.DecodeString(id.EncodeToString()))
require.True(t, id2.Equals(id))
})
}
func TestMakeZero(t *testing.T) {
var id subnetid.ID
id.SetNumeric(13)
require.False(t, subnetid.IsZero(id))
subnetid.MakeZero(&id)
require.True(t, subnetid.IsZero(id))
require.Equal(t, subnetid.ID{}, id)
}

13
subnet/id/test/doc.go
View file

@ -1,13 +0,0 @@
/*
Package subnetidtest provides functions for convenient testing of subnetid package API.
Note that importing the package into source files is highly discouraged.
Random instance generation functions can be useful when testing expects any value, e.g.:
import subnetidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/suibnet/id/test"
value := subnetidtest.ID()
// test the value
*/
package subnetidtest

13
subnet/id/test/id.go
View file

@ -1,13 +0,0 @@
package subnetidtest
import (
"math/rand"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
)
// ID generates and returns random subnetid.ID.
func ID() (x subnetid.ID) {
x.SetNumeric(rand.Uint32())
return
}

109
subnet/subnet.go
View file

@ -1,109 +0,0 @@
package subnet
import (
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/subnet"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
// Info represents information about FrostFS subnet.
//
// Instances can be created using built-in var declaration.
type Info struct {
id subnetid.ID
owner user.ID
}
// Marshal encodes Info into a binary format of the FrostFS API protocol
// (Protocol Buffers with direct field order).
//
// See also Unmarshal.
func (x Info) Marshal() []byte {
var id refs.SubnetID
x.id.WriteToV2(&id)
var owner refs.OwnerID
x.owner.WriteToV2(&owner)
var m subnet.Info
m.SetID(&id)
m.SetOwner(&owner)
return m.StableMarshal(nil)
}
// Unmarshal decodes binary Info calculated using Marshal. Returns an error
// describing a format violation.
func (x *Info) Unmarshal(data []byte) error {
var m subnet.Info
err := m.Unmarshal(data)
if err != nil {
return err
}
id := m.ID()
if id != nil {
err = x.id.ReadFromV2(*id)
if err != nil {
return fmt.Errorf("invalid ID: %w", err)
}
} else {
subnetid.MakeZero(&x.id)
}
owner := m.Owner()
if owner != nil {
err = x.owner.ReadFromV2(*owner)
if err != nil {
return fmt.Errorf("invalid owner: %w", err)
}
} else {
x.owner = user.ID{}
}
return nil
}
// SetID sets the identifier of the subnet that Info describes.
//
// See also ID.
func (x *Info) SetID(id subnetid.ID) {
x.id = id
}
// ID returns subnet identifier set using SetID.
//
// Zero Info refers to the zero subnet.
func (x Info) ID() subnetid.ID {
return x.id
}
// SetOwner sets identifier of the subnet owner.
func (x *Info) SetOwner(id user.ID) {
x.owner = id
}
// Owner returns subnet owner set using SetOwner.
//
// Zero Info has no owner which is incorrect according to the
// FrostFS API protocol.
func (x Info) Owner() user.ID {
return x.owner
}
// AssertOwnership checks if the given info describes the subnet owned by the
// given user.
func AssertOwnership(info Info, id user.ID) bool {
return id.Equals(info.Owner())
}
// AssertReference checks if the given info describes the subnet referenced by
// the given id.
func AssertReference(info Info, id subnetid.ID) bool {
return id.Equals(info.ID())
}

48
subnet/subnet_test.go
View file

@ -1,48 +0,0 @@
package subnet_test
import (
"testing"
. "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet"
subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id"
subnetidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id/test"
subnettest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
"github.com/stretchr/testify/require"
)
func TestInfoZero(t *testing.T) {
var info Info
require.Zero(t, info.ID())
require.True(t, subnetid.IsZero(info.ID()))
}
func TestInfo_SetID(t *testing.T) {
id := subnetidtest.ID()
var info Info
info.SetID(id)
require.Equal(t, id, info.ID())
require.True(t, AssertReference(info, id))
}
func TestInfo_SetOwner(t *testing.T) {
id := *usertest.ID()
var info Info
info.SetOwner(id)
require.Equal(t, id, info.Owner())
require.True(t, AssertOwnership(info, id))
}
func TestInfo_Marshal(t *testing.T) {
info := subnettest.Info()
var info2 Info
require.NoError(t, info2.Unmarshal(info.Marshal()))
require.Equal(t, info, info2)
}

13
subnet/test/doc.go
View file

@ -1,13 +0,0 @@
/*
Package subnettest provides functions for convenient testing of subnet package API.
Note that importing the package into source files is highly discouraged.
Random instance generation functions can be useful when testing expects any value, e.g.:
import subnettest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/suibnet/test"
value := subnettest.Info()
// test the value
*/
package subnettest

14
subnet/test/subnet.go
View file

@ -1,14 +0,0 @@
package subnettest
import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet"
subnetidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id/test"
usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test"
)
// Info generates and returns random subnet.Info.
func Info() (x subnet.Info) {
x.SetID(subnetidtest.ID())
x.SetOwner(*usertest.ID())
return
}