Use new protobuf marshaler #1322

Merged
fyrchik merged 1 commit from fyrchik/frostfs-node:update-marshaling into master 2024-08-22 07:17:44 +00:00
29 changed files with 17 additions and 751 deletions

View file

@ -11,7 +11,6 @@ GO_VERSION ?= 1.22
LINT_VERSION ?= 1.56.1
TRUECLOUDLAB_LINT_VERSION ?= 0.0.5
PROTOC_VERSION ?= 25.0
PROTOC_GEN_GO_VERSION ?= $(shell go list -f '{{.Version}}' -m google.golang.org/protobuf)
PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2)
PROTOC_OS_VERSION=osx-x86_64
ifeq ($(shell uname), Linux)
@ -39,7 +38,6 @@ LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT
TMP_DIR := .cache
PROTOBUF_DIR ?= $(abspath $(BIN))/protobuf
PROTOC_DIR ?= $(PROTOBUF_DIR)/protoc-v$(PROTOC_VERSION)
PROTOC_GEN_GO_DIR ?= $(PROTOBUF_DIR)/protoc-gen-go-$(PROTOC_GEN_GO_VERSION)
PROTOGEN_FROSTFS_DIR ?= $(PROTOBUF_DIR)/protogen-$(PROTOGEN_FROSTFS_VERSION)
STATICCHECK_DIR ?= $(abspath $(BIN))/staticcheck
STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION)
@ -107,17 +105,15 @@ export-metrics: dep
# Regenerate proto files:
protoc:
@if [ ! -d "$(PROTOC_DIR)" ] || [ ! -d "$(PROTOC_GEN_GO_DIR)" ] || [ ! -d "$(PROTOGEN_FROSTFS_DIR)" ]; then \
@if [ ! -d "$(PROTOC_DIR)" ] || [ ! -d "$(PROTOGEN_FROSTFS_DIR)" ]; then \
make protoc-install; \
fi
@for f in `find . -type f -name '*.proto' -not -path './bin/*'`; do \
echo "⇒ Processing $$f "; \
$(PROTOC_DIR)/bin/protoc \
--proto_path=.:$(PROTOC_DIR)/include:/usr/local/include \
--plugin=protoc-gen-go=$(PROTOC_GEN_GO_DIR)/protoc-gen-go \
dstepanov-yadro marked this conversation as resolved Outdated

If it not required, then need to drop this variable.

If it not required, then need to drop this variable.
Review

fixed

fixed
--plugin=protoc-gen-go-frostfs=$(PROTOGEN_FROSTFS_DIR)/protogen \
--go-frostfs_out=. --go-frostfs_opt=paths=source_relative \
--go_out=. --go_opt=paths=source_relative \
--go-grpc_opt=require_unimplemented_servers=false \
--go-grpc_out=. --go-grpc_opt=paths=source_relative $$f; \
done
@ -130,8 +126,6 @@ protoc-install:
@wget -q -O $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip 'https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS_VERSION).zip'
@unzip -q -o $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip -d $(PROTOC_DIR)
@rm $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip
@echo "⇒ Installing protoc-gen-go..."
@GOBIN=$(PROTOC_GEN_GO_DIR) go install -v google.golang.org/protobuf/...@$(PROTOC_GEN_GO_VERSION)
@echo "⇒ Instaling protogen FrostFS plugin..."
@GOBIN=$(PROTOGEN_FROSTFS_DIR) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/protogen@$(PROTOGEN_FROSTFS_VERSION)

View file

@ -84,7 +84,7 @@ func setNetmapStatus(cmd *cobra.Command, _ []string) {
body.SetStatus(control.NetmapStatus_MAINTENANCE)
if force {
body.SetForceMaintenance()
body.SetForceMaintenance(true)
common.PrintVerbose(cmd, "Local maintenance will be forced.")
}
targetStatus = control.NetmapStatus_MAINTENANCE

View file

@ -117,10 +117,10 @@ func setShardMode(cmd *cobra.Command, _ []string) {
req.SetBody(body)
body.SetMode(mode)
body.SetShardIDList(getShardIDList(cmd))
body.SetShard_ID(getShardIDList(cmd))
reset, _ := cmd.Flags().GetBool(shardClearErrorsFlag)
body.ClearErrorCounter(reset)
body.SetResetErrorCounter(reset)
signRequest(cmd, pk, req)

View file

@ -44,7 +44,7 @@ func verifyResponse(cmd *cobra.Command,
GetSign() []byte
},
body interface {
StableMarshal([]byte) []byte
MarshalProtobuf([]byte) []byte
},
) {
if sigControl == nil {
@ -60,7 +60,7 @@ func verifyResponse(cmd *cobra.Command,
var sig frostfscrypto.Signature
commonCmd.ExitOnErr(cmd, "can't read signature: %w", sig.ReadFromV2(sigV2))
aarifullin marked this conversation as resolved Outdated

Disabled for debug?

Disabled for debug?

oops, yes

oops, yes

moved to WIP, will revisit everything

moved to WIP, will revisit everything

Fixed, retested on dev-env

Fixed, retested on dev-env
if !sig.Verify(body.StableMarshal(nil)) {
if !sig.Verify(body.MarshalProtobuf(nil)) {
commonCmd.ExitOnErr(cmd, "", errors.New("invalid response signature"))
}
}

View file

@ -84,7 +84,7 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) {
}
}
s.setControlNetmapStatus(ctrlNetSt)
s.setControlNetmapStatus(control.NetmapStatus(ctrlNetSt))
}
// sets the current node state to the given value. Subsequent cfg.bootstrap

7
go.mod
View file

@ -4,16 +4,17 @@ go 1.21
require (
code.gitea.io/sdk/gitea v0.17.1
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240813155151-d112a28d382f
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240819074700-a43110e36326
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0
git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240813155821-98aabc45a720
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820072028-6dd7be11d13b
git.frostfs.info/TrueCloudLab/hrw v1.2.1
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88
git.frostfs.info/TrueCloudLab/tzhash v1.8.0
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
github.com/VictoriaMetrics/easyproto v0.1.4
github.com/cheggaaa/pb v1.0.29
github.com/chzyer/readline v1.5.1
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
@ -21,6 +22,7 @@ require (
github.com/google/uuid v1.6.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/klauspost/compress v1.17.4
github.com/mailru/easyjson v0.7.7
github.com/mitchellh/go-homedir v1.1.0
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-multiaddr v0.12.1
@ -84,7 +86,6 @@ require (
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/klauspost/reedsolomon v1.12.1 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect

BIN
go.sum

Binary file not shown.

View file

@ -1,6 +1,7 @@
package meta
import (
"bytes"
"context"
"fmt"
"time"
@ -107,7 +108,7 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b
// check in primary index
data := getFromBucket(tx, primaryBucketName(cnr, bucketName), key)
if len(data) != 0 {
return obj, obj.Unmarshal(data)
return obj, obj.Unmarshal(bytes.Clone(data))
}
data = getFromBucket(tx, ecInfoBucketName(cnr, bucketName), key)
@ -118,13 +119,13 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b
// if not found then check in tombstone index
data = getFromBucket(tx, tombstoneBucketName(cnr, bucketName), key)
if len(data) != 0 {
return obj, obj.Unmarshal(data)
return obj, obj.Unmarshal(bytes.Clone(data))
}
// if not found then check in locker index
data = getFromBucket(tx, bucketNameLockers(cnr, bucketName), key)
if len(data) != 0 {
return obj, obj.Unmarshal(data)
return obj, obj.Unmarshal(bytes.Clone(data))
dstepanov-yadro marked this conversation as resolved Outdated

What is the benefit of using a new marshalled in the end?

What is the benefit of using a new marshalled in the end?
  1. We clone a single slice here vs cloning allocating multiple times for each slice field we use.
  2. If we unmarshal from then FSTree, no clone is done.
  3. Later we will add idiomatic []Attribute fields, that will allow us to gradually remove layers in ToGRPCMessage() conversion functions from the api-go.
1. We clone a single slice here vs cloning allocating multiple times for each slice field we use. 2. If we unmarshal from then FSTree, no clone is done. 3. Later we will add idiomatic `[]Attribute` fields, that will allow us to gradually remove layers in `ToGRPCMessage()` conversion functions from the api-go.
}
// if not found then check if object is a virtual

View file

@ -1,33 +0,0 @@
package control_test
import (
"crypto/rand"
"testing"
"github.com/mr-tron/base58"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
)
type protoMessage interface {
StableMarshal([]byte) []byte
proto.Message
}
func testStableMarshal(t *testing.T, m1, m2 protoMessage, cmp func(m1, m2 protoMessage) bool) {
require.NoError(t, proto.Unmarshal(m1.StableMarshal(nil), m2))
require.True(t, cmp(m1, m2))
}
func testData(sz int) []byte {
d := make([]byte, sz)
_, _ = rand.Read(d)
return d
}
func testString() string {
return base58.Encode(testData(10))
}

View file

@ -1,46 +0,0 @@
package control
// SetBody sets health check request body.
func (x *HealthCheckRequest) SetBody(v *HealthCheckRequest_Body) {
if x != nil {
x.Body = v
}
}
// SetHealthStatus sets health status of the IR application.
func (x *HealthCheckResponse_Body) SetHealthStatus(v HealthStatus) {
if x != nil {
x.HealthStatus = v
}
}
// SetBody sets health check response body.
func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) {
if x != nil {
x.Body = v
}
}
func (x *TickEpochRequest) SetBody(v *TickEpochRequest_Body) {
if x != nil {
x.Body = v
}
}
func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) {
if x != nil {
x.Body = v
}
}
func (x *RemoveNodeRequest) SetBody(v *RemoveNodeRequest_Body) {
if x != nil {
x.Body = v
}
}
func (x *RemoveNodeResponse) SetBody(v *RemoveNodeResponse_Body) {
if x != nil {
x.Body = v
}
}

Binary file not shown.

Binary file not shown.

View file

@ -1,44 +0,0 @@
package control_test
import (
"testing"
control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
)
type protoMessage interface {
StableMarshal([]byte) []byte
proto.Message
}
func testStableMarshal(t *testing.T, m1, m2 protoMessage, cmp func(m1, m2 protoMessage) bool) {
require.NoError(t, proto.Unmarshal(m1.StableMarshal(nil), m2))
require.True(t, cmp(m1, m2))
}
func TestHealthCheckResponse_Body_StableMarshal(t *testing.T) {
testStableMarshal(t,
generateHealthCheckResponseBody(),
new(control.HealthCheckResponse_Body),
func(m1, m2 protoMessage) bool {
return equalHealthCheckResponseBodies(
m1.(*control.HealthCheckResponse_Body),
m2.(*control.HealthCheckResponse_Body),
)
},
)
}
func generateHealthCheckResponseBody() *control.HealthCheckResponse_Body {
body := new(control.HealthCheckResponse_Body)
body.SetHealthStatus(control.HealthStatus_SHUTTING_DOWN)
return body
}
func equalHealthCheckResponseBodies(b1, b2 *control.HealthCheckResponse_Body) bool {
return b1.GetHealthStatus() == b2.GetHealthStatus()
}

View file

@ -1,15 +0,0 @@
package control
// SetKey sets public key used for signing.
func (x *Signature) SetKey(v []byte) {
if x != nil {
x.Key = v
}
}
// SetSign sets binary signature.
func (x *Signature) SetSign(v []byte) {
if x != nil {
x.Sign = v
}
}

Binary file not shown.

Binary file not shown.

View file

@ -30,10 +30,10 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
for _, sh := range info.Shards {
si := new(control.ShardInfo)
si.SetID(*sh.ID)
si.SetShard_ID(*sh.ID)
si.SetMetabasePath(sh.MetaBaseInfo.Path)
si.Blobstor = blobstorInfoToProto(sh.BlobStorInfo)
si.SetWriteCachePath(sh.WriteCacheInfo.Path)
si.SetWritecachePath(sh.WriteCacheInfo.Path)
si.SetPiloramaPath(sh.PiloramaInfo.Path)
var m control.ShardMode

View file

@ -1,142 +0,0 @@
package control
// SetBody sets health check request body.
func (x *HealthCheckRequest) SetBody(v *HealthCheckRequest_Body) {
if x != nil {
x.Body = v
}
}
// SetNetmapStatus sets status of the storage node in FrostFS network map.
func (x *HealthCheckResponse_Body) SetNetmapStatus(v NetmapStatus) {
if x != nil {
x.NetmapStatus = v
}
}
// SetHealthStatus sets health status of the storage node application.
func (x *HealthCheckResponse_Body) SetHealthStatus(v HealthStatus) {
if x != nil {
x.HealthStatus = v
}
}
// SetBody sets health check response body.
func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) {
if x != nil {
x.Body = v
}
}
// SetStatus sets new storage node status in FrostFS network map.
func (x *SetNetmapStatusRequest_Body) SetStatus(v NetmapStatus) {
if x != nil {
x.Status = v
}
}
// SetForceMaintenance sets force_maintenance flag in the message.
func (x *SetNetmapStatusRequest_Body) SetForceMaintenance() {
x.ForceMaintenance = true
}
// SetBody sets body of the set netmap status request .
func (x *SetNetmapStatusRequest) SetBody(v *SetNetmapStatusRequest_Body) {
if x != nil {
x.Body = v
}
}
// SetBody sets set body of the netmap status response.
func (x *SetNetmapStatusResponse) SetBody(v *SetNetmapStatusResponse_Body) {
if x != nil {
x.Body = v
}
}
// SetAddressList sets list of objects to be removed in FrostFS API binary format.
func (x *DropObjectsRequest_Body) SetAddressList(v [][]byte) {
if x != nil {
x.AddressList = v
}
}
// SetBody sets body of the set "Drop objects" request.
func (x *DropObjectsRequest) SetBody(v *DropObjectsRequest_Body) {
if x != nil {
x.Body = v
}
}
// SetBody sets set body of the "Drop objects" response.
func (x *DropObjectsResponse) SetBody(v *DropObjectsResponse_Body) {
if x != nil {
x.Body = v
}
}
// SetBody sets list shards request body.
func (x *ListShardsRequest) SetBody(v *ListShardsRequest_Body) {
if x != nil {
x.Body = v
}
}
// SetShards sets shards of the storage node.
func (x *ListShardsResponse_Body) SetShards(v []*ShardInfo) {
if x != nil {
x.Shards = v
}
}
// SetBody sets list shards response body.
func (x *ListShardsResponse) SetBody(v *ListShardsResponse_Body) {
if x != nil {
x.Body = v
}
}
// SetShardIDList sets shard ID whose mode is requested to be set.
func (x *SetShardModeRequest_Body) SetShardIDList(v [][]byte) {
if v != nil {
x.Shard_ID = v
}
}
// SetMode sets mode of the shard.
func (x *SetShardModeRequest_Body) SetMode(v ShardMode) {
x.Mode = v
}
// ClearErrorCounter sets flag signifying whether error counter for shard should be cleared.
func (x *SetShardModeRequest_Body) ClearErrorCounter(reset bool) {
x.ResetErrorCounter = reset
}
// SetBody sets request body.
func (x *SetShardModeRequest) SetBody(v *SetShardModeRequest_Body) {
if x != nil {
x.Body = v
}
}
// SetBody sets body of the set shard mode response.
func (x *SetShardModeResponse) SetBody(v *SetShardModeResponse_Body) {
if x != nil {
x.Body = v
}
}
// SetBody sets list shards request body.
func (x *SynchronizeTreeRequest) SetBody(v *SynchronizeTreeRequest_Body) {
if x != nil {
x.Body = v
}
}
// SetBody sets list shards response body.
func (x *SynchronizeTreeResponse) SetBody(v *SynchronizeTreeResponse_Body) {
if x != nil {
x.Body = v
}
}

Binary file not shown.

Binary file not shown.

View file

@ -1,181 +0,0 @@
package control_test
import (
"bytes"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
)
func TestHealthCheckResponse_Body_StableMarshal(t *testing.T) {
testStableMarshal(t,
generateHealthCheckResponseBody(),
new(control.HealthCheckResponse_Body),
func(m1, m2 protoMessage) bool {
return equalHealthCheckResponseBodies(
m1.(*control.HealthCheckResponse_Body),
m2.(*control.HealthCheckResponse_Body),
)
},
)
}
func generateHealthCheckResponseBody() *control.HealthCheckResponse_Body {
body := new(control.HealthCheckResponse_Body)
body.SetNetmapStatus(control.NetmapStatus_ONLINE)
body.SetHealthStatus(control.HealthStatus_SHUTTING_DOWN)
return body
}
func equalHealthCheckResponseBodies(b1, b2 *control.HealthCheckResponse_Body) bool {
return b1.GetNetmapStatus() == b2.GetNetmapStatus() &&
b1.GetHealthStatus() == b2.GetHealthStatus()
}
func TestSetNetmapStatusRequest_Body_StableMarshal(t *testing.T) {
testStableMarshal(t,
generateSetNetmapStatusRequestBody(),
new(control.SetNetmapStatusRequest_Body),
func(m1, m2 protoMessage) bool {
return equalSetnetmapStatusRequestBodies(
m1.(*control.SetNetmapStatusRequest_Body),
m2.(*control.SetNetmapStatusRequest_Body),
)
},
)
}
func generateSetNetmapStatusRequestBody() *control.SetNetmapStatusRequest_Body {
body := new(control.SetNetmapStatusRequest_Body)
body.SetStatus(control.NetmapStatus_ONLINE)
return body
}
func equalSetnetmapStatusRequestBodies(b1, b2 *control.SetNetmapStatusRequest_Body) bool {
return b1.GetStatus() == b2.GetStatus()
}
func TestListShardsResponse_Body_StableMarshal(t *testing.T) {
testStableMarshal(t,
generateListShardsResponseBody(),
new(control.ListShardsResponse_Body),
func(m1, m2 protoMessage) bool {
return equalListShardResponseBodies(
m1.(*control.ListShardsResponse_Body),
m2.(*control.ListShardsResponse_Body),
)
},
)
}
func equalListShardResponseBodies(b1, b2 *control.ListShardsResponse_Body) bool {
if len(b1.Shards) != len(b2.Shards) {
return false
}
for i := range b1.Shards {
if b1.Shards[i].GetMetabasePath() != b2.Shards[i].GetMetabasePath() ||
b1.Shards[i].GetWritecachePath() != b2.Shards[i].GetWritecachePath() ||
b1.Shards[i].GetPiloramaPath() != b2.Shards[i].GetPiloramaPath() ||
!bytes.Equal(b1.Shards[i].GetShard_ID(), b2.Shards[i].GetShard_ID()) {
return false
}
info1 := b1.Shards[i].GetBlobstor()
info2 := b2.Shards[i].GetBlobstor()
if !compareBlobstorInfo(info1, info2) {
return false
}
}
for i := range b1.Shards {
for j := i + 1; j < len(b1.Shards); j++ {
if b1.Shards[i].GetMetabasePath() == b2.Shards[j].GetMetabasePath() ||
!compareBlobstorInfo(b1.Shards[i].Blobstor, b2.Shards[i].Blobstor) ||
b1.Shards[i].GetWritecachePath() == b2.Shards[j].GetWritecachePath() ||
bytes.Equal(b1.Shards[i].GetShard_ID(), b2.Shards[j].GetShard_ID()) {
return false
}
}
}
return true
}
func compareBlobstorInfo(a, b []*control.BlobstorInfo) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i].Type != b[i].Type ||
a[i].Path != b[i].Path {
return false
}
}
return true
}
func generateListShardsResponseBody() *control.ListShardsResponse_Body {
body := new(control.ListShardsResponse_Body)
body.SetShards([]*control.ShardInfo{
generateShardInfo(0),
generateShardInfo(1),
})
return body
}
func TestSetShardModeRequest_Body_StableMarshal(t *testing.T) {
testStableMarshal(t,
generateSetShardModeRequestBody(),
new(control.SetShardModeRequest_Body),
func(m1, m2 protoMessage) bool {
return equalSetShardModeRequestBodies(
m1.(*control.SetShardModeRequest_Body),
m2.(*control.SetShardModeRequest_Body),
)
},
)
}
func generateSetShardModeRequestBody() *control.SetShardModeRequest_Body {
body := new(control.SetShardModeRequest_Body)
body.SetShardIDList([][]byte{{0, 1, 2, 3, 4}})
body.SetMode(control.ShardMode_READ_WRITE)
return body
}
func equalSetShardModeRequestBodies(b1, b2 *control.SetShardModeRequest_Body) bool {
if b1.GetMode() != b2.GetMode() || len(b1.Shard_ID) != len(b2.Shard_ID) {
return false
}
for i := range b1.Shard_ID {
if !bytes.Equal(b1.Shard_ID[i], b2.Shard_ID[i]) {
return false
}
}
return true
}
func TestSynchronizeTreeRequest_Body_StableMarshal(t *testing.T) {
testStableMarshal(t,
&control.SynchronizeTreeRequest_Body{
ContainerId: []byte{1, 2, 3, 4, 5, 6, 7},
TreeId: "someID",
Height: 42,
},
new(control.SynchronizeTreeRequest_Body),
func(m1, m2 protoMessage) bool {
b1 := m1.(*control.SynchronizeTreeRequest_Body)
b2 := m2.(*control.SynchronizeTreeRequest_Body)
return bytes.Equal(b1.GetContainerId(), b2.GetContainerId()) &&
b1.GetTreeId() == b2.GetTreeId() &&
b1.GetHeight() == b2.GetHeight()
},
)
}

View file

@ -1,118 +0,0 @@
package control
import (
"google.golang.org/protobuf/encoding/protojson"
)
// SetKey sets public key used for signing.
func (x *Signature) SetKey(v []byte) {
if x != nil {
x.Key = v
}
}
// SetSign sets binary signature.
func (x *Signature) SetSign(v []byte) {
if x != nil {
x.Sign = v
}
}
// SetKey sets key of the node attribute.
func (x *NodeInfo_Attribute) SetKey(v string) {
if x != nil {
x.Key = v
}
}
// SetValue sets value of the node attribute.
func (x *NodeInfo_Attribute) SetValue(v string) {
if x != nil {
x.Value = v
}
}
// SetParents sets parent keys.
func (x *NodeInfo_Attribute) SetParents(v []string) {
if x != nil {
x.Parents = v
}
}
// SetPublicKey sets public key of the FrostFS node in a binary format.
func (x *NodeInfo) SetPublicKey(v []byte) {
if x != nil {
x.PublicKey = v
}
}
// SetAddresses sets ways to connect to a node.
func (x *NodeInfo) SetAddresses(v []string) {
if x != nil {
x.Addresses = v
}
}
// SetAttributes sets attributes of the FrostFS Storage Node.
func (x *NodeInfo) SetAttributes(v []*NodeInfo_Attribute) {
if x != nil {
x.Attributes = v
}
}
// SetState sets state of the FrostFS node.
func (x *NodeInfo) SetState(v NetmapStatus) {
if x != nil {
x.State = v
}
}
// SetEpoch sets revision number of the network map.
func (x *Netmap) SetEpoch(v uint64) {
if x != nil {
x.Epoch = v
}
}
// SetNodes sets nodes presented in network.
func (x *Netmap) SetNodes(v []*NodeInfo) {
if x != nil {
x.Nodes = v
}
}
func (x *Netmap) MarshalJSON() ([]byte, error) {
return protojson.MarshalOptions{
EmitUnpopulated: true,
}.Marshal(x)
}
// SetID sets identificator of the shard.
func (x *ShardInfo) SetID(v []byte) {
x.Shard_ID = v
}
// SetMetabasePath sets path to shard's metabase.
func (x *ShardInfo) SetMetabasePath(v string) {
x.MetabasePath = v
}
// SetWriteCachePath sets path to shard's write-cache.
func (x *ShardInfo) SetWriteCachePath(v string) {
x.WritecachePath = v
}
// SetPiloramaPath sets path to shard's pilorama.
func (x *ShardInfo) SetPiloramaPath(v string) {
x.PiloramaPath = v
}
// SetMode sets path to shard's work mode.
func (x *ShardInfo) SetMode(v ShardMode) {
x.Mode = v
}
// SetErrorCount sets shard's error counter.
func (x *ShardInfo) SetErrorCount(count uint32) {
x.ErrorCount = count
}

Binary file not shown.

Binary file not shown.

View file

@ -1,151 +0,0 @@
package control_test
import (
"bytes"
"path/filepath"
"strconv"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
"github.com/google/uuid"
)
func TestNetmap_StableMarshal(t *testing.T) {
testStableMarshal(t, generateNetmap(), new(control.Netmap), func(m1, m2 protoMessage) bool {
return equalNetmaps(m1.(*control.Netmap), m2.(*control.Netmap))
})
}
func generateNetmap() *control.Netmap {
nm := new(control.Netmap)
nm.SetEpoch(13)
const nodeCount = 2
nodes := make([]*control.NodeInfo, 0, nodeCount)
for i := 0; i < nodeCount; i++ {
n := new(control.NodeInfo)
n.SetPublicKey(testData(33))
n.SetAddresses([]string{testString(), testString()})
n.SetState(control.NetmapStatus_ONLINE)
const attrCount = 2
attrs := make([]*control.NodeInfo_Attribute, 0, attrCount)
for j := 0; j < attrCount; j++ {
a := new(control.NodeInfo_Attribute)
a.SetKey(testString())
a.SetValue(testString())
const parentsCount = 2
parents := make([]string, 0, parentsCount)
for k := 0; k < parentsCount; k++ {
parents = append(parents, testString())
}
a.SetParents(parents)
attrs = append(attrs, a)
}
n.SetAttributes(attrs)
nodes = append(nodes, n)
}
nm.SetNodes(nodes)
return nm
}
func equalNetmaps(nm1, nm2 *control.Netmap) bool {
if nm1.GetEpoch() != nm2.GetEpoch() {
return false
}
n1, n2 := nm1.GetNodes(), nm2.GetNodes()
if len(n1) != len(n2) {
return false
}
for i := range n1 {
if !equalNodeInfos(n1[i], n2[i]) {
return false
}
}
return true
}
func equalNodeInfos(n1, n2 *control.NodeInfo) bool {
if !bytes.Equal(n1.GetPublicKey(), n2.GetPublicKey()) ||
n1.GetState() != n2.GetState() {
return false
}
na1, na2 := n1.GetAddresses(), n2.GetAddresses()
if len(na1) != len(na2) {
return false
}
for i := range na1 {
if na1[i] != na2[i] {
return false
}
}
a1, a2 := n1.GetAttributes(), n2.GetAttributes()
if len(a1) != len(a2) {
return false
}
for i := range a1 {
if a1[i].GetKey() != a2[i].GetKey() || a1[i].GetValue() != a2[i].GetValue() {
return false
}
p1, p2 := a1[i].GetParents(), a2[i].GetParents()
if len(p1) != len(p2) {
return false
}
for j := range p1 {
if p1[j] != p2[j] {
return false
}
}
}
return true
}
func generateShardInfo(id int) *control.ShardInfo {
si := new(control.ShardInfo)
path := "/nice/dir/awesome/files/" + strconv.Itoa(id)
uid, _ := uuid.NewRandom()
bin, _ := uid.MarshalBinary()
si.SetID(bin)
si.SetMode(control.ShardMode_READ_WRITE)
si.SetMetabasePath(filepath.Join(path, "meta"))
si.Blobstor = []*control.BlobstorInfo{
{Type: fstree.Type, Path: filepath.Join(path, "fstree")},
{Type: blobovniczatree.Type, Path: filepath.Join(path, "blobtree")},
}
si.SetWriteCachePath(filepath.Join(path, "writecache"))
si.SetPiloramaPath(filepath.Join(path, "pilorama"))
return si
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.