Use new protobuf marshaler #1322
29 changed files with 17 additions and 751 deletions
8
Makefile
8
Makefile
|
@ -11,7 +11,6 @@ GO_VERSION ?= 1.22
|
|||
LINT_VERSION ?= 1.56.1
|
||||
TRUECLOUDLAB_LINT_VERSION ?= 0.0.5
|
||||
PROTOC_VERSION ?= 25.0
|
||||
PROTOC_GEN_GO_VERSION ?= $(shell go list -f '{{.Version}}' -m google.golang.org/protobuf)
|
||||
PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2)
|
||||
PROTOC_OS_VERSION=osx-x86_64
|
||||
ifeq ($(shell uname), Linux)
|
||||
|
@ -39,7 +38,6 @@ LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT
|
|||
TMP_DIR := .cache
|
||||
PROTOBUF_DIR ?= $(abspath $(BIN))/protobuf
|
||||
PROTOC_DIR ?= $(PROTOBUF_DIR)/protoc-v$(PROTOC_VERSION)
|
||||
PROTOC_GEN_GO_DIR ?= $(PROTOBUF_DIR)/protoc-gen-go-$(PROTOC_GEN_GO_VERSION)
|
||||
PROTOGEN_FROSTFS_DIR ?= $(PROTOBUF_DIR)/protogen-$(PROTOGEN_FROSTFS_VERSION)
|
||||
STATICCHECK_DIR ?= $(abspath $(BIN))/staticcheck
|
||||
STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION)
|
||||
|
@ -107,17 +105,15 @@ export-metrics: dep
|
|||
|
||||
# Regenerate proto files:
|
||||
protoc:
|
||||
@if [ ! -d "$(PROTOC_DIR)" ] || [ ! -d "$(PROTOC_GEN_GO_DIR)" ] || [ ! -d "$(PROTOGEN_FROSTFS_DIR)" ]; then \
|
||||
@if [ ! -d "$(PROTOC_DIR)" ] || [ ! -d "$(PROTOGEN_FROSTFS_DIR)" ]; then \
|
||||
make protoc-install; \
|
||||
fi
|
||||
@for f in `find . -type f -name '*.proto' -not -path './bin/*'`; do \
|
||||
echo "⇒ Processing $$f "; \
|
||||
$(PROTOC_DIR)/bin/protoc \
|
||||
--proto_path=.:$(PROTOC_DIR)/include:/usr/local/include \
|
||||
--plugin=protoc-gen-go=$(PROTOC_GEN_GO_DIR)/protoc-gen-go \
|
||||
dstepanov-yadro marked this conversation as resolved
Outdated
|
||||
--plugin=protoc-gen-go-frostfs=$(PROTOGEN_FROSTFS_DIR)/protogen \
|
||||
--go-frostfs_out=. --go-frostfs_opt=paths=source_relative \
|
||||
--go_out=. --go_opt=paths=source_relative \
|
||||
--go-grpc_opt=require_unimplemented_servers=false \
|
||||
--go-grpc_out=. --go-grpc_opt=paths=source_relative $$f; \
|
||||
done
|
||||
|
@ -130,8 +126,6 @@ protoc-install:
|
|||
@wget -q -O $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip 'https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS_VERSION).zip'
|
||||
@unzip -q -o $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip -d $(PROTOC_DIR)
|
||||
@rm $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip
|
||||
@echo "⇒ Installing protoc-gen-go..."
|
||||
@GOBIN=$(PROTOC_GEN_GO_DIR) go install -v google.golang.org/protobuf/...@$(PROTOC_GEN_GO_VERSION)
|
||||
@echo "⇒ Instaling protogen FrostFS plugin..."
|
||||
@GOBIN=$(PROTOGEN_FROSTFS_DIR) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/protogen@$(PROTOGEN_FROSTFS_VERSION)
|
||||
|
||||
|
|
|
@ -84,7 +84,7 @@ func setNetmapStatus(cmd *cobra.Command, _ []string) {
|
|||
body.SetStatus(control.NetmapStatus_MAINTENANCE)
|
||||
|
||||
if force {
|
||||
body.SetForceMaintenance()
|
||||
body.SetForceMaintenance(true)
|
||||
common.PrintVerbose(cmd, "Local maintenance will be forced.")
|
||||
}
|
||||
targetStatus = control.NetmapStatus_MAINTENANCE
|
||||
|
|
|
@ -117,10 +117,10 @@ func setShardMode(cmd *cobra.Command, _ []string) {
|
|||
req.SetBody(body)
|
||||
|
||||
body.SetMode(mode)
|
||||
body.SetShardIDList(getShardIDList(cmd))
|
||||
body.SetShard_ID(getShardIDList(cmd))
|
||||
|
||||
reset, _ := cmd.Flags().GetBool(shardClearErrorsFlag)
|
||||
body.ClearErrorCounter(reset)
|
||||
body.SetResetErrorCounter(reset)
|
||||
|
||||
signRequest(cmd, pk, req)
|
||||
|
||||
|
|
|
@ -44,7 +44,7 @@ func verifyResponse(cmd *cobra.Command,
|
|||
GetSign() []byte
|
||||
},
|
||||
body interface {
|
||||
StableMarshal([]byte) []byte
|
||||
MarshalProtobuf([]byte) []byte
|
||||
},
|
||||
) {
|
||||
if sigControl == nil {
|
||||
|
@ -60,7 +60,7 @@ func verifyResponse(cmd *cobra.Command,
|
|||
var sig frostfscrypto.Signature
|
||||
commonCmd.ExitOnErr(cmd, "can't read signature: %w", sig.ReadFromV2(sigV2))
|
||||
aarifullin marked this conversation as resolved
Outdated
aarifullin
commented
Disabled for debug? Disabled for debug?
fyrchik
commented
oops, yes oops, yes
fyrchik
commented
moved to WIP, will revisit everything moved to WIP, will revisit everything
fyrchik
commented
Fixed, retested on dev-env Fixed, retested on dev-env
|
||||
|
||||
if !sig.Verify(body.StableMarshal(nil)) {
|
||||
if !sig.Verify(body.MarshalProtobuf(nil)) {
|
||||
commonCmd.ExitOnErr(cmd, "", errors.New("invalid response signature"))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -84,7 +84,7 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) {
|
|||
}
|
||||
}
|
||||
|
||||
s.setControlNetmapStatus(ctrlNetSt)
|
||||
s.setControlNetmapStatus(control.NetmapStatus(ctrlNetSt))
|
||||
}
|
||||
|
||||
// sets the current node state to the given value. Subsequent cfg.bootstrap
|
||||
|
|
7
go.mod
7
go.mod
|
@ -4,16 +4,17 @@ go 1.21
|
|||
|
||||
require (
|
||||
code.gitea.io/sdk/gitea v0.17.1
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240813155151-d112a28d382f
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240819074700-a43110e36326
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0
|
||||
git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240813155821-98aabc45a720
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820072028-6dd7be11d13b
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1
|
||||
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88
|
||||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0
|
||||
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
||||
github.com/VictoriaMetrics/easyproto v0.1.4
|
||||
github.com/cheggaaa/pb v1.0.29
|
||||
github.com/chzyer/readline v1.5.1
|
||||
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
|
||||
|
@ -21,6 +22,7 @@ require (
|
|||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/klauspost/compress v1.17.4
|
||||
github.com/mailru/easyjson v0.7.7
|
||||
github.com/mitchellh/go-homedir v1.1.0
|
||||
github.com/mr-tron/base58 v1.2.0
|
||||
github.com/multiformats/go-multiaddr v0.12.1
|
||||
|
@ -84,7 +86,6 @@ require (
|
|||
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
||||
github.com/klauspost/reedsolomon v1.12.1 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/minio/sha256-simd v1.0.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
|
|
BIN
go.sum
BIN
go.sum
Binary file not shown.
|
@ -1,6 +1,7 @@
|
|||
package meta
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
@ -107,7 +108,7 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b
|
|||
// check in primary index
|
||||
data := getFromBucket(tx, primaryBucketName(cnr, bucketName), key)
|
||||
if len(data) != 0 {
|
||||
return obj, obj.Unmarshal(data)
|
||||
return obj, obj.Unmarshal(bytes.Clone(data))
|
||||
}
|
||||
|
||||
data = getFromBucket(tx, ecInfoBucketName(cnr, bucketName), key)
|
||||
|
@ -118,13 +119,13 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b
|
|||
// if not found then check in tombstone index
|
||||
data = getFromBucket(tx, tombstoneBucketName(cnr, bucketName), key)
|
||||
if len(data) != 0 {
|
||||
return obj, obj.Unmarshal(data)
|
||||
return obj, obj.Unmarshal(bytes.Clone(data))
|
||||
}
|
||||
|
||||
// if not found then check in locker index
|
||||
data = getFromBucket(tx, bucketNameLockers(cnr, bucketName), key)
|
||||
if len(data) != 0 {
|
||||
return obj, obj.Unmarshal(data)
|
||||
return obj, obj.Unmarshal(bytes.Clone(data))
|
||||
dstepanov-yadro marked this conversation as resolved
Outdated
dstepanov-yadro
commented
What is the benefit of using a new marshalled in the end? What is the benefit of using a new marshalled in the end?
fyrchik
commented
1. We clone a single slice here vs cloning allocating multiple times for each slice field we use.
2. If we unmarshal from then FSTree, no clone is done.
3. Later we will add idiomatic `[]Attribute` fields, that will allow us to gradually remove layers in `ToGRPCMessage()` conversion functions from the api-go.
|
||||
}
|
||||
|
||||
// if not found then check if object is a virtual
|
||||
|
|
|
@ -1,33 +0,0 @@
|
|||
package control_test
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/mr-tron/base58"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
type protoMessage interface {
|
||||
StableMarshal([]byte) []byte
|
||||
proto.Message
|
||||
}
|
||||
|
||||
func testStableMarshal(t *testing.T, m1, m2 protoMessage, cmp func(m1, m2 protoMessage) bool) {
|
||||
require.NoError(t, proto.Unmarshal(m1.StableMarshal(nil), m2))
|
||||
|
||||
require.True(t, cmp(m1, m2))
|
||||
}
|
||||
|
||||
func testData(sz int) []byte {
|
||||
d := make([]byte, sz)
|
||||
|
||||
_, _ = rand.Read(d)
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
func testString() string {
|
||||
return base58.Encode(testData(10))
|
||||
}
|
|
@ -1,46 +0,0 @@
|
|||
package control
|
||||
|
||||
// SetBody sets health check request body.
|
||||
func (x *HealthCheckRequest) SetBody(v *HealthCheckRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetHealthStatus sets health status of the IR application.
|
||||
func (x *HealthCheckResponse_Body) SetHealthStatus(v HealthStatus) {
|
||||
if x != nil {
|
||||
x.HealthStatus = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets health check response body.
|
||||
func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
func (x *TickEpochRequest) SetBody(v *TickEpochRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
func (x *RemoveNodeRequest) SetBody(v *RemoveNodeRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
func (x *RemoveNodeResponse) SetBody(v *RemoveNodeResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
BIN
pkg/services/control/ir/service.pb.go
generated
BIN
pkg/services/control/ir/service.pb.go
generated
Binary file not shown.
BIN
pkg/services/control/ir/service_frostfs.pb.go
generated
BIN
pkg/services/control/ir/service_frostfs.pb.go
generated
Binary file not shown.
|
@ -1,44 +0,0 @@
|
|||
package control_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
type protoMessage interface {
|
||||
StableMarshal([]byte) []byte
|
||||
proto.Message
|
||||
}
|
||||
|
||||
func testStableMarshal(t *testing.T, m1, m2 protoMessage, cmp func(m1, m2 protoMessage) bool) {
|
||||
require.NoError(t, proto.Unmarshal(m1.StableMarshal(nil), m2))
|
||||
|
||||
require.True(t, cmp(m1, m2))
|
||||
}
|
||||
|
||||
func TestHealthCheckResponse_Body_StableMarshal(t *testing.T) {
|
||||
testStableMarshal(t,
|
||||
generateHealthCheckResponseBody(),
|
||||
new(control.HealthCheckResponse_Body),
|
||||
func(m1, m2 protoMessage) bool {
|
||||
return equalHealthCheckResponseBodies(
|
||||
m1.(*control.HealthCheckResponse_Body),
|
||||
m2.(*control.HealthCheckResponse_Body),
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func generateHealthCheckResponseBody() *control.HealthCheckResponse_Body {
|
||||
body := new(control.HealthCheckResponse_Body)
|
||||
body.SetHealthStatus(control.HealthStatus_SHUTTING_DOWN)
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
func equalHealthCheckResponseBodies(b1, b2 *control.HealthCheckResponse_Body) bool {
|
||||
return b1.GetHealthStatus() == b2.GetHealthStatus()
|
||||
}
|
|
@ -1,15 +0,0 @@
|
|||
package control
|
||||
|
||||
// SetKey sets public key used for signing.
|
||||
func (x *Signature) SetKey(v []byte) {
|
||||
if x != nil {
|
||||
x.Key = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetSign sets binary signature.
|
||||
func (x *Signature) SetSign(v []byte) {
|
||||
if x != nil {
|
||||
x.Sign = v
|
||||
}
|
||||
}
|
BIN
pkg/services/control/ir/types.pb.go
generated
BIN
pkg/services/control/ir/types.pb.go
generated
Binary file not shown.
BIN
pkg/services/control/ir/types_frostfs.pb.go
generated
BIN
pkg/services/control/ir/types_frostfs.pb.go
generated
Binary file not shown.
|
@ -30,10 +30,10 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) (
|
|||
for _, sh := range info.Shards {
|
||||
si := new(control.ShardInfo)
|
||||
|
||||
si.SetID(*sh.ID)
|
||||
si.SetShard_ID(*sh.ID)
|
||||
si.SetMetabasePath(sh.MetaBaseInfo.Path)
|
||||
si.Blobstor = blobstorInfoToProto(sh.BlobStorInfo)
|
||||
si.SetWriteCachePath(sh.WriteCacheInfo.Path)
|
||||
si.SetWritecachePath(sh.WriteCacheInfo.Path)
|
||||
si.SetPiloramaPath(sh.PiloramaInfo.Path)
|
||||
|
||||
var m control.ShardMode
|
||||
|
|
|
@ -1,142 +0,0 @@
|
|||
package control
|
||||
|
||||
// SetBody sets health check request body.
|
||||
func (x *HealthCheckRequest) SetBody(v *HealthCheckRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetNetmapStatus sets status of the storage node in FrostFS network map.
|
||||
func (x *HealthCheckResponse_Body) SetNetmapStatus(v NetmapStatus) {
|
||||
if x != nil {
|
||||
x.NetmapStatus = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetHealthStatus sets health status of the storage node application.
|
||||
func (x *HealthCheckResponse_Body) SetHealthStatus(v HealthStatus) {
|
||||
if x != nil {
|
||||
x.HealthStatus = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets health check response body.
|
||||
func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetStatus sets new storage node status in FrostFS network map.
|
||||
func (x *SetNetmapStatusRequest_Body) SetStatus(v NetmapStatus) {
|
||||
if x != nil {
|
||||
x.Status = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetForceMaintenance sets force_maintenance flag in the message.
|
||||
func (x *SetNetmapStatusRequest_Body) SetForceMaintenance() {
|
||||
x.ForceMaintenance = true
|
||||
}
|
||||
|
||||
// SetBody sets body of the set netmap status request .
|
||||
func (x *SetNetmapStatusRequest) SetBody(v *SetNetmapStatusRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets set body of the netmap status response.
|
||||
func (x *SetNetmapStatusResponse) SetBody(v *SetNetmapStatusResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetAddressList sets list of objects to be removed in FrostFS API binary format.
|
||||
func (x *DropObjectsRequest_Body) SetAddressList(v [][]byte) {
|
||||
if x != nil {
|
||||
x.AddressList = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets body of the set "Drop objects" request.
|
||||
func (x *DropObjectsRequest) SetBody(v *DropObjectsRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets set body of the "Drop objects" response.
|
||||
func (x *DropObjectsResponse) SetBody(v *DropObjectsResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets list shards request body.
|
||||
func (x *ListShardsRequest) SetBody(v *ListShardsRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetShards sets shards of the storage node.
|
||||
func (x *ListShardsResponse_Body) SetShards(v []*ShardInfo) {
|
||||
if x != nil {
|
||||
x.Shards = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets list shards response body.
|
||||
func (x *ListShardsResponse) SetBody(v *ListShardsResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetShardIDList sets shard ID whose mode is requested to be set.
|
||||
func (x *SetShardModeRequest_Body) SetShardIDList(v [][]byte) {
|
||||
if v != nil {
|
||||
x.Shard_ID = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetMode sets mode of the shard.
|
||||
func (x *SetShardModeRequest_Body) SetMode(v ShardMode) {
|
||||
x.Mode = v
|
||||
}
|
||||
|
||||
// ClearErrorCounter sets flag signifying whether error counter for shard should be cleared.
|
||||
func (x *SetShardModeRequest_Body) ClearErrorCounter(reset bool) {
|
||||
x.ResetErrorCounter = reset
|
||||
}
|
||||
|
||||
// SetBody sets request body.
|
||||
func (x *SetShardModeRequest) SetBody(v *SetShardModeRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets body of the set shard mode response.
|
||||
func (x *SetShardModeResponse) SetBody(v *SetShardModeResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets list shards request body.
|
||||
func (x *SynchronizeTreeRequest) SetBody(v *SynchronizeTreeRequest_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetBody sets list shards response body.
|
||||
func (x *SynchronizeTreeResponse) SetBody(v *SynchronizeTreeResponse_Body) {
|
||||
if x != nil {
|
||||
x.Body = v
|
||||
}
|
||||
}
|
BIN
pkg/services/control/service.pb.go
generated
BIN
pkg/services/control/service.pb.go
generated
Binary file not shown.
BIN
pkg/services/control/service_frostfs.pb.go
generated
BIN
pkg/services/control/service_frostfs.pb.go
generated
Binary file not shown.
|
@ -1,181 +0,0 @@
|
|||
package control_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
)
|
||||
|
||||
func TestHealthCheckResponse_Body_StableMarshal(t *testing.T) {
|
||||
testStableMarshal(t,
|
||||
generateHealthCheckResponseBody(),
|
||||
new(control.HealthCheckResponse_Body),
|
||||
func(m1, m2 protoMessage) bool {
|
||||
return equalHealthCheckResponseBodies(
|
||||
m1.(*control.HealthCheckResponse_Body),
|
||||
m2.(*control.HealthCheckResponse_Body),
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func generateHealthCheckResponseBody() *control.HealthCheckResponse_Body {
|
||||
body := new(control.HealthCheckResponse_Body)
|
||||
body.SetNetmapStatus(control.NetmapStatus_ONLINE)
|
||||
body.SetHealthStatus(control.HealthStatus_SHUTTING_DOWN)
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
func equalHealthCheckResponseBodies(b1, b2 *control.HealthCheckResponse_Body) bool {
|
||||
return b1.GetNetmapStatus() == b2.GetNetmapStatus() &&
|
||||
b1.GetHealthStatus() == b2.GetHealthStatus()
|
||||
}
|
||||
|
||||
func TestSetNetmapStatusRequest_Body_StableMarshal(t *testing.T) {
|
||||
testStableMarshal(t,
|
||||
generateSetNetmapStatusRequestBody(),
|
||||
new(control.SetNetmapStatusRequest_Body),
|
||||
func(m1, m2 protoMessage) bool {
|
||||
return equalSetnetmapStatusRequestBodies(
|
||||
m1.(*control.SetNetmapStatusRequest_Body),
|
||||
m2.(*control.SetNetmapStatusRequest_Body),
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func generateSetNetmapStatusRequestBody() *control.SetNetmapStatusRequest_Body {
|
||||
body := new(control.SetNetmapStatusRequest_Body)
|
||||
body.SetStatus(control.NetmapStatus_ONLINE)
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
func equalSetnetmapStatusRequestBodies(b1, b2 *control.SetNetmapStatusRequest_Body) bool {
|
||||
return b1.GetStatus() == b2.GetStatus()
|
||||
}
|
||||
|
||||
func TestListShardsResponse_Body_StableMarshal(t *testing.T) {
|
||||
testStableMarshal(t,
|
||||
generateListShardsResponseBody(),
|
||||
new(control.ListShardsResponse_Body),
|
||||
func(m1, m2 protoMessage) bool {
|
||||
return equalListShardResponseBodies(
|
||||
m1.(*control.ListShardsResponse_Body),
|
||||
m2.(*control.ListShardsResponse_Body),
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func equalListShardResponseBodies(b1, b2 *control.ListShardsResponse_Body) bool {
|
||||
if len(b1.Shards) != len(b2.Shards) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range b1.Shards {
|
||||
if b1.Shards[i].GetMetabasePath() != b2.Shards[i].GetMetabasePath() ||
|
||||
b1.Shards[i].GetWritecachePath() != b2.Shards[i].GetWritecachePath() ||
|
||||
b1.Shards[i].GetPiloramaPath() != b2.Shards[i].GetPiloramaPath() ||
|
||||
!bytes.Equal(b1.Shards[i].GetShard_ID(), b2.Shards[i].GetShard_ID()) {
|
||||
return false
|
||||
}
|
||||
|
||||
info1 := b1.Shards[i].GetBlobstor()
|
||||
info2 := b2.Shards[i].GetBlobstor()
|
||||
if !compareBlobstorInfo(info1, info2) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
for i := range b1.Shards {
|
||||
for j := i + 1; j < len(b1.Shards); j++ {
|
||||
if b1.Shards[i].GetMetabasePath() == b2.Shards[j].GetMetabasePath() ||
|
||||
!compareBlobstorInfo(b1.Shards[i].Blobstor, b2.Shards[i].Blobstor) ||
|
||||
b1.Shards[i].GetWritecachePath() == b2.Shards[j].GetWritecachePath() ||
|
||||
bytes.Equal(b1.Shards[i].GetShard_ID(), b2.Shards[j].GetShard_ID()) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func compareBlobstorInfo(a, b []*control.BlobstorInfo) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i].Type != b[i].Type ||
|
||||
a[i].Path != b[i].Path {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func generateListShardsResponseBody() *control.ListShardsResponse_Body {
|
||||
body := new(control.ListShardsResponse_Body)
|
||||
body.SetShards([]*control.ShardInfo{
|
||||
generateShardInfo(0),
|
||||
generateShardInfo(1),
|
||||
})
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
func TestSetShardModeRequest_Body_StableMarshal(t *testing.T) {
|
||||
testStableMarshal(t,
|
||||
generateSetShardModeRequestBody(),
|
||||
new(control.SetShardModeRequest_Body),
|
||||
func(m1, m2 protoMessage) bool {
|
||||
return equalSetShardModeRequestBodies(
|
||||
m1.(*control.SetShardModeRequest_Body),
|
||||
m2.(*control.SetShardModeRequest_Body),
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func generateSetShardModeRequestBody() *control.SetShardModeRequest_Body {
|
||||
body := new(control.SetShardModeRequest_Body)
|
||||
body.SetShardIDList([][]byte{{0, 1, 2, 3, 4}})
|
||||
body.SetMode(control.ShardMode_READ_WRITE)
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
func equalSetShardModeRequestBodies(b1, b2 *control.SetShardModeRequest_Body) bool {
|
||||
if b1.GetMode() != b2.GetMode() || len(b1.Shard_ID) != len(b2.Shard_ID) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range b1.Shard_ID {
|
||||
if !bytes.Equal(b1.Shard_ID[i], b2.Shard_ID[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func TestSynchronizeTreeRequest_Body_StableMarshal(t *testing.T) {
|
||||
testStableMarshal(t,
|
||||
&control.SynchronizeTreeRequest_Body{
|
||||
ContainerId: []byte{1, 2, 3, 4, 5, 6, 7},
|
||||
TreeId: "someID",
|
||||
Height: 42,
|
||||
},
|
||||
new(control.SynchronizeTreeRequest_Body),
|
||||
func(m1, m2 protoMessage) bool {
|
||||
b1 := m1.(*control.SynchronizeTreeRequest_Body)
|
||||
b2 := m2.(*control.SynchronizeTreeRequest_Body)
|
||||
return bytes.Equal(b1.GetContainerId(), b2.GetContainerId()) &&
|
||||
b1.GetTreeId() == b2.GetTreeId() &&
|
||||
b1.GetHeight() == b2.GetHeight()
|
||||
},
|
||||
)
|
||||
}
|
|
@ -1,118 +0,0 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
|
||||
// SetKey sets public key used for signing.
|
||||
func (x *Signature) SetKey(v []byte) {
|
||||
if x != nil {
|
||||
x.Key = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetSign sets binary signature.
|
||||
func (x *Signature) SetSign(v []byte) {
|
||||
if x != nil {
|
||||
x.Sign = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetKey sets key of the node attribute.
|
||||
func (x *NodeInfo_Attribute) SetKey(v string) {
|
||||
if x != nil {
|
||||
x.Key = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetValue sets value of the node attribute.
|
||||
func (x *NodeInfo_Attribute) SetValue(v string) {
|
||||
if x != nil {
|
||||
x.Value = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetParents sets parent keys.
|
||||
func (x *NodeInfo_Attribute) SetParents(v []string) {
|
||||
if x != nil {
|
||||
x.Parents = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetPublicKey sets public key of the FrostFS node in a binary format.
|
||||
func (x *NodeInfo) SetPublicKey(v []byte) {
|
||||
if x != nil {
|
||||
x.PublicKey = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetAddresses sets ways to connect to a node.
|
||||
func (x *NodeInfo) SetAddresses(v []string) {
|
||||
if x != nil {
|
||||
x.Addresses = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetAttributes sets attributes of the FrostFS Storage Node.
|
||||
func (x *NodeInfo) SetAttributes(v []*NodeInfo_Attribute) {
|
||||
if x != nil {
|
||||
x.Attributes = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetState sets state of the FrostFS node.
|
||||
func (x *NodeInfo) SetState(v NetmapStatus) {
|
||||
if x != nil {
|
||||
x.State = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetEpoch sets revision number of the network map.
|
||||
func (x *Netmap) SetEpoch(v uint64) {
|
||||
if x != nil {
|
||||
x.Epoch = v
|
||||
}
|
||||
}
|
||||
|
||||
// SetNodes sets nodes presented in network.
|
||||
func (x *Netmap) SetNodes(v []*NodeInfo) {
|
||||
if x != nil {
|
||||
x.Nodes = v
|
||||
}
|
||||
}
|
||||
|
||||
func (x *Netmap) MarshalJSON() ([]byte, error) {
|
||||
return protojson.MarshalOptions{
|
||||
EmitUnpopulated: true,
|
||||
}.Marshal(x)
|
||||
}
|
||||
|
||||
// SetID sets identificator of the shard.
|
||||
func (x *ShardInfo) SetID(v []byte) {
|
||||
x.Shard_ID = v
|
||||
}
|
||||
|
||||
// SetMetabasePath sets path to shard's metabase.
|
||||
func (x *ShardInfo) SetMetabasePath(v string) {
|
||||
x.MetabasePath = v
|
||||
}
|
||||
|
||||
// SetWriteCachePath sets path to shard's write-cache.
|
||||
func (x *ShardInfo) SetWriteCachePath(v string) {
|
||||
x.WritecachePath = v
|
||||
}
|
||||
|
||||
// SetPiloramaPath sets path to shard's pilorama.
|
||||
func (x *ShardInfo) SetPiloramaPath(v string) {
|
||||
x.PiloramaPath = v
|
||||
}
|
||||
|
||||
// SetMode sets path to shard's work mode.
|
||||
func (x *ShardInfo) SetMode(v ShardMode) {
|
||||
x.Mode = v
|
||||
}
|
||||
|
||||
// SetErrorCount sets shard's error counter.
|
||||
func (x *ShardInfo) SetErrorCount(count uint32) {
|
||||
x.ErrorCount = count
|
||||
}
|
BIN
pkg/services/control/types.pb.go
generated
BIN
pkg/services/control/types.pb.go
generated
Binary file not shown.
BIN
pkg/services/control/types_frostfs.pb.go
generated
BIN
pkg/services/control/types_frostfs.pb.go
generated
Binary file not shown.
|
@ -1,151 +0,0 @@
|
|||
package control_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestNetmap_StableMarshal(t *testing.T) {
|
||||
testStableMarshal(t, generateNetmap(), new(control.Netmap), func(m1, m2 protoMessage) bool {
|
||||
return equalNetmaps(m1.(*control.Netmap), m2.(*control.Netmap))
|
||||
})
|
||||
}
|
||||
|
||||
func generateNetmap() *control.Netmap {
|
||||
nm := new(control.Netmap)
|
||||
nm.SetEpoch(13)
|
||||
|
||||
const nodeCount = 2
|
||||
|
||||
nodes := make([]*control.NodeInfo, 0, nodeCount)
|
||||
|
||||
for i := 0; i < nodeCount; i++ {
|
||||
n := new(control.NodeInfo)
|
||||
n.SetPublicKey(testData(33))
|
||||
n.SetAddresses([]string{testString(), testString()})
|
||||
n.SetState(control.NetmapStatus_ONLINE)
|
||||
|
||||
const attrCount = 2
|
||||
|
||||
attrs := make([]*control.NodeInfo_Attribute, 0, attrCount)
|
||||
|
||||
for j := 0; j < attrCount; j++ {
|
||||
a := new(control.NodeInfo_Attribute)
|
||||
a.SetKey(testString())
|
||||
a.SetValue(testString())
|
||||
|
||||
const parentsCount = 2
|
||||
|
||||
parents := make([]string, 0, parentsCount)
|
||||
|
||||
for k := 0; k < parentsCount; k++ {
|
||||
parents = append(parents, testString())
|
||||
}
|
||||
|
||||
a.SetParents(parents)
|
||||
|
||||
attrs = append(attrs, a)
|
||||
}
|
||||
|
||||
n.SetAttributes(attrs)
|
||||
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
|
||||
nm.SetNodes(nodes)
|
||||
|
||||
return nm
|
||||
}
|
||||
|
||||
func equalNetmaps(nm1, nm2 *control.Netmap) bool {
|
||||
if nm1.GetEpoch() != nm2.GetEpoch() {
|
||||
return false
|
||||
}
|
||||
|
||||
n1, n2 := nm1.GetNodes(), nm2.GetNodes()
|
||||
|
||||
if len(n1) != len(n2) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range n1 {
|
||||
if !equalNodeInfos(n1[i], n2[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func equalNodeInfos(n1, n2 *control.NodeInfo) bool {
|
||||
if !bytes.Equal(n1.GetPublicKey(), n2.GetPublicKey()) ||
|
||||
n1.GetState() != n2.GetState() {
|
||||
return false
|
||||
}
|
||||
|
||||
na1, na2 := n1.GetAddresses(), n2.GetAddresses()
|
||||
|
||||
if len(na1) != len(na2) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range na1 {
|
||||
if na1[i] != na2[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
a1, a2 := n1.GetAttributes(), n2.GetAttributes()
|
||||
|
||||
if len(a1) != len(a2) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range a1 {
|
||||
if a1[i].GetKey() != a2[i].GetKey() || a1[i].GetValue() != a2[i].GetValue() {
|
||||
return false
|
||||
}
|
||||
|
||||
p1, p2 := a1[i].GetParents(), a2[i].GetParents()
|
||||
|
||||
if len(p1) != len(p2) {
|
||||
return false
|
||||
}
|
||||
|
||||
for j := range p1 {
|
||||
if p1[j] != p2[j] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func generateShardInfo(id int) *control.ShardInfo {
|
||||
si := new(control.ShardInfo)
|
||||
|
||||
path := "/nice/dir/awesome/files/" + strconv.Itoa(id)
|
||||
|
||||
uid, _ := uuid.NewRandom()
|
||||
bin, _ := uid.MarshalBinary()
|
||||
|
||||
si.SetID(bin)
|
||||
si.SetMode(control.ShardMode_READ_WRITE)
|
||||
si.SetMetabasePath(filepath.Join(path, "meta"))
|
||||
si.Blobstor = []*control.BlobstorInfo{
|
||||
{Type: fstree.Type, Path: filepath.Join(path, "fstree")},
|
||||
{Type: blobovniczatree.Type, Path: filepath.Join(path, "blobtree")},
|
||||
}
|
||||
si.SetWriteCachePath(filepath.Join(path, "writecache"))
|
||||
si.SetPiloramaPath(filepath.Join(path, "pilorama"))
|
||||
|
||||
return si
|
||||
}
|
BIN
pkg/services/tree/service.pb.go
generated
BIN
pkg/services/tree/service.pb.go
generated
Binary file not shown.
BIN
pkg/services/tree/service_frostfs.pb.go
generated
BIN
pkg/services/tree/service_frostfs.pb.go
generated
Binary file not shown.
BIN
pkg/services/tree/types.pb.go
generated
BIN
pkg/services/tree/types.pb.go
generated
Binary file not shown.
BIN
pkg/services/tree/types_frostfs.pb.go
generated
BIN
pkg/services/tree/types_frostfs.pb.go
generated
Binary file not shown.
Loading…
Reference in a new issue
If it not required, then need to drop this variable.
fixed