forked from TrueCloudLab/frostfs-node
[#1324] services/tree: Implement Object Tree Service
Object Tree Service allows changing trees assotiated with the container in runtime. Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov
parent
46f4ce2773
commit
62154da17c
18 changed files with 900 additions and 0 deletions
39
cmd/neofs-node/config/tree/config.go
Normal file
39
cmd/neofs-node/config/tree/config.go
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
package treeconfig
|
||||
|
||||
import (
|
||||
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
|
||||
)
|
||||
|
||||
// GRPCConfig is a wrapper over "grpc" config section which provides access
|
||||
// to gRPC configuration of control service.
|
||||
type GRPCConfig struct {
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
const (
|
||||
subsection = "tree"
|
||||
grpcSubsection = "grpc"
|
||||
|
||||
// GRPCEndpointDefault is a default endpoint of gRPC Control service.
|
||||
GRPCEndpointDefault = ""
|
||||
)
|
||||
|
||||
// GRPC returns structure that provides access to "grpc" subsection of
|
||||
// "tree" section.
|
||||
func GRPC(c *config.Config) GRPCConfig {
|
||||
return GRPCConfig{
|
||||
c.Sub(subsection).Sub(grpcSubsection),
|
||||
}
|
||||
}
|
||||
|
||||
// Endpoint returns value of "endpoint" config parameter.
|
||||
//
|
||||
// Returns GRPCEndpointDefault if value is not a non-empty string.
|
||||
func (g GRPCConfig) Endpoint() string {
|
||||
v := config.String(g.cfg, "endpoint")
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
|
||||
return GRPCEndpointDefault
|
||||
}
|
||||
30
cmd/neofs-node/config/tree/config_test.go
Normal file
30
cmd/neofs-node/config/tree/config_test.go
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
package treeconfig_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
|
||||
configtest "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/test"
|
||||
treeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/tree"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestControlSection(t *testing.T) {
|
||||
t.Run("defaults", func(t *testing.T) {
|
||||
empty := configtest.EmptyConfig()
|
||||
|
||||
require.Equal(t, treeconfig.GRPCEndpointDefault, treeconfig.GRPC(empty).Endpoint())
|
||||
})
|
||||
|
||||
const path = "../../../../config/example/node"
|
||||
|
||||
var fileConfigTest = func(c *config.Config) {
|
||||
require.Equal(t, "127.0.0.1:8091", treeconfig.GRPC(c).Endpoint())
|
||||
}
|
||||
|
||||
configtest.ForEachFileType(path, fileConfigTest)
|
||||
|
||||
t.Run("ENV", func(t *testing.T) {
|
||||
configtest.ForEnvFileType(path, fileConfigTest)
|
||||
})
|
||||
}
|
||||
|
|
@ -81,6 +81,7 @@ func initApp(c *cfg) {
|
|||
initAndLog(c, "pprof", initProfiler)
|
||||
initAndLog(c, "prometheus", initMetrics)
|
||||
initAndLog(c, "control", initControlService)
|
||||
initAndLog(c, "tree", initTreeService)
|
||||
|
||||
initAndLog(c, "storage engine", func(c *cfg) {
|
||||
fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Open())
|
||||
|
|
|
|||
41
cmd/neofs-node/tree.go
Normal file
41
cmd/neofs-node/tree.go
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
treeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/tree"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/services/tree"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func initTreeService(c *cfg) {
|
||||
endpoint := treeconfig.GRPC(c.appCfg).Endpoint()
|
||||
if endpoint == treeconfig.GRPCEndpointDefault {
|
||||
return
|
||||
}
|
||||
|
||||
treeSvc := tree.New(
|
||||
tree.WithContainerSource(c.cfgObject.cnrSource),
|
||||
tree.WithNetmapSource(c.netMapSource),
|
||||
tree.WithPrivateKey(&c.key.PrivateKey),
|
||||
tree.WithLogger(c.log),
|
||||
tree.WithStorage(c.cfgObject.cfgLocalStorage.localStorage))
|
||||
|
||||
treeServer := grpc.NewServer()
|
||||
|
||||
c.onShutdown(func() {
|
||||
stopGRPC("NeoFS Tree Service API", treeServer, c.log)
|
||||
treeSvc.Shutdown()
|
||||
})
|
||||
|
||||
lis, err := net.Listen("tcp", endpoint)
|
||||
fatalOnErr(err)
|
||||
|
||||
tree.RegisterTreeServiceServer(treeServer, treeSvc)
|
||||
|
||||
c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) {
|
||||
treeSvc.Start(ctx)
|
||||
fatalOnErr(treeServer.Serve(lis))
|
||||
}))
|
||||
}
|
||||
|
|
@ -46,6 +46,9 @@ NEOFS_GRPC_1_TLS_ENABLED=false
|
|||
NEOFS_CONTROL_AUTHORIZED_KEYS="035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6"
|
||||
NEOFS_CONTROL_GRPC_ENDPOINT=localhost:8090
|
||||
|
||||
# Tree service section
|
||||
NEOFS_TREE_GRPC_ENDPOINT=127.0.0.1:8091
|
||||
|
||||
# Contracts section
|
||||
NEOFS_CONTRACTS_BALANCE=5263abba1abedbf79bb57f3e40b50b4425d2d6cd
|
||||
NEOFS_CONTRACTS_CONTAINER=5d084790d7aa36cea7b53fe897380dab11d2cd3c
|
||||
|
|
|
|||
|
|
@ -84,6 +84,11 @@
|
|||
"endpoint": "localhost:8090"
|
||||
}
|
||||
},
|
||||
"tree": {
|
||||
"grpc": {
|
||||
"endpoint": "127.0.0.1:8091"
|
||||
}
|
||||
},
|
||||
"contracts": {
|
||||
"balance": "5263abba1abedbf79bb57f3e40b50b4425d2d6cd",
|
||||
"container": "5d084790d7aa36cea7b53fe897380dab11d2cd3c",
|
||||
|
|
|
|||
|
|
@ -66,6 +66,10 @@ control:
|
|||
grpc:
|
||||
endpoint: localhost:8090 # endpoint that is listened by the Control Service
|
||||
|
||||
tree:
|
||||
grpc:
|
||||
endpoint: 127.0.0.1:8091 # endpoint that is listened by the Tree Service
|
||||
|
||||
contracts: # side chain NEOFS contract script hashes; optional, override values retrieved from NNS contract
|
||||
balance: 5263abba1abedbf79bb57f3e40b50b4425d2d6cd
|
||||
container: 5d084790d7aa36cea7b53fe897380dab11d2cd3c
|
||||
|
|
|
|||
8
go.mod
8
go.mod
|
|
@ -38,7 +38,10 @@ require (
|
|||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require google.golang.org/api v0.44.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.81.0 // indirect
|
||||
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/btcsuite/btcd v0.22.0-beta // indirect
|
||||
|
|
@ -47,8 +50,10 @@ require (
|
|||
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/golang/snappy v0.0.3 // indirect
|
||||
github.com/google/go-cmp v0.5.6 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/gorilla/websocket v1.4.2 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
|
|
@ -87,12 +92,15 @@ require (
|
|||
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect
|
||||
github.com/twmb/murmur3 v1.1.5 // indirect
|
||||
github.com/urfave/cli v1.22.5 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
|
||||
gopkg.in/ini.v1 v1.62.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -17,6 +17,7 @@ cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKP
|
|||
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
|
||||
cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
|
||||
cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
|
||||
cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
|
||||
cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
|
|
@ -161,6 +162,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
|
|||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
|
|
@ -559,6 +561,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
|||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||
go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
|
||||
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
|
|
@ -687,6 +690,7 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ
|
|||
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc=
|
||||
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -871,6 +875,7 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q
|
|||
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
|
||||
google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
|
||||
google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
|
||||
google.golang.org/api v0.44.0 h1:URs6qR1lAxDsqWITsQXI4ZkGiYJ5dHtRNiCpfs2OeKA=
|
||||
google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
|
@ -878,6 +883,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
|
|||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
|
|
|||
59
pkg/services/tree/options.go
Normal file
59
pkg/services/tree/options.go
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
package tree
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
|
||||
"github.com/nspcc-dev/neofs-node/pkg/core/container"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type cfg struct {
|
||||
log *zap.Logger
|
||||
key *ecdsa.PrivateKey
|
||||
nmSource netmap.Source
|
||||
cnrSource container.Source
|
||||
forest pilorama.Forest
|
||||
}
|
||||
|
||||
// Option represents configuration option for a tree service.
|
||||
type Option func(*cfg)
|
||||
|
||||
// WithContainerSource sets a container source for a tree service.
|
||||
// This option is required.
|
||||
func WithContainerSource(src container.Source) Option {
|
||||
return func(c *cfg) {
|
||||
c.cnrSource = src
|
||||
}
|
||||
}
|
||||
|
||||
// WithNetmapSource sets a netmap source for a tree service.
|
||||
// This option is required.
|
||||
func WithNetmapSource(src netmap.Source) Option {
|
||||
return func(c *cfg) {
|
||||
c.nmSource = src
|
||||
}
|
||||
}
|
||||
|
||||
// WithPrivateKey sets a netmap source for a tree service.
|
||||
// This option is required.
|
||||
func WithPrivateKey(key *ecdsa.PrivateKey) Option {
|
||||
return func(c *cfg) {
|
||||
c.key = key
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogger sets logger for a tree service.
|
||||
func WithLogger(log *zap.Logger) Option {
|
||||
return func(c *cfg) {
|
||||
c.log = log
|
||||
}
|
||||
}
|
||||
|
||||
// WithStorage sets tree storage for a service.
|
||||
func WithStorage(s pilorama.Forest) Option {
|
||||
return func(c *cfg) {
|
||||
c.forest = s
|
||||
}
|
||||
}
|
||||
145
pkg/services/tree/replicator.go
Normal file
145
pkg/services/tree/replicator.go
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
package tree
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
clientcore "github.com/nspcc-dev/neofs-node/pkg/core/client"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/network"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
|
||||
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
netmapSDK "github.com/nspcc-dev/neofs-sdk-go/netmap"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/api/option"
|
||||
"google.golang.org/api/transport/grpc"
|
||||
)
|
||||
|
||||
type movePair struct {
|
||||
cid cidSDK.ID
|
||||
treeID string
|
||||
op *pilorama.LogMove
|
||||
}
|
||||
|
||||
const (
|
||||
defaultReplicatorCapacity = 64
|
||||
defaultReplicatorTimeout = time.Second * 2
|
||||
)
|
||||
|
||||
func (s *Service) replicateLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-s.closeCh:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case op := <-s.replicateCh:
|
||||
ctx, cancel := context.WithTimeout(ctx, defaultReplicatorTimeout)
|
||||
err := s.replicate(ctx, op)
|
||||
cancel()
|
||||
|
||||
if err != nil {
|
||||
s.log.Error("error during replication",
|
||||
zap.String("err", err.Error()),
|
||||
zap.Stringer("cid", op.cid),
|
||||
zap.String("treeID", op.treeID))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) replicate(ctx context.Context, op movePair) error {
|
||||
req := newApplyRequest(&op)
|
||||
// TODO(@fyrchik): #1328 access control
|
||||
//err := signature.SignDataWithHandler(s.key, req, func(key, sign []byte) {
|
||||
// req.Signature = &Signature{
|
||||
// Key: key,
|
||||
// Sign: sign,
|
||||
// }
|
||||
//})
|
||||
//if err != nil {
|
||||
// return fmt.Errorf("can't sign data: %w", err)
|
||||
//}
|
||||
|
||||
nodes, err := s.getContainerNodes(op.cid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't get container nodes: %w", err)
|
||||
}
|
||||
|
||||
var node clientcore.NodeInfo
|
||||
for _, n := range nodes {
|
||||
var lastErr error
|
||||
|
||||
n.IterateNetworkEndpoints(func(addr string) bool {
|
||||
cc, err := grpc.Dial(ctx, option.WithEndpoint(addr))
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
return false
|
||||
}
|
||||
|
||||
// TODO cache clients
|
||||
c := NewTreeServiceClient(cc)
|
||||
|
||||
_, lastErr = c.Apply(ctx, req)
|
||||
return lastErr == nil
|
||||
})
|
||||
|
||||
if lastErr != nil {
|
||||
s.log.Warn("failed to sent update to the node",
|
||||
zap.String("last_error", lastErr.Error()),
|
||||
zap.String("address", network.StringifyGroup(node.AddressGroup())),
|
||||
zap.String("key", base64.StdEncoding.EncodeToString(node.PublicKey())))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) pushToQueue(cid cidSDK.ID, treeID string, op *pilorama.LogMove) {
|
||||
s.replicateCh <- movePair{
|
||||
cid: cid,
|
||||
treeID: treeID,
|
||||
op: op,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) getContainerNodes(cid cidSDK.ID) ([]netmapSDK.NodeInfo, error) {
|
||||
nm, err := s.nmSource.GetNetMap(0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't get netmap: %w", err)
|
||||
}
|
||||
|
||||
cnr, err := s.cnrSource.Get(cid)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't get container: %w", err)
|
||||
}
|
||||
|
||||
policy := cnr.Value.PlacementPolicy()
|
||||
rawCID := make([]byte, sha256.Size)
|
||||
cid.Encode(rawCID)
|
||||
|
||||
nodes, err := nm.ContainerNodes(policy, rawCID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return placement.FlattenNodes(nodes), nil
|
||||
}
|
||||
|
||||
func newApplyRequest(op *movePair) *ApplyRequest {
|
||||
rawCID := make([]byte, sha256.Size)
|
||||
op.cid.Encode(rawCID)
|
||||
|
||||
return &ApplyRequest{
|
||||
Body: &ApplyRequest_Body{
|
||||
ContainerId: rawCID,
|
||||
TreeId: op.treeID,
|
||||
Operation: &LogMove{
|
||||
ParentId: op.op.Parent,
|
||||
Meta: op.op.Meta.Bytes(),
|
||||
ChildId: op.op.Child,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
294
pkg/services/tree/service.go
Normal file
294
pkg/services/tree/service.go
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
package tree
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nspcc-dev/neofs-api-go/v2/signature"
|
||||
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
|
||||
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Service represents tree-service capable of working with multiple
|
||||
// instances of CRDT trees.
|
||||
type Service struct {
|
||||
cfg
|
||||
|
||||
replicateCh chan movePair
|
||||
closeCh chan struct{}
|
||||
}
|
||||
|
||||
var _ TreeServiceServer = (*Service)(nil)
|
||||
|
||||
// New creates new tree service instance.
|
||||
func New(opts ...Option) *Service {
|
||||
var s Service
|
||||
for i := range opts {
|
||||
opts[i](&s.cfg)
|
||||
}
|
||||
|
||||
if s.log == nil {
|
||||
s.log = zap.NewNop()
|
||||
}
|
||||
|
||||
s.closeCh = make(chan struct{})
|
||||
s.replicateCh = make(chan movePair, defaultReplicatorCapacity)
|
||||
|
||||
return &s
|
||||
}
|
||||
|
||||
// Start starts the service.
|
||||
func (s *Service) Start(ctx context.Context) {
|
||||
go s.replicateLoop(ctx)
|
||||
}
|
||||
|
||||
// Shutdown shutdowns the service.
|
||||
func (s *Service) Shutdown() {
|
||||
close(s.closeCh)
|
||||
}
|
||||
|
||||
func (s *Service) Add(_ context.Context, req *AddRequest) (*AddResponse, error) {
|
||||
b := req.GetBody()
|
||||
|
||||
var cid cidSDK.ID
|
||||
if err := cid.Decode(b.GetContainerId()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err := s.verifyClient(req, cid, req.GetSignature().GetKey())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{
|
||||
Parent: b.GetParentId(),
|
||||
Child: pilorama.RootID,
|
||||
Meta: pilorama.Meta{Items: constructMeta(b.GetMeta())},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.pushToQueue(cid, b.GetTreeId(), log)
|
||||
return &AddResponse{
|
||||
Body: &AddResponse_Body{
|
||||
NodeId: log.Child,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) AddByPath(_ context.Context, req *AddByPathRequest) (*AddByPathResponse, error) {
|
||||
b := req.GetBody()
|
||||
|
||||
var cid cidSDK.ID
|
||||
if err := cid.Decode(b.GetContainerId()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err := s.verifyClient(req, cid, req.GetSignature().GetKey())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
meta := constructMeta(b.GetMeta())
|
||||
|
||||
attr := b.GetPathAttribute()
|
||||
if len(attr) == 0 {
|
||||
attr = pilorama.AttributeFilename
|
||||
}
|
||||
|
||||
logs, err := s.forest.TreeAddByPath(cid, b.GetTreeId(), attr, b.GetPath(), meta)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range logs {
|
||||
s.pushToQueue(cid, b.GetTreeId(), &logs[i])
|
||||
}
|
||||
|
||||
nodes := make([]uint64, len(logs))
|
||||
nodes[0] = logs[len(logs)-1].Child
|
||||
for i, l := range logs[:len(logs)-1] {
|
||||
nodes[i+1] = l.Child
|
||||
}
|
||||
|
||||
return &AddByPathResponse{
|
||||
Body: &AddByPathResponse_Body{
|
||||
Nodes: nodes,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) Remove(_ context.Context, req *RemoveRequest) (*RemoveResponse, error) {
|
||||
b := req.GetBody()
|
||||
|
||||
var cid cidSDK.ID
|
||||
if err := cid.Decode(b.GetContainerId()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err := s.verifyClient(req, cid, req.GetSignature().GetKey())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if b.GetNodeId() == pilorama.RootID {
|
||||
return nil, fmt.Errorf("node with ID %d is root and can't be removed", b.GetNodeId())
|
||||
}
|
||||
|
||||
log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{
|
||||
Parent: pilorama.TrashID,
|
||||
Child: b.GetNodeId(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.pushToQueue(cid, b.GetTreeId(), log)
|
||||
return new(RemoveResponse), nil
|
||||
}
|
||||
|
||||
// Move applies client operation to the specified tree and pushes in queue
|
||||
// for replication on other nodes.
|
||||
func (s *Service) Move(_ context.Context, req *MoveRequest) (*MoveResponse, error) {
|
||||
b := req.GetBody()
|
||||
|
||||
var cid cidSDK.ID
|
||||
if err := cid.Decode(b.GetContainerId()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err := s.verifyClient(req, cid, req.GetSignature().GetKey())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if b.GetNodeId() == pilorama.RootID {
|
||||
return nil, fmt.Errorf("node with ID %d is root and can't be moved", b.GetNodeId())
|
||||
}
|
||||
|
||||
log, err := s.forest.TreeMove(cid, b.GetTreeId(), &pilorama.Move{
|
||||
Parent: b.GetParentId(),
|
||||
Child: b.GetNodeId(),
|
||||
Meta: pilorama.Meta{Items: constructMeta(b.GetMeta())},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.pushToQueue(cid, b.GetTreeId(), log)
|
||||
return new(MoveResponse), nil
|
||||
}
|
||||
|
||||
func (s *Service) GetNodeByPath(_ context.Context, req *GetNodeByPathRequest) (*GetNodeByPathResponse, error) {
|
||||
b := req.GetBody()
|
||||
|
||||
var cid cidSDK.ID
|
||||
if err := cid.Decode(b.GetContainerId()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
attr := b.GetPathAttribute()
|
||||
if len(attr) == 0 {
|
||||
attr = pilorama.AttributeFilename
|
||||
}
|
||||
|
||||
nodes, err := s.forest.TreeGetByPath(cid, b.GetTreeId(), attr, b.GetPath(), b.GetLatestOnly())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := make([]*GetNodeByPathResponse_Info, 0, len(nodes))
|
||||
for _, node := range nodes {
|
||||
m, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var x GetNodeByPathResponse_Info
|
||||
x.NodeId = node
|
||||
x.Timestamp = m.Time
|
||||
for _, kv := range m.Items {
|
||||
needAttr := b.AllAttributes
|
||||
if !needAttr {
|
||||
for _, attr := range b.GetAttributes() {
|
||||
if kv.Key == attr {
|
||||
needAttr = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if needAttr {
|
||||
x.Meta = append(x.Meta, &KeyValue{
|
||||
Key: kv.Key,
|
||||
Value: kv.Value,
|
||||
})
|
||||
}
|
||||
}
|
||||
info = append(info, &x)
|
||||
}
|
||||
|
||||
return &GetNodeByPathResponse{
|
||||
Body: &GetNodeByPathResponse_Body{
|
||||
Nodes: info,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetSubTree(_ context.Context, req *GetSubTreeRequest) (*GetSubTreeResponse, error) {
|
||||
return nil, errors.New("GetSubTree is unimplemented")
|
||||
}
|
||||
|
||||
// Apply locally applies operation from the remote node to the tree.
|
||||
func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) {
|
||||
err := signature.VerifyServiceMessage(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var cid cidSDK.ID
|
||||
if err := cid.Decode(req.GetBody().GetContainerId()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
found := false
|
||||
key := req.GetSignature().GetKey()
|
||||
nodes, _ := s.getContainerNodes(cid)
|
||||
|
||||
loop:
|
||||
for _, n := range nodes {
|
||||
if bytes.Equal(key, n.PublicKey()) {
|
||||
found = true
|
||||
break loop
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, errors.New("`Apply` request must be signed by a container node")
|
||||
}
|
||||
|
||||
op := req.GetBody().GetOperation()
|
||||
|
||||
var meta pilorama.Meta
|
||||
if err := meta.FromBytes(op.GetMeta()); err != nil {
|
||||
return nil, fmt.Errorf("can't parse meta-information: %w", err)
|
||||
}
|
||||
|
||||
return nil, s.forest.TreeApply(cid, req.GetBody().GetTreeId(), &pilorama.Move{
|
||||
Parent: op.GetParentId(),
|
||||
Child: op.GetChildId(),
|
||||
Meta: meta,
|
||||
})
|
||||
}
|
||||
|
||||
func constructMeta(arr []*KeyValue) []pilorama.KeyValue {
|
||||
meta := make([]pilorama.KeyValue, len(arr))
|
||||
for i, kv := range arr {
|
||||
meta[i].Key = kv.Key
|
||||
meta[i].Value = kv.Value
|
||||
}
|
||||
return meta
|
||||
}
|
||||
BIN
pkg/services/tree/service.pb.go
generated
Normal file
BIN
pkg/services/tree/service.pb.go
generated
Normal file
Binary file not shown.
194
pkg/services/tree/service.proto
Normal file
194
pkg/services/tree/service.proto
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
syntax = "proto3";
|
||||
|
||||
package tree;
|
||||
|
||||
import "pkg/services/tree/types.proto";
|
||||
|
||||
option go_package = "github.com/nspcc-dev/neofs-node/pkg/services/tree";
|
||||
|
||||
// `TreeService` provides an interface for working with distributed tree.
|
||||
service TreeService {
|
||||
/* Client API */
|
||||
|
||||
// Add adds new node to the tree. Invoked by a client.
|
||||
rpc Add (AddRequest) returns (AddResponse);
|
||||
// AddByPath adds new node to the tree by path. Invoked by a client.
|
||||
rpc AddByPath (AddByPathRequest) returns (AddByPathResponse);
|
||||
// Remove removes node from the tree. Invoked by a client.
|
||||
rpc Remove (RemoveRequest) returns (RemoveResponse);
|
||||
// Move moves node from one parent to another. Invoked by a client.
|
||||
rpc Move (MoveRequest) returns (MoveResponse);
|
||||
// GetNodeByPath returns list of IDs corresponding to a specific filepath.
|
||||
rpc GetNodeByPath (GetNodeByPathRequest) returns (GetNodeByPathResponse);
|
||||
// GetSubTree returns tree corresponding to a specific node.
|
||||
rpc GetSubTree (GetSubTreeRequest) returns (GetSubTreeResponse);
|
||||
|
||||
/* Synchronization API */
|
||||
|
||||
// Apply pushes log operation from another node to the current.
|
||||
// The request must be signed by a container node.
|
||||
rpc Apply (ApplyRequest) returns (ApplyResponse);
|
||||
}
|
||||
|
||||
message AddRequest {
|
||||
message Body {
|
||||
bytes container_id = 1;
|
||||
string tree_id = 2;
|
||||
uint64 parent_id = 3;
|
||||
repeated KeyValue meta = 4;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
}
|
||||
|
||||
message AddResponse {
|
||||
message Body {
|
||||
uint64 node_id = 1;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
};
|
||||
|
||||
|
||||
message AddByPathRequest {
|
||||
message Body {
|
||||
bytes container_id = 1;
|
||||
string tree_id = 2;
|
||||
string path_attribute = 3;
|
||||
repeated string path = 4;
|
||||
repeated KeyValue meta = 5;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
}
|
||||
|
||||
message AddByPathResponse {
|
||||
message Body {
|
||||
repeated uint64 nodes = 1;
|
||||
uint64 parent_id = 2;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
};
|
||||
|
||||
|
||||
message RemoveRequest {
|
||||
message Body {
|
||||
bytes container_id = 1;
|
||||
string tree_id = 2;
|
||||
uint64 node_id = 3;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
}
|
||||
|
||||
message RemoveResponse {
|
||||
message Body {
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
};
|
||||
|
||||
|
||||
message MoveRequest {
|
||||
message Body {
|
||||
// TODO import neo.fs.v2.refs.ContainerID directly.
|
||||
bytes container_id = 1;
|
||||
string tree_id = 2;
|
||||
uint64 parent_id = 3;
|
||||
uint64 node_id = 4;
|
||||
repeated KeyValue meta = 5;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
}
|
||||
|
||||
message MoveResponse {
|
||||
message Body {
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
};
|
||||
|
||||
|
||||
message GetNodeByPathRequest {
|
||||
message Body {
|
||||
bytes container_id = 1;
|
||||
string tree_id = 2;
|
||||
string path_attribute = 3;
|
||||
repeated string path = 4;
|
||||
repeated string attributes = 5;
|
||||
bool latest_only = 6;
|
||||
bool all_attributes = 7;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
}
|
||||
|
||||
message GetNodeByPathResponse {
|
||||
message Info {
|
||||
uint64 node_id = 1;
|
||||
uint64 timestamp = 2;
|
||||
repeated KeyValue meta = 3;
|
||||
}
|
||||
message Body {
|
||||
repeated Info nodes = 1;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
};
|
||||
|
||||
|
||||
message GetSubTreeRequest {
|
||||
message Body {
|
||||
bytes container_id = 1;
|
||||
string tree_id = 2;
|
||||
repeated uint64 nodes = 3;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
}
|
||||
|
||||
message GetSubTreeResponse {
|
||||
message Info {
|
||||
uint64 node_id = 1;
|
||||
repeated KeyValue meta = 2;
|
||||
}
|
||||
message Body {
|
||||
repeated Info info = 1;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
};
|
||||
|
||||
|
||||
message ApplyRequest {
|
||||
message Body {
|
||||
bytes container_id = 1;
|
||||
string tree_id = 2;
|
||||
LogMove operation = 3;
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
}
|
||||
|
||||
message ApplyResponse {
|
||||
message Body {
|
||||
}
|
||||
|
||||
Body body = 1;
|
||||
Signature signature = 2;
|
||||
};
|
||||
BIN
pkg/services/tree/service_grpc.pb.go
generated
Normal file
BIN
pkg/services/tree/service_grpc.pb.go
generated
Normal file
Binary file not shown.
44
pkg/services/tree/signature.go
Normal file
44
pkg/services/tree/signature.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package tree
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/nspcc-dev/neofs-api-go/v2/signature"
|
||||
cidSDK "github.com/nspcc-dev/neofs-sdk-go/container/id"
|
||||
"github.com/nspcc-dev/neofs-sdk-go/user"
|
||||
)
|
||||
|
||||
func (s *Service) verifyClient(req interface{}, cid cidSDK.ID, rawKey []byte) error {
|
||||
// TODO(@fyrchik): #1328 access control
|
||||
return nil
|
||||
//nolint:govet
|
||||
err := signature.VerifyServiceMessage(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cnr, err := s.cnrSource.Get(cid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't get container %s: %w", cid, err)
|
||||
}
|
||||
|
||||
ownerID := cnr.Value.Owner()
|
||||
|
||||
pub, err := keys.NewPublicKeyFromBytes(rawKey, elliptic.P256())
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid public key: %w", err)
|
||||
}
|
||||
|
||||
var actualID user.ID
|
||||
user.IDFromKey(&actualID, (ecdsa.PublicKey)(*pub))
|
||||
|
||||
if !actualID.Equals(ownerID) {
|
||||
return errors.New("`Move` request must be signed by a container owner")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
BIN
pkg/services/tree/types.pb.go
generated
Normal file
BIN
pkg/services/tree/types.pb.go
generated
Normal file
Binary file not shown.
27
pkg/services/tree/types.proto
Normal file
27
pkg/services/tree/types.proto
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
syntax = "proto3";
|
||||
|
||||
package tree;
|
||||
|
||||
option go_package = "github.com/nspcc-dev/neofs-node/pkg/services/tree";
|
||||
|
||||
// KeyValue represents key-value pair attached to an object.
|
||||
message KeyValue {
|
||||
string key = 1 [json_name = "key"];
|
||||
bytes value = 2 [json_name = "value"];
|
||||
}
|
||||
|
||||
// LogMove represents log-entry for a single move operation.
|
||||
message LogMove {
|
||||
// ID of the parent node.
|
||||
uint64 parent_id = 2 [json_name = "parentID"];
|
||||
// Node meta information, including operation timestamp.
|
||||
bytes meta = 3 [json_name = "meta"];
|
||||
// ID of the node to move.
|
||||
uint64 child_id = 4 [json_name = "childID"];
|
||||
}
|
||||
|
||||
// Signature of a message.
|
||||
message Signature {
|
||||
bytes key = 1 [json_name = "key"];
|
||||
bytes sign = 2 [json_name = "signature"];
|
||||
}
|
||||
Loading…
Reference in a new issue