[#59] tree: Make interface for tree service client #62
8 changed files with 995 additions and 419 deletions
|
@ -17,6 +17,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
|
@ -88,7 +89,7 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
|
|||
Caches: layer.DefaultCachesConfigs(zap.NewExample()),
|
||||
AnonKey: layer.AnonymousKey{Key: key},
|
||||
Resolver: testResolver,
|
||||
TreeService: layer.NewTreeService(),
|
||||
TreeService: NewTreeServiceMock(t),
|
||||
}
|
||||
|
||||
var pp netmap.PlacementPolicy
|
||||
|
@ -113,6 +114,12 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
|
|||
}
|
||||
}
|
||||
|
||||
func NewTreeServiceMock(t *testing.T) *tree.Tree {
|
||||
memCli, err := tree.NewTreeServiceClientMemory()
|
||||
require.NoError(t, err)
|
||||
return tree.NewTree(memCli)
|
||||
}
|
||||
|
||||
func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo {
|
||||
_, err := hc.MockedPool().CreateContainer(hc.Context(), layer.PrmContainerCreate{
|
||||
Creator: hc.owner,
|
||||
|
|
|
@ -25,12 +25,15 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/xml"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/spf13/viper"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
type (
|
||||
|
@ -110,10 +113,12 @@ func (a *App) initLayer(ctx context.Context) {
|
|||
a.initResolver()
|
||||
|
||||
treeServiceEndpoint := a.cfg.GetString(cfgTreeServiceEndpoint)
|
||||
treeService, err := frostfs.NewTreeClient(ctx, treeServiceEndpoint, a.key)
|
||||
grpcDialOpt := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
treeGRPCClient, err := tree.NewTreeServiceClientGRPC(ctx, treeServiceEndpoint, a.key, grpcDialOpt)
|
||||
if err != nil {
|
||||
a.log.Fatal("failed to create tree service", zap.Error(err))
|
||||
}
|
||||
treeService := tree.NewTree(treeGRPCClient)
|
||||
a.log.Info("init tree service", zap.String("endpoint", treeServiceEndpoint))
|
||||
|
||||
// prepare random key for anonymous requests
|
||||
|
|
File diff suppressed because it is too large
Load diff
311
pkg/service/tree/tree_client_grpc.go
Normal file
311
pkg/service/tree/tree_client_grpc.go
Normal file
|
@ -0,0 +1,311 @@
|
|||
package tree
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type GetNodeByPathResponseInfoWrapper struct {
|
||||
response *tree.GetNodeByPathResponse_Info
|
||||
}
|
||||
|
||||
func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 {
|
||||
return n.response.GetNodeId()
|
||||
}
|
||||
|
||||
func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 {
|
||||
return n.response.GetParentId()
|
||||
}
|
||||
|
||||
func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 {
|
||||
return n.response.GetTimestamp()
|
||||
}
|
||||
|
||||
func (n GetNodeByPathResponseInfoWrapper) GetMeta() []Meta {
|
||||
res := make([]Meta, len(n.response.Meta))
|
||||
for i, value := range n.response.Meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
type GetSubTreeResponseBodyWrapper struct {
|
||||
response *tree.GetSubTreeResponse_Body
|
||||
}
|
||||
|
||||
func (n GetSubTreeResponseBodyWrapper) GetNodeID() uint64 {
|
||||
return n.response.GetNodeId()
|
||||
}
|
||||
|
||||
func (n GetSubTreeResponseBodyWrapper) GetParentID() uint64 {
|
||||
return n.response.GetParentId()
|
||||
}
|
||||
|
||||
func (n GetSubTreeResponseBodyWrapper) GetTimestamp() uint64 {
|
||||
return n.response.GetTimestamp()
|
||||
}
|
||||
|
||||
func (n GetSubTreeResponseBodyWrapper) GetMeta() []Meta {
|
||||
res := make([]Meta, len(n.response.Meta))
|
||||
for i, value := range n.response.Meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
type ServiceClientGRPC struct {
|
||||
key *keys.PrivateKey
|
||||
conn *grpc.ClientConn
|
||||
service tree.TreeServiceClient
|
||||
}
|
||||
|
||||
func NewTreeServiceClientGRPC(ctx context.Context, addr string, key *keys.PrivateKey, grpcOpts ...grpc.DialOption) (*ServiceClientGRPC, error) {
|
||||
conn, err := grpc.Dial(addr, grpcOpts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("did not connect: %v", err)
|
||||
}
|
||||
|
||||
c := tree.NewTreeServiceClient(conn)
|
||||
if _, err = c.Healthcheck(ctx, &tree.HealthcheckRequest{}); err != nil {
|
||||
return nil, fmt.Errorf("healthcheck: %w", err)
|
||||
}
|
||||
|
||||
return &ServiceClientGRPC{
|
||||
key: key,
|
||||
conn: conn,
|
||||
service: c,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientGRPC) GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) {
|
||||
request := &tree.GetNodeByPathRequest{
|
||||
Body: &tree.GetNodeByPathRequest_Body{
|
||||
ContainerId: p.BktInfo.CID[:],
|
||||
TreeId: p.TreeID,
|
||||
Path: p.Path,
|
||||
Attributes: p.Meta,
|
||||
PathAttribute: FileNameKey,
|
||||
LatestOnly: p.LatestOnly,
|
||||
AllAttributes: p.AllAttrs,
|
||||
BearerToken: getBearer(ctx, p.BktInfo),
|
||||
},
|
||||
}
|
||||
|
||||
if err := c.signRequest(request.Body, func(key, sign []byte) {
|
||||
request.Signature = &tree.Signature{
|
||||
Key: key,
|
||||
Sign: sign,
|
||||
}
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := c.service.GetNodeByPath(ctx, request)
|
||||
if err != nil {
|
||||
return nil, handleError("failed to get node by path", err)
|
||||
}
|
||||
|
||||
res := make([]NodeResponse, len(resp.GetBody().GetNodes()))
|
||||
for i, info := range resp.GetBody().GetNodes() {
|
||||
res[i] = GetNodeByPathResponseInfoWrapper{info}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientGRPC) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error) {
|
||||
request := &tree.GetSubTreeRequest{
|
||||
Body: &tree.GetSubTreeRequest_Body{
|
||||
ContainerId: bktInfo.CID[:],
|
||||
TreeId: treeID,
|
||||
RootId: rootID,
|
||||
Depth: depth,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
},
|
||||
}
|
||||
|
||||
if err := c.signRequest(request.Body, func(key, sign []byte) {
|
||||
request.Signature = &tree.Signature{
|
||||
Key: key,
|
||||
Sign: sign,
|
||||
}
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cli, err := c.service.GetSubTree(ctx, request)
|
||||
if err != nil {
|
||||
return nil, handleError("failed to get sub tree client", err)
|
||||
}
|
||||
|
||||
var subtree []NodeResponse
|
||||
for {
|
||||
resp, err := cli.Recv()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, handleError("failed to get sub tree", err)
|
||||
}
|
||||
subtree = append(subtree, GetSubTreeResponseBodyWrapper{resp.Body})
|
||||
}
|
||||
|
||||
return subtree, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientGRPC) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) {
|
||||
request := &tree.AddRequest{
|
||||
Body: &tree.AddRequest_Body{
|
||||
ContainerId: bktInfo.CID[:],
|
||||
TreeId: treeID,
|
||||
ParentId: parent,
|
||||
Meta: metaToKV(meta),
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
},
|
||||
}
|
||||
if err := c.signRequest(request.Body, func(key, sign []byte) {
|
||||
request.Signature = &tree.Signature{
|
||||
Key: key,
|
||||
Sign: sign,
|
||||
}
|
||||
}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
resp, err := c.service.Add(ctx, request)
|
||||
if err != nil {
|
||||
return 0, handleError("failed to add node", err)
|
||||
}
|
||||
|
||||
return resp.GetBody().GetNodeId(), nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientGRPC) AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) {
|
||||
request := &tree.AddByPathRequest{
|
||||
Body: &tree.AddByPathRequest_Body{
|
||||
ContainerId: bktInfo.CID[:],
|
||||
TreeId: treeID,
|
||||
Path: path,
|
||||
Meta: metaToKV(meta),
|
||||
PathAttribute: FileNameKey,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
},
|
||||
}
|
||||
|
||||
if err := c.signRequest(request.Body, func(key, sign []byte) {
|
||||
request.Signature = &tree.Signature{
|
||||
Key: key,
|
||||
Sign: sign,
|
||||
}
|
||||
}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
resp, err := c.service.AddByPath(ctx, request)
|
||||
if err != nil {
|
||||
return 0, handleError("failed to add node by path", err)
|
||||
}
|
||||
|
||||
body := resp.GetBody()
|
||||
if body == nil {
|
||||
return 0, errors.New("nil body in tree service response")
|
||||
} else if len(body.Nodes) == 0 {
|
||||
return 0, errors.New("empty list of added nodes in tree service response")
|
||||
}
|
||||
|
||||
// The first node is the leaf that we add, according to tree service docs.
|
||||
return body.Nodes[0], nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientGRPC) MoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error {
|
||||
request := &tree.MoveRequest{
|
||||
Body: &tree.MoveRequest_Body{
|
||||
ContainerId: bktInfo.CID[:],
|
||||
TreeId: treeID,
|
||||
NodeId: nodeID,
|
||||
ParentId: parentID,
|
||||
Meta: metaToKV(meta),
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
},
|
||||
}
|
||||
|
||||
if err := c.signRequest(request.Body, func(key, sign []byte) {
|
||||
request.Signature = &tree.Signature{
|
||||
Key: key,
|
||||
Sign: sign,
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := c.service.Move(ctx, request); err != nil {
|
||||
return handleError("failed to move node", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientGRPC) RemoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error {
|
||||
request := &tree.RemoveRequest{
|
||||
Body: &tree.RemoveRequest_Body{
|
||||
ContainerId: bktInfo.CID[:],
|
||||
TreeId: treeID,
|
||||
NodeId: nodeID,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
},
|
||||
}
|
||||
if err := c.signRequest(request.Body, func(key, sign []byte) {
|
||||
request.Signature = &tree.Signature{
|
||||
Key: key,
|
||||
Sign: sign,
|
||||
}
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := c.service.Remove(ctx, request); err != nil {
|
||||
return handleError("failed to remove node", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func metaToKV(meta map[string]string) []*tree.KeyValue {
|
||||
result := make([]*tree.KeyValue, 0, len(meta))
|
||||
|
||||
for key, value := range meta {
|
||||
result = append(result, &tree.KeyValue{Key: key, Value: []byte(value)})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func getBearer(ctx context.Context, bktInfo *data.BucketInfo) []byte {
|
||||
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil {
|
||||
if bd.Gate.BearerToken != nil {
|
||||
if bktInfo.Owner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) {
|
||||
return bd.Gate.BearerToken.Marshal()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleError(msg string, err error) error {
|
||||
if strings.Contains(err.Error(), "not found") {
|
||||
return fmt.Errorf("%w: %s", ErrNodeNotFound, err.Error())
|
||||
} else if strings.Contains(err.Error(), "is denied by") {
|
||||
return fmt.Errorf("%w: %s", ErrNodeAccessDenied, err.Error())
|
||||
}
|
||||
return fmt.Errorf("%s: %w", msg, err)
|
||||
}
|
|
@ -1,12 +1,12 @@
|
|||
/*REMOVE THIS AFTER SIGNATURE WILL BE AVAILABLE IN TREE CLIENT FROM FROSTFS NODE*/
|
||||
package frostfs
|
||||
package tree
|
||||
|
||||
import (
|
||||
crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func (c *TreeClient) signData(buf []byte, f func(key, sign []byte)) error {
|
||||
func (c *ServiceClientGRPC) signData(buf []byte, f func(key, sign []byte)) error {
|
||||
// crypto package should not be used outside of API libraries (see neofs-node#491).
|
||||
// For now tree service does not include into SDK Client nor SDK Pool, so there is no choice.
|
||||
// When SDK library adopts Tree service client, this should be dropped.
|
||||
|
@ -19,7 +19,7 @@ func (c *TreeClient) signData(buf []byte, f func(key, sign []byte)) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *TreeClient) signRequest(requestBody proto.Message, f func(key, sign []byte)) error {
|
||||
func (c *ServiceClientGRPC) signRequest(requestBody proto.Message, f func(key, sign []byte)) error {
|
||||
buf, err := proto.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return err
|
35
pkg/service/tree/tree_client_grpc_test.go
Normal file
35
pkg/service/tree/tree_client_grpc_test.go
Normal file
|
@ -0,0 +1,35 @@
|
|||
package tree
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHandleError(t *testing.T) {
|
||||
defaultError := errors.New("default error")
|
||||
for _, tc := range []struct {
|
||||
err error
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
err: defaultError,
|
||||
expectedError: defaultError,
|
||||
},
|
||||
{
|
||||
err: errors.New("something not found"),
|
||||
expectedError: layer.ErrNodeNotFound,
|
||||
},
|
||||
{
|
||||
err: errors.New("something is denied by some acl rule"),
|
||||
expectedError: layer.ErrNodeAccessDenied,
|
||||
},
|
||||
} {
|
||||
t.Run("", func(t *testing.T) {
|
||||
err := handleError("err message", tc.err)
|
||||
require.True(t, errors.Is(err, tc.expectedError))
|
||||
})
|
||||
}
|
||||
}
|
393
pkg/service/tree/tree_client_in_memory.go
Normal file
393
pkg/service/tree/tree_client_in_memory.go
Normal file
|
@ -0,0 +1,393 @@
|
|||
package tree
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
)
|
||||
|
||||
type nodeMeta struct {
|
||||
key string
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (m nodeMeta) GetKey() string {
|
||||
return m.key
|
||||
}
|
||||
|
||||
func (m nodeMeta) GetValue() []byte {
|
||||
return m.value
|
||||
}
|
||||
|
||||
type nodeResponse struct {
|
||||
meta []nodeMeta
|
||||
nodeID uint64
|
||||
parentID uint64
|
||||
timestamp uint64
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetNodeID() uint64 {
|
||||
return n.nodeID
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetParentID() uint64 {
|
||||
return n.parentID
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetTimestamp() uint64 {
|
||||
return n.timestamp
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetMeta() []Meta {
|
||||
res := make([]Meta, len(n.meta))
|
||||
for i, value := range n.meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (n nodeResponse) getValue(key string) string {
|
||||
for _, value := range n.meta {
|
||||
if value.key == key {
|
||||
return string(value.value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type ServiceClientMemory struct {
|
||||
containers map[string]containerInfo
|
||||
}
|
||||
|
||||
type containerInfo struct {
|
||||
bkt *data.BucketInfo
|
||||
trees map[string]memoryTree
|
||||
}
|
||||
|
||||
type memoryTree struct {
|
||||
idCounter uint64
|
||||
treeData *treeNodeMemory
|
||||
}
|
||||
|
||||
type treeNodeMemory struct {
|
||||
data nodeResponse
|
||||
parent *treeNodeMemory
|
||||
children []*treeNodeMemory
|
||||
}
|
||||
|
||||
func (t *treeNodeMemory) getNode(nodeID uint64) *treeNodeMemory {
|
||||
if t.data.nodeID == nodeID {
|
||||
return t
|
||||
}
|
||||
|
||||
for _, child := range t.children {
|
||||
if node := child.getNode(nodeID); node != nil {
|
||||
return node
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *memoryTree) getNodesByPath(path []string) []nodeResponse {
|
||||
if len(path) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var res []nodeResponse
|
||||
for _, child := range t.treeData.children {
|
||||
res = child.listNodesByPath(res, path)
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (t *treeNodeMemory) listNodesByPath(res []nodeResponse, path []string) []nodeResponse {
|
||||
if len(path) == 0 || t.data.getValue(FileNameKey) != path[0] {
|
||||
return res
|
||||
}
|
||||
|
||||
if len(path) == 1 {
|
||||
return append(res, t.data)
|
||||
}
|
||||
|
||||
for _, ch := range t.children {
|
||||
res = ch.listNodesByPath(res, path[1:])
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (t *memoryTree) createPathIfNotExist(parent *treeNodeMemory, path []string) *treeNodeMemory {
|
||||
if len(path) == 0 {
|
||||
return parent
|
||||
}
|
||||
|
||||
var node *treeNodeMemory
|
||||
for _, child := range parent.children {
|
||||
if len(child.data.meta) == 1 && child.data.getValue(FileNameKey) == path[0] {
|
||||
node = child
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if node == nil {
|
||||
node = &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
meta: []nodeMeta{{key: FileNameKey, value: []byte(path[0])}},
|
||||
nodeID: t.idCounter,
|
||||
parentID: parent.data.nodeID,
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
parent: parent,
|
||||
}
|
||||
t.idCounter++
|
||||
parent.children = append(parent.children, node)
|
||||
}
|
||||
|
||||
return t.createPathIfNotExist(node, path[1:])
|
||||
}
|
||||
|
||||
func (t *treeNodeMemory) removeChild(nodeID uint64) {
|
||||
ind := -1
|
||||
for i, ch := range t.children {
|
||||
if ch.data.nodeID == nodeID {
|
||||
ind = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if ind != -1 {
|
||||
t.children = append(t.children[:ind], t.children[ind+1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *treeNodeMemory) listNodes(res []NodeResponse, depth uint32) []NodeResponse {
|
||||
res = append(res, t.data)
|
||||
|
||||
if depth == 0 {
|
||||
return res
|
||||
}
|
||||
|
||||
for _, ch := range t.children {
|
||||
res = ch.listNodes(res, depth-1)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func NewTreeServiceClientMemory() (*ServiceClientMemory, error) {
|
||||
return &ServiceClientMemory{
|
||||
containers: make(map[string]containerInfo),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) {
|
||||
cnr, ok := c.containers[p.BktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[p.TreeID]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
res := tr.getNodesByPath(p.Path)
|
||||
sort.Slice(res, func(i, j int) bool {
|
||||
return res[i].timestamp < res[j].timestamp
|
||||
})
|
||||
|
||||
if p.LatestOnly && len(res) != 0 {
|
||||
res = res[len(res)-1:]
|
||||
}
|
||||
|
||||
res2 := make([]NodeResponse, len(res))
|
||||
for i, n := range res {
|
||||
res2[i] = n
|
||||
}
|
||||
|
||||
return res2, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
node := tr.treeData.getNode(rootID)
|
||||
if node == nil {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
return node.listNodes(nil, depth-1), nil
|
||||
}
|
||||
|
||||
func newContainerInfo(bktInfo *data.BucketInfo, treeID string) containerInfo {
|
||||
return containerInfo{
|
||||
bkt: bktInfo,
|
||||
trees: map[string]memoryTree{
|
||||
treeID: {
|
||||
idCounter: 1,
|
||||
treeData: &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newMemoryTree() memoryTree {
|
||||
return memoryTree{
|
||||
idCounter: 1,
|
||||
treeData: &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
cnr = newContainerInfo(bktInfo, treeID)
|
||||
c.containers[bktInfo.CID.EncodeToString()] = cnr
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
tr = newMemoryTree()
|
||||
cnr.trees[treeID] = tr
|
||||
}
|
||||
|
||||
parentNode := tr.treeData.getNode(parent)
|
||||
if parentNode == nil {
|
||||
return 0, ErrNodeNotFound
|
||||
}
|
||||
|
||||
newID := tr.idCounter
|
||||
tr.idCounter++
|
||||
|
||||
tn := &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
meta: metaToNodeMeta(meta),
|
||||
nodeID: newID,
|
||||
parentID: parent,
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
parent: parentNode,
|
||||
}
|
||||
|
||||
parentNode.children = append(parentNode.children, tn)
|
||||
cnr.trees[treeID] = tr
|
||||
|
||||
return newID, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
cnr = newContainerInfo(bktInfo, treeID)
|
||||
c.containers[bktInfo.CID.EncodeToString()] = cnr
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
tr = newMemoryTree()
|
||||
cnr.trees[treeID] = tr
|
||||
}
|
||||
|
||||
parentNode := tr.createPathIfNotExist(tr.treeData, path)
|
||||
if parentNode == nil {
|
||||
return 0, fmt.Errorf("create path '%s'", path)
|
||||
}
|
||||
|
||||
newID := tr.idCounter
|
||||
tr.idCounter++
|
||||
|
||||
tn := &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
meta: metaToNodeMeta(meta),
|
||||
nodeID: newID,
|
||||
parentID: parentNode.data.nodeID,
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
parent: parentNode,
|
||||
}
|
||||
|
||||
parentNode.children = append(parentNode.children, tn)
|
||||
cnr.trees[treeID] = tr
|
||||
|
||||
return newID, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) MoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
node := tr.treeData.getNode(nodeID)
|
||||
if node == nil {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
newParent := tr.treeData.getNode(parentID)
|
||||
if newParent == nil {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
node.data.meta = metaToNodeMeta(meta)
|
||||
node.data.parentID = parentID
|
||||
|
||||
newParent.children = append(newParent.children, node)
|
||||
node.parent.removeChild(nodeID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) RemoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
node := tr.treeData.getNode(nodeID)
|
||||
if node == nil {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
node.parent.removeChild(nodeID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func metaToNodeMeta(m map[string]string) []nodeMeta {
|
||||
result := make([]nodeMeta, 0, len(m))
|
||||
|
||||
for key, value := range m {
|
||||
result = append(result, nodeMeta{key: key, value: []byte(value)})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
|
@ -1,11 +1,12 @@
|
|||
package frostfs
|
||||
package tree
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -96,28 +97,73 @@ func TestLockConfigurationEncoding(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestHandleError(t *testing.T) {
|
||||
defaultError := errors.New("default error")
|
||||
for _, tc := range []struct {
|
||||
err error
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
err: defaultError,
|
||||
expectedError: defaultError,
|
||||
},
|
||||
{
|
||||
err: errors.New("something not found"),
|
||||
expectedError: layer.ErrNodeNotFound,
|
||||
},
|
||||
{
|
||||
err: errors.New("something is denied by some acl rule"),
|
||||
expectedError: layer.ErrNodeAccessDenied,
|
||||
},
|
||||
} {
|
||||
t.Run("", func(t *testing.T) {
|
||||
err := handleError("err message", tc.err)
|
||||
require.True(t, errors.Is(err, tc.expectedError))
|
||||
})
|
||||
func TestTreeServiceSettings(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
memCli, err := NewTreeServiceClientMemory()
|
||||
require.NoError(t, err)
|
||||
treeService := NewTree(memCli)
|
||||
|
||||
bktInfo := &data.BucketInfo{
|
||||
CID: cidtest.ID(),
|
||||
}
|
||||
|
||||
settings := &data.BucketSettings{
|
||||
Versioning: "Versioning",
|
||||
LockConfiguration: &data.ObjectLockConfiguration{
|
||||
ObjectLockEnabled: "Enabled",
|
||||
Rule: &data.ObjectLockRule{
|
||||
DefaultRetention: &data.DefaultRetention{
|
||||
Days: 1,
|
||||
Mode: "mode",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
err = treeService.PutSettingsNode(ctx, bktInfo, settings)
|
||||
require.NoError(t, err)
|
||||
|
||||
storedSettings, err := treeService.GetSettingsNode(ctx, bktInfo)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, settings, storedSettings)
|
||||
}
|
||||
|
||||
func TestTreeServiceAddVersion(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
memCli, err := NewTreeServiceClientMemory()
|
||||
require.NoError(t, err)
|
||||
treeService := NewTree(memCli)
|
||||
|
||||
bktInfo := &data.BucketInfo{
|
||||
CID: cidtest.ID(),
|
||||
}
|
||||
|
||||
version := &data.NodeVersion{
|
||||
BaseNodeVersion: data.BaseNodeVersion{
|
||||
OID: oidtest.ID(),
|
||||
Size: 10,
|
||||
ETag: "etag",
|
||||
FilePath: "path/to/version",
|
||||
},
|
||||
IsUnversioned: true,
|
||||
}
|
||||
|
||||
nodeID, err := treeService.AddVersion(ctx, bktInfo, version)
|
||||
require.NoError(t, err)
|
||||
|
||||
storedNode, err := treeService.GetUnversioned(ctx, bktInfo, "path/to/version")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, nodeID, storedNode.ID)
|
||||
require.Equal(t, version.BaseNodeVersion.Size, storedNode.Size)
|
||||
require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag)
|
||||
require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag)
|
||||
require.Equal(t, version.BaseNodeVersion.FilePath, storedNode.FilePath)
|
||||
require.Equal(t, version.BaseNodeVersion.OID, storedNode.OID)
|
||||
|
||||
versions, err := treeService.GetVersions(ctx, bktInfo, "path/to/version")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, versions, 1)
|
||||
require.Equal(t, storedNode, versions[0])
|
||||
}
|
Loading…
Reference in a new issue