From a025f2e9c59a5b2146077df65de085e0fb9fa52f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 14 Mar 2023 17:31:15 +0300 Subject: [PATCH] [#59] tree: Make interface for tree service client Signed-off-by: Denis Kirillov --- api/handler/handlers_test.go | 9 +- cmd/s3-gw/app.go | 7 +- .../frostfs => pkg/service/tree}/tree.go | 555 ++++++------------ pkg/service/tree/tree_client_grpc.go | 311 ++++++++++ .../tree/tree_client_grpc_signature.go | 6 +- pkg/service/tree/tree_client_grpc_test.go | 35 ++ pkg/service/tree/tree_client_in_memory.go | 393 +++++++++++++ .../frostfs => pkg/service/tree}/tree_test.go | 98 +++- 8 files changed, 995 insertions(+), 419 deletions(-) rename {internal/frostfs => pkg/service/tree}/tree.go (57%) create mode 100644 pkg/service/tree/tree_client_grpc.go rename internal/frostfs/tree_signature.go => pkg/service/tree/tree_client_grpc_signature.go (77%) create mode 100644 pkg/service/tree/tree_client_grpc_test.go create mode 100644 pkg/service/tree/tree_client_in_memory.go rename {internal/frostfs => pkg/service/tree}/tree_test.go (51%) diff --git a/api/handler/handlers_test.go b/api/handler/handlers_test.go index 1f7d0a0d..8c1816f0 100644 --- a/api/handler/handlers_test.go +++ b/api/handler/handlers_test.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -88,7 +89,7 @@ func prepareHandlerContext(t *testing.T) *handlerContext { Caches: layer.DefaultCachesConfigs(zap.NewExample()), AnonKey: layer.AnonymousKey{Key: key}, Resolver: testResolver, - TreeService: layer.NewTreeService(), + TreeService: NewTreeServiceMock(t), } var pp netmap.PlacementPolicy @@ -113,6 +114,12 @@ func prepareHandlerContext(t *testing.T) *handlerContext { } } +func NewTreeServiceMock(t *testing.T) *tree.Tree { + memCli, err := tree.NewTreeServiceClientMemory() + require.NoError(t, err) + return tree.NewTree(memCli) +} + func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo { _, err := hc.MockedPool().CreateContainer(hc.Context(), layer.PrmContainerCreate{ Creator: hc.owner, diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index 7c04876c..6b38a950 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -25,12 +25,15 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/xml" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/gorilla/mux" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/viper" "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) type ( @@ -110,10 +113,12 @@ func (a *App) initLayer(ctx context.Context) { a.initResolver() treeServiceEndpoint := a.cfg.GetString(cfgTreeServiceEndpoint) - treeService, err := frostfs.NewTreeClient(ctx, treeServiceEndpoint, a.key) + grpcDialOpt := grpc.WithTransportCredentials(insecure.NewCredentials()) + treeGRPCClient, err := tree.NewTreeServiceClientGRPC(ctx, treeServiceEndpoint, a.key, grpcDialOpt) if err != nil { a.log.Fatal("failed to create tree service", zap.Error(err)) } + treeService := tree.NewTree(treeGRPCClient) a.log.Info("init tree service", zap.String("endpoint", treeServiceEndpoint)) // prepare random key for anonymous requests diff --git a/internal/frostfs/tree.go b/pkg/service/tree/tree.go similarity index 57% rename from internal/frostfs/tree.go rename to pkg/service/tree/tree.go index cca1959d..43c59602 100644 --- a/internal/frostfs/tree.go +++ b/pkg/service/tree/tree.go @@ -1,35 +1,36 @@ -package frostfs +package tree import ( "context" "errors" "fmt" - "io" "strconv" "strings" "time" - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services/tree" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) type ( - TreeClient struct { - key *keys.PrivateKey - conn *grpc.ClientConn - service tree.TreeServiceClient + Tree struct { + service ServiceClient } - TreeNode struct { + // ServiceClient is a client to interact with tree service. + // Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant. + ServiceClient interface { + GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) + GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error) + AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) + AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) + MoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error + RemoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error + } + + treeNode struct { ID uint64 ParentID uint64 ObjID oid.ID @@ -38,7 +39,7 @@ type ( Meta map[string]string } - getNodesParams struct { + GetNodesParams struct { BktInfo *data.BucketInfo TreeID string Path []string @@ -48,17 +49,29 @@ type ( } ) +const ( + FileNameKey = "FileName" +) + +var ( + // ErrNodeNotFound is returned from ServiceClient in case of not found error. + ErrNodeNotFound = layer.ErrNodeNotFound + + // ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error. + ErrNodeAccessDenied = layer.ErrNodeAccessDenied +) + const ( versioningKV = "Versioning" lockConfigurationKV = "LockConfiguration" oidKV = "OID" - fileNameKV = "FileName" - isUnversionedKV = "IsUnversioned" - isTagKV = "IsTag" - uploadIDKV = "UploadId" - partNumberKV = "Number" - sizeKV = "Size" - etagKV = "ETag" + + isUnversionedKV = "IsUnversioned" + isTagKV = "IsTag" + uploadIDKV = "UploadId" + partNumberKV = "Number" + sizeKV = "Size" + etagKV = "ETag" // keys for lock. isLockKV = "IsLock" @@ -90,36 +103,27 @@ const ( maxGetSubTreeDepth = 0 // means all subTree ) -// NewTreeClient creates instance of TreeClient using provided address and create grpc connection. -func NewTreeClient(ctx context.Context, addr string, key *keys.PrivateKey) (*TreeClient, error) { - conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - return nil, fmt.Errorf("did not connect: %v", err) - } +// NewTree creates instance of Tree using provided address and create grpc connection. +func NewTree(service ServiceClient) *Tree { + return &Tree{service: service} +} - c := tree.NewTreeServiceClient(conn) - if _, err = c.Healthcheck(ctx, &tree.HealthcheckRequest{}); err != nil { - return nil, fmt.Errorf("healthcheck: %w", err) - } - - return &TreeClient{ - key: key, - conn: conn, - service: c, - }, nil +type Meta interface { + GetKey() string + GetValue() []byte } type NodeResponse interface { - GetMeta() []*tree.KeyValue - GetNodeId() uint64 - GetParentId() uint64 + GetMeta() []Meta + GetNodeID() uint64 + GetParentID() uint64 GetTimestamp() uint64 } -func newTreeNode(nodeInfo NodeResponse) (*TreeNode, error) { - treeNode := &TreeNode{ - ID: nodeInfo.GetNodeId(), - ParentID: nodeInfo.GetParentId(), +func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) { + treeNode := &treeNode{ + ID: nodeInfo.GetNodeID(), + ParentID: nodeInfo.GetParentID(), TimeStamp: nodeInfo.GetTimestamp(), Meta: make(map[string]string, len(nodeInfo.GetMeta())), } @@ -145,13 +149,13 @@ func newTreeNode(nodeInfo NodeResponse) (*TreeNode, error) { return treeNode, nil } -func (n *TreeNode) Get(key string) (string, bool) { +func (n *treeNode) Get(key string) (string, bool) { value, ok := n.Meta[key] return value, ok } -func (n *TreeNode) FileName() (string, bool) { - value, ok := n.Meta[fileNameKV] +func (n *treeNode) FileName() (string, bool) { + value, ok := n.Meta[FileNameKey] return value, ok } @@ -164,7 +168,7 @@ func newNodeVersion(filePath string, node NodeResponse) (*data.NodeVersion, erro return newNodeVersionFromTreeNode(filePath, treeNode), nil } -func newNodeVersionFromTreeNode(filePath string, treeNode *TreeNode) *data.NodeVersion { +func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeVersion { _, isUnversioned := treeNode.Get(isUnversionedKV) _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) eTag, _ := treeNode.Get(etagKV) @@ -205,7 +209,7 @@ func newNodeVersionFromTreeNode(filePath string, treeNode *TreeNode) *data.NodeV func newMultipartInfo(node NodeResponse) (*data.MultipartInfo, error) { multipartInfo := &data.MultipartInfo{ - ID: node.GetNodeId(), + ID: node.GetNodeID(), Meta: make(map[string]string, len(node.GetMeta())), } @@ -213,7 +217,7 @@ func newMultipartInfo(node NodeResponse) (*data.MultipartInfo, error) { switch kv.GetKey() { case uploadIDKV: multipartInfo.UploadID = string(kv.GetValue()) - case fileNameKV: + case FileNameKey: multipartInfo.Key = string(kv.GetValue()) case createdKV: if utcMilli, err := strconv.ParseInt(string(kv.GetValue()), 10, 64); err == nil { @@ -270,7 +274,7 @@ func newPartInfo(node NodeResponse) (*data.PartInfo, error) { return partInfo, nil } -func (c *TreeClient) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { +func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { keysToReturn := []string{versioningKV, lockConfigurationKV} node, err := c.getSystemNode(ctx, bktInfo, []string{settingsFileName}, keysToReturn) if err != nil { @@ -291,7 +295,7 @@ func (c *TreeClient) GetSettingsNode(ctx context.Context, bktInfo *data.BucketIn return settings, nil } -func (c *TreeClient) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error { +func (c *Tree) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error { node, err := c.getSystemNode(ctx, bktInfo, []string{settingsFileName}, []string{}) isErrNotFound := errors.Is(err, layer.ErrNodeNotFound) if err != nil && !isErrNotFound { @@ -301,14 +305,14 @@ func (c *TreeClient) PutSettingsNode(ctx context.Context, bktInfo *data.BucketIn meta := metaFromSettings(settings) if isErrNotFound { - _, err = c.addNode(ctx, bktInfo, systemTree, 0, meta) + _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta) return err } - return c.moveNode(ctx, bktInfo, systemTree, node.ID, 0, meta) + return c.service.MoveNode(ctx, bktInfo, systemTree, node.ID, 0, meta) } -func (c *TreeClient) GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) { +func (c *Tree) GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) { node, err := c.getSystemNode(ctx, bktInfo, []string{notifConfFileName}, []string{oidKV}) if err != nil { return oid.ID{}, err @@ -317,7 +321,7 @@ func (c *TreeClient) GetNotificationConfigurationNode(ctx context.Context, bktIn return node.ObjID, nil } -func (c *TreeClient) PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) { +func (c *Tree) PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) { node, err := c.getSystemNode(ctx, bktInfo, []string{notifConfFileName}, []string{oidKV}) isErrNotFound := errors.Is(err, layer.ErrNodeNotFound) if err != nil && !isErrNotFound { @@ -325,20 +329,20 @@ func (c *TreeClient) PutNotificationConfigurationNode(ctx context.Context, bktIn } meta := make(map[string]string) - meta[fileNameKV] = notifConfFileName + meta[FileNameKey] = notifConfFileName meta[oidKV] = objID.EncodeToString() if isErrNotFound { - if _, err = c.addNode(ctx, bktInfo, systemTree, 0, meta); err != nil { + if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil { return oid.ID{}, err } return oid.ID{}, layer.ErrNoNodeToRemove } - return node.ObjID, c.moveNode(ctx, bktInfo, systemTree, node.ID, 0, meta) + return node.ObjID, c.service.MoveNode(ctx, bktInfo, systemTree, node.ID, 0, meta) } -func (c *TreeClient) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) { +func (c *Tree) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) { node, err := c.getSystemNode(ctx, bktInfo, []string{corsFilename}, []string{oidKV}) if err != nil { return oid.ID{}, err @@ -347,7 +351,7 @@ func (c *TreeClient) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo return node.ObjID, nil } -func (c *TreeClient) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) { +func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) { node, err := c.getSystemNode(ctx, bktInfo, []string{corsFilename}, []string{oidKV}) isErrNotFound := errors.Is(err, layer.ErrNodeNotFound) if err != nil && !isErrNotFound { @@ -355,33 +359,33 @@ func (c *TreeClient) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo } meta := make(map[string]string) - meta[fileNameKV] = corsFilename + meta[FileNameKey] = corsFilename meta[oidKV] = objID.EncodeToString() if isErrNotFound { - if _, err = c.addNode(ctx, bktInfo, systemTree, 0, meta); err != nil { + if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil { return oid.ID{}, err } return oid.ID{}, layer.ErrNoNodeToRemove } - return node.ObjID, c.moveNode(ctx, bktInfo, systemTree, node.ID, 0, meta) + return node.ObjID, c.service.MoveNode(ctx, bktInfo, systemTree, node.ID, 0, meta) } -func (c *TreeClient) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) { +func (c *Tree) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) { node, err := c.getSystemNode(ctx, bktInfo, []string{corsFilename}, []string{oidKV}) if err != nil && !errors.Is(err, layer.ErrNodeNotFound) { return oid.ID{}, err } if node != nil { - return node.ObjID, c.removeNode(ctx, bktInfo, systemTree, node.ID) + return node.ObjID, c.service.RemoveNode(ctx, bktInfo, systemTree, node.ID) } return oid.ID{}, layer.ErrNoNodeToRemove } -func (c *TreeClient) GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error) { +func (c *Tree) GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error) { tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV) if err != nil { return nil, err @@ -390,7 +394,7 @@ func (c *TreeClient) GetObjectTagging(ctx context.Context, bktInfo *data.BucketI return getObjectTagging(tagNode), nil } -func getObjectTagging(tagNode *TreeNode) map[string]string { +func getObjectTagging(tagNode *treeNode) map[string]string { if tagNode == nil { return nil } @@ -406,7 +410,7 @@ func getObjectTagging(tagNode *TreeNode) map[string]string { return meta } -func (c *TreeClient) PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error { +func (c *Tree) PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error { tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV) if err != nil { return err @@ -420,15 +424,15 @@ func (c *TreeClient) PutObjectTagging(ctx context.Context, bktInfo *data.BucketI } if tagNode == nil { - _, err = c.addNode(ctx, bktInfo, versionTree, objVersion.ID, treeTagSet) + _, err = c.service.AddNode(ctx, bktInfo, versionTree, objVersion.ID, treeTagSet) } else { - err = c.moveNode(ctx, bktInfo, versionTree, tagNode.ID, objVersion.ID, treeTagSet) + err = c.service.MoveNode(ctx, bktInfo, versionTree, tagNode.ID, objVersion.ID, treeTagSet) } return err } -func (c *TreeClient) DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error { +func (c *Tree) DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error { tagNode, err := c.getTreeNode(ctx, bktInfo, objVersion.ID, isTagKV) if err != nil { return err @@ -438,10 +442,10 @@ func (c *TreeClient) DeleteObjectTagging(ctx context.Context, bktInfo *data.Buck return nil } - return c.removeNode(ctx, bktInfo, versionTree, tagNode.ID) + return c.service.RemoveNode(ctx, bktInfo, versionTree, tagNode.ID) } -func (c *TreeClient) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) { +func (c *Tree) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) { node, err := c.getSystemNodeWithAllAttributes(ctx, bktInfo, []string{bucketTaggingFilename}) if err != nil { return nil, err @@ -458,7 +462,7 @@ func (c *TreeClient) GetBucketTagging(ctx context.Context, bktInfo *data.BucketI return tags, nil } -func (c *TreeClient) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error { +func (c *Tree) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error { node, err := c.getSystemNode(ctx, bktInfo, []string{bucketTaggingFilename}, []string{}) isErrNotFound := errors.Is(err, layer.ErrNodeNotFound) if err != nil && !isErrNotFound { @@ -466,35 +470,35 @@ func (c *TreeClient) PutBucketTagging(ctx context.Context, bktInfo *data.BucketI } treeTagSet := make(map[string]string) - treeTagSet[fileNameKV] = bucketTaggingFilename + treeTagSet[FileNameKey] = bucketTaggingFilename for key, val := range tagSet { treeTagSet[userDefinedTagPrefix+key] = val } if isErrNotFound { - _, err = c.addNode(ctx, bktInfo, systemTree, 0, treeTagSet) + _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, treeTagSet) } else { - err = c.moveNode(ctx, bktInfo, systemTree, node.ID, 0, treeTagSet) + err = c.service.MoveNode(ctx, bktInfo, systemTree, node.ID, 0, treeTagSet) } return err } -func (c *TreeClient) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error { +func (c *Tree) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error { node, err := c.getSystemNode(ctx, bktInfo, []string{bucketTaggingFilename}, nil) if err != nil && !errors.Is(err, layer.ErrNodeNotFound) { return err } if node != nil { - return c.removeNode(ctx, bktInfo, systemTree, node.ID) + return c.service.RemoveNode(ctx, bktInfo, systemTree, node.ID) } return nil } -func (c *TreeClient) getTreeNode(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, key string) (*TreeNode, error) { +func (c *Tree) getTreeNode(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, key string) (*treeNode, error) { nodes, err := c.getTreeNodes(ctx, bktInfo, nodeID, key) if err != nil { return nil, err @@ -504,13 +508,13 @@ func (c *TreeClient) getTreeNode(ctx context.Context, bktInfo *data.BucketInfo, return nodes[key], nil } -func (c *TreeClient) getTreeNodes(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, keys ...string) (map[string]*TreeNode, error) { - subtree, err := c.getSubTree(ctx, bktInfo, versionTree, nodeID, 2) +func (c *Tree) getTreeNodes(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, keys ...string) (map[string]*treeNode, error) { + subtree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, nodeID, 2) if err != nil { return nil, err } - treeNodes := make(map[string]*TreeNode, len(keys)) + treeNodes := make(map[string]*treeNode, len(keys)) for _, s := range subtree { node, err := newTreeNode(s) @@ -531,15 +535,15 @@ func (c *TreeClient) getTreeNodes(ctx context.Context, bktInfo *data.BucketInfo, return treeNodes, nil } -func (c *TreeClient) GetVersions(ctx context.Context, bktInfo *data.BucketInfo, filepath string) ([]*data.NodeVersion, error) { +func (c *Tree) GetVersions(ctx context.Context, bktInfo *data.BucketInfo, filepath string) ([]*data.NodeVersion, error) { return c.getVersions(ctx, bktInfo, versionTree, filepath, false) } -func (c *TreeClient) GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) { +func (c *Tree) GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) { meta := []string{oidKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV} path := pathFromName(objectName) - p := &getNodesParams{ + p := &GetNodesParams{ BktInfo: bktInfo, TreeID: versionTree, Path: path, @@ -547,7 +551,7 @@ func (c *TreeClient) GetLatestVersion(ctx context.Context, bktInfo *data.BucketI LatestOnly: true, AllAttrs: false, } - nodes, err := c.getNodes(ctx, p) + nodes, err := c.service.GetNodes(ctx, p) if err != nil { return nil, err } @@ -564,11 +568,11 @@ func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } -func (c *TreeClient) GetLatestVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) { +func (c *Tree) GetLatestVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) { return c.getVersionsByPrefix(ctx, bktInfo, prefix, true) } -func (c *TreeClient) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) (uint64, string, error) { +func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) (uint64, string, error) { var rootID uint64 path := strings.Split(prefix, separator) tailPrefix := path[len(path)-1] @@ -584,15 +588,15 @@ func (c *TreeClient) determinePrefixNode(ctx context.Context, bktInfo *data.Buck return rootID, tailPrefix, nil } -func (c *TreeClient) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) (uint64, error) { - p := &getNodesParams{ +func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) (uint64, error) { + p := &GetNodesParams{ BktInfo: bktInfo, TreeID: treeID, Path: prefixPath, LatestOnly: false, AllAttrs: true, } - nodes, err := c.getNodes(ctx, p) + nodes, err := c.service.GetNodes(ctx, p) if err != nil { return 0, err } @@ -600,7 +604,7 @@ func (c *TreeClient) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketIn var intermediateNodes []uint64 for _, node := range nodes { if isIntermediate(node) { - intermediateNodes = append(intermediateNodes, node.GetNodeId()) + intermediateNodes = append(intermediateNodes, node.GetNodeID()) } } @@ -614,7 +618,7 @@ func (c *TreeClient) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketIn return intermediateNodes[0], nil } -func (c *TreeClient) getSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string, latestOnly bool) ([]*tree.GetSubTreeResponse_Body, string, error) { +func (c *Tree) getSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string, latestOnly bool) ([]NodeResponse, string, error) { rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, treeID, prefix) if err != nil { if errors.Is(err, layer.ErrNodeNotFound) { @@ -623,7 +627,7 @@ func (c *TreeClient) getSubTreeByPrefix(ctx context.Context, bktInfo *data.Bucke return nil, "", err } - subTree, err := c.getSubTree(ctx, bktInfo, treeID, rootID, 2) + subTree, err := c.service.GetSubTree(ctx, bktInfo, treeID, rootID, 2) if err != nil { if errors.Is(err, layer.ErrNodeNotFound) { return nil, "", nil @@ -631,9 +635,9 @@ func (c *TreeClient) getSubTreeByPrefix(ctx context.Context, bktInfo *data.Bucke return nil, "", err } - nodesMap := make(map[string][]*tree.GetSubTreeResponse_Body, len(subTree)) + nodesMap := make(map[string][]NodeResponse, len(subTree)) for _, node := range subTree { - if node.GetNodeId() == rootID { + if node.GetNodeID() == rootID { continue } @@ -648,11 +652,11 @@ func (c *TreeClient) getSubTreeByPrefix(ctx context.Context, bktInfo *data.Bucke // Add all intermediate nodes (actually should be exactly one intermediate node with the same name) // and only latest leaf (object) nodes. To do this store and replace last leaf (object) node in nodes[0] if len(nodes) == 0 { - nodes = []*tree.GetSubTreeResponse_Body{node} + nodes = []NodeResponse{node} } else if !latestOnly || isIntermediate(node) { nodes = append(nodes, node) } else if isIntermediate(nodes[0]) { - nodes = append([]*tree.GetSubTreeResponse_Body{node}, nodes...) + nodes = append([]NodeResponse{node}, nodes...) } else if node.GetTimestamp() > nodes[0].GetTimestamp() { nodes[0] = node } @@ -660,7 +664,7 @@ func (c *TreeClient) getSubTreeByPrefix(ctx context.Context, bktInfo *data.Bucke nodesMap[fileName] = nodes } - result := make([]*tree.GetSubTreeResponse_Body, 0, len(subTree)) + result := make([]NodeResponse, 0, len(subTree)) for _, nodes := range nodesMap { result = append(result, nodes...) } @@ -668,9 +672,9 @@ func (c *TreeClient) getSubTreeByPrefix(ctx context.Context, bktInfo *data.Bucke return result, strings.TrimSuffix(prefix, tailPrefix), nil } -func getFilename(node *tree.GetSubTreeResponse_Body) string { +func getFilename(node NodeResponse) string { for _, kv := range node.GetMeta() { - if kv.GetKey() == fileNameKV { + if kv.GetKey() == FileNameKey { return string(kv.GetValue()) } } @@ -683,11 +687,11 @@ func isIntermediate(node NodeResponse) bool { return false } - return node.GetMeta()[0].GetKey() == fileNameKV + return node.GetMeta()[0].GetKey() == FileNameKey } -func (c *TreeClient) getSubTreeVersions(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, parentFilePath string, latestOnly bool) ([]*data.NodeVersion, error) { - subTree, err := c.getSubTree(ctx, bktInfo, versionTree, nodeID, maxGetSubTreeDepth) +func (c *Tree) getSubTreeVersions(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, parentFilePath string, latestOnly bool) ([]*data.NodeVersion, error) { + subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, nodeID, maxGetSubTreeDepth) if err != nil { return nil, err } @@ -721,7 +725,7 @@ func (c *TreeClient) getSubTreeVersions(ctx context.Context, bktInfo *data.Bucke continue } - key := formLatestNodeKey(node.GetParentId(), fileName) + key := formLatestNodeKey(node.GetParentID(), fileName) versionNodes, ok := versions[key] if !ok { versionNodes = []*data.NodeVersion{newNodeVersionFromTreeNode(filepath, treeNode)} @@ -745,19 +749,19 @@ func (c *TreeClient) getSubTreeVersions(ctx context.Context, bktInfo *data.Bucke return result, nil } -func formFilePath(node *tree.GetSubTreeResponse_Body, fileName string, namesMap map[uint64]string) (string, error) { - parentPath, ok := namesMap[node.GetParentId()] +func formFilePath(node NodeResponse, fileName string, namesMap map[uint64]string) (string, error) { + parentPath, ok := namesMap[node.GetParentID()] if !ok { return "", fmt.Errorf("couldn't get parent path") } filepath := parentPath + separator + fileName - namesMap[node.GetNodeId()] = filepath + namesMap[node.GetNodeID()] = filepath return filepath, nil } -func parseTreeNode(node *tree.GetSubTreeResponse_Body) (*TreeNode, string, error) { +func parseTreeNode(node NodeResponse) (*treeNode, string, error) { treeNode, err := newTreeNode(node) if err != nil { // invalid OID attribute return nil, "", err @@ -775,11 +779,11 @@ func formLatestNodeKey(parentID uint64, fileName string) string { return strconv.FormatUint(parentID, 10) + "." + fileName } -func (c *TreeClient) GetAllVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) { +func (c *Tree) GetAllVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) { return c.getVersionsByPrefix(ctx, bktInfo, prefix, false) } -func (c *TreeClient) getVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]*data.NodeVersion, error) { +func (c *Tree) getVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]*data.NodeVersion, error) { prefixNodes, headPrefix, err := c.getSubTreeByPrefix(ctx, bktInfo, versionTree, prefix, latestOnly) if err != nil { return nil, err @@ -787,7 +791,7 @@ func (c *TreeClient) getVersionsByPrefix(ctx context.Context, bktInfo *data.Buck var result []*data.NodeVersion for _, node := range prefixNodes { - versions, err := c.getSubTreeVersions(ctx, bktInfo, node.GetNodeId(), headPrefix, latestOnly) + versions, err := c.getSubTreeVersions(ctx, bktInfo, node.GetNodeID(), headPrefix, latestOnly) if err != nil { return nil, err } @@ -797,11 +801,11 @@ func (c *TreeClient) getVersionsByPrefix(ctx context.Context, bktInfo *data.Buck return result, nil } -func (c *TreeClient) GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, filepath string) (*data.NodeVersion, error) { +func (c *Tree) GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, filepath string) (*data.NodeVersion, error) { return c.getUnversioned(ctx, bktInfo, versionTree, filepath) } -func (c *TreeClient) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, treeID, filepath string) (*data.NodeVersion, error) { +func (c *Tree) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, treeID, filepath string) (*data.NodeVersion, error) { nodes, err := c.getVersions(ctx, bktInfo, treeID, filepath, true) if err != nil { return nil, err @@ -818,23 +822,23 @@ func (c *TreeClient) getUnversioned(ctx context.Context, bktInfo *data.BucketInf return nodes[0], nil } -func (c *TreeClient) AddVersion(ctx context.Context, bktInfo *data.BucketInfo, version *data.NodeVersion) (uint64, error) { +func (c *Tree) AddVersion(ctx context.Context, bktInfo *data.BucketInfo, version *data.NodeVersion) (uint64, error) { return c.addVersion(ctx, bktInfo, versionTree, version) } -func (c *TreeClient) RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, id uint64) error { - return c.removeNode(ctx, bktInfo, versionTree, id) +func (c *Tree) RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, id uint64) error { + return c.service.RemoveNode(ctx, bktInfo, versionTree, id) } -func (c *TreeClient) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error { +func (c *Tree) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error { path := pathFromName(info.Key) meta := metaFromMultipart(info, path[len(path)-1]) - _, err := c.addNodeByPath(ctx, bktInfo, systemTree, path[:len(path)-1], meta) + _, err := c.service.AddNodeByPath(ctx, bktInfo, systemTree, path[:len(path)-1], meta) return err } -func (c *TreeClient) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) { +func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) { subTreeNodes, _, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false) if err != nil { return nil, err @@ -842,7 +846,7 @@ func (c *TreeClient) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *d var result []*data.MultipartInfo for _, node := range subTreeNodes { - multipartUploads, err := c.getSubTreeMultipartUploads(ctx, bktInfo, node.GetNodeId()) + multipartUploads, err := c.getSubTreeMultipartUploads(ctx, bktInfo, node.GetNodeID()) if err != nil { return nil, err } @@ -852,8 +856,8 @@ func (c *TreeClient) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *d return result, nil } -func (c *TreeClient) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) ([]*data.MultipartInfo, error) { - subTree, err := c.getSubTree(ctx, bktInfo, systemTree, nodeID, maxGetSubTreeDepth) +func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) ([]*data.MultipartInfo, error) { + subTree, err := c.service.GetSubTree(ctx, bktInfo, systemTree, nodeID, maxGetSubTreeDepth) if err != nil { return nil, err } @@ -870,16 +874,16 @@ func (c *TreeClient) getSubTreeMultipartUploads(ctx context.Context, bktInfo *da return result, nil } -func (c *TreeClient) GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error) { +func (c *Tree) GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error) { path := pathFromName(objectName) - p := &getNodesParams{ + p := &GetNodesParams{ BktInfo: bktInfo, TreeID: systemTree, Path: path, AllAttrs: true, } - nodes, err := c.getNodes(ctx, p) + nodes, err := c.service.GetNodes(ctx, p) if err != nil { return nil, err } @@ -897,8 +901,8 @@ func (c *TreeClient) GetMultipartUpload(ctx context.Context, bktInfo *data.Bucke return nil, layer.ErrNodeNotFound } -func (c *TreeClient) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error) { - parts, err := c.getSubTree(ctx, bktInfo, systemTree, multipartNodeID, 2) +func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error) { + parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, multipartNodeID, 2) if err != nil { return oid.ID{}, err } @@ -913,7 +917,7 @@ func (c *TreeClient) AddPart(ctx context.Context, bktInfo *data.BucketInfo, mult var foundPartID uint64 for _, part := range parts { - if part.GetNodeId() == multipartNodeID { + if part.GetNodeID() == multipartNodeID { continue } partInfo, err := newPartInfo(part) @@ -921,31 +925,31 @@ func (c *TreeClient) AddPart(ctx context.Context, bktInfo *data.BucketInfo, mult continue } if partInfo.Number == info.Number { - foundPartID = part.GetNodeId() + foundPartID = part.GetNodeID() oldObjIDToDelete = partInfo.OID break } } if foundPartID != multipartNodeID { - if _, err = c.addNode(ctx, bktInfo, systemTree, multipartNodeID, meta); err != nil { + if _, err = c.service.AddNode(ctx, bktInfo, systemTree, multipartNodeID, meta); err != nil { return oid.ID{}, err } return oid.ID{}, layer.ErrNoNodeToRemove } - return oldObjIDToDelete, c.moveNode(ctx, bktInfo, systemTree, foundPartID, multipartNodeID, meta) + return oldObjIDToDelete, c.service.MoveNode(ctx, bktInfo, systemTree, foundPartID, multipartNodeID, meta) } -func (c *TreeClient) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error) { - parts, err := c.getSubTree(ctx, bktInfo, systemTree, multipartNodeID, 2) +func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error) { + parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, multipartNodeID, 2) if err != nil { return nil, err } result := make([]*data.PartInfo, 0, len(parts)) for _, part := range parts { - if part.GetNodeId() == multipartNodeID { + if part.GetNodeID() == multipartNodeID { continue } partInfo, err := newPartInfo(part) @@ -958,11 +962,11 @@ func (c *TreeClient) GetParts(ctx context.Context, bktInfo *data.BucketInfo, mul return result, nil } -func (c *TreeClient) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) error { - return c.removeNode(ctx, bktInfo, systemTree, multipartNodeID) +func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) error { + return c.service.RemoveNode(ctx, bktInfo, systemTree, multipartNodeID) } -func (c *TreeClient) PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error { +func (c *Tree) PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error { meta := map[string]string{isLockKV: "true"} if lock.IsLegalHoldSet() { @@ -977,14 +981,14 @@ func (c *TreeClient) PutLock(ctx context.Context, bktInfo *data.BucketInfo, node } if lock.ID() == 0 { - _, err := c.addNode(ctx, bktInfo, versionTree, nodeID, meta) + _, err := c.service.AddNode(ctx, bktInfo, versionTree, nodeID, meta) return err } - return c.moveNode(ctx, bktInfo, versionTree, lock.ID(), nodeID, meta) + return c.service.MoveNode(ctx, bktInfo, versionTree, lock.ID(), nodeID, meta) } -func (c *TreeClient) GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) { +func (c *Tree) GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) { lockNode, err := c.getTreeNode(ctx, bktInfo, nodeID, isLockKV) if err != nil { return nil, err @@ -993,7 +997,7 @@ func (c *TreeClient) GetLock(ctx context.Context, bktInfo *data.BucketInfo, node return getLock(lockNode) } -func getLock(lockNode *TreeNode) (*data.LockInfo, error) { +func getLock(lockNode *treeNode) (*data.LockInfo, error) { if lockNode == nil { return &data.LockInfo{}, nil } @@ -1020,7 +1024,7 @@ func getLock(lockNode *TreeNode) (*data.LockInfo, error) { return lockInfo, nil } -func (c *TreeClient) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) { +func (c *Tree) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) { nodes, err := c.getTreeNodes(ctx, bktInfo, objVersion.ID, isTagKV, isLockKV) if err != nil { return nil, nil, err @@ -1034,19 +1038,11 @@ func (c *TreeClient) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data. return getObjectTagging(nodes[isTagKV]), lockInfo, nil } -func (c *TreeClient) Close() error { - if c.conn != nil { - return c.conn.Close() - } - - return nil -} - -func (c *TreeClient) addVersion(ctx context.Context, bktInfo *data.BucketInfo, treeID string, version *data.NodeVersion) (uint64, error) { +func (c *Tree) addVersion(ctx context.Context, bktInfo *data.BucketInfo, treeID string, version *data.NodeVersion) (uint64, error) { path := pathFromName(version.FilePath) meta := map[string]string{ - oidKV: version.OID.EncodeToString(), - fileNameKV: path[len(path)-1], + oidKV: version.OID.EncodeToString(), + FileNameKey: path[len(path)-1], } if version.Size > 0 { @@ -1067,7 +1063,7 @@ func (c *TreeClient) addVersion(ctx context.Context, bktInfo *data.BucketInfo, t node, err := c.getUnversioned(ctx, bktInfo, treeID, version.FilePath) if err == nil { - if err = c.moveNode(ctx, bktInfo, treeID, node.ID, node.ParenID, meta); err != nil { + if err = c.service.MoveNode(ctx, bktInfo, treeID, node.ID, node.ParenID, meta); err != nil { return 0, err } @@ -1079,25 +1075,25 @@ func (c *TreeClient) addVersion(ctx context.Context, bktInfo *data.BucketInfo, t } } - return c.addNodeByPath(ctx, bktInfo, treeID, path[:len(path)-1], meta) + return c.service.AddNodeByPath(ctx, bktInfo, treeID, path[:len(path)-1], meta) } -func (c *TreeClient) clearOutdatedVersionInfo(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error { +func (c *Tree) clearOutdatedVersionInfo(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error { taggingNode, err := c.getTreeNode(ctx, bktInfo, nodeID, isTagKV) if err != nil { return err } if taggingNode != nil { - return c.removeNode(ctx, bktInfo, treeID, taggingNode.ID) + return c.service.RemoveNode(ctx, bktInfo, treeID, taggingNode.ID) } return nil } -func (c *TreeClient) getVersions(ctx context.Context, bktInfo *data.BucketInfo, treeID, filepath string, onlyUnversioned bool) ([]*data.NodeVersion, error) { +func (c *Tree) getVersions(ctx context.Context, bktInfo *data.BucketInfo, treeID, filepath string, onlyUnversioned bool) ([]*data.NodeVersion, error) { keysToReturn := []string{oidKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV} path := pathFromName(filepath) - p := &getNodesParams{ + p := &GetNodesParams{ BktInfo: bktInfo, TreeID: treeID, Path: path, @@ -1105,7 +1101,7 @@ func (c *TreeClient) getVersions(ctx context.Context, bktInfo *data.BucketInfo, LatestOnly: false, AllAttrs: false, } - nodes, err := c.getNodes(ctx, p) + nodes, err := c.service.GetNodes(ctx, p) if err != nil { if errors.Is(err, layer.ErrNodeNotFound) { return nil, nil @@ -1130,49 +1126,10 @@ func (c *TreeClient) getVersions(ctx context.Context, bktInfo *data.BucketInfo, return result, nil } -func (c *TreeClient) getSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]*tree.GetSubTreeResponse_Body, error) { - request := &tree.GetSubTreeRequest{ - Body: &tree.GetSubTreeRequest_Body{ - ContainerId: bktInfo.CID[:], - TreeId: treeID, - RootId: rootID, - Depth: depth, - BearerToken: getBearer(ctx, bktInfo), - }, - } - - if err := c.signRequest(request.Body, func(key, sign []byte) { - request.Signature = &tree.Signature{ - Key: key, - Sign: sign, - } - }); err != nil { - return nil, err - } - - cli, err := c.service.GetSubTree(ctx, request) - if err != nil { - return nil, handleError("failed to get sub tree client", err) - } - - var subtree []*tree.GetSubTreeResponse_Body - for { - resp, err := cli.Recv() - if err == io.EOF { - break - } else if err != nil { - return nil, handleError("failed to get sub tree", err) - } - subtree = append(subtree, resp.Body) - } - - return subtree, nil -} - func metaFromSettings(settings *data.BucketSettings) map[string]string { results := make(map[string]string, 3) - results[fileNameKV] = settingsFileName + results[FileNameKey] = settingsFileName results[versioningKV] = settings.Versioning results[lockConfigurationKV] = encodeLockConfiguration(settings.LockConfiguration) @@ -1180,7 +1137,7 @@ func metaFromSettings(settings *data.BucketSettings) map[string]string { } func metaFromMultipart(info *data.MultipartInfo, fileName string) map[string]string { - info.Meta[fileNameKV] = fileName + info.Meta[FileNameKey] = fileName info.Meta[uploadIDKV] = info.UploadID info.Meta[ownerKV] = info.Owner.EncodeToString() info.Meta[createdKV] = strconv.FormatInt(info.Created.UTC().UnixMilli(), 10) @@ -1188,16 +1145,16 @@ func metaFromMultipart(info *data.MultipartInfo, fileName string) map[string]str return info.Meta } -func (c *TreeClient) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, path, meta []string) (*TreeNode, error) { +func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, path, meta []string) (*treeNode, error) { return c.getNode(ctx, bktInfo, systemTree, path, meta, false) } -func (c *TreeClient) getSystemNodeWithAllAttributes(ctx context.Context, bktInfo *data.BucketInfo, path []string) (*TreeNode, error) { +func (c *Tree) getSystemNodeWithAllAttributes(ctx context.Context, bktInfo *data.BucketInfo, path []string) (*treeNode, error) { return c.getNode(ctx, bktInfo, systemTree, path, []string{}, true) } -func (c *TreeClient) getNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path, meta []string, allAttrs bool) (*TreeNode, error) { - p := &getNodesParams{ +func (c *Tree) getNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path, meta []string, allAttrs bool) (*treeNode, error) { + p := &GetNodesParams{ BktInfo: bktInfo, TreeID: treeID, Path: path, @@ -1205,7 +1162,7 @@ func (c *TreeClient) getNode(ctx context.Context, bktInfo *data.BucketInfo, tree LatestOnly: false, AllAttrs: allAttrs, } - nodes, err := c.getNodes(ctx, p) + nodes, err := c.service.GetNodes(ctx, p) if err != nil { return nil, err } @@ -1219,184 +1176,6 @@ func (c *TreeClient) getNode(ctx context.Context, bktInfo *data.BucketInfo, tree return newTreeNode(nodes[0]) } -func (c *TreeClient) getNodes(ctx context.Context, p *getNodesParams) ([]*tree.GetNodeByPathResponse_Info, error) { - request := &tree.GetNodeByPathRequest{ - Body: &tree.GetNodeByPathRequest_Body{ - ContainerId: p.BktInfo.CID[:], - TreeId: p.TreeID, - Path: p.Path, - Attributes: p.Meta, - PathAttribute: fileNameKV, - LatestOnly: p.LatestOnly, - AllAttributes: p.AllAttrs, - BearerToken: getBearer(ctx, p.BktInfo), - }, - } - - if err := c.signRequest(request.Body, func(key, sign []byte) { - request.Signature = &tree.Signature{ - Key: key, - Sign: sign, - } - }); err != nil { - return nil, err - } - - resp, err := c.service.GetNodeByPath(ctx, request) - if err != nil { - return nil, handleError("failed to get node by path", err) - } - - return resp.GetBody().GetNodes(), nil -} - -func handleError(msg string, err error) error { - if strings.Contains(err.Error(), "not found") { - return fmt.Errorf("%w: %s", layer.ErrNodeNotFound, err.Error()) - } else if strings.Contains(err.Error(), "is denied by") { - return fmt.Errorf("%w: %s", layer.ErrNodeAccessDenied, err.Error()) - } - return fmt.Errorf("%s: %w", msg, err) -} - -func getBearer(ctx context.Context, bktInfo *data.BucketInfo) []byte { - if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil { - if bd.Gate.BearerToken != nil { - if bktInfo.Owner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) { - return bd.Gate.BearerToken.Marshal() - } - } - } - return nil -} - -func (c *TreeClient) addNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) { - request := &tree.AddRequest{ - Body: &tree.AddRequest_Body{ - ContainerId: bktInfo.CID[:], - TreeId: treeID, - ParentId: parent, - Meta: metaToKV(meta), - BearerToken: getBearer(ctx, bktInfo), - }, - } - if err := c.signRequest(request.Body, func(key, sign []byte) { - request.Signature = &tree.Signature{ - Key: key, - Sign: sign, - } - }); err != nil { - return 0, err - } - - resp, err := c.service.Add(ctx, request) - if err != nil { - return 0, handleError("failed to add node", err) - } - - return resp.GetBody().GetNodeId(), nil -} - -func (c *TreeClient) addNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) { - request := &tree.AddByPathRequest{ - Body: &tree.AddByPathRequest_Body{ - ContainerId: bktInfo.CID[:], - TreeId: treeID, - Path: path, - Meta: metaToKV(meta), - PathAttribute: fileNameKV, - BearerToken: getBearer(ctx, bktInfo), - }, - } - - if err := c.signRequest(request.Body, func(key, sign []byte) { - request.Signature = &tree.Signature{ - Key: key, - Sign: sign, - } - }); err != nil { - return 0, err - } - - resp, err := c.service.AddByPath(ctx, request) - if err != nil { - return 0, handleError("failed to add node by path", err) - } - - body := resp.GetBody() - if body == nil { - return 0, errors.New("nil body in tree service response") - } else if len(body.Nodes) == 0 { - return 0, errors.New("empty list of added nodes in tree service response") - } - - // The first node is the leaf that we add, according to tree service docs. - return body.Nodes[0], nil -} - -func (c *TreeClient) moveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error { - request := &tree.MoveRequest{ - Body: &tree.MoveRequest_Body{ - ContainerId: bktInfo.CID[:], - TreeId: treeID, - NodeId: nodeID, - ParentId: parentID, - Meta: metaToKV(meta), - BearerToken: getBearer(ctx, bktInfo), - }, - } - - if err := c.signRequest(request.Body, func(key, sign []byte) { - request.Signature = &tree.Signature{ - Key: key, - Sign: sign, - } - }); err != nil { - return err - } - - if _, err := c.service.Move(ctx, request); err != nil { - return handleError("failed to move node", err) - } - - return nil -} - -func (c *TreeClient) removeNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error { - request := &tree.RemoveRequest{ - Body: &tree.RemoveRequest_Body{ - ContainerId: bktInfo.CID[:], - TreeId: treeID, - NodeId: nodeID, - BearerToken: getBearer(ctx, bktInfo), - }, - } - if err := c.signRequest(request.Body, func(key, sign []byte) { - request.Signature = &tree.Signature{ - Key: key, - Sign: sign, - } - }); err != nil { - return err - } - - if _, err := c.service.Remove(ctx, request); err != nil { - return handleError("failed to remove node", err) - } - - return nil -} - -func metaToKV(meta map[string]string) []*tree.KeyValue { - result := make([]*tree.KeyValue, 0, len(meta)) - - for key, value := range meta { - result = append(result, &tree.KeyValue{Key: key, Value: []byte(value)}) - } - - return result -} - func parseLockConfiguration(value string) (*data.ObjectLockConfiguration, error) { result := &data.ObjectLockConfiguration{} if len(value) == 0 { diff --git a/pkg/service/tree/tree_client_grpc.go b/pkg/service/tree/tree_client_grpc.go new file mode 100644 index 00000000..e447627a --- /dev/null +++ b/pkg/service/tree/tree_client_grpc.go @@ -0,0 +1,311 @@ +package tree + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services/tree" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "google.golang.org/grpc" +) + +type GetNodeByPathResponseInfoWrapper struct { + response *tree.GetNodeByPathResponse_Info +} + +func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 { + return n.response.GetNodeId() +} + +func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 { + return n.response.GetParentId() +} + +func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 { + return n.response.GetTimestamp() +} + +func (n GetNodeByPathResponseInfoWrapper) GetMeta() []Meta { + res := make([]Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} + +type GetSubTreeResponseBodyWrapper struct { + response *tree.GetSubTreeResponse_Body +} + +func (n GetSubTreeResponseBodyWrapper) GetNodeID() uint64 { + return n.response.GetNodeId() +} + +func (n GetSubTreeResponseBodyWrapper) GetParentID() uint64 { + return n.response.GetParentId() +} + +func (n GetSubTreeResponseBodyWrapper) GetTimestamp() uint64 { + return n.response.GetTimestamp() +} + +func (n GetSubTreeResponseBodyWrapper) GetMeta() []Meta { + res := make([]Meta, len(n.response.Meta)) + for i, value := range n.response.Meta { + res[i] = value + } + return res +} + +type ServiceClientGRPC struct { + key *keys.PrivateKey + conn *grpc.ClientConn + service tree.TreeServiceClient +} + +func NewTreeServiceClientGRPC(ctx context.Context, addr string, key *keys.PrivateKey, grpcOpts ...grpc.DialOption) (*ServiceClientGRPC, error) { + conn, err := grpc.Dial(addr, grpcOpts...) + if err != nil { + return nil, fmt.Errorf("did not connect: %v", err) + } + + c := tree.NewTreeServiceClient(conn) + if _, err = c.Healthcheck(ctx, &tree.HealthcheckRequest{}); err != nil { + return nil, fmt.Errorf("healthcheck: %w", err) + } + + return &ServiceClientGRPC{ + key: key, + conn: conn, + service: c, + }, nil +} + +func (c *ServiceClientGRPC) GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) { + request := &tree.GetNodeByPathRequest{ + Body: &tree.GetNodeByPathRequest_Body{ + ContainerId: p.BktInfo.CID[:], + TreeId: p.TreeID, + Path: p.Path, + Attributes: p.Meta, + PathAttribute: FileNameKey, + LatestOnly: p.LatestOnly, + AllAttributes: p.AllAttrs, + BearerToken: getBearer(ctx, p.BktInfo), + }, + } + + if err := c.signRequest(request.Body, func(key, sign []byte) { + request.Signature = &tree.Signature{ + Key: key, + Sign: sign, + } + }); err != nil { + return nil, err + } + + resp, err := c.service.GetNodeByPath(ctx, request) + if err != nil { + return nil, handleError("failed to get node by path", err) + } + + res := make([]NodeResponse, len(resp.GetBody().GetNodes())) + for i, info := range resp.GetBody().GetNodes() { + res[i] = GetNodeByPathResponseInfoWrapper{info} + } + + return res, nil +} + +func (c *ServiceClientGRPC) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error) { + request := &tree.GetSubTreeRequest{ + Body: &tree.GetSubTreeRequest_Body{ + ContainerId: bktInfo.CID[:], + TreeId: treeID, + RootId: rootID, + Depth: depth, + BearerToken: getBearer(ctx, bktInfo), + }, + } + + if err := c.signRequest(request.Body, func(key, sign []byte) { + request.Signature = &tree.Signature{ + Key: key, + Sign: sign, + } + }); err != nil { + return nil, err + } + + cli, err := c.service.GetSubTree(ctx, request) + if err != nil { + return nil, handleError("failed to get sub tree client", err) + } + + var subtree []NodeResponse + for { + resp, err := cli.Recv() + if err == io.EOF { + break + } else if err != nil { + return nil, handleError("failed to get sub tree", err) + } + subtree = append(subtree, GetSubTreeResponseBodyWrapper{resp.Body}) + } + + return subtree, nil +} + +func (c *ServiceClientGRPC) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) { + request := &tree.AddRequest{ + Body: &tree.AddRequest_Body{ + ContainerId: bktInfo.CID[:], + TreeId: treeID, + ParentId: parent, + Meta: metaToKV(meta), + BearerToken: getBearer(ctx, bktInfo), + }, + } + if err := c.signRequest(request.Body, func(key, sign []byte) { + request.Signature = &tree.Signature{ + Key: key, + Sign: sign, + } + }); err != nil { + return 0, err + } + + resp, err := c.service.Add(ctx, request) + if err != nil { + return 0, handleError("failed to add node", err) + } + + return resp.GetBody().GetNodeId(), nil +} + +func (c *ServiceClientGRPC) AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) { + request := &tree.AddByPathRequest{ + Body: &tree.AddByPathRequest_Body{ + ContainerId: bktInfo.CID[:], + TreeId: treeID, + Path: path, + Meta: metaToKV(meta), + PathAttribute: FileNameKey, + BearerToken: getBearer(ctx, bktInfo), + }, + } + + if err := c.signRequest(request.Body, func(key, sign []byte) { + request.Signature = &tree.Signature{ + Key: key, + Sign: sign, + } + }); err != nil { + return 0, err + } + + resp, err := c.service.AddByPath(ctx, request) + if err != nil { + return 0, handleError("failed to add node by path", err) + } + + body := resp.GetBody() + if body == nil { + return 0, errors.New("nil body in tree service response") + } else if len(body.Nodes) == 0 { + return 0, errors.New("empty list of added nodes in tree service response") + } + + // The first node is the leaf that we add, according to tree service docs. + return body.Nodes[0], nil +} + +func (c *ServiceClientGRPC) MoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error { + request := &tree.MoveRequest{ + Body: &tree.MoveRequest_Body{ + ContainerId: bktInfo.CID[:], + TreeId: treeID, + NodeId: nodeID, + ParentId: parentID, + Meta: metaToKV(meta), + BearerToken: getBearer(ctx, bktInfo), + }, + } + + if err := c.signRequest(request.Body, func(key, sign []byte) { + request.Signature = &tree.Signature{ + Key: key, + Sign: sign, + } + }); err != nil { + return err + } + + if _, err := c.service.Move(ctx, request); err != nil { + return handleError("failed to move node", err) + } + + return nil +} + +func (c *ServiceClientGRPC) RemoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error { + request := &tree.RemoveRequest{ + Body: &tree.RemoveRequest_Body{ + ContainerId: bktInfo.CID[:], + TreeId: treeID, + NodeId: nodeID, + BearerToken: getBearer(ctx, bktInfo), + }, + } + if err := c.signRequest(request.Body, func(key, sign []byte) { + request.Signature = &tree.Signature{ + Key: key, + Sign: sign, + } + }); err != nil { + return err + } + + if _, err := c.service.Remove(ctx, request); err != nil { + return handleError("failed to remove node", err) + } + + return nil +} + +func metaToKV(meta map[string]string) []*tree.KeyValue { + result := make([]*tree.KeyValue, 0, len(meta)) + + for key, value := range meta { + result = append(result, &tree.KeyValue{Key: key, Value: []byte(value)}) + } + + return result +} + +func getBearer(ctx context.Context, bktInfo *data.BucketInfo) []byte { + if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil { + if bd.Gate.BearerToken != nil { + if bktInfo.Owner.Equals(bearer.ResolveIssuer(*bd.Gate.BearerToken)) { + return bd.Gate.BearerToken.Marshal() + } + } + } + return nil +} + +func handleError(msg string, err error) error { + if strings.Contains(err.Error(), "not found") { + return fmt.Errorf("%w: %s", ErrNodeNotFound, err.Error()) + } else if strings.Contains(err.Error(), "is denied by") { + return fmt.Errorf("%w: %s", ErrNodeAccessDenied, err.Error()) + } + return fmt.Errorf("%s: %w", msg, err) +} diff --git a/internal/frostfs/tree_signature.go b/pkg/service/tree/tree_client_grpc_signature.go similarity index 77% rename from internal/frostfs/tree_signature.go rename to pkg/service/tree/tree_client_grpc_signature.go index 410a5920..7eb65643 100644 --- a/internal/frostfs/tree_signature.go +++ b/pkg/service/tree/tree_client_grpc_signature.go @@ -1,12 +1,12 @@ /*REMOVE THIS AFTER SIGNATURE WILL BE AVAILABLE IN TREE CLIENT FROM FROSTFS NODE*/ -package frostfs +package tree import ( crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto" "google.golang.org/protobuf/proto" ) -func (c *TreeClient) signData(buf []byte, f func(key, sign []byte)) error { +func (c *ServiceClientGRPC) signData(buf []byte, f func(key, sign []byte)) error { // crypto package should not be used outside of API libraries (see neofs-node#491). // For now tree service does not include into SDK Client nor SDK Pool, so there is no choice. // When SDK library adopts Tree service client, this should be dropped. @@ -19,7 +19,7 @@ func (c *TreeClient) signData(buf []byte, f func(key, sign []byte)) error { return nil } -func (c *TreeClient) signRequest(requestBody proto.Message, f func(key, sign []byte)) error { +func (c *ServiceClientGRPC) signRequest(requestBody proto.Message, f func(key, sign []byte)) error { buf, err := proto.Marshal(requestBody) if err != nil { return err diff --git a/pkg/service/tree/tree_client_grpc_test.go b/pkg/service/tree/tree_client_grpc_test.go new file mode 100644 index 00000000..b9f0f8c3 --- /dev/null +++ b/pkg/service/tree/tree_client_grpc_test.go @@ -0,0 +1,35 @@ +package tree + +import ( + "errors" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" + "github.com/stretchr/testify/require" +) + +func TestHandleError(t *testing.T) { + defaultError := errors.New("default error") + for _, tc := range []struct { + err error + expectedError error + }{ + { + err: defaultError, + expectedError: defaultError, + }, + { + err: errors.New("something not found"), + expectedError: layer.ErrNodeNotFound, + }, + { + err: errors.New("something is denied by some acl rule"), + expectedError: layer.ErrNodeAccessDenied, + }, + } { + t.Run("", func(t *testing.T) { + err := handleError("err message", tc.err) + require.True(t, errors.Is(err, tc.expectedError)) + }) + } +} diff --git a/pkg/service/tree/tree_client_in_memory.go b/pkg/service/tree/tree_client_in_memory.go new file mode 100644 index 00000000..740c4823 --- /dev/null +++ b/pkg/service/tree/tree_client_in_memory.go @@ -0,0 +1,393 @@ +package tree + +import ( + "context" + "fmt" + "sort" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" +) + +type nodeMeta struct { + key string + value []byte +} + +func (m nodeMeta) GetKey() string { + return m.key +} + +func (m nodeMeta) GetValue() []byte { + return m.value +} + +type nodeResponse struct { + meta []nodeMeta + nodeID uint64 + parentID uint64 + timestamp uint64 +} + +func (n nodeResponse) GetNodeID() uint64 { + return n.nodeID +} + +func (n nodeResponse) GetParentID() uint64 { + return n.parentID +} + +func (n nodeResponse) GetTimestamp() uint64 { + return n.timestamp +} + +func (n nodeResponse) GetMeta() []Meta { + res := make([]Meta, len(n.meta)) + for i, value := range n.meta { + res[i] = value + } + return res +} + +func (n nodeResponse) getValue(key string) string { + for _, value := range n.meta { + if value.key == key { + return string(value.value) + } + } + return "" +} + +type ServiceClientMemory struct { + containers map[string]containerInfo +} + +type containerInfo struct { + bkt *data.BucketInfo + trees map[string]memoryTree +} + +type memoryTree struct { + idCounter uint64 + treeData *treeNodeMemory +} + +type treeNodeMemory struct { + data nodeResponse + parent *treeNodeMemory + children []*treeNodeMemory +} + +func (t *treeNodeMemory) getNode(nodeID uint64) *treeNodeMemory { + if t.data.nodeID == nodeID { + return t + } + + for _, child := range t.children { + if node := child.getNode(nodeID); node != nil { + return node + } + } + + return nil +} + +func (t *memoryTree) getNodesByPath(path []string) []nodeResponse { + if len(path) == 0 { + return nil + } + + var res []nodeResponse + for _, child := range t.treeData.children { + res = child.listNodesByPath(res, path) + } + + return res +} + +func (t *treeNodeMemory) listNodesByPath(res []nodeResponse, path []string) []nodeResponse { + if len(path) == 0 || t.data.getValue(FileNameKey) != path[0] { + return res + } + + if len(path) == 1 { + return append(res, t.data) + } + + for _, ch := range t.children { + res = ch.listNodesByPath(res, path[1:]) + } + + return res +} + +func (t *memoryTree) createPathIfNotExist(parent *treeNodeMemory, path []string) *treeNodeMemory { + if len(path) == 0 { + return parent + } + + var node *treeNodeMemory + for _, child := range parent.children { + if len(child.data.meta) == 1 && child.data.getValue(FileNameKey) == path[0] { + node = child + break + } + } + + if node == nil { + node = &treeNodeMemory{ + data: nodeResponse{ + meta: []nodeMeta{{key: FileNameKey, value: []byte(path[0])}}, + nodeID: t.idCounter, + parentID: parent.data.nodeID, + timestamp: uint64(time.Now().UnixMicro()), + }, + parent: parent, + } + t.idCounter++ + parent.children = append(parent.children, node) + } + + return t.createPathIfNotExist(node, path[1:]) +} + +func (t *treeNodeMemory) removeChild(nodeID uint64) { + ind := -1 + for i, ch := range t.children { + if ch.data.nodeID == nodeID { + ind = i + break + } + } + if ind != -1 { + t.children = append(t.children[:ind], t.children[ind+1:]...) + } +} + +func (t *treeNodeMemory) listNodes(res []NodeResponse, depth uint32) []NodeResponse { + res = append(res, t.data) + + if depth == 0 { + return res + } + + for _, ch := range t.children { + res = ch.listNodes(res, depth-1) + } + return res +} + +func NewTreeServiceClientMemory() (*ServiceClientMemory, error) { + return &ServiceClientMemory{ + containers: make(map[string]containerInfo), + }, nil +} + +func (c *ServiceClientMemory) GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) { + cnr, ok := c.containers[p.BktInfo.CID.EncodeToString()] + if !ok { + return nil, nil + } + + tr, ok := cnr.trees[p.TreeID] + if !ok { + return nil, nil + } + + res := tr.getNodesByPath(p.Path) + sort.Slice(res, func(i, j int) bool { + return res[i].timestamp < res[j].timestamp + }) + + if p.LatestOnly && len(res) != 0 { + res = res[len(res)-1:] + } + + res2 := make([]NodeResponse, len(res)) + for i, n := range res { + res2[i] = n + } + + return res2, nil +} + +func (c *ServiceClientMemory) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error) { + cnr, ok := c.containers[bktInfo.CID.EncodeToString()] + if !ok { + return nil, nil + } + + tr, ok := cnr.trees[treeID] + if !ok { + return nil, ErrNodeNotFound + } + + node := tr.treeData.getNode(rootID) + if node == nil { + return nil, ErrNodeNotFound + } + + return node.listNodes(nil, depth-1), nil +} + +func newContainerInfo(bktInfo *data.BucketInfo, treeID string) containerInfo { + return containerInfo{ + bkt: bktInfo, + trees: map[string]memoryTree{ + treeID: { + idCounter: 1, + treeData: &treeNodeMemory{ + data: nodeResponse{ + timestamp: uint64(time.Now().UnixMicro()), + }, + }, + }, + }, + } +} + +func newMemoryTree() memoryTree { + return memoryTree{ + idCounter: 1, + treeData: &treeNodeMemory{ + data: nodeResponse{ + timestamp: uint64(time.Now().UnixMicro()), + }, + }, + } +} + +func (c *ServiceClientMemory) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) { + cnr, ok := c.containers[bktInfo.CID.EncodeToString()] + if !ok { + cnr = newContainerInfo(bktInfo, treeID) + c.containers[bktInfo.CID.EncodeToString()] = cnr + } + + tr, ok := cnr.trees[treeID] + if !ok { + tr = newMemoryTree() + cnr.trees[treeID] = tr + } + + parentNode := tr.treeData.getNode(parent) + if parentNode == nil { + return 0, ErrNodeNotFound + } + + newID := tr.idCounter + tr.idCounter++ + + tn := &treeNodeMemory{ + data: nodeResponse{ + meta: metaToNodeMeta(meta), + nodeID: newID, + parentID: parent, + timestamp: uint64(time.Now().UnixMicro()), + }, + parent: parentNode, + } + + parentNode.children = append(parentNode.children, tn) + cnr.trees[treeID] = tr + + return newID, nil +} + +func (c *ServiceClientMemory) AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) { + cnr, ok := c.containers[bktInfo.CID.EncodeToString()] + if !ok { + cnr = newContainerInfo(bktInfo, treeID) + c.containers[bktInfo.CID.EncodeToString()] = cnr + } + + tr, ok := cnr.trees[treeID] + if !ok { + tr = newMemoryTree() + cnr.trees[treeID] = tr + } + + parentNode := tr.createPathIfNotExist(tr.treeData, path) + if parentNode == nil { + return 0, fmt.Errorf("create path '%s'", path) + } + + newID := tr.idCounter + tr.idCounter++ + + tn := &treeNodeMemory{ + data: nodeResponse{ + meta: metaToNodeMeta(meta), + nodeID: newID, + parentID: parentNode.data.nodeID, + timestamp: uint64(time.Now().UnixMicro()), + }, + parent: parentNode, + } + + parentNode.children = append(parentNode.children, tn) + cnr.trees[treeID] = tr + + return newID, nil +} + +func (c *ServiceClientMemory) MoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error { + cnr, ok := c.containers[bktInfo.CID.EncodeToString()] + if !ok { + return ErrNodeNotFound + } + + tr, ok := cnr.trees[treeID] + if !ok { + return ErrNodeNotFound + } + + node := tr.treeData.getNode(nodeID) + if node == nil { + return ErrNodeNotFound + } + + newParent := tr.treeData.getNode(parentID) + if newParent == nil { + return ErrNodeNotFound + } + + node.data.meta = metaToNodeMeta(meta) + node.data.parentID = parentID + + newParent.children = append(newParent.children, node) + node.parent.removeChild(nodeID) + + return nil +} + +func (c *ServiceClientMemory) RemoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error { + cnr, ok := c.containers[bktInfo.CID.EncodeToString()] + if !ok { + return ErrNodeNotFound + } + + tr, ok := cnr.trees[treeID] + if !ok { + return ErrNodeNotFound + } + + node := tr.treeData.getNode(nodeID) + if node == nil { + return ErrNodeNotFound + } + + node.parent.removeChild(nodeID) + + return nil +} + +func metaToNodeMeta(m map[string]string) []nodeMeta { + result := make([]nodeMeta, 0, len(m)) + + for key, value := range m { + result = append(result, nodeMeta{key: key, value: []byte(value)}) + } + + return result +} diff --git a/internal/frostfs/tree_test.go b/pkg/service/tree/tree_test.go similarity index 51% rename from internal/frostfs/tree_test.go rename to pkg/service/tree/tree_test.go index 58aa9bf2..271027f9 100644 --- a/internal/frostfs/tree_test.go +++ b/pkg/service/tree/tree_test.go @@ -1,11 +1,12 @@ -package frostfs +package tree import ( - "errors" + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" - "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -96,28 +97,73 @@ func TestLockConfigurationEncoding(t *testing.T) { } } -func TestHandleError(t *testing.T) { - defaultError := errors.New("default error") - for _, tc := range []struct { - err error - expectedError error - }{ - { - err: defaultError, - expectedError: defaultError, - }, - { - err: errors.New("something not found"), - expectedError: layer.ErrNodeNotFound, - }, - { - err: errors.New("something is denied by some acl rule"), - expectedError: layer.ErrNodeAccessDenied, - }, - } { - t.Run("", func(t *testing.T) { - err := handleError("err message", tc.err) - require.True(t, errors.Is(err, tc.expectedError)) - }) +func TestTreeServiceSettings(t *testing.T) { + ctx := context.Background() + + memCli, err := NewTreeServiceClientMemory() + require.NoError(t, err) + treeService := NewTree(memCli) + + bktInfo := &data.BucketInfo{ + CID: cidtest.ID(), } + + settings := &data.BucketSettings{ + Versioning: "Versioning", + LockConfiguration: &data.ObjectLockConfiguration{ + ObjectLockEnabled: "Enabled", + Rule: &data.ObjectLockRule{ + DefaultRetention: &data.DefaultRetention{ + Days: 1, + Mode: "mode", + }, + }, + }, + } + + err = treeService.PutSettingsNode(ctx, bktInfo, settings) + require.NoError(t, err) + + storedSettings, err := treeService.GetSettingsNode(ctx, bktInfo) + require.NoError(t, err) + require.Equal(t, settings, storedSettings) +} + +func TestTreeServiceAddVersion(t *testing.T) { + ctx := context.Background() + + memCli, err := NewTreeServiceClientMemory() + require.NoError(t, err) + treeService := NewTree(memCli) + + bktInfo := &data.BucketInfo{ + CID: cidtest.ID(), + } + + version := &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + OID: oidtest.ID(), + Size: 10, + ETag: "etag", + FilePath: "path/to/version", + }, + IsUnversioned: true, + } + + nodeID, err := treeService.AddVersion(ctx, bktInfo, version) + require.NoError(t, err) + + storedNode, err := treeService.GetUnversioned(ctx, bktInfo, "path/to/version") + require.NoError(t, err) + require.Equal(t, nodeID, storedNode.ID) + require.Equal(t, version.BaseNodeVersion.Size, storedNode.Size) + require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag) + require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag) + require.Equal(t, version.BaseNodeVersion.FilePath, storedNode.FilePath) + require.Equal(t, version.BaseNodeVersion.OID, storedNode.OID) + + versions, err := treeService.GetVersions(ctx, bktInfo, "path/to/version") + require.NoError(t, err) + require.Len(t, versions, 1) + require.Equal(t, storedNode, versions[0]) }