411 lines
10 KiB
Go
411 lines
10 KiB
Go
|
package tree
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||
|
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||
|
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||
|
|
"go.uber.org/zap"
|
||
|
|
)
|
||
|
|
|
||
|
|
type (
|
||
|
|
Tree struct {
|
||
|
|
service ServiceClient
|
||
|
|
log *zap.Logger
|
||
|
|
}
|
||
|
|
|
||
|
|
// ServiceClient is a client to interact with tree service.
|
||
|
|
// Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant.
|
||
|
|
ServiceClient interface {
|
||
|
|
GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error)
|
||
|
|
GetSubTree(ctx context.Context, containerID cid.ID, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error)
|
||
|
|
AddNodeByPath(ctx context.Context, containerID cid.ID, treeID string, path []string, meta map[string]string) (uint64, error)
|
||
|
|
RemoveNode(ctx context.Context, containerID cid.ID, treeID string, nodeID uint64) error
|
||
|
|
}
|
||
|
|
|
||
|
|
SubTreeStream interface {
|
||
|
|
Next() (NodeResponse, error)
|
||
|
|
}
|
||
|
|
|
||
|
|
TreeNode struct {
|
||
|
|
ID uint64
|
||
|
|
ParentID uint64
|
||
|
|
ObjID oid.ID
|
||
|
|
TimeStamp uint64
|
||
|
|
ModificationTime time.Time
|
||
|
|
PayloadSize uint64
|
||
|
|
FullPath string
|
||
|
|
Meta map[string]string
|
||
|
|
}
|
||
|
|
|
||
|
|
GetNodesParams struct {
|
||
|
|
ContainerID cid.ID
|
||
|
|
TreeID string
|
||
|
|
Path []string
|
||
|
|
Meta []string
|
||
|
|
LatestOnly bool
|
||
|
|
AllAttrs bool
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
type Meta interface {
|
||
|
|
GetKey() string
|
||
|
|
GetValue() []byte
|
||
|
|
}
|
||
|
|
|
||
|
|
type NodeResponse interface {
|
||
|
|
GetMeta() []Meta
|
||
|
|
GetNodeID() uint64
|
||
|
|
GetParentID() uint64
|
||
|
|
GetTimestamp() uint64
|
||
|
|
}
|
||
|
|
|
||
|
|
const (
|
||
|
|
FileNameKey = "FileName"
|
||
|
|
oidKV = "OID"
|
||
|
|
splitIDKV = "SplitID"
|
||
|
|
payloadSize = "PayloadSize"
|
||
|
|
fullPath = "FullPath"
|
||
|
|
modificationTime = "ModificationTime"
|
||
|
|
|
||
|
|
objectTree = "distribution-object"
|
||
|
|
splitTree = "distribution-split"
|
||
|
|
|
||
|
|
separator = "/"
|
||
|
|
|
||
|
|
maxDepth = 0
|
||
|
|
)
|
||
|
|
|
||
|
|
var (
|
||
|
|
// ErrNodeNotFound is returned from ServiceClient in case of not found error.
|
||
|
|
ErrNodeNotFound = errors.New("not found")
|
||
|
|
|
||
|
|
ErrOnlyDirFound = errors.New("only directory found")
|
||
|
|
|
||
|
|
// ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error.
|
||
|
|
ErrNodeAccessDenied = errors.New("access denied")
|
||
|
|
|
||
|
|
// ErrGatewayTimeout is returned from ServiceClient service in case of timeout error.
|
||
|
|
ErrGatewayTimeout = errors.New("gateway timeout")
|
||
|
|
)
|
||
|
|
|
||
|
|
func NewTree(service ServiceClient, log *zap.Logger) *Tree {
|
||
|
|
return &Tree{
|
||
|
|
service: service,
|
||
|
|
log: log,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) GetObjectByPath(ctx context.Context, containerID cid.ID, path string) (*TreeNode, error) {
|
||
|
|
newPath := pathFromName(path)
|
||
|
|
p := &GetNodesParams{
|
||
|
|
ContainerID: containerID,
|
||
|
|
TreeID: objectTree,
|
||
|
|
Path: newPath,
|
||
|
|
LatestOnly: false,
|
||
|
|
AllAttrs: true,
|
||
|
|
}
|
||
|
|
nodes, err := c.service.GetNodes(ctx, p)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
var result NodeResponse
|
||
|
|
|
||
|
|
switch {
|
||
|
|
case len(nodes) == 0 || len(nodes) == 1 && isIntermediate(nodes[0]):
|
||
|
|
return nil, ErrNodeNotFound
|
||
|
|
case len(nodes) > 2:
|
||
|
|
return nil, fmt.Errorf("found more than two nodes")
|
||
|
|
default:
|
||
|
|
result = nodes[0]
|
||
|
|
if len(nodes) == 2 && isIntermediate(result) {
|
||
|
|
result = nodes[1]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
treeNode, err := newTreeNode(result)
|
||
|
|
if err != nil {
|
||
|
|
return nil, fmt.Errorf("error parse tree node: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
return treeNode, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) GetObjectByPathDir(ctx context.Context, containerID cid.ID, path string) (*TreeNode, error) {
|
||
|
|
newPath := pathFromName(path)
|
||
|
|
p := &GetNodesParams{
|
||
|
|
ContainerID: containerID,
|
||
|
|
TreeID: objectTree,
|
||
|
|
Path: newPath,
|
||
|
|
LatestOnly: false,
|
||
|
|
AllAttrs: true,
|
||
|
|
}
|
||
|
|
nodes, err := c.service.GetNodes(ctx, p)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
var result NodeResponse
|
||
|
|
|
||
|
|
switch {
|
||
|
|
case len(nodes) == 0:
|
||
|
|
return nil, ErrNodeNotFound
|
||
|
|
case len(nodes) == 1 && isIntermediate(nodes[0]):
|
||
|
|
return nil, ErrOnlyDirFound
|
||
|
|
case len(nodes) > 2:
|
||
|
|
return nil, fmt.Errorf("found more than two nodes")
|
||
|
|
default:
|
||
|
|
result = nodes[0]
|
||
|
|
if len(nodes) == 2 && isIntermediate(result) {
|
||
|
|
result = nodes[1]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
treeNode, err := newTreeNode(result)
|
||
|
|
if err != nil {
|
||
|
|
return nil, fmt.Errorf("error parse tree node: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
return treeNode, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) GetListObjectByPrefix(ctx context.Context, containerID cid.ID, prefixPath string) ([]TreeNode, error) {
|
||
|
|
rootID, err := c.determinePrefixNode(ctx, containerID, objectTree, prefixPath)
|
||
|
|
if err != nil {
|
||
|
|
if errors.Is(err, ErrNodeNotFound) {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
subTree, err := c.service.GetSubTree(ctx, containerID, objectTree, rootID, maxDepth)
|
||
|
|
if err != nil {
|
||
|
|
if errors.Is(err, ErrNodeNotFound) {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
nodes := make([]NodeResponse, 0)
|
||
|
|
|
||
|
|
for _, node := range subTree {
|
||
|
|
if node.GetNodeID() == rootID {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if !isIntermediate(node) {
|
||
|
|
nodes = append(nodes, node)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
result := make([]TreeNode, 0, len(nodes))
|
||
|
|
for _, v := range nodes {
|
||
|
|
node, err := newTreeNode(v)
|
||
|
|
if err != nil {
|
||
|
|
return nil, fmt.Errorf("error parse tree node: %w", err)
|
||
|
|
}
|
||
|
|
result = append(result, *node)
|
||
|
|
}
|
||
|
|
|
||
|
|
return result, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) GetListOIDBySplitID(ctx context.Context, containerID cid.ID, path string, splitID *object.SplitID) ([]oid.ID, error) {
|
||
|
|
nodes, err := c.getNodesBySplitID(ctx, containerID, path, splitID)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
ids := make([]oid.ID, len(nodes))
|
||
|
|
for i, node := range nodes {
|
||
|
|
ids[i] = node.ObjID
|
||
|
|
}
|
||
|
|
|
||
|
|
return ids, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) AddObject(ctx context.Context, containerID cid.ID, filePath string, oid oid.ID, size uint64) (uint64, error) {
|
||
|
|
path := pathFromName(filePath)
|
||
|
|
meta := map[string]string{
|
||
|
|
oidKV: oid.String(),
|
||
|
|
FileNameKey: path[len(path)-1],
|
||
|
|
payloadSize: strconv.FormatUint(size, 10),
|
||
|
|
fullPath: filePath,
|
||
|
|
modificationTime: strconv.FormatInt(time.Now().Unix(), 10),
|
||
|
|
}
|
||
|
|
|
||
|
|
nodeID, err := c.service.AddNodeByPath(ctx, containerID, objectTree, path[:len(path)-1], meta)
|
||
|
|
|
||
|
|
return nodeID, err
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) AddPHYObject(ctx context.Context, containerID cid.ID, filePath string, oid oid.ID, splitID *object.SplitID) (uint64, error) {
|
||
|
|
meta := map[string]string{
|
||
|
|
oidKV: oid.String(),
|
||
|
|
splitIDKV: splitID.String(),
|
||
|
|
}
|
||
|
|
|
||
|
|
nodeID, err := c.service.AddNodeByPath(ctx, containerID, splitTree, pathFromName(filePath), meta)
|
||
|
|
|
||
|
|
return nodeID, err
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) DeleteObject(ctx context.Context, containerID cid.ID, nodeID uint64) error {
|
||
|
|
return c.service.RemoveNode(ctx, containerID, objectTree, nodeID)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) DeleteObjectsBySplitID(ctx context.Context, containerID cid.ID, path string, splitID *object.SplitID) error {
|
||
|
|
splitNodes, err := c.getNodesBySplitID(ctx, containerID, path, splitID)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, node := range splitNodes {
|
||
|
|
err = c.service.RemoveNode(ctx, containerID, splitTree, node.ID)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) determinePrefixNode(ctx context.Context, containerID cid.ID, treeID, prefixPath string) (uint64, error) {
|
||
|
|
var rootID uint64
|
||
|
|
path := pathFromName(prefixPath)
|
||
|
|
|
||
|
|
if len(path) > 1 {
|
||
|
|
var err error
|
||
|
|
rootID, err = c.getPrefixNodeID(ctx, containerID, treeID, path)
|
||
|
|
if err != nil {
|
||
|
|
return 0, err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return rootID, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) getPrefixNodeID(ctx context.Context, containerID cid.ID, treeID string, prefixPath []string) (uint64, error) {
|
||
|
|
p := &GetNodesParams{
|
||
|
|
ContainerID: containerID,
|
||
|
|
TreeID: treeID,
|
||
|
|
Path: prefixPath,
|
||
|
|
LatestOnly: false,
|
||
|
|
AllAttrs: true,
|
||
|
|
}
|
||
|
|
nodes, err := c.service.GetNodes(ctx, p)
|
||
|
|
if err != nil {
|
||
|
|
return 0, err
|
||
|
|
}
|
||
|
|
|
||
|
|
var intermediateNodes []uint64
|
||
|
|
for _, node := range nodes {
|
||
|
|
if isIntermediate(node) {
|
||
|
|
intermediateNodes = append(intermediateNodes, node.GetNodeID())
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(intermediateNodes) == 0 {
|
||
|
|
return 0, ErrNodeNotFound
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(intermediateNodes) > 1 {
|
||
|
|
return 0, fmt.Errorf("found more than one intermediate nodes")
|
||
|
|
}
|
||
|
|
|
||
|
|
return intermediateNodes[0], nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *Tree) getNodesBySplitID(ctx context.Context, containerID cid.ID, filePath string, splitID *object.SplitID) ([]TreeNode, error) {
|
||
|
|
rootID, err := c.determinePrefixNode(ctx, containerID, splitTree, filePath)
|
||
|
|
if err != nil {
|
||
|
|
if errors.Is(err, ErrNodeNotFound) {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
subTree, err := c.service.GetSubTree(ctx, containerID, splitTree, rootID, 2)
|
||
|
|
if err != nil {
|
||
|
|
if errors.Is(err, ErrNodeNotFound) {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
nodes := make([]TreeNode, 0, len(subTree))
|
||
|
|
for _, v := range subTree {
|
||
|
|
node, err := newTreeNode(v)
|
||
|
|
if err != nil {
|
||
|
|
return nil, fmt.Errorf("error parse tree node: %w", err)
|
||
|
|
}
|
||
|
|
nodes = append(nodes, *node)
|
||
|
|
}
|
||
|
|
|
||
|
|
result := make([]TreeNode, 0)
|
||
|
|
for _, node := range nodes {
|
||
|
|
if val, ok := node.Meta[splitIDKV]; ok {
|
||
|
|
if val == splitID.String() {
|
||
|
|
result = append(result, node)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return result, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func isIntermediate(node NodeResponse) bool {
|
||
|
|
if len(node.GetMeta()) != 1 {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
return node.GetMeta()[0].GetKey() == FileNameKey
|
||
|
|
}
|
||
|
|
|
||
|
|
func newTreeNode(nodeInfo NodeResponse) (*TreeNode, error) {
|
||
|
|
treeNode := &TreeNode{
|
||
|
|
ID: nodeInfo.GetNodeID(),
|
||
|
|
ParentID: nodeInfo.GetParentID(),
|
||
|
|
TimeStamp: nodeInfo.GetTimestamp(),
|
||
|
|
Meta: make(map[string]string, len(nodeInfo.GetMeta())),
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, kv := range nodeInfo.GetMeta() {
|
||
|
|
switch kv.GetKey() {
|
||
|
|
case oidKV:
|
||
|
|
if err := treeNode.ObjID.DecodeString(string(kv.GetValue())); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
case payloadSize:
|
||
|
|
if sizeStr := string(kv.GetValue()); len(sizeStr) > 0 {
|
||
|
|
var err error
|
||
|
|
if treeNode.PayloadSize, err = strconv.ParseUint(sizeStr, 10, 64); err != nil {
|
||
|
|
return nil, fmt.Errorf("invalid payload size value '%s': %w", sizeStr, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
case modificationTime:
|
||
|
|
timestamp, err := strconv.ParseInt(string(kv.GetValue()), 10, 64)
|
||
|
|
if err != nil {
|
||
|
|
return nil, fmt.Errorf("invalid modification time value '%s': %w", string(kv.GetValue()), err)
|
||
|
|
}
|
||
|
|
treeNode.ModificationTime = time.Unix(timestamp, 0)
|
||
|
|
case fullPath:
|
||
|
|
treeNode.FullPath = string(kv.GetValue())
|
||
|
|
default:
|
||
|
|
treeNode.Meta[kv.GetKey()] = string(kv.GetValue())
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return treeNode, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func pathFromName(objectName string) []string {
|
||
|
|
return strings.Split(objectName, separator)
|
||
|
|
}
|