TrueCloudLab/frostfs-http-gw
17
1
Fork 17
Code Issues 12 Pull requests 7 Projects Releases 9 Activity Actions

[#166] Change the check of protocol during get object request
All checks were successful
/ DCO (pull_request) Successful in 2m16s
Details
/ Vulncheck (pull_request) Successful in 2m21s
Details
/ Builds (pull_request) Successful in 1m39s
Details
/ Lint (pull_request) Successful in 2m39s
Details
/ Tests (pull_request) Successful in 1m42s
Details

Browse source 
Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this methods and GetLatestVersion.

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-12-04 12:44:43 +03:00
parent 762bbe86b7
commit 2e71755d69
Signed by: nzinkevich
GPG key ID: 748EA1D0B2E6420A
15 changed files with 388 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/http-gw/app.go
View file

@ -499,10 +499,10 @@ func (a *app) Serve() {
close(a.webDone)
}()
handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool)
handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool)
// Configure router.
a.configureRouter(handler)
a.configureRouter(handle)
a.startServices()
a.initServers(a.ctx)

18
internal/api/tree.go
View file

@ -1,18 +0,0 @@
package api
import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
// NodeVersion represent node from tree service.
type NodeVersion struct {
BaseNodeVersion
DeleteMarker bool
IsPrefixNode bool
}
// BaseNodeVersion is minimal node info from tree service.
// Basically used for "system" object.
type BaseNodeVersion struct {
OID oid.ID
}

12
internal/data/bucket.go
View file

@ -1,12 +0,0 @@
package data
import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
type BucketInfo struct {
Name string // container name from system attribute
Zone string // container zone from system attribute
CID cid.ID
HomomorphicHashDisabled bool
}

27
internal/data/info.go Normal file
View file

@ -0,0 +1,27 @@
package data
import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
)
const (
VersioningUnversioned = "Unversioned"
VersioningEnabled = "Enabled"
VersioningSuspended = "Suspended"
)
type BucketInfo struct {
Name string // container name from system attribute
Zone string // container zone from system attribute
CID cid.ID
HomomorphicHashDisabled bool
}
// BucketSettings stores settings such as versioning.
type BucketSettings struct {
Versioning string
LockConfiguration *ObjectLockConfiguration
CannedACL string
OwnerKey *keys.PublicKey
}

21
internal/data/locking.go Normal file
View file

@ -0,0 +1,21 @@
package data
import "encoding/xml"
type (
ObjectLockConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ObjectLockConfiguration" json:"-"`
ObjectLockEnabled string `xml:"ObjectLockEnabled" json:"ObjectLockEnabled"`
Rule *ObjectLockRule `xml:"Rule" json:"Rule"`
}
ObjectLockRule struct {
DefaultRetention *DefaultRetention `xml:"DefaultRetention" json:"DefaultRetention"`
}
DefaultRetention struct {
Days int64 `xml:"Days" json:"Days"`
Mode string `xml:"Mode" json:"Mode"`
Years int64 `xml:"Years" json:"Years"`
}
)

45
internal/data/tree.go Normal file
View file

@ -0,0 +1,45 @@
package data
import (
"time"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
// NodeVersion represent node from tree service.
type NodeVersion struct {
BaseNodeVersion
DeleteMarker bool
IsPrefixNode bool
IsUnversioned bool
}
type NodeResponse interface {
GetMeta() []Meta
GetTimestamp() []uint64
GetNodeID() []uint64
GetParentID() []uint64
}
type Meta interface {
GetKey() string
GetValue() []byte
}
// BaseNodeVersion is minimal node info from tree service.
// Basically used for "system" object.
type BaseNodeVersion struct {
ID uint64
ParentID uint64
OID oid.ID
Timestamp uint64
Size uint64
ETag string
MD5 string
FilePath string
Created *time.Time
Owner *user.ID
IsDeleteMarker bool
CreationEpoch uint64
}

2
internal/handler/download.go
View file

@ -37,7 +37,7 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
return
}
s3checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo)
_, s3checkErr := h.tree.GetSettingsNode(ctx, bktInfo)
if s3checkErr != nil && !strings.Contains(s3checkErr.Error(), "tree not found") {
logAndSendBucketError(c, log, s3checkErr)
return

8
internal/handler/handler.go
View file

@ -11,9 +11,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
@ -164,7 +164,7 @@ type Handler struct {
ownerID *user.ID
config Config
containerResolver ContainerResolver
tree *tree.Tree
tree layer.TreeService
cache *cache.BucketCache
workerPool *ants.Pool
}
@ -177,7 +177,7 @@ type AppParams struct {
Cache *cache.BucketCache
}
func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler {
func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
return &Handler{
log: params.Logger,
frostfs: params.FrostFS,
@ -217,7 +217,7 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(co
logAndSendBucketError(c, log, err)
return
}
if foundOID.DeleteMarker {
if foundOID.IsDeleteMarker {
log.Error(logs.ObjectWasDeleted)
response.Error(c, "object deleted", fasthttp.StatusNotFound)
return

113
internal/handler/handler_test.go
View file

@ -5,9 +5,11 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"mime/multipart"
"net/http"
"sort"
"testing"
"time"
@ -32,15 +34,98 @@ import (
"go.uber.org/zap"
)
type treeClientMock struct {
type treeServiceMock struct {
settings map[string]*data.BucketSettings
versions map[string]map[string][]*data.NodeVersion
system map[string]map[string]*data.BaseNodeVersion
}
func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) {
return nil, nil
func newTreeService() *treeServiceMock {
return &treeServiceMock{
settings: make(map[string]*data.BucketSettings),
versions: make(map[string]map[string][]*data.NodeVersion),
system: make(map[string]map[string]*data.BaseNodeVersion),
}
}
func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) {
return nil, nil
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeResponse, string, error) {
return nil, "", nil
}
func (t *treeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
settings, ok := t.settings[bktInfo.CID.EncodeToString()]
if !ok {
return nil, errors.New("tree not found")
}
return settings, nil
}
func (t *treeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
t.settings[bktInfo.CID.EncodeToString()] = settings
return nil
}
func (t *treeServiceMock) GetLatestVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
if !ok {
return nil, tree.ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, tree.ErrNodeNotFound
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
return versions[len(versions)-1], nil
}
return nil, tree.ErrNodeNotFound
}
func (t *treeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{
newVersion.FilePath: {newVersion},
}
return newVersion.ID, nil
}
versions, ok := cnrVersionsMap[newVersion.FilePath]
if !ok {
cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion}
return newVersion.ID, nil
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
newVersion.ID = versions[len(versions)-1].ID + 1
newVersion.Timestamp = versions[len(versions)-1].Timestamp + 1
}
result := versions
if newVersion.IsUnversioned {
result = make([]*data.NodeVersion, 0, len(versions))
for _, node := range versions {
if !node.IsUnversioned {
result = append(result, node)
}
}
}
cnrVersionsMap[newVersion.FilePath] = append(result, newVersion)
return newVersion.ID, nil
}
type configMock struct {
@ -84,7 +169,7 @@ type handlerContext struct {
h *Handler
frostfs *TestFrostFS
tree *treeClientMock
tree *treeServiceMock
cfg *configMock
}
@ -125,14 +210,14 @@ func prepareHandlerContext() (*handlerContext, error) {
}),
}
treeMock := &treeClientMock{}
treeMock := newTreeService()
cfgMock := &configMock{}
workerPool, err := ants.NewPool(1)
if err != nil {
return nil, err
}
handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool)
handler := New(params, cfgMock, treeMock, workerPool)
return &handlerContext{
key: key,
@ -199,6 +284,18 @@ func TestBasic(t *testing.T) {
require.NoError(t, err)
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
objID, ok := obj.ID()
require.True(t, ok)
_, err = hc.tree.AddVersion(context.TODO(), &data.BucketInfo{CID: cnrID}, &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: objID,
FilePath: objFileName,
},
})
require.NoError(t, err)
attr := object.NewAttribute()
attr.SetKey(object.AttributeFilePath)
attr.SetValue(objFileName)

2
internal/handler/head.go
View file

@ -117,7 +117,7 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
logAndSendBucketError(c, log, err)
return
}
checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo)
_, checkErr := h.tree.GetSettingsNode(ctx, bktInfo)
if checkErr != nil && !strings.Contains(checkErr.Error(), "tree not found") {
logAndSendBucketError(c, log, checkErr)
return

8
internal/api/layer/tree_service.go → internal/layer/tree_service.go
View file

@ -4,13 +4,17 @@ import (
"context"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error)
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
// GetSettingsNode retrieves the settings node from the tree service and form data.BucketSettings.
// If tree node is not found returns ErrNodeNotFound error.
GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeResponse, string, error)
}
var (

5
internal/logs/logs.go
View file

@ -79,6 +79,11 @@ const (
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
FailedToUnescapeQuery = "failed to unescape query"
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
SystemNodeHasMultipleIDs = "system node has multiple ids"
FailedToRemoveOldSystemNode = "failed to remove old system node"
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
ServerReconnecting = "reconnecting server..."
ServerReconnectedSuccessfully = "server reconnected successfully"
ServerReconnectFailed = "failed to reconnect server"

16
internal/service/frostfs/pool_wrapper.go → internal/service/frostfs/tree_pool_wrapper.go
View file

@ -29,8 +29,8 @@ func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 {
return []uint64{n.response.GetTimestamp()}
}
func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.response.Meta))
func (n GetNodeByPathResponseInfoWrapper) GetMeta() []data.Meta {
res := make([]data.Meta, len(n.response.Meta))
for i, value := range n.response.Meta {
res[i] = value
}
@ -45,7 +45,7 @@ func NewPoolWrapper(p *treepool.Pool) *PoolWrapper {
return &PoolWrapper{p: p}
}
func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) {
func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]data.NodeResponse, error) {
poolPrm := treepool.GetNodesParams{
CID: prm.CnrID,
TreeID: prm.TreeID,
@ -62,7 +62,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([
return nil, handleError(err)
}
res := make([]tree.NodeResponse, len(nodes))
res := make([]data.NodeResponse, len(nodes))
for i, info := range nodes {
res[i] = GetNodeByPathResponseInfoWrapper{info}
}
@ -92,7 +92,7 @@ func handleError(err error) error {
return err
}
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) {
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]data.NodeResponse, error) {
order := treepool.NoneOrder
if sort {
order = treepool.AscendingOrder
@ -118,7 +118,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
return nil, handleError(err)
}
var subtree []tree.NodeResponse
var subtree []data.NodeResponse
node, err := subTreeReader.Next()
for err == nil {
@ -154,8 +154,8 @@ func (n GetSubTreeResponseBodyWrapper) GetTimestamp() []uint64 {
return n.response.GetTimestamp()
}
func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.response.Meta))
func (n GetSubTreeResponseBodyWrapper) GetMeta() []data.Meta {
res := make([]data.Meta, len(n.response.Meta))
for i, value := range n.response.Meta {
res[i] = value
}

196
tree/tree.go
View file

@ -4,29 +4,37 @@ import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"go.uber.org/zap"
)
type (
Tree struct {
service ServiceClient
log *zap.Logger
}
// ServiceClient is a client to interact with tree service.
// Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant.
ServiceClient interface {
GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error)
GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]NodeResponse, error)
GetNodes(ctx context.Context, p *GetNodesParams) ([]data.NodeResponse, error)
GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]data.NodeResponse, error)
}
treeNode struct {
ID []uint64
ParentID []uint64
ObjID oid.ID
TimeStamp []uint64
Size uint64
Meta map[string]string
}
@ -56,40 +64,38 @@ var (
const (
FileNameKey = "FileName"
settingsFileName = "bucket-settings"
versioningKV = "Versioning"
cannedACLKV = "cannedACL"
ownerKeyKV = "ownerKey"
lockConfigurationKV = "LockConfiguration"
oidKV = "OID"
cidKV = "CID"
isUnversionedKV = "IsUnversioned"
uploadIDKV = "UploadId"
sizeKV = "Size"
// keys for delete marker nodes.
isDeleteMarkerKV = "IsDeleteMarker"
settingsFileName = "bucket-settings"
// versionTree -- ID of a tree with object versions.
versionTree = "version"
// systemTree -- ID of a tree with system objects
// i.e. bucket settings with versioning and lock configuration, cors.
systemTree = "system"
separator = "/"
)
// NewTree creates instance of Tree using provided address and create grpc connection.
func NewTree(service ServiceClient) *Tree {
return &Tree{service: service}
func NewTree(service ServiceClient, log *zap.Logger) *Tree {
return &Tree{service: service, log: log}
}
type Meta interface {
GetKey() string
GetValue() []byte
}
type NodeResponse interface {
GetMeta() []Meta
GetTimestamp() []uint64
GetNodeID() []uint64
GetParentID() []uint64
}
func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) {
func newTreeNode(nodeInfo data.NodeResponse) (*treeNode, error) {
tNode := &treeNode{
Meta: make(map[string]string, len(nodeInfo.GetMeta())),
}
@ -118,30 +124,54 @@ func (n *treeNode) FileName() (string, bool) {
return value, ok
}
func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) {
func (n *treeNode) GetLatestNodeIndex() int {
var (
maxTimestamp uint64
index int
)
for i, timestamp := range n.TimeStamp {
if timestamp > maxTimestamp {
maxTimestamp = timestamp
index = i
}
}
return index
}
func (n *treeNode) IsSplit() bool {
return len(n.ID) != 1 || len(n.ParentID) != 1 || len(n.TimeStamp) != 1
}
func newNodeVersion(node data.NodeResponse, objectName string) (*data.NodeVersion, error) {
tNode, err := newTreeNode(node)
if err != nil {
return nil, fmt.Errorf("invalid tree node: %w", err)
}
return newNodeVersionFromTreeNode(tNode), nil
return newNodeVersionFromTreeNode(objectName, tNode), nil
}
func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion {
func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeVersion {
_, isDeleteMarker := treeNode.Get(isDeleteMarkerKV)
_, isUnversioned := treeNode.Get(isUnversionedKV)
size, _ := treeNode.Get(sizeKV)
version := &api.NodeVersion{
BaseNodeVersion: api.BaseNodeVersion{
version := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: treeNode.ObjID,
FilePath: filePath,
},
DeleteMarker: isDeleteMarker,
IsUnversioned: isUnversioned,
IsPrefixNode: size == "",
}
return version
}
func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) {
func newMultiNode(nodes []data.NodeResponse) (*multiSystemNode, error) {
var (
err error
index int
@ -180,7 +210,7 @@ func (m *multiSystemNode) Old() []*treeNode {
return m.nodes[1:]
}
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) {
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
nodes, err := c.GetVersions(ctx, cnrID, objectName)
if err != nil {
return nil, err
@ -191,10 +221,10 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s
return nil, err
}
return newNodeVersion(latestNode)
return newNodeVersion(latestNode, objectName)
}
func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) {
func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]data.NodeResponse, error) {
meta := []string{oidKV, isDeleteMarkerKV, sizeKV}
path := pathFromName(objectName)
@ -210,15 +240,6 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
return c.service.GetNodes(ctx, p)
}
func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error {
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
if err != nil {
return err
}
return nil
}
func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) {
p := &GetNodesParams{
CnrID: bktInfo.CID,
@ -242,8 +263,8 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
return newMultiNode(nodes)
}
func filterMultipartNodes(nodes []NodeResponse) []NodeResponse {
res := make([]NodeResponse, 0, len(nodes))
func filterMultipartNodes(nodes []data.NodeResponse) []data.NodeResponse {
res := make([]data.NodeResponse, 0, len(nodes))
LOOP:
for _, node := range nodes {
@ -259,7 +280,7 @@ LOOP:
return res
}
func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
func getLatestVersionNode(nodes []data.NodeResponse) (data.NodeResponse, error) {
var (
maxCreationTime uint64
targetIndexNode = -1
@ -283,7 +304,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
return nodes[targetIndexNode], nil
}
func checkExistOID(meta []Meta) bool {
func checkExistOID(meta []data.Meta) bool {
for _, kv := range meta {
if kv.GetKey() == "OID" {
return true
@ -298,7 +319,7 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator)
}
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]NodeResponse, string, error) {
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeResponse, string, error) {
rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix)
if err != nil {
return nil, "", err
@ -311,7 +332,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
return nil, "", err
}
nodesMap := make(map[string][]NodeResponse, len(subTree))
nodesMap := make(map[string][]data.NodeResponse, len(subTree))
for _, node := range subTree {
if MultiID(rootID).Equal(node.GetNodeID()) {
continue
@ -328,11 +349,11 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
// Add all intermediate nodes
// and only latest leaf (object) nodes. To do this store and replace last leaf (object) node in nodes[0]
if len(nodes) == 0 {
nodes = []NodeResponse{node}
nodes = []data.NodeResponse{node}
} else if !latestOnly || isIntermediate(node) {
nodes = append(nodes, node)
} else if isIntermediate(nodes[0]) {
nodes = append([]NodeResponse{node}, nodes...)
nodes = append([]data.NodeResponse{node}, nodes...)
} else if getMaxTimestamp(node) > getMaxTimestamp(nodes[0]) {
nodes[0] = node
}
@ -340,7 +361,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
nodesMap[fileName] = nodes
}
result := make([]NodeResponse, 0, len(subTree))
result := make([]data.NodeResponse, 0, len(subTree))
for _, nodes := range nodesMap {
result = append(result, nodes...)
}
@ -392,7 +413,37 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
return intermediateNodes, nil
}
func GetFilename(node NodeResponse) string {
func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
if err != nil {
return nil, fmt.Errorf("couldn't get node: %w", err)
}
node := multiNode.Latest()
settings := &data.BucketSettings{Versioning: data.VersioningUnversioned}
if versioningValue, ok := node.Get(versioningKV); ok {
settings.Versioning = versioningValue
}
if lockConfigurationValue, ok := node.Get(lockConfigurationKV); ok {
if settings.LockConfiguration, err = parseLockConfiguration(lockConfigurationValue); err != nil {
return nil, fmt.Errorf("settings node: invalid lock configuration: %w", err)
}
}
settings.CannedACL, _ = node.Get(cannedACLKV)
if ownerKeyHex, ok := node.Get(ownerKeyKV); ok {
if settings.OwnerKey, err = keys.NewPublicKeyFromString(ownerKeyHex); err != nil {
c.log.Error(logs.SettingsNodeInvalidOwnerKey, zap.Error(err))
}
}
return settings, nil
}
func GetFilename(node data.NodeResponse) string {
for _, kv := range node.GetMeta() {
if kv.GetKey() == FileNameKey {
return string(kv.GetValue())
@ -402,7 +453,7 @@ func GetFilename(node NodeResponse) string {
return ""
}
func isIntermediate(node NodeResponse) bool {
func isIntermediate(node data.NodeResponse) bool {
if len(node.GetMeta()) != 1 {
return false
}
@ -410,7 +461,7 @@ func isIntermediate(node NodeResponse) bool {
return node.GetMeta()[0].GetKey() == FileNameKey
}
func getMaxTimestamp(node NodeResponse) uint64 {
func getMaxTimestamp(node data.NodeResponse) uint64 {
var maxTimestamp uint64
for _, timestamp := range node.GetTimestamp() {
@ -439,3 +490,46 @@ func (m MultiID) Equal(id MultiID) bool {
return true
}
func parseLockConfiguration(value string) (*data.ObjectLockConfiguration, error) {
result := &data.ObjectLockConfiguration{}
if len(value) == 0 {
return result, nil
}
lockValues := strings.Split(value, ",")
result.ObjectLockEnabled = lockValues[0]
if len(lockValues) == 1 {
return result, nil
}
if len(lockValues) != 4 {
return nil, fmt.Errorf("invalid lock configuration: %s", value)
}
var err error
var days, years int64
if len(lockValues[1]) > 0 {
if days, err = strconv.ParseInt(lockValues[1], 10, 64); err != nil {
return nil, fmt.Errorf("invalid lock configuration: %s", value)
}
}
if len(lockValues[3]) > 0 {
if years, err = strconv.ParseInt(lockValues[3], 10, 64); err != nil {
return nil, fmt.Errorf("invalid lock configuration: %s", value)
}
}
result.Rule = &data.ObjectLockRule{
DefaultRetention: &data.DefaultRetention{
Days: days,
Mode: lockValues[2],
Years: years,
},
}
return result, nil
}

17
tree/tree_test.go
View file

@ -3,6 +3,7 @@ package tree
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"github.com/stretchr/testify/require"
)
@ -28,8 +29,8 @@ func (n nodeResponse) GetTimestamp() []uint64 {
return n.timestamp
}
func (n nodeResponse) GetMeta() []Meta {
res := make([]Meta, len(n.meta))
func (n nodeResponse) GetMeta() []data.Meta {
res := make([]data.Meta, len(n.meta))
for i, value := range n.meta {
res[i] = value
}
@ -46,18 +47,18 @@ func (n nodeResponse) GetParentID() []uint64 {
func TestGetLatestNode(t *testing.T) {
for _, tc := range []struct {
name string
nodes []NodeResponse
nodes []data.NodeResponse
exceptedOID string
error bool
}{
{
name: "empty",
nodes: []NodeResponse{},
nodes: []data.NodeResponse{},
error: true,
},
{
name: "one node of the object version",
nodes: []NodeResponse{
nodes: []data.NodeResponse{
nodeResponse{
timestamp: []uint64{1},
meta: []nodeMeta{
@ -72,7 +73,7 @@ func TestGetLatestNode(t *testing.T) {
},
{
name: "one node of the object version and one node of the secondary object",
nodes: []NodeResponse{
nodes: []data.NodeResponse{
nodeResponse{
timestamp: []uint64{3},
meta: []nodeMeta{},
@ -91,7 +92,7 @@ func TestGetLatestNode(t *testing.T) {
},
{
name: "all nodes represent a secondary object",
nodes: []NodeResponse{
nodes: []data.NodeResponse{
nodeResponse{
timestamp: []uint64{3},
meta: []nodeMeta{},
@ -105,7 +106,7 @@ func TestGetLatestNode(t *testing.T) {
},
{
name: "several nodes of different types and with different timestamp",
nodes: []NodeResponse{
nodes: []data.NodeResponse{
nodeResponse{
timestamp: []uint64{1},
meta: []nodeMeta{