[#166] Change the check of protocol during get object request
All checks were successful
/ DCO (pull_request) Successful in 2m16s
/ Vulncheck (pull_request) Successful in 2m21s
/ Builds (pull_request) Successful in 1m39s
/ Lint (pull_request) Successful in 2m39s
/ Tests (pull_request) Successful in 1m42s

Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this methods and GetLatestVersion.

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-12-04 12:44:43 +03:00
parent 762bbe86b7
commit 2e71755d69
Signed by: nzinkevich
GPG key ID: 748EA1D0B2E6420A
15 changed files with 388 additions and 124 deletions

View file

@ -499,10 +499,10 @@ func (a *app) Serve() {
close(a.webDone) close(a.webDone)
}() }()
handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool)
// Configure router. // Configure router.
a.configureRouter(handler) a.configureRouter(handle)
a.startServices() a.startServices()
a.initServers(a.ctx) a.initServers(a.ctx)

View file

@ -1,18 +0,0 @@
package api
import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
// NodeVersion represent node from tree service.
type NodeVersion struct {
BaseNodeVersion
DeleteMarker bool
IsPrefixNode bool
}
// BaseNodeVersion is minimal node info from tree service.
// Basically used for "system" object.
type BaseNodeVersion struct {
OID oid.ID
}

View file

@ -1,12 +0,0 @@
package data
import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
type BucketInfo struct {
Name string // container name from system attribute
Zone string // container zone from system attribute
CID cid.ID
HomomorphicHashDisabled bool
}

27
internal/data/info.go Normal file
View file

@ -0,0 +1,27 @@
package data
import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
)
const (
VersioningUnversioned = "Unversioned"
VersioningEnabled = "Enabled"
VersioningSuspended = "Suspended"
)
type BucketInfo struct {
Name string // container name from system attribute
Zone string // container zone from system attribute
CID cid.ID
HomomorphicHashDisabled bool
}
// BucketSettings stores settings such as versioning.
type BucketSettings struct {
Versioning string
LockConfiguration *ObjectLockConfiguration
CannedACL string
OwnerKey *keys.PublicKey
}

21
internal/data/locking.go Normal file
View file

@ -0,0 +1,21 @@
package data
import "encoding/xml"
type (
ObjectLockConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ObjectLockConfiguration" json:"-"`
ObjectLockEnabled string `xml:"ObjectLockEnabled" json:"ObjectLockEnabled"`
Rule *ObjectLockRule `xml:"Rule" json:"Rule"`
}
ObjectLockRule struct {
DefaultRetention *DefaultRetention `xml:"DefaultRetention" json:"DefaultRetention"`
}
DefaultRetention struct {
Days int64 `xml:"Days" json:"Days"`
Mode string `xml:"Mode" json:"Mode"`
Years int64 `xml:"Years" json:"Years"`
}
)

45
internal/data/tree.go Normal file
View file

@ -0,0 +1,45 @@
package data
import (
"time"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
)
// NodeVersion represent node from tree service.
type NodeVersion struct {
BaseNodeVersion
DeleteMarker bool
IsPrefixNode bool
IsUnversioned bool
}
type NodeResponse interface {
GetMeta() []Meta
GetTimestamp() []uint64
GetNodeID() []uint64
GetParentID() []uint64
}
type Meta interface {
GetKey() string
GetValue() []byte
}
// BaseNodeVersion is minimal node info from tree service.
// Basically used for "system" object.
type BaseNodeVersion struct {
ID uint64
ParentID uint64
OID oid.ID
Timestamp uint64
Size uint64
ETag string
MD5 string
FilePath string
Created *time.Time
Owner *user.ID
IsDeleteMarker bool
CreationEpoch uint64
}

View file

@ -37,7 +37,7 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
return return
} }
s3checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo) _, s3checkErr := h.tree.GetSettingsNode(ctx, bktInfo)
if s3checkErr != nil && !strings.Contains(s3checkErr.Error(), "tree not found") { if s3checkErr != nil && !strings.Contains(s3checkErr.Error(), "tree not found") {
logAndSendBucketError(c, log, s3checkErr) logAndSendBucketError(c, log, s3checkErr)
return return

View file

@ -11,9 +11,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
@ -164,7 +164,7 @@ type Handler struct {
ownerID *user.ID ownerID *user.ID
config Config config Config
containerResolver ContainerResolver containerResolver ContainerResolver
tree *tree.Tree tree layer.TreeService
cache *cache.BucketCache cache *cache.BucketCache
workerPool *ants.Pool workerPool *ants.Pool
} }
@ -177,7 +177,7 @@ type AppParams struct {
Cache *cache.BucketCache Cache *cache.BucketCache
} }
func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
return &Handler{ return &Handler{
log: params.Logger, log: params.Logger,
frostfs: params.FrostFS, frostfs: params.FrostFS,
@ -217,7 +217,7 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(co
logAndSendBucketError(c, log, err) logAndSendBucketError(c, log, err)
return return
} }
if foundOID.DeleteMarker { if foundOID.IsDeleteMarker {
log.Error(logs.ObjectWasDeleted) log.Error(logs.ObjectWasDeleted)
response.Error(c, "object deleted", fasthttp.StatusNotFound) response.Error(c, "object deleted", fasthttp.StatusNotFound)
return return

View file

@ -5,9 +5,11 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"io" "io"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
"sort"
"testing" "testing"
"time" "time"
@ -32,15 +34,98 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
) )
type treeClientMock struct { type treeServiceMock struct {
settings map[string]*data.BucketSettings
versions map[string]map[string][]*data.NodeVersion
system map[string]map[string]*data.BaseNodeVersion
} }
func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) { func newTreeService() *treeServiceMock {
return nil, nil return &treeServiceMock{
settings: make(map[string]*data.BucketSettings),
versions: make(map[string]map[string][]*data.NodeVersion),
system: make(map[string]map[string]*data.BaseNodeVersion),
}
} }
func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeResponse, string, error) {
return nil, nil return nil, "", nil
}
func (t *treeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
settings, ok := t.settings[bktInfo.CID.EncodeToString()]
if !ok {
return nil, errors.New("tree not found")
}
return settings, nil
}
func (t *treeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
t.settings[bktInfo.CID.EncodeToString()] = settings
return nil
}
func (t *treeServiceMock) GetLatestVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
if !ok {
return nil, tree.ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, tree.ErrNodeNotFound
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
return versions[len(versions)-1], nil
}
return nil, tree.ErrNodeNotFound
}
func (t *treeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{
newVersion.FilePath: {newVersion},
}
return newVersion.ID, nil
}
versions, ok := cnrVersionsMap[newVersion.FilePath]
if !ok {
cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion}
return newVersion.ID, nil
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
newVersion.ID = versions[len(versions)-1].ID + 1
newVersion.Timestamp = versions[len(versions)-1].Timestamp + 1
}
result := versions
if newVersion.IsUnversioned {
result = make([]*data.NodeVersion, 0, len(versions))
for _, node := range versions {
if !node.IsUnversioned {
result = append(result, node)
}
}
}
cnrVersionsMap[newVersion.FilePath] = append(result, newVersion)
return newVersion.ID, nil
} }
type configMock struct { type configMock struct {
@ -84,7 +169,7 @@ type handlerContext struct {
h *Handler h *Handler
frostfs *TestFrostFS frostfs *TestFrostFS
tree *treeClientMock tree *treeServiceMock
cfg *configMock cfg *configMock
} }
@ -125,14 +210,14 @@ func prepareHandlerContext() (*handlerContext, error) {
}), }),
} }
treeMock := &treeClientMock{} treeMock := newTreeService()
cfgMock := &configMock{} cfgMock := &configMock{}
workerPool, err := ants.NewPool(1) workerPool, err := ants.NewPool(1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) handler := New(params, cfgMock, treeMock, workerPool)
return &handlerContext{ return &handlerContext{
key: key, key: key,
@ -199,6 +284,18 @@ func TestBasic(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
objID, ok := obj.ID()
require.True(t, ok)
_, err = hc.tree.AddVersion(context.TODO(), &data.BucketInfo{CID: cnrID}, &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: objID,
FilePath: objFileName,
},
})
require.NoError(t, err)
attr := object.NewAttribute() attr := object.NewAttribute()
attr.SetKey(object.AttributeFilePath) attr.SetKey(object.AttributeFilePath)
attr.SetValue(objFileName) attr.SetValue(objFileName)

View file

@ -117,7 +117,7 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
logAndSendBucketError(c, log, err) logAndSendBucketError(c, log, err)
return return
} }
checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo) _, checkErr := h.tree.GetSettingsNode(ctx, bktInfo)
if checkErr != nil && !strings.Contains(checkErr.Error(), "tree not found") { if checkErr != nil && !strings.Contains(checkErr.Error(), "tree not found") {
logAndSendBucketError(c, log, checkErr) logAndSendBucketError(c, log, checkErr)
return return

View file

@ -4,13 +4,17 @@ import (
"context" "context"
"errors" "errors"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
) )
// TreeService provide interface to interact with tree service using s3 data models. // TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface { type TreeService interface {
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
// GetSettingsNode retrieves the settings node from the tree service and form data.BucketSettings.
// If tree node is not found returns ErrNodeNotFound error.
GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeResponse, string, error)
} }
var ( var (

View file

@ -79,6 +79,11 @@ const (
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
FailedToUnescapeQuery = "failed to unescape query" FailedToUnescapeQuery = "failed to unescape query"
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
SystemNodeHasMultipleIDs = "system node has multiple ids"
FailedToRemoveOldSystemNode = "failed to remove old system node"
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
ServerReconnecting = "reconnecting server..." ServerReconnecting = "reconnecting server..."
ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectedSuccessfully = "server reconnected successfully"
ServerReconnectFailed = "failed to reconnect server" ServerReconnectFailed = "failed to reconnect server"

View file

@ -29,8 +29,8 @@ func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 {
return []uint64{n.response.GetTimestamp()} return []uint64{n.response.GetTimestamp()}
} }
func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { func (n GetNodeByPathResponseInfoWrapper) GetMeta() []data.Meta {
res := make([]tree.Meta, len(n.response.Meta)) res := make([]data.Meta, len(n.response.Meta))
for i, value := range n.response.Meta { for i, value := range n.response.Meta {
res[i] = value res[i] = value
} }
@ -45,7 +45,7 @@ func NewPoolWrapper(p *treepool.Pool) *PoolWrapper {
return &PoolWrapper{p: p} return &PoolWrapper{p: p}
} }
func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) { func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]data.NodeResponse, error) {
poolPrm := treepool.GetNodesParams{ poolPrm := treepool.GetNodesParams{
CID: prm.CnrID, CID: prm.CnrID,
TreeID: prm.TreeID, TreeID: prm.TreeID,
@ -62,7 +62,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([
return nil, handleError(err) return nil, handleError(err)
} }
res := make([]tree.NodeResponse, len(nodes)) res := make([]data.NodeResponse, len(nodes))
for i, info := range nodes { for i, info := range nodes {
res[i] = GetNodeByPathResponseInfoWrapper{info} res[i] = GetNodeByPathResponseInfoWrapper{info}
} }
@ -92,7 +92,7 @@ func handleError(err error) error {
return err return err
} }
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]data.NodeResponse, error) {
order := treepool.NoneOrder order := treepool.NoneOrder
if sort { if sort {
order = treepool.AscendingOrder order = treepool.AscendingOrder
@ -118,7 +118,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
return nil, handleError(err) return nil, handleError(err)
} }
var subtree []tree.NodeResponse var subtree []data.NodeResponse
node, err := subTreeReader.Next() node, err := subTreeReader.Next()
for err == nil { for err == nil {
@ -154,8 +154,8 @@ func (n GetSubTreeResponseBodyWrapper) GetTimestamp() []uint64 {
return n.response.GetTimestamp() return n.response.GetTimestamp()
} }
func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { func (n GetSubTreeResponseBodyWrapper) GetMeta() []data.Meta {
res := make([]tree.Meta, len(n.response.Meta)) res := make([]data.Meta, len(n.response.Meta))
for i, value := range n.response.Meta { for i, value := range n.response.Meta {
res[i] = value res[i] = value
} }

View file

@ -4,30 +4,38 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"strconv"
"strings" "strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"go.uber.org/zap"
) )
type ( type (
Tree struct { Tree struct {
service ServiceClient service ServiceClient
log *zap.Logger
} }
// ServiceClient is a client to interact with tree service. // ServiceClient is a client to interact with tree service.
// Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant. // Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant.
ServiceClient interface { ServiceClient interface {
GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) GetNodes(ctx context.Context, p *GetNodesParams) ([]data.NodeResponse, error)
GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]NodeResponse, error) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]data.NodeResponse, error)
} }
treeNode struct { treeNode struct {
ObjID oid.ID ID []uint64
Meta map[string]string ParentID []uint64
ObjID oid.ID
TimeStamp []uint64
Size uint64
Meta map[string]string
} }
multiSystemNode struct { multiSystemNode struct {
@ -55,41 +63,39 @@ var (
) )
const ( const (
FileNameKey = "FileName" FileNameKey = "FileName"
settingsFileName = "bucket-settings"
oidKV = "OID" versioningKV = "Versioning"
uploadIDKV = "UploadId" cannedACLKV = "cannedACL"
sizeKV = "Size" ownerKeyKV = "ownerKey"
lockConfigurationKV = "LockConfiguration"
oidKV = "OID"
cidKV = "CID"
isUnversionedKV = "IsUnversioned"
uploadIDKV = "UploadId"
sizeKV = "Size"
// keys for delete marker nodes. // keys for delete marker nodes.
isDeleteMarkerKV = "IsDeleteMarker" isDeleteMarkerKV = "IsDeleteMarker"
settingsFileName = "bucket-settings"
// versionTree -- ID of a tree with object versions. // versionTree -- ID of a tree with object versions.
versionTree = "version" versionTree = "version"
systemTree = "system"
// systemTree -- ID of a tree with system objects
// i.e. bucket settings with versioning and lock configuration, cors.
systemTree = "system"
separator = "/" separator = "/"
) )
// NewTree creates instance of Tree using provided address and create grpc connection. // NewTree creates instance of Tree using provided address and create grpc connection.
func NewTree(service ServiceClient) *Tree { func NewTree(service ServiceClient, log *zap.Logger) *Tree {
return &Tree{service: service} return &Tree{service: service, log: log}
} }
type Meta interface { func newTreeNode(nodeInfo data.NodeResponse) (*treeNode, error) {
GetKey() string
GetValue() []byte
}
type NodeResponse interface {
GetMeta() []Meta
GetTimestamp() []uint64
GetNodeID() []uint64
GetParentID() []uint64
}
func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) {
tNode := &treeNode{ tNode := &treeNode{
Meta: make(map[string]string, len(nodeInfo.GetMeta())), Meta: make(map[string]string, len(nodeInfo.GetMeta())),
} }
@ -118,30 +124,54 @@ func (n *treeNode) FileName() (string, bool) {
return value, ok return value, ok
} }
func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { func (n *treeNode) GetLatestNodeIndex() int {
var (
maxTimestamp uint64
index int
)
for i, timestamp := range n.TimeStamp {
if timestamp > maxTimestamp {
maxTimestamp = timestamp
index = i
}
}
return index
}
func (n *treeNode) IsSplit() bool {
return len(n.ID) != 1 || len(n.ParentID) != 1 || len(n.TimeStamp) != 1
}
func newNodeVersion(node data.NodeResponse, objectName string) (*data.NodeVersion, error) {
tNode, err := newTreeNode(node) tNode, err := newTreeNode(node)
if err != nil { if err != nil {
return nil, fmt.Errorf("invalid tree node: %w", err) return nil, fmt.Errorf("invalid tree node: %w", err)
} }
return newNodeVersionFromTreeNode(tNode), nil return newNodeVersionFromTreeNode(objectName, tNode), nil
} }
func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeVersion {
_, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV)
_, isUnversioned := treeNode.Get(isUnversionedKV)
size, _ := treeNode.Get(sizeKV) size, _ := treeNode.Get(sizeKV)
version := &api.NodeVersion{ version := &data.NodeVersion{
BaseNodeVersion: api.BaseNodeVersion{ BaseNodeVersion: data.BaseNodeVersion{
OID: treeNode.ObjID, OID: treeNode.ObjID,
FilePath: filePath,
}, },
DeleteMarker: isDeleteMarker, DeleteMarker: isDeleteMarker,
IsPrefixNode: size == "", IsUnversioned: isUnversioned,
IsPrefixNode: size == "",
} }
return version return version
} }
func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) { func newMultiNode(nodes []data.NodeResponse) (*multiSystemNode, error) {
var ( var (
err error err error
index int index int
@ -180,7 +210,7 @@ func (m *multiSystemNode) Old() []*treeNode {
return m.nodes[1:] return m.nodes[1:]
} }
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
nodes, err := c.GetVersions(ctx, cnrID, objectName) nodes, err := c.GetVersions(ctx, cnrID, objectName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -191,10 +221,10 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s
return nil, err return nil, err
} }
return newNodeVersion(latestNode) return newNodeVersion(latestNode, objectName)
} }
func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) { func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]data.NodeResponse, error) {
meta := []string{oidKV, isDeleteMarkerKV, sizeKV} meta := []string{oidKV, isDeleteMarkerKV, sizeKV}
path := pathFromName(objectName) path := pathFromName(objectName)
@ -210,15 +240,6 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
return c.service.GetNodes(ctx, p) return c.service.GetNodes(ctx, p)
} }
func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error {
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
if err != nil {
return err
}
return nil
}
func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) { func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) {
p := &GetNodesParams{ p := &GetNodesParams{
CnrID: bktInfo.CID, CnrID: bktInfo.CID,
@ -242,8 +263,8 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
return newMultiNode(nodes) return newMultiNode(nodes)
} }
func filterMultipartNodes(nodes []NodeResponse) []NodeResponse { func filterMultipartNodes(nodes []data.NodeResponse) []data.NodeResponse {
res := make([]NodeResponse, 0, len(nodes)) res := make([]data.NodeResponse, 0, len(nodes))
LOOP: LOOP:
for _, node := range nodes { for _, node := range nodes {
@ -259,7 +280,7 @@ LOOP:
return res return res
} }
func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { func getLatestVersionNode(nodes []data.NodeResponse) (data.NodeResponse, error) {
var ( var (
maxCreationTime uint64 maxCreationTime uint64
targetIndexNode = -1 targetIndexNode = -1
@ -283,7 +304,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
return nodes[targetIndexNode], nil return nodes[targetIndexNode], nil
} }
func checkExistOID(meta []Meta) bool { func checkExistOID(meta []data.Meta) bool {
for _, kv := range meta { for _, kv := range meta {
if kv.GetKey() == "OID" { if kv.GetKey() == "OID" {
return true return true
@ -298,7 +319,7 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator) return strings.Split(objectName, separator)
} }
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]NodeResponse, string, error) { func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeResponse, string, error) {
rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix)
if err != nil { if err != nil {
return nil, "", err return nil, "", err
@ -311,7 +332,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
return nil, "", err return nil, "", err
} }
nodesMap := make(map[string][]NodeResponse, len(subTree)) nodesMap := make(map[string][]data.NodeResponse, len(subTree))
for _, node := range subTree { for _, node := range subTree {
if MultiID(rootID).Equal(node.GetNodeID()) { if MultiID(rootID).Equal(node.GetNodeID()) {
continue continue
@ -328,11 +349,11 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
// Add all intermediate nodes // Add all intermediate nodes
// and only latest leaf (object) nodes. To do this store and replace last leaf (object) node in nodes[0] // and only latest leaf (object) nodes. To do this store and replace last leaf (object) node in nodes[0]
if len(nodes) == 0 { if len(nodes) == 0 {
nodes = []NodeResponse{node} nodes = []data.NodeResponse{node}
} else if !latestOnly || isIntermediate(node) { } else if !latestOnly || isIntermediate(node) {
nodes = append(nodes, node) nodes = append(nodes, node)
} else if isIntermediate(nodes[0]) { } else if isIntermediate(nodes[0]) {
nodes = append([]NodeResponse{node}, nodes...) nodes = append([]data.NodeResponse{node}, nodes...)
} else if getMaxTimestamp(node) > getMaxTimestamp(nodes[0]) { } else if getMaxTimestamp(node) > getMaxTimestamp(nodes[0]) {
nodes[0] = node nodes[0] = node
} }
@ -340,7 +361,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
nodesMap[fileName] = nodes nodesMap[fileName] = nodes
} }
result := make([]NodeResponse, 0, len(subTree)) result := make([]data.NodeResponse, 0, len(subTree))
for _, nodes := range nodesMap { for _, nodes := range nodesMap {
result = append(result, nodes...) result = append(result, nodes...)
} }
@ -392,7 +413,37 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
return intermediateNodes, nil return intermediateNodes, nil
} }
func GetFilename(node NodeResponse) string { func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
if err != nil {
return nil, fmt.Errorf("couldn't get node: %w", err)
}
node := multiNode.Latest()
settings := &data.BucketSettings{Versioning: data.VersioningUnversioned}
if versioningValue, ok := node.Get(versioningKV); ok {
settings.Versioning = versioningValue
}
if lockConfigurationValue, ok := node.Get(lockConfigurationKV); ok {
if settings.LockConfiguration, err = parseLockConfiguration(lockConfigurationValue); err != nil {
return nil, fmt.Errorf("settings node: invalid lock configuration: %w", err)
}
}
settings.CannedACL, _ = node.Get(cannedACLKV)
if ownerKeyHex, ok := node.Get(ownerKeyKV); ok {
if settings.OwnerKey, err = keys.NewPublicKeyFromString(ownerKeyHex); err != nil {
c.log.Error(logs.SettingsNodeInvalidOwnerKey, zap.Error(err))
}
}
return settings, nil
}
func GetFilename(node data.NodeResponse) string {
for _, kv := range node.GetMeta() { for _, kv := range node.GetMeta() {
if kv.GetKey() == FileNameKey { if kv.GetKey() == FileNameKey {
return string(kv.GetValue()) return string(kv.GetValue())
@ -402,7 +453,7 @@ func GetFilename(node NodeResponse) string {
return "" return ""
} }
func isIntermediate(node NodeResponse) bool { func isIntermediate(node data.NodeResponse) bool {
if len(node.GetMeta()) != 1 { if len(node.GetMeta()) != 1 {
return false return false
} }
@ -410,7 +461,7 @@ func isIntermediate(node NodeResponse) bool {
return node.GetMeta()[0].GetKey() == FileNameKey return node.GetMeta()[0].GetKey() == FileNameKey
} }
func getMaxTimestamp(node NodeResponse) uint64 { func getMaxTimestamp(node data.NodeResponse) uint64 {
var maxTimestamp uint64 var maxTimestamp uint64
for _, timestamp := range node.GetTimestamp() { for _, timestamp := range node.GetTimestamp() {
@ -439,3 +490,46 @@ func (m MultiID) Equal(id MultiID) bool {
return true return true
} }
func parseLockConfiguration(value string) (*data.ObjectLockConfiguration, error) {
result := &data.ObjectLockConfiguration{}
if len(value) == 0 {
return result, nil
}
lockValues := strings.Split(value, ",")
result.ObjectLockEnabled = lockValues[0]
if len(lockValues) == 1 {
return result, nil
}
if len(lockValues) != 4 {
return nil, fmt.Errorf("invalid lock configuration: %s", value)
}
var err error
var days, years int64
if len(lockValues[1]) > 0 {
if days, err = strconv.ParseInt(lockValues[1], 10, 64); err != nil {
return nil, fmt.Errorf("invalid lock configuration: %s", value)
}
}
if len(lockValues[3]) > 0 {
if years, err = strconv.ParseInt(lockValues[3], 10, 64); err != nil {
return nil, fmt.Errorf("invalid lock configuration: %s", value)
}
}
result.Rule = &data.ObjectLockRule{
DefaultRetention: &data.DefaultRetention{
Days: days,
Mode: lockValues[2],
Years: years,
},
}
return result, nil
}

View file

@ -3,6 +3,7 @@ package tree
import ( import (
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -28,8 +29,8 @@ func (n nodeResponse) GetTimestamp() []uint64 {
return n.timestamp return n.timestamp
} }
func (n nodeResponse) GetMeta() []Meta { func (n nodeResponse) GetMeta() []data.Meta {
res := make([]Meta, len(n.meta)) res := make([]data.Meta, len(n.meta))
for i, value := range n.meta { for i, value := range n.meta {
res[i] = value res[i] = value
} }
@ -46,18 +47,18 @@ func (n nodeResponse) GetParentID() []uint64 {
func TestGetLatestNode(t *testing.T) { func TestGetLatestNode(t *testing.T) {
for _, tc := range []struct { for _, tc := range []struct {
name string name string
nodes []NodeResponse nodes []data.NodeResponse
exceptedOID string exceptedOID string
error bool error bool
}{ }{
{ {
name: "empty", name: "empty",
nodes: []NodeResponse{}, nodes: []data.NodeResponse{},
error: true, error: true,
}, },
{ {
name: "one node of the object version", name: "one node of the object version",
nodes: []NodeResponse{ nodes: []data.NodeResponse{
nodeResponse{ nodeResponse{
timestamp: []uint64{1}, timestamp: []uint64{1},
meta: []nodeMeta{ meta: []nodeMeta{
@ -72,7 +73,7 @@ func TestGetLatestNode(t *testing.T) {
}, },
{ {
name: "one node of the object version and one node of the secondary object", name: "one node of the object version and one node of the secondary object",
nodes: []NodeResponse{ nodes: []data.NodeResponse{
nodeResponse{ nodeResponse{
timestamp: []uint64{3}, timestamp: []uint64{3},
meta: []nodeMeta{}, meta: []nodeMeta{},
@ -91,7 +92,7 @@ func TestGetLatestNode(t *testing.T) {
}, },
{ {
name: "all nodes represent a secondary object", name: "all nodes represent a secondary object",
nodes: []NodeResponse{ nodes: []data.NodeResponse{
nodeResponse{ nodeResponse{
timestamp: []uint64{3}, timestamp: []uint64{3},
meta: []nodeMeta{}, meta: []nodeMeta{},
@ -105,7 +106,7 @@ func TestGetLatestNode(t *testing.T) {
}, },
{ {
name: "several nodes of different types and with different timestamp", name: "several nodes of different types and with different timestamp",
nodes: []NodeResponse{ nodes: []data.NodeResponse{
nodeResponse{ nodeResponse{
timestamp: []uint64{1}, timestamp: []uint64{1},
meta: []nodeMeta{ meta: []nodeMeta{