From 2e71755d692a8aa08fc0f4d31c1e2ba85353231d Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 4 Dec 2024 12:44:43 +0300 Subject: [PATCH] [#166] Change the check of protocol during get object request Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this methods and GetLatestVersion. Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 4 +- internal/api/tree.go | 18 -- internal/data/bucket.go | 12 - internal/data/info.go | 27 +++ internal/data/locking.go | 21 ++ internal/data/tree.go | 45 ++++ internal/handler/download.go | 2 +- internal/handler/handler.go | 8 +- internal/handler/handler_test.go | 113 ++++++++- internal/handler/head.go | 2 +- internal/{api => }/layer/tree_service.go | 8 +- internal/logs/logs.go | 5 + .../{pool_wrapper.go => tree_pool_wrapper.go} | 16 +- tree/tree.go | 214 +++++++++++++----- tree/tree_test.go | 17 +- 15 files changed, 388 insertions(+), 124 deletions(-) delete mode 100644 internal/api/tree.go delete mode 100644 internal/data/bucket.go create mode 100644 internal/data/info.go create mode 100644 internal/data/locking.go create mode 100644 internal/data/tree.go rename internal/{api => }/layer/tree_service.go (53%) rename internal/service/frostfs/{pool_wrapper.go => tree_pool_wrapper.go} (90%) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 0dd53a6..da40e71 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -499,10 +499,10 @@ func (a *app) Serve() { close(a.webDone) }() - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) + handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool) // Configure router. - a.configureRouter(handler) + a.configureRouter(handle) a.startServices() a.initServers(a.ctx) diff --git a/internal/api/tree.go b/internal/api/tree.go deleted file mode 100644 index 5b1d608..0000000 --- a/internal/api/tree.go +++ /dev/null @@ -1,18 +0,0 @@ -package api - -import ( - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -// NodeVersion represent node from tree service. -type NodeVersion struct { - BaseNodeVersion - DeleteMarker bool - IsPrefixNode bool -} - -// BaseNodeVersion is minimal node info from tree service. -// Basically used for "system" object. -type BaseNodeVersion struct { - OID oid.ID -} diff --git a/internal/data/bucket.go b/internal/data/bucket.go deleted file mode 100644 index d99ca49..0000000 --- a/internal/data/bucket.go +++ /dev/null @@ -1,12 +0,0 @@ -package data - -import ( - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" -) - -type BucketInfo struct { - Name string // container name from system attribute - Zone string // container zone from system attribute - CID cid.ID - HomomorphicHashDisabled bool -} diff --git a/internal/data/info.go b/internal/data/info.go new file mode 100644 index 0000000..3934b0c --- /dev/null +++ b/internal/data/info.go @@ -0,0 +1,27 @@ +package data + +import ( + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +const ( + VersioningUnversioned = "Unversioned" + VersioningEnabled = "Enabled" + VersioningSuspended = "Suspended" +) + +type BucketInfo struct { + Name string // container name from system attribute + Zone string // container zone from system attribute + CID cid.ID + HomomorphicHashDisabled bool +} + +// BucketSettings stores settings such as versioning. +type BucketSettings struct { + Versioning string + LockConfiguration *ObjectLockConfiguration + CannedACL string + OwnerKey *keys.PublicKey +} diff --git a/internal/data/locking.go b/internal/data/locking.go new file mode 100644 index 0000000..227f26f --- /dev/null +++ b/internal/data/locking.go @@ -0,0 +1,21 @@ +package data + +import "encoding/xml" + +type ( + ObjectLockConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ObjectLockConfiguration" json:"-"` + ObjectLockEnabled string `xml:"ObjectLockEnabled" json:"ObjectLockEnabled"` + Rule *ObjectLockRule `xml:"Rule" json:"Rule"` + } + + ObjectLockRule struct { + DefaultRetention *DefaultRetention `xml:"DefaultRetention" json:"DefaultRetention"` + } + + DefaultRetention struct { + Days int64 `xml:"Days" json:"Days"` + Mode string `xml:"Mode" json:"Mode"` + Years int64 `xml:"Years" json:"Years"` + } +) diff --git a/internal/data/tree.go b/internal/data/tree.go new file mode 100644 index 0000000..b006456 --- /dev/null +++ b/internal/data/tree.go @@ -0,0 +1,45 @@ +package data + +import ( + "time" + + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" +) + +// NodeVersion represent node from tree service. +type NodeVersion struct { + BaseNodeVersion + DeleteMarker bool + IsPrefixNode bool + IsUnversioned bool +} + +type NodeResponse interface { + GetMeta() []Meta + GetTimestamp() []uint64 + GetNodeID() []uint64 + GetParentID() []uint64 +} + +type Meta interface { + GetKey() string + GetValue() []byte +} + +// BaseNodeVersion is minimal node info from tree service. +// Basically used for "system" object. +type BaseNodeVersion struct { + ID uint64 + ParentID uint64 + OID oid.ID + Timestamp uint64 + Size uint64 + ETag string + MD5 string + FilePath string + Created *time.Time + Owner *user.ID + IsDeleteMarker bool + CreationEpoch uint64 +} diff --git a/internal/handler/download.go b/internal/handler/download.go index 074c3fd..ce58bf1 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -37,7 +37,7 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { return } - s3checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo) + _, s3checkErr := h.tree.GetSettingsNode(ctx, bktInfo) if s3checkErr != nil && !strings.Contains(s3checkErr.Error(), "tree not found") { logAndSendBucketError(c, log, s3checkErr) return diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 53aad8e..acc053e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -11,9 +11,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -164,7 +164,7 @@ type Handler struct { ownerID *user.ID config Config containerResolver ContainerResolver - tree *tree.Tree + tree layer.TreeService cache *cache.BucketCache workerPool *ants.Pool } @@ -177,7 +177,7 @@ type AppParams struct { Cache *cache.BucketCache } -func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -217,7 +217,7 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(co logAndSendBucketError(c, log, err) return } - if foundOID.DeleteMarker { + if foundOID.IsDeleteMarker { log.Error(logs.ObjectWasDeleted) response.Error(c, "object deleted", fasthttp.StatusNotFound) return diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index c5a347f..51cefca 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -5,9 +5,11 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "mime/multipart" "net/http" + "sort" "testing" "time" @@ -32,15 +34,98 @@ import ( "go.uber.org/zap" ) -type treeClientMock struct { +type treeServiceMock struct { + settings map[string]*data.BucketSettings + versions map[string]map[string][]*data.NodeVersion + system map[string]map[string]*data.BaseNodeVersion } -func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) { - return nil, nil +func newTreeService() *treeServiceMock { + return &treeServiceMock{ + settings: make(map[string]*data.BucketSettings), + versions: make(map[string]map[string][]*data.NodeVersion), + system: make(map[string]map[string]*data.BaseNodeVersion), + } } -func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { - return nil, nil +func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeResponse, string, error) { + return nil, "", nil +} + +func (t *treeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { + settings, ok := t.settings[bktInfo.CID.EncodeToString()] + if !ok { + return nil, errors.New("tree not found") + } + + return settings, nil +} + +func (t *treeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error { + t.settings[bktInfo.CID.EncodeToString()] = settings + return nil +} + +func (t *treeServiceMock) GetLatestVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { + cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()] + if !ok { + return nil, tree.ErrNodeNotFound + } + + versions, ok := cnrVersionsMap[objectName] + if !ok { + return nil, tree.ErrNodeNotFound + } + + sort.Slice(versions, func(i, j int) bool { + return versions[i].ID < versions[j].ID + }) + + if len(versions) != 0 { + return versions[len(versions)-1], nil + } + + return nil, tree.ErrNodeNotFound +} + +func (t *treeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) { + cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] + if !ok { + t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{ + newVersion.FilePath: {newVersion}, + } + return newVersion.ID, nil + } + + versions, ok := cnrVersionsMap[newVersion.FilePath] + if !ok { + cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion} + return newVersion.ID, nil + } + + sort.Slice(versions, func(i, j int) bool { + return versions[i].ID < versions[j].ID + }) + + if len(versions) != 0 { + newVersion.ID = versions[len(versions)-1].ID + 1 + newVersion.Timestamp = versions[len(versions)-1].Timestamp + 1 + } + + result := versions + + if newVersion.IsUnversioned { + result = make([]*data.NodeVersion, 0, len(versions)) + for _, node := range versions { + if !node.IsUnversioned { + result = append(result, node) + } + } + } + + cnrVersionsMap[newVersion.FilePath] = append(result, newVersion) + + return newVersion.ID, nil } type configMock struct { @@ -84,7 +169,7 @@ type handlerContext struct { h *Handler frostfs *TestFrostFS - tree *treeClientMock + tree *treeServiceMock cfg *configMock } @@ -125,14 +210,14 @@ func prepareHandlerContext() (*handlerContext, error) { }), } - treeMock := &treeClientMock{} + treeMock := newTreeService() cfgMock := &configMock{} workerPool, err := ants.NewPool(1) if err != nil { return nil, err } - handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) + handler := New(params, cfgMock, treeMock, workerPool) return &handlerContext{ key: key, @@ -199,6 +284,18 @@ func TestBasic(t *testing.T) { require.NoError(t, err) obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] + + objID, ok := obj.ID() + require.True(t, ok) + + _, err = hc.tree.AddVersion(context.TODO(), &data.BucketInfo{CID: cnrID}, &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + OID: objID, + FilePath: objFileName, + }, + }) + require.NoError(t, err) + attr := object.NewAttribute() attr.SetKey(object.AttributeFilePath) attr.SetValue(objFileName) diff --git a/internal/handler/head.go b/internal/handler/head.go index 6385c37..9d00b20 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -117,7 +117,7 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { logAndSendBucketError(c, log, err) return } - checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo) + _, checkErr := h.tree.GetSettingsNode(ctx, bktInfo) if checkErr != nil && !strings.Contains(checkErr.Error(), "tree not found") { logAndSendBucketError(c, log, checkErr) return diff --git a/internal/api/layer/tree_service.go b/internal/layer/tree_service.go similarity index 53% rename from internal/api/layer/tree_service.go rename to internal/layer/tree_service.go index beb1e7a..a295a68 100644 --- a/internal/api/layer/tree_service.go +++ b/internal/layer/tree_service.go @@ -4,13 +4,17 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) // TreeService provide interface to interact with tree service using s3 data models. type TreeService interface { - GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) + GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) + // GetSettingsNode retrieves the settings node from the tree service and form data.BucketSettings. + // If tree node is not found returns ErrNodeNotFound error. + GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) + GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeResponse, string, error) } var ( diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 4dfa21f..5ba321d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -79,6 +79,11 @@ const ( InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go FailedToUnescapeQuery = "failed to unescape query" + FailedToParseAddressInTreeNode = "failed to parse object addr in tree node" + SettingsNodeInvalidOwnerKey = "settings node: invalid owner key" + SystemNodeHasMultipleIDs = "system node has multiple ids" + FailedToRemoveOldSystemNode = "failed to remove old system node" + BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" diff --git a/internal/service/frostfs/pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go similarity index 90% rename from internal/service/frostfs/pool_wrapper.go rename to internal/service/frostfs/tree_pool_wrapper.go index b978d73..d38901e 100644 --- a/internal/service/frostfs/pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -29,8 +29,8 @@ func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() []uint64 { return []uint64{n.response.GetTimestamp()} } -func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) +func (n GetNodeByPathResponseInfoWrapper) GetMeta() []data.Meta { + res := make([]data.Meta, len(n.response.Meta)) for i, value := range n.response.Meta { res[i] = value } @@ -45,7 +45,7 @@ func NewPoolWrapper(p *treepool.Pool) *PoolWrapper { return &PoolWrapper{p: p} } -func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) { +func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]data.NodeResponse, error) { poolPrm := treepool.GetNodesParams{ CID: prm.CnrID, TreeID: prm.TreeID, @@ -62,7 +62,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([ return nil, handleError(err) } - res := make([]tree.NodeResponse, len(nodes)) + res := make([]data.NodeResponse, len(nodes)) for i, info := range nodes { res[i] = GetNodeByPathResponseInfoWrapper{info} } @@ -92,7 +92,7 @@ func handleError(err error) error { return err } -func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { +func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]data.NodeResponse, error) { order := treepool.NoneOrder if sort { order = treepool.AscendingOrder @@ -118,7 +118,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, return nil, handleError(err) } - var subtree []tree.NodeResponse + var subtree []data.NodeResponse node, err := subTreeReader.Next() for err == nil { @@ -154,8 +154,8 @@ func (n GetSubTreeResponseBodyWrapper) GetTimestamp() []uint64 { return n.response.GetTimestamp() } -func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta { - res := make([]tree.Meta, len(n.response.Meta)) +func (n GetSubTreeResponseBodyWrapper) GetMeta() []data.Meta { + res := make([]data.Meta, len(n.response.Meta)) for i, value := range n.response.Meta { res[i] = value } diff --git a/tree/tree.go b/tree/tree.go index 40209a5..bed5711 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -4,30 +4,38 @@ import ( "context" "errors" "fmt" + "strconv" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "go.uber.org/zap" ) type ( Tree struct { service ServiceClient + log *zap.Logger } // ServiceClient is a client to interact with tree service. // Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant. ServiceClient interface { - GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error) - GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]NodeResponse, error) + GetNodes(ctx context.Context, p *GetNodesParams) ([]data.NodeResponse, error) + GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]data.NodeResponse, error) } treeNode struct { - ObjID oid.ID - Meta map[string]string + ID []uint64 + ParentID []uint64 + ObjID oid.ID + TimeStamp []uint64 + Size uint64 + Meta map[string]string } multiSystemNode struct { @@ -55,41 +63,39 @@ var ( ) const ( - FileNameKey = "FileName" - settingsFileName = "bucket-settings" + FileNameKey = "FileName" - oidKV = "OID" - uploadIDKV = "UploadId" - sizeKV = "Size" + versioningKV = "Versioning" + cannedACLKV = "cannedACL" + ownerKeyKV = "ownerKey" + lockConfigurationKV = "LockConfiguration" + oidKV = "OID" + cidKV = "CID" + isUnversionedKV = "IsUnversioned" + uploadIDKV = "UploadId" + sizeKV = "Size" // keys for delete marker nodes. isDeleteMarkerKV = "IsDeleteMarker" + settingsFileName = "bucket-settings" + // versionTree -- ID of a tree with object versions. versionTree = "version" - systemTree = "system" + + // systemTree -- ID of a tree with system objects + // i.e. bucket settings with versioning and lock configuration, cors. + systemTree = "system" separator = "/" ) // NewTree creates instance of Tree using provided address and create grpc connection. -func NewTree(service ServiceClient) *Tree { - return &Tree{service: service} +func NewTree(service ServiceClient, log *zap.Logger) *Tree { + return &Tree{service: service, log: log} } -type Meta interface { - GetKey() string - GetValue() []byte -} - -type NodeResponse interface { - GetMeta() []Meta - GetTimestamp() []uint64 - GetNodeID() []uint64 - GetParentID() []uint64 -} - -func newTreeNode(nodeInfo NodeResponse) (*treeNode, error) { +func newTreeNode(nodeInfo data.NodeResponse) (*treeNode, error) { tNode := &treeNode{ Meta: make(map[string]string, len(nodeInfo.GetMeta())), } @@ -118,30 +124,54 @@ func (n *treeNode) FileName() (string, bool) { return value, ok } -func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { +func (n *treeNode) GetLatestNodeIndex() int { + var ( + maxTimestamp uint64 + index int + ) + + for i, timestamp := range n.TimeStamp { + if timestamp > maxTimestamp { + maxTimestamp = timestamp + index = i + } + } + + return index +} + +func (n *treeNode) IsSplit() bool { + return len(n.ID) != 1 || len(n.ParentID) != 1 || len(n.TimeStamp) != 1 +} + +func newNodeVersion(node data.NodeResponse, objectName string) (*data.NodeVersion, error) { tNode, err := newTreeNode(node) if err != nil { return nil, fmt.Errorf("invalid tree node: %w", err) } - return newNodeVersionFromTreeNode(tNode), nil + return newNodeVersionFromTreeNode(objectName, tNode), nil } -func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { +func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeVersion { _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) + _, isUnversioned := treeNode.Get(isUnversionedKV) + size, _ := treeNode.Get(sizeKV) - version := &api.NodeVersion{ - BaseNodeVersion: api.BaseNodeVersion{ - OID: treeNode.ObjID, + version := &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + OID: treeNode.ObjID, + FilePath: filePath, }, - DeleteMarker: isDeleteMarker, - IsPrefixNode: size == "", + DeleteMarker: isDeleteMarker, + IsUnversioned: isUnversioned, + IsPrefixNode: size == "", } return version } -func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) { +func newMultiNode(nodes []data.NodeResponse) (*multiSystemNode, error) { var ( err error index int @@ -180,7 +210,7 @@ func (m *multiSystemNode) Old() []*treeNode { return m.nodes[1:] } -func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { +func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { return nil, err @@ -191,10 +221,10 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s return nil, err } - return newNodeVersion(latestNode) + return newNodeVersion(latestNode, objectName) } -func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) { +func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]data.NodeResponse, error) { meta := []string{oidKV, isDeleteMarkerKV, sizeKV} path := pathFromName(objectName) @@ -210,15 +240,6 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string return c.service.GetNodes(ctx, p) } -func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error { - _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) - if err != nil { - return err - } - - return nil -} - func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) { p := &GetNodesParams{ CnrID: bktInfo.CID, @@ -242,8 +263,8 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name return newMultiNode(nodes) } -func filterMultipartNodes(nodes []NodeResponse) []NodeResponse { - res := make([]NodeResponse, 0, len(nodes)) +func filterMultipartNodes(nodes []data.NodeResponse) []data.NodeResponse { + res := make([]data.NodeResponse, 0, len(nodes)) LOOP: for _, node := range nodes { @@ -259,7 +280,7 @@ LOOP: return res } -func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { +func getLatestVersionNode(nodes []data.NodeResponse) (data.NodeResponse, error) { var ( maxCreationTime uint64 targetIndexNode = -1 @@ -283,7 +304,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { return nodes[targetIndexNode], nil } -func checkExistOID(meta []Meta) bool { +func checkExistOID(meta []data.Meta) bool { for _, kv := range meta { if kv.GetKey() == "OID" { return true @@ -298,7 +319,7 @@ func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } -func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]NodeResponse, string, error) { +func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeResponse, string, error) { rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix) if err != nil { return nil, "", err @@ -311,7 +332,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, return nil, "", err } - nodesMap := make(map[string][]NodeResponse, len(subTree)) + nodesMap := make(map[string][]data.NodeResponse, len(subTree)) for _, node := range subTree { if MultiID(rootID).Equal(node.GetNodeID()) { continue @@ -328,11 +349,11 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, // Add all intermediate nodes // and only latest leaf (object) nodes. To do this store and replace last leaf (object) node in nodes[0] if len(nodes) == 0 { - nodes = []NodeResponse{node} + nodes = []data.NodeResponse{node} } else if !latestOnly || isIntermediate(node) { nodes = append(nodes, node) } else if isIntermediate(nodes[0]) { - nodes = append([]NodeResponse{node}, nodes...) + nodes = append([]data.NodeResponse{node}, nodes...) } else if getMaxTimestamp(node) > getMaxTimestamp(nodes[0]) { nodes[0] = node } @@ -340,7 +361,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, nodesMap[fileName] = nodes } - result := make([]NodeResponse, 0, len(subTree)) + result := make([]data.NodeResponse, 0, len(subTree)) for _, nodes := range nodesMap { result = append(result, nodes...) } @@ -392,7 +413,37 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr return intermediateNodes, nil } -func GetFilename(node NodeResponse) string { +func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { + multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName) + if err != nil { + return nil, fmt.Errorf("couldn't get node: %w", err) + } + + node := multiNode.Latest() + + settings := &data.BucketSettings{Versioning: data.VersioningUnversioned} + if versioningValue, ok := node.Get(versioningKV); ok { + settings.Versioning = versioningValue + } + + if lockConfigurationValue, ok := node.Get(lockConfigurationKV); ok { + if settings.LockConfiguration, err = parseLockConfiguration(lockConfigurationValue); err != nil { + return nil, fmt.Errorf("settings node: invalid lock configuration: %w", err) + } + } + + settings.CannedACL, _ = node.Get(cannedACLKV) + + if ownerKeyHex, ok := node.Get(ownerKeyKV); ok { + if settings.OwnerKey, err = keys.NewPublicKeyFromString(ownerKeyHex); err != nil { + c.log.Error(logs.SettingsNodeInvalidOwnerKey, zap.Error(err)) + } + } + + return settings, nil +} + +func GetFilename(node data.NodeResponse) string { for _, kv := range node.GetMeta() { if kv.GetKey() == FileNameKey { return string(kv.GetValue()) @@ -402,7 +453,7 @@ func GetFilename(node NodeResponse) string { return "" } -func isIntermediate(node NodeResponse) bool { +func isIntermediate(node data.NodeResponse) bool { if len(node.GetMeta()) != 1 { return false } @@ -410,7 +461,7 @@ func isIntermediate(node NodeResponse) bool { return node.GetMeta()[0].GetKey() == FileNameKey } -func getMaxTimestamp(node NodeResponse) uint64 { +func getMaxTimestamp(node data.NodeResponse) uint64 { var maxTimestamp uint64 for _, timestamp := range node.GetTimestamp() { @@ -439,3 +490,46 @@ func (m MultiID) Equal(id MultiID) bool { return true } + +func parseLockConfiguration(value string) (*data.ObjectLockConfiguration, error) { + result := &data.ObjectLockConfiguration{} + if len(value) == 0 { + return result, nil + } + + lockValues := strings.Split(value, ",") + result.ObjectLockEnabled = lockValues[0] + + if len(lockValues) == 1 { + return result, nil + } + + if len(lockValues) != 4 { + return nil, fmt.Errorf("invalid lock configuration: %s", value) + } + + var err error + var days, years int64 + + if len(lockValues[1]) > 0 { + if days, err = strconv.ParseInt(lockValues[1], 10, 64); err != nil { + return nil, fmt.Errorf("invalid lock configuration: %s", value) + } + } + + if len(lockValues[3]) > 0 { + if years, err = strconv.ParseInt(lockValues[3], 10, 64); err != nil { + return nil, fmt.Errorf("invalid lock configuration: %s", value) + } + } + + result.Rule = &data.ObjectLockRule{ + DefaultRetention: &data.DefaultRetention{ + Days: days, + Mode: lockValues[2], + Years: years, + }, + } + + return result, nil +} diff --git a/tree/tree_test.go b/tree/tree_test.go index 62f9914..e3b3950 100644 --- a/tree/tree_test.go +++ b/tree/tree_test.go @@ -3,6 +3,7 @@ package tree import ( "testing" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "github.com/stretchr/testify/require" ) @@ -28,8 +29,8 @@ func (n nodeResponse) GetTimestamp() []uint64 { return n.timestamp } -func (n nodeResponse) GetMeta() []Meta { - res := make([]Meta, len(n.meta)) +func (n nodeResponse) GetMeta() []data.Meta { + res := make([]data.Meta, len(n.meta)) for i, value := range n.meta { res[i] = value } @@ -46,18 +47,18 @@ func (n nodeResponse) GetParentID() []uint64 { func TestGetLatestNode(t *testing.T) { for _, tc := range []struct { name string - nodes []NodeResponse + nodes []data.NodeResponse exceptedOID string error bool }{ { name: "empty", - nodes: []NodeResponse{}, + nodes: []data.NodeResponse{}, error: true, }, { name: "one node of the object version", - nodes: []NodeResponse{ + nodes: []data.NodeResponse{ nodeResponse{ timestamp: []uint64{1}, meta: []nodeMeta{ @@ -72,7 +73,7 @@ func TestGetLatestNode(t *testing.T) { }, { name: "one node of the object version and one node of the secondary object", - nodes: []NodeResponse{ + nodes: []data.NodeResponse{ nodeResponse{ timestamp: []uint64{3}, meta: []nodeMeta{}, @@ -91,7 +92,7 @@ func TestGetLatestNode(t *testing.T) { }, { name: "all nodes represent a secondary object", - nodes: []NodeResponse{ + nodes: []data.NodeResponse{ nodeResponse{ timestamp: []uint64{3}, meta: []nodeMeta{}, @@ -105,7 +106,7 @@ func TestGetLatestNode(t *testing.T) { }, { name: "several nodes of different types and with different timestamp", - nodes: []NodeResponse{ + nodes: []data.NodeResponse{ nodeResponse{ timestamp: []uint64{1}, meta: []nodeMeta{