[#166] Change the check of protocol during get object request
All checks were successful
/ DCO (pull_request) Successful in 2m33s
/ Vulncheck (pull_request) Successful in 2m46s
/ Builds (pull_request) Successful in 1m43s
/ Lint (pull_request) Successful in 2m43s
/ Tests (pull_request) Successful in 1m45s

Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this and GetLatestVersion methods.

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-12-04 12:44:43 +03:00
parent 762bbe86b7
commit 5e4a68105d
Signed by: nzinkevich
GPG key ID: 748EA1D0B2E6420A
12 changed files with 261 additions and 131 deletions

View file

@ -499,10 +499,10 @@ func (a *app) Serve() {
close(a.webDone) close(a.webDone)
}() }()
handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool)
// Configure router. // Configure router.
a.configureRouter(handler) a.configureRouter(handle)
a.startServices() a.startServices()
a.initServers(a.ctx) a.initServers(a.ctx)

View file

@ -1,22 +0,0 @@
package layer
import (
"context"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error)
}
var (
// ErrNodeNotFound is returned from Tree service in case of not found error.
ErrNodeNotFound = errors.New("not found")
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
ErrNodeAccessDenied = errors.New("access denied")
)

View file

@ -4,9 +4,20 @@ import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
) )
const (
VersioningUnversioned = "Unversioned"
VersioningEnabled = "Enabled"
VersioningSuspended = "Suspended"
)
type BucketInfo struct { type BucketInfo struct {
Name string // container name from system attribute Name string // container name from system attribute
Zone string // container zone from system attribute Zone string // container zone from system attribute
CID cid.ID CID cid.ID
HomomorphicHashDisabled bool HomomorphicHashDisabled bool
} }
// BucketSettings stores settings such as versioning.
type BucketSettings struct {
Versioning string
}

View file

@ -1,4 +1,4 @@
package api package data
import ( import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -7,12 +7,14 @@ import (
// NodeVersion represent node from tree service. // NodeVersion represent node from tree service.
type NodeVersion struct { type NodeVersion struct {
BaseNodeVersion BaseNodeVersion
DeleteMarker bool IsUnversioned bool
IsPrefixNode bool
} }
// BaseNodeVersion is minimal node info from tree service. // BaseNodeVersion is minimal node info from tree service.
// Basically used for "system" object. // Basically used for "system" object.
type BaseNodeVersion struct { type BaseNodeVersion struct {
OID oid.ID ID uint64
OID oid.ID
FilePath string
IsDeleteMarker bool
} }

View file

@ -4,11 +4,11 @@ import (
"archive/zip" "archive/zip"
"bufio" "bufio"
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"strings"
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
@ -18,6 +18,7 @@ import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -29,7 +30,10 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
downloadParam := c.QueryArgs().GetBool("download") downloadParam := c.QueryArgs().GetBool("download")
ctx := utils.GetContextFromRequest(c) ctx := utils.GetContextFromRequest(c)
log := utils.GetReqLogOrDefault(ctx, h.log) log := utils.GetReqLogOrDefault(ctx, h.log).With(
zap.String("cid", cidParam),
zap.String("oid", oidParam),
)
bktInfo, err := h.getBucketInfo(ctx, cidParam, log) bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
if err != nil { if err != nil {
@ -37,20 +41,19 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
return return
} }
s3checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo) _, s3checkErr := h.tree.GetSettingsNode(ctx, bktInfo)
if s3checkErr != nil && !strings.Contains(s3checkErr.Error(), "tree not found") { if s3checkErr != nil && !errors.Is(s3checkErr, tree.ErrNodeNotFound) {
logAndSendBucketError(c, log, s3checkErr) logAndSendBucketError(c, log, s3checkErr)
return return
} }
req := h.newRequest(c, log)
var objID oid.ID var objID oid.ID
if s3checkErr == nil && shouldDownload(oidParam, downloadParam) { if s3checkErr == nil && shouldDownload(oidParam, downloadParam) {
// Receive file via S3 h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile)
h.byS3Path(c, bktInfo.CID, h.receiveFile)
} else if err = objID.DecodeString(oidParam); err == nil { } else if err = objID.DecodeString(oidParam); err == nil {
// Receive file via native protocol h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile)
addr := newAddress(bktInfo.CID, objID)
h.receiveFile(ctx, h.newRequest(c, log), addr)
} else { } else {
h.browseIndex(c, s3checkErr != nil) h.browseIndex(c, s3checkErr != nil)
} }

View file

@ -6,14 +6,13 @@ import (
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
@ -21,6 +20,7 @@ import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/panjf2000/ants/v2" "github.com/panjf2000/ants/v2"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
@ -164,7 +164,7 @@ type Handler struct {
ownerID *user.ID ownerID *user.ID
config Config config Config
containerResolver ContainerResolver containerResolver ContainerResolver
tree *tree.Tree tree layer.TreeService
cache *cache.BucketCache cache *cache.BucketCache
workerPool *ants.Pool workerPool *ants.Pool
} }
@ -177,7 +177,7 @@ type AppParams struct {
Cache *cache.BucketCache Cache *cache.BucketCache
} }
func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
return &Handler{ return &Handler{
log: params.Logger, log: params.Logger,
frostfs: params.FrostFS, frostfs: params.FrostFS,
@ -197,27 +197,22 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci
err := cnrID.DecodeString(containerID) err := cnrID.DecodeString(containerID)
if err != nil { if err != nil {
cnrID, err = h.containerResolver.Resolve(ctx, containerID) cnrID, err = h.containerResolver.Resolve(ctx, containerID)
if err != nil && strings.Contains(err.Error(), "not found") { if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error()) err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error())
} }
} }
return cnrID, err return cnrID, err
} }
func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(context.Context, request, oid.Address)) { func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) {
oidParam := c.UserValue("oid").(string) c, log := req.RequestCtx, req.log
ctx := utils.GetContextFromRequest(c)
log := utils.GetReqLogOrDefault(ctx, h.log).With(
zap.String("cid", cnrID.EncodeToString()),
zap.String("oid", oidParam),
)
foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, oidParam) foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path)
if err != nil { if err != nil {
logAndSendBucketError(c, log, err) logAndSendBucketError(c, log, err)
return return
} }
if foundOID.DeleteMarker { if foundOID.IsDeleteMarker {
log.Error(logs.ObjectWasDeleted) log.Error(logs.ObjectWasDeleted)
response.Error(c, "object deleted", fasthttp.StatusNotFound) response.Error(c, "object deleted", fasthttp.StatusNotFound)
return return
@ -227,6 +222,11 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(co
handler(ctx, h.newRequest(c, log), addr) handler(ctx, h.newRequest(c, log), addr)
} }
func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) {
addr := newAddress(cnrID, objID)
handler(ctx, req, addr)
}
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) { func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) {
cidParam, _ := c.UserValue("cid").(string) cidParam, _ := c.UserValue("cid").(string)
key, _ := c.UserValue("attr_key").(string) key, _ := c.UserValue("attr_key").(string)

View file

@ -8,6 +8,7 @@ import (
"io" "io"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
"sort"
"testing" "testing"
"time" "time"
@ -24,6 +25,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/panjf2000/ants/v2" "github.com/panjf2000/ants/v2"
@ -32,15 +34,97 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
) )
type treeClientMock struct { type treeServiceMock struct {
settings map[string]*data.BucketSettings
versions map[string]map[string][]*data.NodeVersion
system map[string]map[string]*data.BaseNodeVersion
} }
func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) { func newTreeService() *treeServiceMock {
return nil, nil return &treeServiceMock{
settings: make(map[string]*data.BucketSettings),
versions: make(map[string]map[string][]*data.NodeVersion),
system: make(map[string]map[string]*data.BaseNodeVersion),
}
} }
func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]tree.NodeResponse, string, error) {
return nil, nil return nil, "", nil
}
func (t *treeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
settings, ok := t.settings[bktInfo.CID.EncodeToString()]
if !ok {
return nil, treepool.ErrNodeNotFound
}
return settings, nil
}
func (t *treeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
t.settings[bktInfo.CID.EncodeToString()] = settings
return nil
}
func (t *treeServiceMock) GetLatestVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
if !ok {
return nil, treepool.ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, treepool.ErrNodeNotFound
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
return versions[len(versions)-1], nil
}
return nil, treepool.ErrNodeNotFound
}
func (t *treeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{
newVersion.FilePath: {newVersion},
}
return newVersion.ID, nil
}
versions, ok := cnrVersionsMap[newVersion.FilePath]
if !ok {
cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion}
return newVersion.ID, nil
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
newVersion.ID = versions[len(versions)-1].ID + 1
}
result := versions
if newVersion.IsUnversioned {
result = make([]*data.NodeVersion, 0, len(versions))
for _, node := range versions {
if !node.IsUnversioned {
result = append(result, node)
}
}
}
cnrVersionsMap[newVersion.FilePath] = append(result, newVersion)
return newVersion.ID, nil
} }
type configMock struct { type configMock struct {
@ -84,7 +168,7 @@ type handlerContext struct {
h *Handler h *Handler
frostfs *TestFrostFS frostfs *TestFrostFS
tree *treeClientMock tree *treeServiceMock
cfg *configMock cfg *configMock
} }
@ -125,14 +209,14 @@ func prepareHandlerContext() (*handlerContext, error) {
}), }),
} }
treeMock := &treeClientMock{} treeMock := newTreeService()
cfgMock := &configMock{} cfgMock := &configMock{}
workerPool, err := ants.NewPool(1) workerPool, err := ants.NewPool(1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) handler := New(params, cfgMock, treeMock, workerPool)
return &handlerContext{ return &handlerContext{
key: key, key: key,
@ -199,6 +283,18 @@ func TestBasic(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
objID, ok := obj.ID()
require.True(t, ok)
_, err = hc.tree.AddVersion(context.TODO(), &data.BucketInfo{CID: cnrID}, &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: objID,
FilePath: objFileName,
},
})
require.NoError(t, err)
attr := object.NewAttribute() attr := object.NewAttribute()
attr.SetKey(object.AttributeFilePath) attr.SetKey(object.AttributeFilePath)
attr.SetValue(objFileName) attr.SetValue(objFileName)

View file

@ -2,16 +2,17 @@ package handler
import ( import (
"context" "context"
"errors"
"io" "io"
"net/http" "net/http"
"strconv" "strconv"
"strings"
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -117,20 +118,19 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
logAndSendBucketError(c, log, err) logAndSendBucketError(c, log, err)
return return
} }
checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo) _, checkErr := h.tree.GetSettingsNode(ctx, bktInfo)
if checkErr != nil && !strings.Contains(checkErr.Error(), "tree not found") { if checkErr != nil && !errors.Is(checkErr, tree.ErrNodeNotFound) {
logAndSendBucketError(c, log, checkErr) logAndSendBucketError(c, log, checkErr)
return return
} }
req := h.newRequest(c, log)
var objID oid.ID var objID oid.ID
if checkErr == nil { if checkErr == nil {
// Head object via s3 protocol h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject)
h.byS3Path(c, bktInfo.CID, h.headObject)
} else if err = objID.DecodeString(oidParam); err == nil { } else if err = objID.DecodeString(oidParam); err == nil {
// Head object via native protocol h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject)
addr := newAddress(bktInfo.CID, objID)
h.headObject(ctx, h.newRequest(c, log), addr)
} else { } else {
logAndSendBucketError(c, log, checkErr) logAndSendBucketError(c, log, checkErr)
return return

View file

@ -0,0 +1,18 @@
package layer
import (
"context"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
// GetSettingsNode retrieves the settings node from the tree service and form data.BucketSettings.
// If tree node is not found returns ErrNodeNotFound error.
GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]tree.NodeResponse, string, error)
}

View file

@ -79,6 +79,11 @@ const (
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
FailedToUnescapeQuery = "failed to unescape query" FailedToUnescapeQuery = "failed to unescape query"
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
SystemNodeHasMultipleIDs = "system node has multiple ids"
FailedToRemoveOldSystemNode = "failed to remove old system node"
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
ServerReconnecting = "reconnecting server..." ServerReconnecting = "reconnecting server..."
ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectedSuccessfully = "server reconnected successfully"
ServerReconnectFailed = "failed to reconnect server" ServerReconnectFailed = "failed to reconnect server"

View file

@ -2,8 +2,6 @@ package frostfs
import ( import (
"context" "context"
"errors"
"fmt"
"io" "io"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
@ -59,7 +57,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([
nodes, err := w.p.GetNodes(ctx, poolPrm) nodes, err := w.p.GetNodes(ctx, poolPrm)
if err != nil { if err != nil {
return nil, handleError(err) return nil, err
} }
res := make([]tree.NodeResponse, len(nodes)) res := make([]tree.NodeResponse, len(nodes))
@ -78,20 +76,6 @@ func getBearer(ctx context.Context) []byte {
return token.Marshal() return token.Marshal()
} }
func handleError(err error) error {
if err == nil {
return nil
}
if errors.Is(err, treepool.ErrNodeNotFound) {
return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error())
}
if errors.Is(err, treepool.ErrNodeAccessDenied) {
return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error())
}
return err
}
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) {
order := treepool.NoneOrder order := treepool.NoneOrder
if sort { if sort {
@ -115,7 +99,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
if err != nil { if err != nil {
return nil, handleError(err) return nil, err
} }
var subtree []tree.NodeResponse var subtree []tree.NodeResponse
@ -126,7 +110,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
node, err = subTreeReader.Next() node, err = subTreeReader.Next()
} }
if err != io.EOF { if err != io.EOF {
return nil, handleError(err) return nil, err
} }
return subtree, nil return subtree, nil

View file

@ -6,16 +6,17 @@ import (
"fmt" "fmt"
"strings" "strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"go.uber.org/zap"
) )
type ( type (
Tree struct { Tree struct {
service ServiceClient service ServiceClient
log *zap.Logger
} }
// ServiceClient is a client to interact with tree service. // ServiceClient is a client to interact with tree service.
@ -26,8 +27,12 @@ type (
} }
treeNode struct { treeNode struct {
ObjID oid.ID ID []uint64
Meta map[string]string ParentID []uint64
ObjID oid.ID
TimeStamp []uint64
Size uint64
Meta map[string]string
} }
multiSystemNode struct { multiSystemNode struct {
@ -46,35 +51,36 @@ type (
} }
) )
var (
// ErrNodeNotFound is returned from ServiceClient in case of not found error.
ErrNodeNotFound = layer.ErrNodeNotFound
// ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error.
ErrNodeAccessDenied = layer.ErrNodeAccessDenied
)
const ( const (
FileNameKey = "FileName" FileNameKey = "FileName"
settingsFileName = "bucket-settings"
oidKV = "OID" versioningKV = "Versioning"
uploadIDKV = "UploadId" cannedACLKV = "cannedACL"
sizeKV = "Size" ownerKeyKV = "ownerKey"
lockConfigurationKV = "LockConfiguration"
oidKV = "OID"
isUnversionedKV = "IsUnversioned"
uploadIDKV = "UploadId"
sizeKV = "Size"
// keys for delete marker nodes. // keys for delete marker nodes.
isDeleteMarkerKV = "IsDeleteMarker" isDeleteMarkerKV = "IsDeleteMarker"
settingsFileName = "bucket-settings"
// versionTree -- ID of a tree with object versions. // versionTree -- ID of a tree with object versions.
versionTree = "version" versionTree = "version"
systemTree = "system"
// systemTree -- ID of a tree with system objects
// i.e. bucket settings with versioning and lock configuration, cors.
systemTree = "system"
separator = "/" separator = "/"
) )
// NewTree creates instance of Tree using provided address and create grpc connection. // NewTree creates instance of Tree using provided address and create grpc connection.
func NewTree(service ServiceClient) *Tree { func NewTree(service ServiceClient, log *zap.Logger) *Tree {
return &Tree{service: service} return &Tree{service: service, log: log}
} }
type Meta interface { type Meta interface {
@ -118,24 +124,44 @@ func (n *treeNode) FileName() (string, bool) {
return value, ok return value, ok
} }
func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { func (n *treeNode) GetLatestNodeIndex() int {
var (
maxTimestamp uint64
index int
)
for i, timestamp := range n.TimeStamp {
if timestamp > maxTimestamp {
maxTimestamp = timestamp
index = i
}
}
return index
}
func (n *treeNode) IsSplit() bool {
return len(n.ID) != 1 || len(n.ParentID) != 1 || len(n.TimeStamp) != 1
}
func newNodeVersion(node NodeResponse, objectName string) (*data.NodeVersion, error) {
tNode, err := newTreeNode(node) tNode, err := newTreeNode(node)
if err != nil { if err != nil {
return nil, fmt.Errorf("invalid tree node: %w", err) return nil, fmt.Errorf("invalid tree node: %w", err)
} }
return newNodeVersionFromTreeNode(tNode), nil return newNodeVersionFromTreeNode(objectName, tNode), nil
} }
func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeVersion {
_, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) _, isUnversioned := treeNode.Get(isUnversionedKV)
size, _ := treeNode.Get(sizeKV)
version := &api.NodeVersion{ version := &data.NodeVersion{
BaseNodeVersion: api.BaseNodeVersion{ BaseNodeVersion: data.BaseNodeVersion{
OID: treeNode.ObjID, OID: treeNode.ObjID,
FilePath: filePath,
}, },
DeleteMarker: isDeleteMarker, IsUnversioned: isUnversioned,
IsPrefixNode: size == "",
} }
return version return version
@ -180,7 +206,7 @@ func (m *multiSystemNode) Old() []*treeNode {
return m.nodes[1:] return m.nodes[1:]
} }
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
nodes, err := c.GetVersions(ctx, cnrID, objectName) nodes, err := c.GetVersions(ctx, cnrID, objectName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -191,7 +217,7 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s
return nil, err return nil, err
} }
return newNodeVersion(latestNode) return newNodeVersion(latestNode, objectName)
} }
func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) { func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) {
@ -210,15 +236,6 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
return c.service.GetNodes(ctx, p) return c.service.GetNodes(ctx, p)
} }
func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error {
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
if err != nil {
return err
}
return nil
}
func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) { func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) {
p := &GetNodesParams{ p := &GetNodesParams{
CnrID: bktInfo.CID, CnrID: bktInfo.CID,
@ -236,7 +253,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
nodes = filterMultipartNodes(nodes) nodes = filterMultipartNodes(nodes)
if len(nodes) == 0 { if len(nodes) == 0 {
return nil, ErrNodeNotFound return nil, tree.ErrNodeNotFound
} }
return newMultiNode(nodes) return newMultiNode(nodes)
@ -277,7 +294,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
} }
if targetIndexNode == -1 { if targetIndexNode == -1 {
return nil, layer.ErrNodeNotFound return nil, tree.ErrNodeNotFound
} }
return nodes[targetIndexNode], nil return nodes[targetIndexNode], nil
@ -305,7 +322,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
} }
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
if err != nil { if err != nil {
if errors.Is(err, layer.ErrNodeNotFound) { if errors.Is(err, tree.ErrNodeNotFound) {
return nil, "", nil return nil, "", nil
} }
return nil, "", err return nil, "", err
@ -386,12 +403,28 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
} }
if len(intermediateNodes) == 0 { if len(intermediateNodes) == 0 {
return nil, layer.ErrNodeNotFound return nil, tree.ErrNodeNotFound
} }
return intermediateNodes, nil return intermediateNodes, nil
} }
func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
if err != nil {
return nil, fmt.Errorf("couldn't get node: %w", err)
}
node := multiNode.Latest()
settings := &data.BucketSettings{Versioning: data.VersioningUnversioned}
if versioningValue, ok := node.Get(versioningKV); ok {
settings.Versioning = versioningValue
}
return settings, nil
}
func GetFilename(node NodeResponse) string { func GetFilename(node NodeResponse) string {
for _, kv := range node.GetMeta() { for _, kv := range node.GetMeta() {
if kv.GetKey() == FileNameKey { if kv.GetKey() == FileNameKey {