[#166] Fix getting s3 object with the FrostFS OID name
All checks were successful
/ DCO (pull_request) Successful in 2m41s
/ Vulncheck (pull_request) Successful in 2m48s
/ Builds (pull_request) Successful in 1m55s
/ Lint (pull_request) Successful in 2m44s
/ Tests (pull_request) Successful in 1m58s

Prioritize getting s3 object with the key, which equals to valid FrostFS OID, rather than getting non-existent object with OID via native protocol for GET and HEAD requests

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2024-12-02 11:45:30 +03:00
parent e81f01c2ab
commit a6bc8208ae
Signed by: nzinkevich
GPG key ID: 748EA1D0B2E6420A
13 changed files with 173 additions and 201 deletions

View file

@ -499,10 +499,10 @@ func (a *app) Serve() {
close(a.webDone) close(a.webDone)
}() }()
handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool)
// Configure router. // Configure router.
a.configureRouter(handler) a.configureRouter(handle)
a.startServices() a.startServices()
a.initServers(a.ctx) a.initServers(a.ctx)

View file

@ -1,22 +0,0 @@
package layer
import (
"context"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error)
}
var (
// ErrNodeNotFound is returned from Tree service in case of not found error.
ErrNodeNotFound = errors.New("not found")
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
ErrNodeAccessDenied = errors.New("access denied")
)

View file

@ -1,4 +1,4 @@
package api package data
import ( import (
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -7,12 +7,13 @@ import (
// NodeVersion represent node from tree service. // NodeVersion represent node from tree service.
type NodeVersion struct { type NodeVersion struct {
BaseNodeVersion BaseNodeVersion
DeleteMarker bool IsUnversioned bool
IsPrefixNode bool
} }
// BaseNodeVersion is minimal node info from tree service. // BaseNodeVersion is minimal node info from tree service.
// Basically used for "system" object. // Basically used for "system" object.
type BaseNodeVersion struct { type BaseNodeVersion struct {
OID oid.ID ID uint64
OID oid.ID
IsDeleteMarker bool
} }

View file

@ -4,6 +4,7 @@ import (
"archive/zip" "archive/zip"
"bufio" "bufio"
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -17,27 +18,53 @@ import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"go.uber.org/zap" "go.uber.org/zap"
) )
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format. // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
oidURLParam := c.UserValue("oid").(string) cidParam := c.UserValue("cid").(string)
downloadQueryParam := c.QueryArgs().GetBool("download") oidParam := c.UserValue("oid").(string)
downloadParam := c.QueryArgs().GetBool("download")
switch { ctx := utils.GetContextFromRequest(c)
case isObjectID(oidURLParam): log := utils.GetReqLogOrDefault(ctx, h.log).With(
h.byNativeAddress(c, h.receiveFile) zap.String("cid", cidParam),
case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)): zap.String("oid", oidParam),
h.byS3Path(c, h.receiveFile) )
default:
h.browseIndex(c) bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
if err != nil {
logAndSendBucketError(c, log, err)
return
}
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) {
logAndSendBucketError(c, log, checkS3Err)
return
}
req := h.newRequest(c, log)
var objID oid.ID
if checkS3Err == nil && shouldDownload(oidParam, downloadParam) {
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile)
} else if err = objID.DecodeString(oidParam); err == nil {
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile)
} else {
h.browseIndex(c, checkS3Err != nil)
} }
} }
func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request { func shouldDownload(oidParam string, downloadParam bool) bool {
return &request{ return !isDir(oidParam) || downloadParam
}
func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request {
return request{
RequestCtx: ctx, RequestCtx: ctx,
log: log, log: log,
} }

View file

@ -6,14 +6,13 @@ import (
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
@ -21,6 +20,7 @@ import (
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/panjf2000/ants/v2" "github.com/panjf2000/ants/v2"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
@ -164,7 +164,7 @@ type Handler struct {
ownerID *user.ID ownerID *user.ID
config Config config Config
containerResolver ContainerResolver containerResolver ContainerResolver
tree *tree.Tree tree layer.TreeService
cache *cache.BucketCache cache *cache.BucketCache
workerPool *ants.Pool workerPool *ants.Pool
} }
@ -177,7 +177,7 @@ type AppParams struct {
Cache *cache.BucketCache Cache *cache.BucketCache
} }
func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
return &Handler{ return &Handler{
log: params.Logger, log: params.Logger,
frostfs: params.FrostFS, frostfs: params.FrostFS,
@ -190,79 +190,45 @@ func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Poo
} }
} }
// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolveContainer decode container id, if it's not a valid container id
// prepares request and object address to it. // then trey to resolve name using provided resolver.
func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) {
idCnr, _ := c.UserValue("cid").(string) cnrID := new(cid.ID)
idObj, _ := url.PathUnescape(c.UserValue("oid").(string)) err := cnrID.DecodeString(containerID)
ctx := utils.GetContextFromRequest(c)
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
log := reqLog.With(zap.String("cid", idCnr), zap.String("oid", idObj))
bktInfo, err := h.getBucketInfo(ctx, idCnr, log)
if err != nil { if err != nil {
logAndSendBucketError(c, log, err) cnrID, err = h.containerResolver.Resolve(ctx, containerID)
return if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error())
}
} }
return cnrID, err
objID := new(oid.ID)
if err = objID.DecodeString(idObj); err != nil {
log.Error(logs.WrongObjectID, zap.Error(err))
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
return
}
addr := newAddress(bktInfo.CID, *objID)
f(ctx, *h.newRequest(c, log), addr)
} }
// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) {
// resolves object address from S3-like path <bucket name>/<object key>. c, log := req.RequestCtx, req.log
func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
bucketname := c.UserValue("cid").(string)
key := c.UserValue("oid").(string)
ctx := utils.GetContextFromRequest(c) foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path)
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
log := reqLog.With(zap.String("bucketname", bucketname), zap.String("key", key))
unescapedKey, err := url.QueryUnescape(key)
if err != nil { if err != nil {
logAndSendBucketError(c, log, err) logAndSendBucketError(c, log, err)
return return
} }
if foundOID.IsDeleteMarker {
bktInfo, err := h.getBucketInfo(ctx, bucketname, log)
if err != nil {
logAndSendBucketError(c, log, err)
return
}
foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey)
if err != nil {
if errors.Is(err, tree.ErrNodeAccessDenied) {
response.Error(c, "Access Denied", fasthttp.StatusForbidden)
} else {
response.Error(c, "object wasn't found", fasthttp.StatusNotFound)
log.Error(logs.GetLatestObjectVersion, zap.Error(err))
}
return
}
if foundOid.DeleteMarker {
log.Error(logs.ObjectWasDeleted) log.Error(logs.ObjectWasDeleted)
response.Error(c, "object deleted", fasthttp.StatusNotFound) response.Error(c, "object deleted", fasthttp.StatusNotFound)
return return
} }
addr := newAddress(bktInfo.CID, foundOid.OID)
f(ctx, *h.newRequest(c, log), addr) addr := newAddress(cnrID, foundOID.OID)
handler(ctx, h.newRequest(c, log), addr)
} }
// byAttribute is a wrapper similar to byNativeAddress. func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) {
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) { addr := newAddress(cnrID, objID)
scid, _ := c.UserValue("cid").(string) handler(ctx, req, addr)
}
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) {
cidParam, _ := c.UserValue("cid").(string)
key, _ := c.UserValue("attr_key").(string) key, _ := c.UserValue("attr_key").(string)
val, _ := c.UserValue("attr_val").(string) val, _ := c.UserValue("attr_val").(string)
@ -271,21 +237,21 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re
key, err := url.QueryUnescape(key) key, err := url.QueryUnescape(key)
if err != nil { if err != nil {
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Error(err)) log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err))
response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest) response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest)
return return
} }
val, err = url.QueryUnescape(val) val, err = url.QueryUnescape(val)
if err != nil { if err != nil {
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Error(err)) log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err))
response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest) response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest)
return return
} }
log = log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val)) log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val))
bktInfo, err := h.getBucketInfo(ctx, scid, log) bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
if err != nil { if err != nil {
logAndSendBucketError(c, log, err) logAndSendBucketError(c, log, err)
return return
@ -315,25 +281,11 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re
return return
} }
var addrObj oid.Address var addr oid.Address
addrObj.SetContainer(bktInfo.CID) addr.SetContainer(bktInfo.CID)
addrObj.SetObject(buf[0]) addr.SetObject(buf[0])
f(ctx, *h.newRequest(c, log), addrObj) handler(ctx, h.newRequest(c, log), addr)
}
// resolveContainer decode container id, if it's not a valid container id
// then trey to resolve name using provided resolver.
func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) {
cnrID := new(cid.ID)
err := cnrID.DecodeString(containerID)
if err != nil {
cnrID, err = h.containerResolver.Resolve(ctx, containerID)
if err != nil && strings.Contains(err.Error(), "not found") {
err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error())
}
}
return cnrID, err
} }
func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *zap.Logger) (*data.BucketInfo, error) { func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *zap.Logger) (*data.BucketInfo, error) {
@ -388,7 +340,7 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
return bktInfo, err return bktInfo, err
} }
func (h *Handler) browseIndex(c *fasthttp.RequestCtx) { func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) {
if !h.config.IndexPageEnabled() { if !h.config.IndexPageEnabled() {
c.SetStatusCode(fasthttp.StatusNotFound) c.SetStatusCode(fasthttp.StatusNotFound)
return return
@ -414,18 +366,9 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx) {
} }
listFunc := h.getDirObjectsS3 listFunc := h.getDirObjectsS3
isNativeList := false if isNativeList {
// tree probe failed, trying to use native
err = h.tree.CheckSettingsNodeExist(ctx, bktInfo) listFunc = h.getDirObjectsNative
if err != nil {
if errors.Is(err, tree.ErrNodeNotFound) {
// tree probe failed, try to use native
listFunc = h.getDirObjectsNative
isNativeList = true
} else {
logAndSendBucketError(c, log, err)
return
}
} }
h.browseObjects(c, browseParams{ h.browseObjects(c, browseParams{

View file

@ -24,6 +24,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/panjf2000/ants/v2" "github.com/panjf2000/ants/v2"
@ -32,14 +33,29 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
) )
type treeClientMock struct { type treeServiceMock struct {
system map[string]map[string]*data.BaseNodeVersion
} }
func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) { func newTreeService() *treeServiceMock {
return nil, nil return &treeServiceMock{
system: make(map[string]map[string]*data.BaseNodeVersion),
}
} }
func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error {
_, ok := t.system["bucket-settings"]
if !ok {
return treepool.ErrNodeNotFound
}
return nil
}
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]tree.NodeResponse, string, error) {
return nil, "", nil
}
func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) {
return nil, nil return nil, nil
} }
@ -84,7 +100,7 @@ type handlerContext struct {
h *Handler h *Handler
frostfs *TestFrostFS frostfs *TestFrostFS
tree *treeClientMock tree *treeServiceMock
cfg *configMock cfg *configMock
} }
@ -125,14 +141,14 @@ func prepareHandlerContext() (*handlerContext, error) {
}), }),
} }
treeMock := &treeClientMock{} treeMock := newTreeService()
cfgMock := &configMock{} cfgMock := &configMock{}
workerPool, err := ants.NewPool(1000) workerPool, err := ants.NewPool(1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) handler := New(params, cfgMock, treeMock, workerPool)
return &handlerContext{ return &handlerContext{
key: key, key: key,

View file

@ -2,6 +2,7 @@ package handler
import ( import (
"context" "context"
"errors"
"io" "io"
"net/http" "net/http"
"strconv" "strconv"
@ -11,6 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -102,14 +104,36 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) {
// HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format. // HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format.
func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
test, _ := c.UserValue("oid").(string) cidParam, _ := c.UserValue("cid").(string)
var id oid.ID oidParam, _ := c.UserValue("oid").(string)
err := id.DecodeString(test) ctx := utils.GetContextFromRequest(c)
log := utils.GetReqLogOrDefault(ctx, h.log).With(
zap.String("cid", cidParam),
zap.String("oid", oidParam),
)
bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
if err != nil { if err != nil {
h.byS3Path(c, h.headObject) logAndSendBucketError(c, log, err)
return
}
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) {
logAndSendBucketError(c, log, checkS3Err)
return
}
req := h.newRequest(c, log)
var objID oid.ID
if checkS3Err == nil {
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject)
} else if err = objID.DecodeString(oidParam); err == nil {
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject)
} else { } else {
h.byNativeAddress(c, h.headObject) logAndSendBucketError(c, log, checkS3Err)
return
} }
} }

View file

@ -42,16 +42,7 @@ func bearerToken(ctx context.Context) *bearer.Token {
} }
func isDir(name string) bool { func isDir(name string) bool {
return strings.HasSuffix(name, "/") return name == "" || strings.HasSuffix(name, "/")
}
func isObjectID(s string) bool {
var objID oid.ID
return objID.DecodeString(s) == nil
}
func isContainerRoot(key string) bool {
return key == ""
} }
func loadAttributes(attrs []object.Attribute) map[string]string { func loadAttributes(attrs []object.Attribute) map[string]string {

View file

@ -0,0 +1,16 @@
package layer
import (
"context"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]tree.NodeResponse, string, error)
CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error
}

View file

@ -79,6 +79,11 @@ const (
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
FailedToUnescapeQuery = "failed to unescape query" FailedToUnescapeQuery = "failed to unescape query"
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
SystemNodeHasMultipleIDs = "system node has multiple ids"
FailedToRemoveOldSystemNode = "failed to remove old system node"
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
ServerReconnecting = "reconnecting server..." ServerReconnecting = "reconnecting server..."
ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectedSuccessfully = "server reconnected successfully"
ServerReconnectFailed = "failed to reconnect server" ServerReconnectFailed = "failed to reconnect server"

View file

@ -2,8 +2,6 @@ package frostfs
import ( import (
"context" "context"
"errors"
"fmt"
"io" "io"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
@ -59,7 +57,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([
nodes, err := w.p.GetNodes(ctx, poolPrm) nodes, err := w.p.GetNodes(ctx, poolPrm)
if err != nil { if err != nil {
return nil, handleError(err) return nil, err
} }
res := make([]tree.NodeResponse, len(nodes)) res := make([]tree.NodeResponse, len(nodes))
@ -78,20 +76,6 @@ func getBearer(ctx context.Context) []byte {
return token.Marshal() return token.Marshal()
} }
func handleError(err error) error {
if err == nil {
return nil
}
if errors.Is(err, treepool.ErrNodeNotFound) {
return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error())
}
if errors.Is(err, treepool.ErrNodeAccessDenied) {
return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error())
}
return err
}
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) {
order := treepool.NoneOrder order := treepool.NoneOrder
if sort { if sort {
@ -115,7 +99,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
if err != nil { if err != nil {
return nil, handleError(err) return nil, err
} }
var subtree []tree.NodeResponse var subtree []tree.NodeResponse
@ -126,7 +110,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
node, err = subTreeReader.Next() node, err = subTreeReader.Next()
} }
if err != io.EOF { if err != io.EOF {
return nil, handleError(err) return nil, err
} }
return subtree, nil return subtree, nil

View file

@ -6,11 +6,10 @@ import (
"fmt" "fmt"
"strings" "strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
) )
type ( type (
@ -46,14 +45,6 @@ type (
} }
) )
var (
// ErrNodeNotFound is returned from ServiceClient in case of not found error.
ErrNodeNotFound = layer.ErrNodeNotFound
// ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error.
ErrNodeAccessDenied = layer.ErrNodeAccessDenied
)
const ( const (
FileNameKey = "FileName" FileNameKey = "FileName"
settingsFileName = "bucket-settings" settingsFileName = "bucket-settings"
@ -118,7 +109,7 @@ func (n *treeNode) FileName() (string, bool) {
return value, ok return value, ok
} }
func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { func newNodeVersion(node NodeResponse) (*data.NodeVersion, error) {
tNode, err := newTreeNode(node) tNode, err := newTreeNode(node)
if err != nil { if err != nil {
return nil, fmt.Errorf("invalid tree node: %w", err) return nil, fmt.Errorf("invalid tree node: %w", err)
@ -127,15 +118,11 @@ func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) {
return newNodeVersionFromTreeNode(tNode), nil return newNodeVersionFromTreeNode(tNode), nil
} }
func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { func newNodeVersionFromTreeNode(treeNode *treeNode) *data.NodeVersion {
_, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) version := &data.NodeVersion{
size, _ := treeNode.Get(sizeKV) BaseNodeVersion: data.BaseNodeVersion{
version := &api.NodeVersion{
BaseNodeVersion: api.BaseNodeVersion{
OID: treeNode.ObjID, OID: treeNode.ObjID,
}, },
DeleteMarker: isDeleteMarker,
IsPrefixNode: size == "",
} }
return version return version
@ -180,7 +167,7 @@ func (m *multiSystemNode) Old() []*treeNode {
return m.nodes[1:] return m.nodes[1:]
} }
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
nodes, err := c.GetVersions(ctx, cnrID, objectName) nodes, err := c.GetVersions(ctx, cnrID, objectName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -210,7 +197,7 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
return c.service.GetNodes(ctx, p) return c.service.GetNodes(ctx, p)
} }
func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error { func (c *Tree) CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error {
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName) _, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
if err != nil { if err != nil {
return err return err
@ -236,7 +223,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
nodes = filterMultipartNodes(nodes) nodes = filterMultipartNodes(nodes)
if len(nodes) == 0 { if len(nodes) == 0 {
return nil, ErrNodeNotFound return nil, tree.ErrNodeNotFound
} }
return newMultiNode(nodes) return newMultiNode(nodes)
@ -277,7 +264,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
} }
if targetIndexNode == -1 { if targetIndexNode == -1 {
return nil, layer.ErrNodeNotFound return nil, tree.ErrNodeNotFound
} }
return nodes[targetIndexNode], nil return nodes[targetIndexNode], nil
@ -305,7 +292,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
} }
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
if err != nil { if err != nil {
if errors.Is(err, layer.ErrNodeNotFound) { if errors.Is(err, tree.ErrNodeNotFound) {
return nil, "", nil return nil, "", nil
} }
return nil, "", err return nil, "", err
@ -386,7 +373,7 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
} }
if len(intermediateNodes) == 0 { if len(intermediateNodes) == 0 {
return nil, layer.ErrNodeNotFound return nil, tree.ErrNodeNotFound
} }
return intermediateNodes, nil return intermediateNodes, nil