[#166] Fix getting s3 object with the FrostFS OID name
Prioritize getting s3 object with the key, which equals to valid FrostFS OID, rather than getting non-existent object with OID via native protocol for GET and HEAD requests Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
parent
e81f01c2ab
commit
4d424383b9
13 changed files with 161 additions and 188 deletions
|
@ -499,10 +499,10 @@ func (a *app) Serve() {
|
||||||
close(a.webDone)
|
close(a.webDone)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool)
|
handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool)
|
||||||
|
|
||||||
// Configure router.
|
// Configure router.
|
||||||
a.configureRouter(handler)
|
a.configureRouter(handle)
|
||||||
|
|
||||||
a.startServices()
|
a.startServices()
|
||||||
a.initServers(a.ctx)
|
a.initServers(a.ctx)
|
||||||
|
|
|
@ -1,22 +0,0 @@
|
||||||
package layer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
|
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TreeService provide interface to interact with tree service using s3 data models.
|
|
||||||
type TreeService interface {
|
|
||||||
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
// ErrNodeNotFound is returned from Tree service in case of not found error.
|
|
||||||
ErrNodeNotFound = errors.New("not found")
|
|
||||||
|
|
||||||
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
|
|
||||||
ErrNodeAccessDenied = errors.New("access denied")
|
|
||||||
)
|
|
|
@ -1,4 +1,4 @@
|
||||||
package api
|
package data
|
||||||
|
|
||||||
import (
|
import (
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
@ -7,12 +7,12 @@ import (
|
||||||
// NodeVersion represent node from tree service.
|
// NodeVersion represent node from tree service.
|
||||||
type NodeVersion struct {
|
type NodeVersion struct {
|
||||||
BaseNodeVersion
|
BaseNodeVersion
|
||||||
DeleteMarker bool
|
|
||||||
IsPrefixNode bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BaseNodeVersion is minimal node info from tree service.
|
// BaseNodeVersion is minimal node info from tree service.
|
||||||
// Basically used for "system" object.
|
// Basically used for "system" object.
|
||||||
type BaseNodeVersion struct {
|
type BaseNodeVersion struct {
|
||||||
OID oid.ID
|
ID uint64
|
||||||
|
OID oid.ID
|
||||||
|
IsDeleteMarker bool
|
||||||
}
|
}
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"archive/zip"
|
"archive/zip"
|
||||||
"bufio"
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -17,27 +18,53 @@ import (
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
|
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
|
||||||
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
||||||
oidURLParam := c.UserValue("oid").(string)
|
cidParam := c.UserValue("cid").(string)
|
||||||
downloadQueryParam := c.QueryArgs().GetBool("download")
|
oidParam := c.UserValue("oid").(string)
|
||||||
|
downloadParam := c.QueryArgs().GetBool("download")
|
||||||
|
|
||||||
switch {
|
ctx := utils.GetContextFromRequest(c)
|
||||||
case isObjectID(oidURLParam):
|
log := utils.GetReqLogOrDefault(ctx, h.log).With(
|
||||||
h.byNativeAddress(c, h.receiveFile)
|
zap.String("cid", cidParam),
|
||||||
case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)):
|
zap.String("oid", oidParam),
|
||||||
h.byS3Path(c, h.receiveFile)
|
)
|
||||||
default:
|
|
||||||
h.browseIndex(c)
|
bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
|
||||||
|
if err != nil {
|
||||||
|
logAndSendBucketError(c, log, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
|
||||||
|
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) {
|
||||||
|
logAndSendBucketError(c, log, checkS3Err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
req := h.newRequest(c, log)
|
||||||
|
|
||||||
|
var objID oid.ID
|
||||||
|
if checkS3Err == nil && shouldDownload(oidParam, downloadParam) {
|
||||||
|
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile)
|
||||||
|
} else if err = objID.DecodeString(oidParam); err == nil {
|
||||||
|
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile)
|
||||||
|
} else {
|
||||||
|
h.browseIndex(c, checkS3Err != nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) *request {
|
func shouldDownload(oidParam string, downloadParam bool) bool {
|
||||||
return &request{
|
return !isDir(oidParam) || downloadParam
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) newRequest(ctx *fasthttp.RequestCtx, log *zap.Logger) request {
|
||||||
|
return request{
|
||||||
RequestCtx: ctx,
|
RequestCtx: ctx,
|
||||||
log: log,
|
log: log,
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,14 +6,13 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
|
@ -21,6 +20,7 @@ import (
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/panjf2000/ants/v2"
|
"github.com/panjf2000/ants/v2"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
|
@ -164,7 +164,7 @@ type Handler struct {
|
||||||
ownerID *user.ID
|
ownerID *user.ID
|
||||||
config Config
|
config Config
|
||||||
containerResolver ContainerResolver
|
containerResolver ContainerResolver
|
||||||
tree *tree.Tree
|
tree layer.TreeService
|
||||||
cache *cache.BucketCache
|
cache *cache.BucketCache
|
||||||
workerPool *ants.Pool
|
workerPool *ants.Pool
|
||||||
}
|
}
|
||||||
|
@ -177,7 +177,7 @@ type AppParams struct {
|
||||||
Cache *cache.BucketCache
|
Cache *cache.BucketCache
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler {
|
func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
|
||||||
return &Handler{
|
return &Handler{
|
||||||
log: params.Logger,
|
log: params.Logger,
|
||||||
frostfs: params.FrostFS,
|
frostfs: params.FrostFS,
|
||||||
|
@ -192,77 +192,31 @@ func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Poo
|
||||||
|
|
||||||
// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
|
// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
|
||||||
// prepares request and object address to it.
|
// prepares request and object address to it.
|
||||||
func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) {
|
||||||
idCnr, _ := c.UserValue("cid").(string)
|
addr := newAddress(cnrID, objID)
|
||||||
idObj, _ := url.PathUnescape(c.UserValue("oid").(string))
|
handler(ctx, req, addr)
|
||||||
|
|
||||||
ctx := utils.GetContextFromRequest(c)
|
|
||||||
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
|
|
||||||
log := reqLog.With(zap.String("cid", idCnr), zap.String("oid", idObj))
|
|
||||||
|
|
||||||
bktInfo, err := h.getBucketInfo(ctx, idCnr, log)
|
|
||||||
if err != nil {
|
|
||||||
logAndSendBucketError(c, log, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
objID := new(oid.ID)
|
|
||||||
if err = objID.DecodeString(idObj); err != nil {
|
|
||||||
log.Error(logs.WrongObjectID, zap.Error(err))
|
|
||||||
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
addr := newAddress(bktInfo.CID, *objID)
|
|
||||||
|
|
||||||
f(ctx, *h.newRequest(c, log), addr)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that
|
func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) {
|
||||||
// resolves object address from S3-like path <bucket name>/<object key>.
|
c, log := req.RequestCtx, req.log
|
||||||
func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
|
||||||
bucketname := c.UserValue("cid").(string)
|
|
||||||
key := c.UserValue("oid").(string)
|
|
||||||
|
|
||||||
ctx := utils.GetContextFromRequest(c)
|
foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path)
|
||||||
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
|
|
||||||
log := reqLog.With(zap.String("bucketname", bucketname), zap.String("key", key))
|
|
||||||
|
|
||||||
unescapedKey, err := url.QueryUnescape(key)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logAndSendBucketError(c, log, err)
|
logAndSendBucketError(c, log, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if foundOID.IsDeleteMarker {
|
||||||
bktInfo, err := h.getBucketInfo(ctx, bucketname, log)
|
|
||||||
if err != nil {
|
|
||||||
logAndSendBucketError(c, log, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, tree.ErrNodeAccessDenied) {
|
|
||||||
response.Error(c, "Access Denied", fasthttp.StatusForbidden)
|
|
||||||
} else {
|
|
||||||
response.Error(c, "object wasn't found", fasthttp.StatusNotFound)
|
|
||||||
log.Error(logs.GetLatestObjectVersion, zap.Error(err))
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if foundOid.DeleteMarker {
|
|
||||||
log.Error(logs.ObjectWasDeleted)
|
log.Error(logs.ObjectWasDeleted)
|
||||||
response.Error(c, "object deleted", fasthttp.StatusNotFound)
|
response.Error(c, "object deleted", fasthttp.StatusNotFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
addr := newAddress(bktInfo.CID, foundOid.OID)
|
|
||||||
|
|
||||||
f(ctx, *h.newRequest(c, log), addr)
|
addr := newAddress(cnrID, foundOID.OID)
|
||||||
|
handler(ctx, h.newRequest(c, log), addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// byAttribute is a wrapper similar to byNativeAddress.
|
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) {
|
||||||
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
cidParam, _ := c.UserValue("cid").(string)
|
||||||
scid, _ := c.UserValue("cid").(string)
|
|
||||||
key, _ := c.UserValue("attr_key").(string)
|
key, _ := c.UserValue("attr_key").(string)
|
||||||
val, _ := c.UserValue("attr_val").(string)
|
val, _ := c.UserValue("attr_val").(string)
|
||||||
|
|
||||||
|
@ -271,21 +225,21 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re
|
||||||
|
|
||||||
key, err := url.QueryUnescape(key)
|
key, err := url.QueryUnescape(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_key", key), zap.Error(err))
|
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err))
|
||||||
response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest)
|
response.Error(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
val, err = url.QueryUnescape(val)
|
val, err = url.QueryUnescape(val)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("attr_val", val), zap.Error(err))
|
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err))
|
||||||
response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest)
|
response.Error(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log = log.With(zap.String("cid", scid), zap.String("attr_key", key), zap.String("attr_val", val))
|
log = log.With(zap.String("cid", cidParam), zap.String("attr_key", key), zap.String("attr_val", val))
|
||||||
|
|
||||||
bktInfo, err := h.getBucketInfo(ctx, scid, log)
|
bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logAndSendBucketError(c, log, err)
|
logAndSendBucketError(c, log, err)
|
||||||
return
|
return
|
||||||
|
@ -315,11 +269,11 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
var addrObj oid.Address
|
var addr oid.Address
|
||||||
addrObj.SetContainer(bktInfo.CID)
|
addr.SetContainer(bktInfo.CID)
|
||||||
addrObj.SetObject(buf[0])
|
addr.SetObject(buf[0])
|
||||||
|
|
||||||
f(ctx, *h.newRequest(c, log), addrObj)
|
handler(ctx, h.newRequest(c, log), addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveContainer decode container id, if it's not a valid container id
|
// resolveContainer decode container id, if it's not a valid container id
|
||||||
|
@ -329,7 +283,7 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci
|
||||||
err := cnrID.DecodeString(containerID)
|
err := cnrID.DecodeString(containerID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cnrID, err = h.containerResolver.Resolve(ctx, containerID)
|
cnrID, err = h.containerResolver.Resolve(ctx, containerID)
|
||||||
if err != nil && strings.Contains(err.Error(), "not found") {
|
if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
|
||||||
err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error())
|
err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -388,7 +342,7 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
|
||||||
return bktInfo, err
|
return bktInfo, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) browseIndex(c *fasthttp.RequestCtx) {
|
func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) {
|
||||||
if !h.config.IndexPageEnabled() {
|
if !h.config.IndexPageEnabled() {
|
||||||
c.SetStatusCode(fasthttp.StatusNotFound)
|
c.SetStatusCode(fasthttp.StatusNotFound)
|
||||||
return
|
return
|
||||||
|
@ -414,18 +368,9 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx) {
|
||||||
}
|
}
|
||||||
|
|
||||||
listFunc := h.getDirObjectsS3
|
listFunc := h.getDirObjectsS3
|
||||||
isNativeList := false
|
if isNativeList {
|
||||||
|
// tree probe failed, trying to use native
|
||||||
err = h.tree.CheckSettingsNodeExist(ctx, bktInfo)
|
listFunc = h.getDirObjectsNative
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
|
||||||
// tree probe failed, try to use native
|
|
||||||
listFunc = h.getDirObjectsNative
|
|
||||||
isNativeList = true
|
|
||||||
} else {
|
|
||||||
logAndSendBucketError(c, log, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
h.browseObjects(c, browseParams{
|
h.browseObjects(c, browseParams{
|
||||||
|
|
|
@ -24,6 +24,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/panjf2000/ants/v2"
|
"github.com/panjf2000/ants/v2"
|
||||||
|
@ -32,14 +33,29 @@ import (
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
type treeClientMock struct {
|
type treeServiceMock struct {
|
||||||
|
system map[string]map[string]*data.BaseNodeVersion
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) {
|
func newTreeService() *treeServiceMock {
|
||||||
return nil, nil
|
return &treeServiceMock{
|
||||||
|
system: make(map[string]map[string]*data.BaseNodeVersion),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) {
|
func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error {
|
||||||
|
_, ok := t.system["bucket-settings"]
|
||||||
|
if !ok {
|
||||||
|
return treepool.ErrNodeNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]tree.NodeResponse, string, error) {
|
||||||
|
return nil, "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -84,7 +100,7 @@ type handlerContext struct {
|
||||||
|
|
||||||
h *Handler
|
h *Handler
|
||||||
frostfs *TestFrostFS
|
frostfs *TestFrostFS
|
||||||
tree *treeClientMock
|
tree *treeServiceMock
|
||||||
cfg *configMock
|
cfg *configMock
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -125,14 +141,14 @@ func prepareHandlerContext() (*handlerContext, error) {
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
treeMock := &treeClientMock{}
|
treeMock := newTreeService()
|
||||||
cfgMock := &configMock{}
|
cfgMock := &configMock{}
|
||||||
|
|
||||||
workerPool, err := ants.NewPool(1000)
|
workerPool, err := ants.NewPool(1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool)
|
handler := New(params, cfgMock, treeMock, workerPool)
|
||||||
|
|
||||||
return &handlerContext{
|
return &handlerContext{
|
||||||
key: key,
|
key: key,
|
||||||
|
|
|
@ -2,6 +2,7 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
@ -11,6 +12,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||||
"github.com/valyala/fasthttp"
|
"github.com/valyala/fasthttp"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -102,14 +104,36 @@ func idsToResponse(resp *fasthttp.Response, obj *object.Object) {
|
||||||
|
|
||||||
// HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format.
|
// HeadByAddressOrBucketName handles head requests using simple cid/oid or bucketname/key format.
|
||||||
func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
||||||
test, _ := c.UserValue("oid").(string)
|
cidParam, _ := c.UserValue("cid").(string)
|
||||||
var id oid.ID
|
oidParam, _ := c.UserValue("oid").(string)
|
||||||
|
|
||||||
err := id.DecodeString(test)
|
ctx := utils.GetContextFromRequest(c)
|
||||||
|
log := utils.GetReqLogOrDefault(ctx, h.log).With(
|
||||||
|
zap.String("cid", cidParam),
|
||||||
|
zap.String("oid", oidParam),
|
||||||
|
)
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.byS3Path(c, h.headObject)
|
logAndSendBucketError(c, log, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
|
||||||
|
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) {
|
||||||
|
logAndSendBucketError(c, log, checkS3Err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
req := h.newRequest(c, log)
|
||||||
|
|
||||||
|
var objID oid.ID
|
||||||
|
if checkS3Err == nil {
|
||||||
|
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject)
|
||||||
|
} else if err = objID.DecodeString(oidParam); err == nil {
|
||||||
|
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject)
|
||||||
} else {
|
} else {
|
||||||
h.byNativeAddress(c, h.headObject)
|
logAndSendBucketError(c, log, checkS3Err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -42,16 +42,7 @@ func bearerToken(ctx context.Context) *bearer.Token {
|
||||||
}
|
}
|
||||||
|
|
||||||
func isDir(name string) bool {
|
func isDir(name string) bool {
|
||||||
return strings.HasSuffix(name, "/")
|
return name == "" || strings.HasSuffix(name, "/")
|
||||||
}
|
|
||||||
|
|
||||||
func isObjectID(s string) bool {
|
|
||||||
var objID oid.ID
|
|
||||||
return objID.DecodeString(s) == nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isContainerRoot(key string) bool {
|
|
||||||
return key == ""
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadAttributes(attrs []object.Attribute) map[string]string {
|
func loadAttributes(attrs []object.Attribute) map[string]string {
|
||||||
|
|
16
internal/layer/tree_service.go
Normal file
16
internal/layer/tree_service.go
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
package layer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||||
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TreeService provide interface to interact with tree service using s3 data models.
|
||||||
|
type TreeService interface {
|
||||||
|
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
|
||||||
|
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]tree.NodeResponse, string, error)
|
||||||
|
CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error
|
||||||
|
}
|
|
@ -79,6 +79,11 @@ const (
|
||||||
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
|
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
|
||||||
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
|
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
|
||||||
FailedToUnescapeQuery = "failed to unescape query"
|
FailedToUnescapeQuery = "failed to unescape query"
|
||||||
|
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
|
||||||
|
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
|
||||||
|
SystemNodeHasMultipleIDs = "system node has multiple ids"
|
||||||
|
FailedToRemoveOldSystemNode = "failed to remove old system node"
|
||||||
|
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
|
||||||
ServerReconnecting = "reconnecting server..."
|
ServerReconnecting = "reconnecting server..."
|
||||||
ServerReconnectedSuccessfully = "server reconnected successfully"
|
ServerReconnectedSuccessfully = "server reconnected successfully"
|
||||||
ServerReconnectFailed = "failed to reconnect server"
|
ServerReconnectFailed = "failed to reconnect server"
|
||||||
|
|
|
@ -2,8 +2,6 @@ package frostfs
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
|
@ -59,7 +57,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([
|
||||||
|
|
||||||
nodes, err := w.p.GetNodes(ctx, poolPrm)
|
nodes, err := w.p.GetNodes(ctx, poolPrm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, handleError(err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
res := make([]tree.NodeResponse, len(nodes))
|
res := make([]tree.NodeResponse, len(nodes))
|
||||||
|
@ -78,20 +76,6 @@ func getBearer(ctx context.Context) []byte {
|
||||||
return token.Marshal()
|
return token.Marshal()
|
||||||
}
|
}
|
||||||
|
|
||||||
func handleError(err error) error {
|
|
||||||
if err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if errors.Is(err, treepool.ErrNodeNotFound) {
|
|
||||||
return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error())
|
|
||||||
}
|
|
||||||
if errors.Is(err, treepool.ErrNodeAccessDenied) {
|
|
||||||
return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) {
|
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) {
|
||||||
order := treepool.NoneOrder
|
order := treepool.NoneOrder
|
||||||
if sort {
|
if sort {
|
||||||
|
@ -115,7 +99,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
|
|
||||||
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
|
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, handleError(err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var subtree []tree.NodeResponse
|
var subtree []tree.NodeResponse
|
||||||
|
@ -126,7 +110,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
node, err = subTreeReader.Next()
|
node, err = subTreeReader.Next()
|
||||||
}
|
}
|
||||||
if err != io.EOF {
|
if err != io.EOF {
|
||||||
return nil, handleError(err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return subtree, nil
|
return subtree, nil
|
35
tree/tree.go
35
tree/tree.go
|
@ -6,11 +6,10 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type (
|
||||||
|
@ -46,14 +45,6 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
// ErrNodeNotFound is returned from ServiceClient in case of not found error.
|
|
||||||
ErrNodeNotFound = layer.ErrNodeNotFound
|
|
||||||
|
|
||||||
// ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error.
|
|
||||||
ErrNodeAccessDenied = layer.ErrNodeAccessDenied
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
FileNameKey = "FileName"
|
FileNameKey = "FileName"
|
||||||
settingsFileName = "bucket-settings"
|
settingsFileName = "bucket-settings"
|
||||||
|
@ -118,7 +109,7 @@ func (n *treeNode) FileName() (string, bool) {
|
||||||
return value, ok
|
return value, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) {
|
func newNodeVersion(node NodeResponse) (*data.NodeVersion, error) {
|
||||||
tNode, err := newTreeNode(node)
|
tNode, err := newTreeNode(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("invalid tree node: %w", err)
|
return nil, fmt.Errorf("invalid tree node: %w", err)
|
||||||
|
@ -127,15 +118,11 @@ func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) {
|
||||||
return newNodeVersionFromTreeNode(tNode), nil
|
return newNodeVersionFromTreeNode(tNode), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion {
|
func newNodeVersionFromTreeNode(treeNode *treeNode) *data.NodeVersion {
|
||||||
_, isDeleteMarker := treeNode.Get(isDeleteMarkerKV)
|
version := &data.NodeVersion{
|
||||||
size, _ := treeNode.Get(sizeKV)
|
BaseNodeVersion: data.BaseNodeVersion{
|
||||||
version := &api.NodeVersion{
|
|
||||||
BaseNodeVersion: api.BaseNodeVersion{
|
|
||||||
OID: treeNode.ObjID,
|
OID: treeNode.ObjID,
|
||||||
},
|
},
|
||||||
DeleteMarker: isDeleteMarker,
|
|
||||||
IsPrefixNode: size == "",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return version
|
return version
|
||||||
|
@ -180,7 +167,7 @@ func (m *multiSystemNode) Old() []*treeNode {
|
||||||
return m.nodes[1:]
|
return m.nodes[1:]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) {
|
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
|
||||||
nodes, err := c.GetVersions(ctx, cnrID, objectName)
|
nodes, err := c.GetVersions(ctx, cnrID, objectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -210,7 +197,7 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
|
||||||
return c.service.GetNodes(ctx, p)
|
return c.service.GetNodes(ctx, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error {
|
func (c *Tree) CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -236,7 +223,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
|
||||||
nodes = filterMultipartNodes(nodes)
|
nodes = filterMultipartNodes(nodes)
|
||||||
|
|
||||||
if len(nodes) == 0 {
|
if len(nodes) == 0 {
|
||||||
return nil, ErrNodeNotFound
|
return nil, tree.ErrNodeNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
return newMultiNode(nodes)
|
return newMultiNode(nodes)
|
||||||
|
@ -277,7 +264,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if targetIndexNode == -1 {
|
if targetIndexNode == -1 {
|
||||||
return nil, layer.ErrNodeNotFound
|
return nil, tree.ErrNodeNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
return nodes[targetIndexNode], nil
|
return nodes[targetIndexNode], nil
|
||||||
|
@ -305,7 +292,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
}
|
}
|
||||||
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
|
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, layer.ErrNodeNotFound) {
|
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||||
return nil, "", nil
|
return nil, "", nil
|
||||||
}
|
}
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
|
@ -386,7 +373,7 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(intermediateNodes) == 0 {
|
if len(intermediateNodes) == 0 {
|
||||||
return nil, layer.ErrNodeNotFound
|
return nil, tree.ErrNodeNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
return intermediateNodes, nil
|
return intermediateNodes, nil
|
||||||
|
|
Loading…
Add table
Reference in a new issue