[#166] Change the check of protocol during get object request
Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this and GetLatestVersion methods. Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
parent
762bbe86b7
commit
5e4a68105d
12 changed files with 261 additions and 131 deletions
|
@ -499,10 +499,10 @@ func (a *app) Serve() {
|
|||
close(a.webDone)
|
||||
}()
|
||||
|
||||
handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool)
|
||||
handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool)
|
||||
|
||||
// Configure router.
|
||||
a.configureRouter(handler)
|
||||
a.configureRouter(handle)
|
||||
|
||||
a.startServices()
|
||||
a.initServers(a.ctx)
|
||||
|
|
|
@ -1,22 +0,0 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
)
|
||||
|
||||
// TreeService provide interface to interact with tree service using s3 data models.
|
||||
type TreeService interface {
|
||||
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error)
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrNodeNotFound is returned from Tree service in case of not found error.
|
||||
ErrNodeNotFound = errors.New("not found")
|
||||
|
||||
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
|
||||
ErrNodeAccessDenied = errors.New("access denied")
|
||||
)
|
|
@ -4,9 +4,20 @@ import (
|
|||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
)
|
||||
|
||||
const (
|
||||
VersioningUnversioned = "Unversioned"
|
||||
VersioningEnabled = "Enabled"
|
||||
VersioningSuspended = "Suspended"
|
||||
)
|
||||
|
||||
type BucketInfo struct {
|
||||
Name string // container name from system attribute
|
||||
Zone string // container zone from system attribute
|
||||
CID cid.ID
|
||||
HomomorphicHashDisabled bool
|
||||
}
|
||||
|
||||
// BucketSettings stores settings such as versioning.
|
||||
type BucketSettings struct {
|
||||
Versioning string
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package api
|
||||
package data
|
||||
|
||||
import (
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
@ -7,12 +7,14 @@ import (
|
|||
// NodeVersion represent node from tree service.
|
||||
type NodeVersion struct {
|
||||
BaseNodeVersion
|
||||
DeleteMarker bool
|
||||
IsPrefixNode bool
|
||||
IsUnversioned bool
|
||||
}
|
||||
|
||||
// BaseNodeVersion is minimal node info from tree service.
|
||||
// Basically used for "system" object.
|
||||
type BaseNodeVersion struct {
|
||||
OID oid.ID
|
||||
ID uint64
|
||||
OID oid.ID
|
||||
FilePath string
|
||||
IsDeleteMarker bool
|
||||
}
|
|
@ -4,11 +4,11 @@ import (
|
|||
"archive/zip"
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
|
@ -18,6 +18,7 @@ import (
|
|||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -29,7 +30,10 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
|||
downloadParam := c.QueryArgs().GetBool("download")
|
||||
|
||||
ctx := utils.GetContextFromRequest(c)
|
||||
log := utils.GetReqLogOrDefault(ctx, h.log)
|
||||
log := utils.GetReqLogOrDefault(ctx, h.log).With(
|
||||
zap.String("cid", cidParam),
|
||||
zap.String("oid", oidParam),
|
||||
)
|
||||
|
||||
bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
|
||||
if err != nil {
|
||||
|
@ -37,20 +41,19 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
|||
return
|
||||
}
|
||||
|
||||
s3checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo)
|
||||
if s3checkErr != nil && !strings.Contains(s3checkErr.Error(), "tree not found") {
|
||||
_, s3checkErr := h.tree.GetSettingsNode(ctx, bktInfo)
|
||||
if s3checkErr != nil && !errors.Is(s3checkErr, tree.ErrNodeNotFound) {
|
||||
logAndSendBucketError(c, log, s3checkErr)
|
||||
return
|
||||
}
|
||||
|
||||
req := h.newRequest(c, log)
|
||||
|
||||
var objID oid.ID
|
||||
if s3checkErr == nil && shouldDownload(oidParam, downloadParam) {
|
||||
// Receive file via S3
|
||||
h.byS3Path(c, bktInfo.CID, h.receiveFile)
|
||||
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile)
|
||||
} else if err = objID.DecodeString(oidParam); err == nil {
|
||||
// Receive file via native protocol
|
||||
addr := newAddress(bktInfo.CID, objID)
|
||||
h.receiveFile(ctx, h.newRequest(c, log), addr)
|
||||
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile)
|
||||
} else {
|
||||
h.browseIndex(c, s3checkErr != nil)
|
||||
}
|
||||
|
|
|
@ -6,14 +6,13 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
|
@ -21,6 +20,7 @@ import (
|
|||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
"github.com/valyala/fasthttp"
|
||||
|
@ -164,7 +164,7 @@ type Handler struct {
|
|||
ownerID *user.ID
|
||||
config Config
|
||||
containerResolver ContainerResolver
|
||||
tree *tree.Tree
|
||||
tree layer.TreeService
|
||||
cache *cache.BucketCache
|
||||
workerPool *ants.Pool
|
||||
}
|
||||
|
@ -177,7 +177,7 @@ type AppParams struct {
|
|||
Cache *cache.BucketCache
|
||||
}
|
||||
|
||||
func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler {
|
||||
func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
|
||||
return &Handler{
|
||||
log: params.Logger,
|
||||
frostfs: params.FrostFS,
|
||||
|
@ -197,27 +197,22 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci
|
|||
err := cnrID.DecodeString(containerID)
|
||||
if err != nil {
|
||||
cnrID, err = h.containerResolver.Resolve(ctx, containerID)
|
||||
if err != nil && strings.Contains(err.Error(), "not found") {
|
||||
if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
|
||||
err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error())
|
||||
}
|
||||
}
|
||||
return cnrID, err
|
||||
}
|
||||
|
||||
func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(context.Context, request, oid.Address)) {
|
||||
oidParam := c.UserValue("oid").(string)
|
||||
ctx := utils.GetContextFromRequest(c)
|
||||
log := utils.GetReqLogOrDefault(ctx, h.log).With(
|
||||
zap.String("cid", cnrID.EncodeToString()),
|
||||
zap.String("oid", oidParam),
|
||||
)
|
||||
func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) {
|
||||
c, log := req.RequestCtx, req.log
|
||||
|
||||
foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, oidParam)
|
||||
foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path)
|
||||
if err != nil {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
if foundOID.DeleteMarker {
|
||||
if foundOID.IsDeleteMarker {
|
||||
log.Error(logs.ObjectWasDeleted)
|
||||
response.Error(c, "object deleted", fasthttp.StatusNotFound)
|
||||
return
|
||||
|
@ -227,6 +222,11 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(co
|
|||
handler(ctx, h.newRequest(c, log), addr)
|
||||
}
|
||||
|
||||
func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) {
|
||||
addr := newAddress(cnrID, objID)
|
||||
handler(ctx, req, addr)
|
||||
}
|
||||
|
||||
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) {
|
||||
cidParam, _ := c.UserValue("cid").(string)
|
||||
key, _ := c.UserValue("attr_key").(string)
|
||||
|
|
|
@ -8,6 +8,7 @@ import (
|
|||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
@ -24,6 +25,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
|
@ -32,15 +34,97 @@ import (
|
|||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type treeClientMock struct {
|
||||
type treeServiceMock struct {
|
||||
settings map[string]*data.BucketSettings
|
||||
versions map[string]map[string][]*data.NodeVersion
|
||||
system map[string]map[string]*data.BaseNodeVersion
|
||||
}
|
||||
|
||||
func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) {
|
||||
return nil, nil
|
||||
func newTreeService() *treeServiceMock {
|
||||
return &treeServiceMock{
|
||||
settings: make(map[string]*data.BucketSettings),
|
||||
versions: make(map[string]map[string][]*data.NodeVersion),
|
||||
system: make(map[string]map[string]*data.BaseNodeVersion),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) {
|
||||
return nil, nil
|
||||
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]tree.NodeResponse, string, error) {
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
func (t *treeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
||||
settings, ok := t.settings[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, treepool.ErrNodeNotFound
|
||||
}
|
||||
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func (t *treeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
|
||||
t.settings[bktInfo.CID.EncodeToString()] = settings
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *treeServiceMock) GetLatestVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
|
||||
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, treepool.ErrNodeNotFound
|
||||
}
|
||||
|
||||
versions, ok := cnrVersionsMap[objectName]
|
||||
if !ok {
|
||||
return nil, treepool.ErrNodeNotFound
|
||||
}
|
||||
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
return versions[i].ID < versions[j].ID
|
||||
})
|
||||
|
||||
if len(versions) != 0 {
|
||||
return versions[len(versions)-1], nil
|
||||
}
|
||||
|
||||
return nil, treepool.ErrNodeNotFound
|
||||
}
|
||||
|
||||
func (t *treeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) {
|
||||
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{
|
||||
newVersion.FilePath: {newVersion},
|
||||
}
|
||||
return newVersion.ID, nil
|
||||
}
|
||||
|
||||
versions, ok := cnrVersionsMap[newVersion.FilePath]
|
||||
if !ok {
|
||||
cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion}
|
||||
return newVersion.ID, nil
|
||||
}
|
||||
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
return versions[i].ID < versions[j].ID
|
||||
})
|
||||
|
||||
if len(versions) != 0 {
|
||||
newVersion.ID = versions[len(versions)-1].ID + 1
|
||||
}
|
||||
|
||||
result := versions
|
||||
|
||||
if newVersion.IsUnversioned {
|
||||
result = make([]*data.NodeVersion, 0, len(versions))
|
||||
for _, node := range versions {
|
||||
if !node.IsUnversioned {
|
||||
result = append(result, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cnrVersionsMap[newVersion.FilePath] = append(result, newVersion)
|
||||
|
||||
return newVersion.ID, nil
|
||||
}
|
||||
|
||||
type configMock struct {
|
||||
|
@ -84,7 +168,7 @@ type handlerContext struct {
|
|||
|
||||
h *Handler
|
||||
frostfs *TestFrostFS
|
||||
tree *treeClientMock
|
||||
tree *treeServiceMock
|
||||
cfg *configMock
|
||||
}
|
||||
|
||||
|
@ -125,14 +209,14 @@ func prepareHandlerContext() (*handlerContext, error) {
|
|||
}),
|
||||
}
|
||||
|
||||
treeMock := &treeClientMock{}
|
||||
treeMock := newTreeService()
|
||||
cfgMock := &configMock{}
|
||||
|
||||
workerPool, err := ants.NewPool(1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool)
|
||||
handler := New(params, cfgMock, treeMock, workerPool)
|
||||
|
||||
return &handlerContext{
|
||||
key: key,
|
||||
|
@ -199,6 +283,18 @@ func TestBasic(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
|
||||
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
|
||||
|
||||
objID, ok := obj.ID()
|
||||
require.True(t, ok)
|
||||
|
||||
_, err = hc.tree.AddVersion(context.TODO(), &data.BucketInfo{CID: cnrID}, &data.NodeVersion{
|
||||
BaseNodeVersion: data.BaseNodeVersion{
|
||||
OID: objID,
|
||||
FilePath: objFileName,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
attr := object.NewAttribute()
|
||||
attr.SetKey(object.AttributeFilePath)
|
||||
attr.SetValue(objFileName)
|
||||
|
|
|
@ -2,16 +2,17 @@ package handler
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -117,20 +118,19 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
|||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo)
|
||||
if checkErr != nil && !strings.Contains(checkErr.Error(), "tree not found") {
|
||||
_, checkErr := h.tree.GetSettingsNode(ctx, bktInfo)
|
||||
if checkErr != nil && !errors.Is(checkErr, tree.ErrNodeNotFound) {
|
||||
logAndSendBucketError(c, log, checkErr)
|
||||
return
|
||||
}
|
||||
|
||||
req := h.newRequest(c, log)
|
||||
|
||||
var objID oid.ID
|
||||
if checkErr == nil {
|
||||
// Head object via s3 protocol
|
||||
h.byS3Path(c, bktInfo.CID, h.headObject)
|
||||
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject)
|
||||
} else if err = objID.DecodeString(oidParam); err == nil {
|
||||
// Head object via native protocol
|
||||
addr := newAddress(bktInfo.CID, objID)
|
||||
h.headObject(ctx, h.newRequest(c, log), addr)
|
||||
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject)
|
||||
} else {
|
||||
logAndSendBucketError(c, log, checkErr)
|
||||
return
|
||||
|
|
18
internal/layer/tree_service.go
Normal file
18
internal/layer/tree_service.go
Normal file
|
@ -0,0 +1,18 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
)
|
||||
|
||||
// TreeService provide interface to interact with tree service using s3 data models.
|
||||
type TreeService interface {
|
||||
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
|
||||
// GetSettingsNode retrieves the settings node from the tree service and form data.BucketSettings.
|
||||
// If tree node is not found returns ErrNodeNotFound error.
|
||||
GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
|
||||
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]tree.NodeResponse, string, error)
|
||||
}
|
|
@ -79,6 +79,11 @@ const (
|
|||
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
|
||||
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
|
||||
FailedToUnescapeQuery = "failed to unescape query"
|
||||
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
|
||||
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
|
||||
SystemNodeHasMultipleIDs = "system node has multiple ids"
|
||||
FailedToRemoveOldSystemNode = "failed to remove old system node"
|
||||
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
|
||||
ServerReconnecting = "reconnecting server..."
|
||||
ServerReconnectedSuccessfully = "server reconnected successfully"
|
||||
ServerReconnectFailed = "failed to reconnect server"
|
||||
|
|
|
@ -2,8 +2,6 @@ package frostfs
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
|
@ -59,7 +57,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([
|
|||
|
||||
nodes, err := w.p.GetNodes(ctx, poolPrm)
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := make([]tree.NodeResponse, len(nodes))
|
||||
|
@ -78,20 +76,6 @@ func getBearer(ctx context.Context) []byte {
|
|||
return token.Marshal()
|
||||
}
|
||||
|
||||
func handleError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, treepool.ErrNodeNotFound) {
|
||||
return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error())
|
||||
}
|
||||
if errors.Is(err, treepool.ErrNodeAccessDenied) {
|
||||
return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error())
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) {
|
||||
order := treepool.NoneOrder
|
||||
if sort {
|
||||
|
@ -115,7 +99,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
|
|||
|
||||
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var subtree []tree.NodeResponse
|
||||
|
@ -126,7 +110,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
|
|||
node, err = subTreeReader.Next()
|
||||
}
|
||||
if err != io.EOF {
|
||||
return nil, handleError(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return subtree, nil
|
123
tree/tree.go
123
tree/tree.go
|
@ -6,16 +6,17 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type (
|
||||
Tree struct {
|
||||
service ServiceClient
|
||||
log *zap.Logger
|
||||
}
|
||||
|
||||
// ServiceClient is a client to interact with tree service.
|
||||
|
@ -26,8 +27,12 @@ type (
|
|||
}
|
||||
|
||||
treeNode struct {
|
||||
ObjID oid.ID
|
||||
Meta map[string]string
|
||||
ID []uint64
|
||||
ParentID []uint64
|
||||
ObjID oid.ID
|
||||
TimeStamp []uint64
|
||||
Size uint64
|
||||
Meta map[string]string
|
||||
}
|
||||
|
||||
multiSystemNode struct {
|
||||
|
@ -46,35 +51,36 @@ type (
|
|||
}
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNodeNotFound is returned from ServiceClient in case of not found error.
|
||||
ErrNodeNotFound = layer.ErrNodeNotFound
|
||||
|
||||
// ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error.
|
||||
ErrNodeAccessDenied = layer.ErrNodeAccessDenied
|
||||
)
|
||||
|
||||
const (
|
||||
FileNameKey = "FileName"
|
||||
settingsFileName = "bucket-settings"
|
||||
FileNameKey = "FileName"
|
||||
|
||||
oidKV = "OID"
|
||||
uploadIDKV = "UploadId"
|
||||
sizeKV = "Size"
|
||||
versioningKV = "Versioning"
|
||||
cannedACLKV = "cannedACL"
|
||||
ownerKeyKV = "ownerKey"
|
||||
lockConfigurationKV = "LockConfiguration"
|
||||
oidKV = "OID"
|
||||
|
||||
isUnversionedKV = "IsUnversioned"
|
||||
uploadIDKV = "UploadId"
|
||||
sizeKV = "Size"
|
||||
// keys for delete marker nodes.
|
||||
isDeleteMarkerKV = "IsDeleteMarker"
|
||||
|
||||
settingsFileName = "bucket-settings"
|
||||
|
||||
// versionTree -- ID of a tree with object versions.
|
||||
versionTree = "version"
|
||||
systemTree = "system"
|
||||
|
||||
// systemTree -- ID of a tree with system objects
|
||||
// i.e. bucket settings with versioning and lock configuration, cors.
|
||||
systemTree = "system"
|
||||
|
||||
separator = "/"
|
||||
)
|
||||
|
||||
// NewTree creates instance of Tree using provided address and create grpc connection.
|
||||
func NewTree(service ServiceClient) *Tree {
|
||||
return &Tree{service: service}
|
||||
func NewTree(service ServiceClient, log *zap.Logger) *Tree {
|
||||
return &Tree{service: service, log: log}
|
||||
}
|
||||
|
||||
type Meta interface {
|
||||
|
@ -118,24 +124,44 @@ func (n *treeNode) FileName() (string, bool) {
|
|||
return value, ok
|
||||
}
|
||||
|
||||
func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) {
|
||||
func (n *treeNode) GetLatestNodeIndex() int {
|
||||
var (
|
||||
maxTimestamp uint64
|
||||
index int
|
||||
)
|
||||
|
||||
for i, timestamp := range n.TimeStamp {
|
||||
if timestamp > maxTimestamp {
|
||||
maxTimestamp = timestamp
|
||||
index = i
|
||||
}
|
||||
}
|
||||
|
||||
return index
|
||||
}
|
||||
|
||||
func (n *treeNode) IsSplit() bool {
|
||||
return len(n.ID) != 1 || len(n.ParentID) != 1 || len(n.TimeStamp) != 1
|
||||
}
|
||||
|
||||
func newNodeVersion(node NodeResponse, objectName string) (*data.NodeVersion, error) {
|
||||
tNode, err := newTreeNode(node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid tree node: %w", err)
|
||||
}
|
||||
|
||||
return newNodeVersionFromTreeNode(tNode), nil
|
||||
return newNodeVersionFromTreeNode(objectName, tNode), nil
|
||||
}
|
||||
|
||||
func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion {
|
||||
_, isDeleteMarker := treeNode.Get(isDeleteMarkerKV)
|
||||
size, _ := treeNode.Get(sizeKV)
|
||||
version := &api.NodeVersion{
|
||||
BaseNodeVersion: api.BaseNodeVersion{
|
||||
OID: treeNode.ObjID,
|
||||
func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeVersion {
|
||||
_, isUnversioned := treeNode.Get(isUnversionedKV)
|
||||
|
||||
version := &data.NodeVersion{
|
||||
BaseNodeVersion: data.BaseNodeVersion{
|
||||
OID: treeNode.ObjID,
|
||||
FilePath: filePath,
|
||||
},
|
||||
DeleteMarker: isDeleteMarker,
|
||||
IsPrefixNode: size == "",
|
||||
IsUnversioned: isUnversioned,
|
||||
}
|
||||
|
||||
return version
|
||||
|
@ -180,7 +206,7 @@ func (m *multiSystemNode) Old() []*treeNode {
|
|||
return m.nodes[1:]
|
||||
}
|
||||
|
||||
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) {
|
||||
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
|
||||
nodes, err := c.GetVersions(ctx, cnrID, objectName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -191,7 +217,7 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s
|
|||
return nil, err
|
||||
}
|
||||
|
||||
return newNodeVersion(latestNode)
|
||||
return newNodeVersion(latestNode, objectName)
|
||||
}
|
||||
|
||||
func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) {
|
||||
|
@ -210,15 +236,6 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
|
|||
return c.service.GetNodes(ctx, p)
|
||||
}
|
||||
|
||||
func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) {
|
||||
p := &GetNodesParams{
|
||||
CnrID: bktInfo.CID,
|
||||
|
@ -236,7 +253,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
|
|||
nodes = filterMultipartNodes(nodes)
|
||||
|
||||
if len(nodes) == 0 {
|
||||
return nil, ErrNodeNotFound
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
return newMultiNode(nodes)
|
||||
|
@ -277,7 +294,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
|
|||
}
|
||||
|
||||
if targetIndexNode == -1 {
|
||||
return nil, layer.ErrNodeNotFound
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
return nodes[targetIndexNode], nil
|
||||
|
@ -305,7 +322,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
|
|||
}
|
||||
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
|
||||
if err != nil {
|
||||
if errors.Is(err, layer.ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, "", nil
|
||||
}
|
||||
return nil, "", err
|
||||
|
@ -386,12 +403,28 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
|
|||
}
|
||||
|
||||
if len(intermediateNodes) == 0 {
|
||||
return nil, layer.ErrNodeNotFound
|
||||
return nil, tree.ErrNodeNotFound
|
||||
}
|
||||
|
||||
return intermediateNodes, nil
|
||||
}
|
||||
|
||||
func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
||||
multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get node: %w", err)
|
||||
}
|
||||
|
||||
node := multiNode.Latest()
|
||||
|
||||
settings := &data.BucketSettings{Versioning: data.VersioningUnversioned}
|
||||
if versioningValue, ok := node.Get(versioningKV); ok {
|
||||
settings.Versioning = versioningValue
|
||||
}
|
||||
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func GetFilename(node NodeResponse) string {
|
||||
for _, kv := range node.GetMeta() {
|
||||
if kv.GetKey() == FileNameKey {
|
||||
|
|
Loading…
Reference in a new issue