From 5e4a68105d9026cf85222d02cb6adf2291f36c98 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Wed, 4 Dec 2024 12:44:43 +0300 Subject: [PATCH] [#166] Change the check of protocol during get object request Add tree service's GetBucketSettings to use them to check for protocol to use (S3 or native). Also add mock implementations for this and GetLatestVersion methods. Signed-off-by: Nikita Zinkevich --- cmd/http-gw/app.go | 4 +- internal/api/layer/tree_service.go | 22 ---- internal/data/{bucket.go => info.go} | 11 ++ internal/{api => data}/tree.go | 10 +- internal/handler/download.go | 21 +-- internal/handler/handler.go | 28 ++-- internal/handler/handler_test.go | 112 ++++++++++++++-- internal/handler/head.go | 16 +-- internal/layer/tree_service.go | 18 +++ internal/logs/logs.go | 5 + .../{pool_wrapper.go => tree_pool_wrapper.go} | 22 +--- tree/tree.go | 123 +++++++++++------- 12 files changed, 261 insertions(+), 131 deletions(-) delete mode 100644 internal/api/layer/tree_service.go rename internal/data/{bucket.go => info.go} (58%) rename internal/{api => data}/tree.go (70%) create mode 100644 internal/layer/tree_service.go rename internal/service/frostfs/{pool_wrapper.go => tree_pool_wrapper.go} (89%) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 0dd53a6..da40e71 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -499,10 +499,10 @@ func (a *app) Serve() { close(a.webDone) }() - handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool) + handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool) // Configure router. - a.configureRouter(handler) + a.configureRouter(handle) a.startServices() a.initServers(a.ctx) diff --git a/internal/api/layer/tree_service.go b/internal/api/layer/tree_service.go deleted file mode 100644 index beb1e7a..0000000 --- a/internal/api/layer/tree_service.go +++ /dev/null @@ -1,22 +0,0 @@ -package layer - -import ( - "context" - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" -) - -// TreeService provide interface to interact with tree service using s3 data models. -type TreeService interface { - GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) -} - -var ( - // ErrNodeNotFound is returned from Tree service in case of not found error. - ErrNodeNotFound = errors.New("not found") - - // ErrNodeAccessDenied is returned from Tree service in case of access denied error. - ErrNodeAccessDenied = errors.New("access denied") -) diff --git a/internal/data/bucket.go b/internal/data/info.go similarity index 58% rename from internal/data/bucket.go rename to internal/data/info.go index d99ca49..9b38dfd 100644 --- a/internal/data/bucket.go +++ b/internal/data/info.go @@ -4,9 +4,20 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) +const ( + VersioningUnversioned = "Unversioned" + VersioningEnabled = "Enabled" + VersioningSuspended = "Suspended" +) + type BucketInfo struct { Name string // container name from system attribute Zone string // container zone from system attribute CID cid.ID HomomorphicHashDisabled bool } + +// BucketSettings stores settings such as versioning. +type BucketSettings struct { + Versioning string +} diff --git a/internal/api/tree.go b/internal/data/tree.go similarity index 70% rename from internal/api/tree.go rename to internal/data/tree.go index 5b1d608..000734a 100644 --- a/internal/api/tree.go +++ b/internal/data/tree.go @@ -1,4 +1,4 @@ -package api +package data import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -7,12 +7,14 @@ import ( // NodeVersion represent node from tree service. type NodeVersion struct { BaseNodeVersion - DeleteMarker bool - IsPrefixNode bool + IsUnversioned bool } // BaseNodeVersion is minimal node info from tree service. // Basically used for "system" object. type BaseNodeVersion struct { - OID oid.ID + ID uint64 + OID oid.ID + FilePath string + IsDeleteMarker bool } diff --git a/internal/handler/download.go b/internal/handler/download.go index 074c3fd..7566a46 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -4,11 +4,11 @@ import ( "archive/zip" "bufio" "context" + "errors" "fmt" "io" "net/http" "net/url" - "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" @@ -18,6 +18,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -29,7 +30,10 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { downloadParam := c.QueryArgs().GetBool("download") ctx := utils.GetContextFromRequest(c) - log := utils.GetReqLogOrDefault(ctx, h.log) + log := utils.GetReqLogOrDefault(ctx, h.log).With( + zap.String("cid", cidParam), + zap.String("oid", oidParam), + ) bktInfo, err := h.getBucketInfo(ctx, cidParam, log) if err != nil { @@ -37,20 +41,19 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) { return } - s3checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo) - if s3checkErr != nil && !strings.Contains(s3checkErr.Error(), "tree not found") { + _, s3checkErr := h.tree.GetSettingsNode(ctx, bktInfo) + if s3checkErr != nil && !errors.Is(s3checkErr, tree.ErrNodeNotFound) { logAndSendBucketError(c, log, s3checkErr) return } + req := h.newRequest(c, log) + var objID oid.ID if s3checkErr == nil && shouldDownload(oidParam, downloadParam) { - // Receive file via S3 - h.byS3Path(c, bktInfo.CID, h.receiveFile) + h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile) } else if err = objID.DecodeString(oidParam); err == nil { - // Receive file via native protocol - addr := newAddress(bktInfo.CID, objID) - h.receiveFile(ctx, h.newRequest(c, log), addr) + h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile) } else { h.browseIndex(c, s3checkErr != nil) } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 53aad8e..e6c32da 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -6,14 +6,13 @@ import ( "fmt" "io" "net/url" - "strings" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/response" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -21,6 +20,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/panjf2000/ants/v2" "github.com/valyala/fasthttp" @@ -164,7 +164,7 @@ type Handler struct { ownerID *user.ID config Config containerResolver ContainerResolver - tree *tree.Tree + tree layer.TreeService cache *cache.BucketCache workerPool *ants.Pool } @@ -177,7 +177,7 @@ type AppParams struct { Cache *cache.BucketCache } -func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler { +func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler { return &Handler{ log: params.Logger, frostfs: params.FrostFS, @@ -197,27 +197,22 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci err := cnrID.DecodeString(containerID) if err != nil { cnrID, err = h.containerResolver.Resolve(ctx, containerID) - if err != nil && strings.Contains(err.Error(), "not found") { + if err != nil && !errors.Is(err, tree.ErrNodeNotFound) { err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error()) } } return cnrID, err } -func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(context.Context, request, oid.Address)) { - oidParam := c.UserValue("oid").(string) - ctx := utils.GetContextFromRequest(c) - log := utils.GetReqLogOrDefault(ctx, h.log).With( - zap.String("cid", cnrID.EncodeToString()), - zap.String("oid", oidParam), - ) +func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path string, handler func(context.Context, request, oid.Address)) { + c, log := req.RequestCtx, req.log - foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, oidParam) + foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) if err != nil { logAndSendBucketError(c, log, err) return } - if foundOID.DeleteMarker { + if foundOID.IsDeleteMarker { log.Error(logs.ObjectWasDeleted) response.Error(c, "object deleted", fasthttp.StatusNotFound) return @@ -227,6 +222,11 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, cnrID cid.ID, handler func(co handler(ctx, h.newRequest(c, log), addr) } +func (h *Handler) byNativeAddress(ctx context.Context, req request, cnrID cid.ID, objID oid.ID, handler func(context.Context, request, oid.Address)) { + addr := newAddress(cnrID, objID) + handler(ctx, req, addr) +} + func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Context, request, oid.Address)) { cidParam, _ := c.UserValue("cid").(string) key, _ := c.UserValue("attr_key").(string) diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index c5a347f..614388c 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -8,6 +8,7 @@ import ( "io" "mime/multipart" "net/http" + "sort" "testing" "time" @@ -24,6 +25,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/panjf2000/ants/v2" @@ -32,15 +34,97 @@ import ( "go.uber.org/zap" ) -type treeClientMock struct { +type treeServiceMock struct { + settings map[string]*data.BucketSettings + versions map[string]map[string][]*data.NodeVersion + system map[string]map[string]*data.BaseNodeVersion } -func (t *treeClientMock) GetNodes(context.Context, *tree.GetNodesParams) ([]tree.NodeResponse, error) { - return nil, nil +func newTreeService() *treeServiceMock { + return &treeServiceMock{ + settings: make(map[string]*data.BucketSettings), + versions: make(map[string]map[string][]*data.NodeVersion), + system: make(map[string]map[string]*data.BaseNodeVersion), + } } -func (t *treeClientMock) GetSubTree(context.Context, *data.BucketInfo, string, []uint64, uint32, bool) ([]tree.NodeResponse, error) { - return nil, nil +func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]tree.NodeResponse, string, error) { + return nil, "", nil +} + +func (t *treeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { + settings, ok := t.settings[bktInfo.CID.EncodeToString()] + if !ok { + return nil, treepool.ErrNodeNotFound + } + + return settings, nil +} + +func (t *treeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error { + t.settings[bktInfo.CID.EncodeToString()] = settings + return nil +} + +func (t *treeServiceMock) GetLatestVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { + cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()] + if !ok { + return nil, treepool.ErrNodeNotFound + } + + versions, ok := cnrVersionsMap[objectName] + if !ok { + return nil, treepool.ErrNodeNotFound + } + + sort.Slice(versions, func(i, j int) bool { + return versions[i].ID < versions[j].ID + }) + + if len(versions) != 0 { + return versions[len(versions)-1], nil + } + + return nil, treepool.ErrNodeNotFound +} + +func (t *treeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) { + cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] + if !ok { + t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{ + newVersion.FilePath: {newVersion}, + } + return newVersion.ID, nil + } + + versions, ok := cnrVersionsMap[newVersion.FilePath] + if !ok { + cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion} + return newVersion.ID, nil + } + + sort.Slice(versions, func(i, j int) bool { + return versions[i].ID < versions[j].ID + }) + + if len(versions) != 0 { + newVersion.ID = versions[len(versions)-1].ID + 1 + } + + result := versions + + if newVersion.IsUnversioned { + result = make([]*data.NodeVersion, 0, len(versions)) + for _, node := range versions { + if !node.IsUnversioned { + result = append(result, node) + } + } + } + + cnrVersionsMap[newVersion.FilePath] = append(result, newVersion) + + return newVersion.ID, nil } type configMock struct { @@ -84,7 +168,7 @@ type handlerContext struct { h *Handler frostfs *TestFrostFS - tree *treeClientMock + tree *treeServiceMock cfg *configMock } @@ -125,14 +209,14 @@ func prepareHandlerContext() (*handlerContext, error) { }), } - treeMock := &treeClientMock{} + treeMock := newTreeService() cfgMock := &configMock{} workerPool, err := ants.NewPool(1) if err != nil { return nil, err } - handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool) + handler := New(params, cfgMock, treeMock, workerPool) return &handlerContext{ key: key, @@ -199,6 +283,18 @@ func TestBasic(t *testing.T) { require.NoError(t, err) obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] + + objID, ok := obj.ID() + require.True(t, ok) + + _, err = hc.tree.AddVersion(context.TODO(), &data.BucketInfo{CID: cnrID}, &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + OID: objID, + FilePath: objFileName, + }, + }) + require.NoError(t, err) + attr := object.NewAttribute() attr.SetKey(object.AttributeFilePath) attr.SetValue(objFileName) diff --git a/internal/handler/head.go b/internal/handler/head.go index 6385c37..4e78927 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -2,16 +2,17 @@ package handler import ( "context" + "errors" "io" "net/http" "strconv" - "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" "github.com/valyala/fasthttp" "go.uber.org/zap" ) @@ -117,20 +118,19 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) { logAndSendBucketError(c, log, err) return } - checkErr := h.tree.CheckSettingsNodeExist(ctx, bktInfo) - if checkErr != nil && !strings.Contains(checkErr.Error(), "tree not found") { + _, checkErr := h.tree.GetSettingsNode(ctx, bktInfo) + if checkErr != nil && !errors.Is(checkErr, tree.ErrNodeNotFound) { logAndSendBucketError(c, log, checkErr) return } + req := h.newRequest(c, log) + var objID oid.ID if checkErr == nil { - // Head object via s3 protocol - h.byS3Path(c, bktInfo.CID, h.headObject) + h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject) } else if err = objID.DecodeString(oidParam); err == nil { - // Head object via native protocol - addr := newAddress(bktInfo.CID, objID) - h.headObject(ctx, h.newRequest(c, log), addr) + h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { logAndSendBucketError(c, log, checkErr) return diff --git a/internal/layer/tree_service.go b/internal/layer/tree_service.go new file mode 100644 index 0000000..eee61a2 --- /dev/null +++ b/internal/layer/tree_service.go @@ -0,0 +1,18 @@ +package layer + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +// TreeService provide interface to interact with tree service using s3 data models. +type TreeService interface { + GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) + // GetSettingsNode retrieves the settings node from the tree service and form data.BucketSettings. + // If tree node is not found returns ErrNodeNotFound error. + GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) + GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]tree.NodeResponse, string, error) +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 4dfa21f..5ba321d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -79,6 +79,11 @@ const ( InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go FailedToUnescapeQuery = "failed to unescape query" + FailedToParseAddressInTreeNode = "failed to parse object addr in tree node" + SettingsNodeInvalidOwnerKey = "settings node: invalid owner key" + SystemNodeHasMultipleIDs = "system node has multiple ids" + FailedToRemoveOldSystemNode = "failed to remove old system node" + BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids" ServerReconnecting = "reconnecting server..." ServerReconnectedSuccessfully = "server reconnected successfully" ServerReconnectFailed = "failed to reconnect server" diff --git a/internal/service/frostfs/pool_wrapper.go b/internal/service/frostfs/tree_pool_wrapper.go similarity index 89% rename from internal/service/frostfs/pool_wrapper.go rename to internal/service/frostfs/tree_pool_wrapper.go index b978d73..56d428e 100644 --- a/internal/service/frostfs/pool_wrapper.go +++ b/internal/service/frostfs/tree_pool_wrapper.go @@ -2,8 +2,6 @@ package frostfs import ( "context" - "errors" - "fmt" "io" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" @@ -59,7 +57,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([ nodes, err := w.p.GetNodes(ctx, poolPrm) if err != nil { - return nil, handleError(err) + return nil, err } res := make([]tree.NodeResponse, len(nodes)) @@ -78,20 +76,6 @@ func getBearer(ctx context.Context) []byte { return token.Marshal() } -func handleError(err error) error { - if err == nil { - return nil - } - if errors.Is(err, treepool.ErrNodeNotFound) { - return fmt.Errorf("%w: %s", tree.ErrNodeNotFound, err.Error()) - } - if errors.Is(err, treepool.ErrNodeAccessDenied) { - return fmt.Errorf("%w: %s", tree.ErrNodeAccessDenied, err.Error()) - } - - return err -} - func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) { order := treepool.NoneOrder if sort { @@ -115,7 +99,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, subTreeReader, err := w.p.GetSubTree(ctx, poolPrm) if err != nil { - return nil, handleError(err) + return nil, err } var subtree []tree.NodeResponse @@ -126,7 +110,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, node, err = subTreeReader.Next() } if err != io.EOF { - return nil, handleError(err) + return nil, err } return subtree, nil diff --git a/tree/tree.go b/tree/tree.go index 40209a5..c678fb0 100644 --- a/tree/tree.go +++ b/tree/tree.go @@ -6,16 +6,17 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" + "go.uber.org/zap" ) type ( Tree struct { service ServiceClient + log *zap.Logger } // ServiceClient is a client to interact with tree service. @@ -26,8 +27,12 @@ type ( } treeNode struct { - ObjID oid.ID - Meta map[string]string + ID []uint64 + ParentID []uint64 + ObjID oid.ID + TimeStamp []uint64 + Size uint64 + Meta map[string]string } multiSystemNode struct { @@ -46,35 +51,36 @@ type ( } ) -var ( - // ErrNodeNotFound is returned from ServiceClient in case of not found error. - ErrNodeNotFound = layer.ErrNodeNotFound - - // ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error. - ErrNodeAccessDenied = layer.ErrNodeAccessDenied -) - const ( - FileNameKey = "FileName" - settingsFileName = "bucket-settings" + FileNameKey = "FileName" - oidKV = "OID" - uploadIDKV = "UploadId" - sizeKV = "Size" + versioningKV = "Versioning" + cannedACLKV = "cannedACL" + ownerKeyKV = "ownerKey" + lockConfigurationKV = "LockConfiguration" + oidKV = "OID" + isUnversionedKV = "IsUnversioned" + uploadIDKV = "UploadId" + sizeKV = "Size" // keys for delete marker nodes. isDeleteMarkerKV = "IsDeleteMarker" + settingsFileName = "bucket-settings" + // versionTree -- ID of a tree with object versions. versionTree = "version" - systemTree = "system" + + // systemTree -- ID of a tree with system objects + // i.e. bucket settings with versioning and lock configuration, cors. + systemTree = "system" separator = "/" ) // NewTree creates instance of Tree using provided address and create grpc connection. -func NewTree(service ServiceClient) *Tree { - return &Tree{service: service} +func NewTree(service ServiceClient, log *zap.Logger) *Tree { + return &Tree{service: service, log: log} } type Meta interface { @@ -118,24 +124,44 @@ func (n *treeNode) FileName() (string, bool) { return value, ok } -func newNodeVersion(node NodeResponse) (*api.NodeVersion, error) { +func (n *treeNode) GetLatestNodeIndex() int { + var ( + maxTimestamp uint64 + index int + ) + + for i, timestamp := range n.TimeStamp { + if timestamp > maxTimestamp { + maxTimestamp = timestamp + index = i + } + } + + return index +} + +func (n *treeNode) IsSplit() bool { + return len(n.ID) != 1 || len(n.ParentID) != 1 || len(n.TimeStamp) != 1 +} + +func newNodeVersion(node NodeResponse, objectName string) (*data.NodeVersion, error) { tNode, err := newTreeNode(node) if err != nil { return nil, fmt.Errorf("invalid tree node: %w", err) } - return newNodeVersionFromTreeNode(tNode), nil + return newNodeVersionFromTreeNode(objectName, tNode), nil } -func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion { - _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) - size, _ := treeNode.Get(sizeKV) - version := &api.NodeVersion{ - BaseNodeVersion: api.BaseNodeVersion{ - OID: treeNode.ObjID, +func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeVersion { + _, isUnversioned := treeNode.Get(isUnversionedKV) + + version := &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + OID: treeNode.ObjID, + FilePath: filePath, }, - DeleteMarker: isDeleteMarker, - IsPrefixNode: size == "", + IsUnversioned: isUnversioned, } return version @@ -180,7 +206,7 @@ func (m *multiSystemNode) Old() []*treeNode { return m.nodes[1:] } -func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) { +func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { nodes, err := c.GetVersions(ctx, cnrID, objectName) if err != nil { return nil, err @@ -191,7 +217,7 @@ func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName s return nil, err } - return newNodeVersion(latestNode) + return newNodeVersion(latestNode, objectName) } func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]NodeResponse, error) { @@ -210,15 +236,6 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string return c.service.GetNodes(ctx, p) } -func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error { - _, err := c.getSystemNode(ctx, bktInfo, settingsFileName) - if err != nil { - return err - } - - return nil -} - func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) { p := &GetNodesParams{ CnrID: bktInfo.CID, @@ -236,7 +253,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name nodes = filterMultipartNodes(nodes) if len(nodes) == 0 { - return nil, ErrNodeNotFound + return nil, tree.ErrNodeNotFound } return newMultiNode(nodes) @@ -277,7 +294,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) { } if targetIndexNode == -1 { - return nil, layer.ErrNodeNotFound + return nil, tree.ErrNodeNotFound } return nodes[targetIndexNode], nil @@ -305,7 +322,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, } subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) if err != nil { - if errors.Is(err, layer.ErrNodeNotFound) { + if errors.Is(err, tree.ErrNodeNotFound) { return nil, "", nil } return nil, "", err @@ -386,12 +403,28 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr } if len(intermediateNodes) == 0 { - return nil, layer.ErrNodeNotFound + return nil, tree.ErrNodeNotFound } return intermediateNodes, nil } +func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) { + multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName) + if err != nil { + return nil, fmt.Errorf("couldn't get node: %w", err) + } + + node := multiNode.Latest() + + settings := &data.BucketSettings{Versioning: data.VersioningUnversioned} + if versioningValue, ok := node.Get(versioningKV); ok { + settings.Versioning = versioningValue + } + + return settings, nil +} + func GetFilename(node NodeResponse) string { for _, kv := range node.GetMeta() { if kv.GetKey() == FileNameKey {