From 9c74cca9af2c3842f2346c144649161d97d83455 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 20 May 2022 18:02:00 +0300 Subject: [PATCH] [#420] Using tree service to list object versions Signed-off-by: Denis Kirillov --- api/cache/objectslist.go | 25 ++++ api/data/info.go | 7 +- api/data/tree.go | 37 +++++ api/layer/layer.go | 17 ++- api/layer/object.go | 59 +++++--- api/layer/system_object.go | 2 +- api/layer/tree_service.go | 24 +--- api/layer/versioning.go | 12 +- api/layer/versioning_test.go | 12 +- internal/neofs/tree.go | 202 +++++++++++++++++++++------ internal/neofstest/tree/tree_mock.go | 112 +++++++++++---- 11 files changed, 374 insertions(+), 135 deletions(-) create mode 100644 api/data/tree.go diff --git a/api/cache/objectslist.go b/api/cache/objectslist.go index 7eed1bd17..6fbcd7498 100644 --- a/api/cache/objectslist.go +++ b/api/cache/objectslist.go @@ -6,6 +6,7 @@ import ( "time" "github.com/bluele/gcache" + "github.com/nspcc-dev/neofs-s3-gw/api/data" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" "go.uber.org/zap" @@ -78,6 +79,21 @@ func (l *ObjectsListCache) Get(key ObjectsListKey) []oid.ID { return result } +// GetVersions returns a list of ObjectInfo. +func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion { + entry, err := l.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.([]*data.NodeVersion) + if !ok { + return nil + } + + return result +} + // Put puts a list of objects to cache. func (l *ObjectsListCache) Put(key ObjectsListKey, oids []oid.ID) error { if len(oids) == 0 { @@ -87,6 +103,15 @@ func (l *ObjectsListCache) Put(key ObjectsListKey, oids []oid.ID) error { return l.cache.Set(key, oids) } +// PutVersions puts a list of object versions to cache. +func (l *ObjectsListCache) PutVersions(key ObjectsListKey, versions []*data.NodeVersion) error { + if len(versions) == 0 { + return fmt.Errorf("list versions is empty, cid: %s, prefix: %s", key.cid, key.prefix) + } + + return l.cache.Set(key, versions) +} + // CleanCacheEntriesContainingObject deletes entries containing specified object. func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string, cnr cid.ID) { cidStr := cnr.EncodeToString() diff --git a/api/data/info.go b/api/data/info.go index 9ca034cee..99f644e01 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -28,9 +28,10 @@ type ( // ObjectInfo holds S3 object data. ObjectInfo struct { - ID oid.ID - CID cid.ID - IsDir bool + ID oid.ID + CID cid.ID + IsDir bool + IsDeleteMarker bool Bucket string Name string diff --git a/api/data/tree.go b/api/data/tree.go new file mode 100644 index 000000000..446286bef --- /dev/null +++ b/api/data/tree.go @@ -0,0 +1,37 @@ +package data + +import ( + "time" + + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + "github.com/nspcc-dev/neofs-sdk-go/user" +) + +// NodeVersion represent node from tree service. +type NodeVersion struct { + BaseNodeVersion + DeleteMarker *DeleteMarkerInfo + IsUnversioned bool +} + +// DeleteMarkerInfo is used to save object info if node in the tree service is delete marker. +// We need this information because the "delete marker" object is no longer stored in NeoFS. +type DeleteMarkerInfo struct { + FilePath string + Created time.Time + Owner user.ID +} + +// ExtendedObjectInfo contains additional node info to be able to sort versions by timestamp. +type ExtendedObjectInfo struct { + ObjectInfo *ObjectInfo + NodeVersion *NodeVersion +} + +// BaseNodeVersion is minimal node info from tree service. +// Basically used for "system" object. +type BaseNodeVersion struct { + ID uint64 + OID oid.ID + Timestamp uint64 +} diff --git a/api/layer/layer.go b/api/layer/layer.go index 5577e043d..09f14ef49 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -8,6 +8,7 @@ import ( "io" "net/url" "strings" + "time" "github.com/nats-io/nats.go" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -562,12 +563,16 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings } obj.DeleteMarkVersion = unversionedObjectVersionID - newVersion := &NodeVersion{ - BaseNodeVersion: BaseNodeVersion{ + newVersion := &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ OID: *randOID, }, - IsDeleteMarker: true, - IsUnversioned: true, + DeleteMarker: &data.DeleteMarkerInfo{ + FilePath: obj.Name, + Created: time.Now(), + Owner: n.Owner(ctx), + }, + IsUnversioned: true, } if len(obj.VersionID) == 0 && settings.VersioningEnabled { @@ -595,14 +600,14 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings return obj } -func (n *layer) removeVersionIfFound(ctx context.Context, bkt *data.BucketInfo, versions []*NodeVersion, obj *VersionedObject) (string, error) { +func (n *layer) removeVersionIfFound(ctx context.Context, bkt *data.BucketInfo, versions []*data.NodeVersion, obj *VersionedObject) (string, error) { for _, version := range versions { if version.OID.EncodeToString() != obj.VersionID { continue } var deleteMarkVersion string - if version.IsDeleteMarker { + if version.DeleteMarker != nil { deleteMarkVersion = obj.VersionID } diff --git a/api/layer/object.go b/api/layer/object.go index 6ffe36f81..3b0c4cec9 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -171,7 +171,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object own := n.Owner(ctx) versioningEnabled := n.isVersioningEnabled(ctx, p.BktInfo) - newVersion := &NodeVersion{IsUnversioned: !versioningEnabled} + newVersion := &data.NodeVersion{IsUnversioned: !versioningEnabled} r := p.Reader if r != nil { @@ -283,7 +283,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke return nil, err } - if node.IsDeleteMarker { + if node.DeleteMarker != nil { return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey) } @@ -365,7 +365,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb return nil, fmt.Errorf("couldn't get versions: %w", err) } - var foundVersion *NodeVersion + var foundVersion *data.NodeVersion for _, version := range versions { if version.OID.EncodeToString() == p.VersionID { foundVersion = version @@ -554,41 +554,56 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, bkt *data.BucketIn return objects, nil } -func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) { +func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, prefix, delimiter string) (map[string][]*data.ExtendedObjectInfo, error) { var err error cacheKey := cache.CreateObjectsListCacheKey(&bkt.CID, prefix, false) - ids := n.listsCache.Get(cacheKey) + nodeVersions := n.listsCache.GetVersions(cacheKey) - if ids == nil { - ids, err = n.objectSearch(ctx, &findParams{bkt: bkt, prefix: prefix}) + if nodeVersions == nil { + nodeVersions, err = n.treeService.GetAllVersionsByPrefix(ctx, &bkt.CID, prefix) if err != nil { return nil, err } - if err = n.listsCache.Put(cacheKey, ids); err != nil { + if err = n.listsCache.PutVersions(cacheKey, nodeVersions); err != nil { n.log.Error("couldn't cache list of objects", zap.Error(err)) } } - versions := make(map[string]*objectVersions, len(ids)/2) + versions := make(map[string][]*data.ExtendedObjectInfo, len(nodeVersions)) - for i := 0; i < len(ids); i++ { - obj := n.objectFromObjectsCacheOrNeoFS(ctx, bkt, ids[i]) - if obj == nil { - continue - } - if oi := objectInfoFromMeta(bkt, obj, prefix, delimiter); oi != nil { - if isSystem(oi) { + for _, nodeVersion := range nodeVersions { + oi := &data.ObjectInfo{} + + if nodeVersion.DeleteMarker != nil { // delete marker does not match any object in NeoFS + oi.ID = nodeVersion.OID + oi.Name = nodeVersion.DeleteMarker.FilePath + oi.Owner = nodeVersion.DeleteMarker.Owner + oi.Created = nodeVersion.DeleteMarker.Created + oi.IsDeleteMarker = true + } else { + obj := n.objectFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion.OID) + if obj == nil { continue } - - objVersions, ok := versions[oi.Name] - if !ok { - objVersions = newObjectVersions(oi.Name) + oi = objectInfoFromMeta(bkt, obj, prefix, delimiter) + if oi == nil { + continue } - objVersions.appendVersion(oi) - versions[oi.Name] = objVersions } + + eoi := &data.ExtendedObjectInfo{ + ObjectInfo: oi, + NodeVersion: nodeVersion, + } + + objVersions, ok := versions[oi.Name] + if !ok { + objVersions = []*data.ExtendedObjectInfo{eoi} + } else if !oi.IsDir { + objVersions = append(objVersions, eoi) + } + versions[oi.Name] = objVersions } return versions, nil diff --git a/api/layer/system_object.go b/api/layer/system_object.go index 29dcafb70..b66ccaf70 100644 --- a/api/layer/system_object.go +++ b/api/layer/system_object.go @@ -122,7 +122,7 @@ func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObject return nil, err } - newVersion := &BaseNodeVersion{OID: *id} + newVersion := &data.BaseNodeVersion{OID: *id} if err = n.treeService.AddSystemVersion(ctx, &p.BktInfo.CID, p.ObjName, newVersion); err != nil { return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err) } diff --git a/api/layer/tree_service.go b/api/layer/tree_service.go index dfbec0bc9..a72f19e3d 100644 --- a/api/layer/tree_service.go +++ b/api/layer/tree_service.go @@ -30,28 +30,18 @@ type TreeService interface { // DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in NeoFS DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.ID, error) - GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]*NodeVersion, error) - GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*NodeVersion, error) + GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]*data.NodeVersion, error) + GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) GetLatestVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]oid.ID, error) - GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*NodeVersion, error) - AddVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *NodeVersion) error + GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]*data.NodeVersion, error) + GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) + AddVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *data.NodeVersion) error RemoveVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error - AddSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *BaseNodeVersion) error - GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*BaseNodeVersion, error) + AddSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *data.BaseNodeVersion) error + GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error } -type NodeVersion struct { - BaseNodeVersion - IsDeleteMarker bool - IsUnversioned bool -} - -type BaseNodeVersion struct { - ID uint64 - OID oid.ID -} - // ErrNodeNotFound is returned from Tree service in case of not found error. var ErrNodeNotFound = errors.New("not found") diff --git a/api/layer/versioning.go b/api/layer/versioning.go index 2ec793b0a..4ffca683a 100644 --- a/api/layer/versioning.go +++ b/api/layer/versioning.go @@ -267,7 +267,6 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar var ( allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys) res = &ListObjectVersionsInfo{} - reverse = true ) versions, err := n.getAllObjectsVersions(ctx, p.BktInfo, p.Prefix, p.Delimiter) @@ -282,7 +281,14 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar sort.Strings(sortedNames) for _, name := range sortedNames { - allObjects = append(allObjects, versions[name].getFiltered(reverse)...) + sortedVersions := versions[name] + sort.Slice(sortedVersions, func(i, j int) bool { + return sortedVersions[j].NodeVersion.Timestamp < sortedVersions[i].NodeVersion.Timestamp // sort in reverse order + }) + + for _, version := range sortedVersions { + allObjects = append(allObjects, version.ObjectInfo) + } } for i, obj := range allObjects { @@ -325,7 +331,7 @@ func triageVersions(objVersions []*ObjectVersionInfo) ([]*ObjectVersionInfo, []* var resDelMarkVersions []*ObjectVersionInfo for _, version := range objVersions { - if version.Object.Headers[VersionsDeleteMarkAttr] == DelMarkFullObject { + if version.Object.IsDeleteMarker { resDelMarkVersions = append(resDelMarkVersions, version) } else { resVersion = append(resVersion, version) diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go index b26a80226..8db34bcd1 100644 --- a/api/layer/versioning_test.go +++ b/api/layer/versioning_test.go @@ -11,6 +11,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/api" "github.com/nspcc-dev/neofs-s3-gw/api/data" "github.com/nspcc-dev/neofs-s3-gw/creds/accessbox" + treetest "github.com/nspcc-dev/neofs-s3-gw/internal/neofstest/tree" bearertest "github.com/nspcc-dev/neofs-sdk-go/bearer/test" "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -164,8 +165,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { } layerCfg := &Config{ - Caches: config, - AnonKey: AnonymousKey{Key: key}, + Caches: config, + AnonKey: AnonymousKey{Key: key}, + TreeService: treetest.NewTreeService(), } return &testContext{ @@ -196,9 +198,8 @@ func TestSimpleVersioning(t *testing.T) { obj1Content2 := []byte("content obj1 v2") obj1v2 := tc.putObject(obj1Content2) - objv2, buffer2 := tc.getObject(tc.obj, "", false) + _, buffer2 := tc.getObject(tc.obj, "", false) require.Equal(t, obj1Content2, buffer2) - require.Contains(t, objv2.Headers[versionsAddAttr], obj1v1.ID.EncodeToString()) _, buffer1 := tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), false) require.Equal(t, obj1Content1, buffer1) @@ -215,9 +216,8 @@ func TestSimpleNoVersioning(t *testing.T) { obj1Content2 := []byte("content obj1 v2") obj1v2 := tc.putObject(obj1Content2) - objv2, buffer2 := tc.getObject(tc.obj, "", false) + _, buffer2 := tc.getObject(tc.obj, "", false) require.Equal(t, obj1Content2, buffer2) - require.Contains(t, objv2.Headers[versionsDelAttr], obj1v1.ID.EncodeToString()) tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), true) tc.checkListObjects(obj1v2.ID) diff --git a/internal/neofs/tree.go b/internal/neofs/tree.go index 6cfd03ea9..d0c8a0b59 100644 --- a/internal/neofs/tree.go +++ b/internal/neofs/tree.go @@ -7,6 +7,7 @@ import ( "io" "strconv" "strings" + "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neofs-s3-gw/api" @@ -16,6 +17,7 @@ import ( "github.com/nspcc-dev/neofs-s3-gw/internal/neofs/services/tree" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + "github.com/nspcc-dev/neofs-sdk-go/user" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -42,7 +44,12 @@ const ( fileNameKV = "FileName" systemNameKV = "SystemName" isUnversionedKV = "IsUnversioned" - isDeleteMarkerKV = "IdDeleteMarker" + + // keys for delete marker nodes + isDeleteMarkerKV = "IdDeleteMarker" + filePathKV = "FilePath" + ownerKV = "Owner" + createdKV = "Created" settingsFileName = "bucket-settings" notifConfFileName = "bucket-notifications" @@ -111,23 +118,50 @@ func (n *TreeNode) Get(key string) (string, bool) { return value, ok } -func newNodeVersion(node NodeResponse) (*layer.NodeVersion, error) { +func newNodeVersion(node NodeResponse) (*data.NodeVersion, error) { treeNode, err := newTreeNode(node) if err != nil { return nil, fmt.Errorf("invalid tree node: %w", err) } + return newNodeVersionFromTreeNode(treeNode), nil +} + +func newNodeVersionFromTreeNode(treeNode *TreeNode) *data.NodeVersion { _, isUnversioned := treeNode.Get(isUnversionedKV) _, isDeleteMarker := treeNode.Get(isDeleteMarkerKV) - return &layer.NodeVersion{ - BaseNodeVersion: layer.BaseNodeVersion{ - ID: treeNode.ID, - OID: treeNode.ObjID, + version := &data.NodeVersion{ + BaseNodeVersion: data.BaseNodeVersion{ + ID: treeNode.ID, + OID: treeNode.ObjID, + Timestamp: treeNode.TimeStamp, }, - IsUnversioned: isUnversioned, - IsDeleteMarker: isDeleteMarker, - }, nil + IsUnversioned: isUnversioned, + } + + if isDeleteMarker { + filePath, _ := treeNode.Get(filePathKV) + + var created time.Time + if createdStr, ok := treeNode.Get(createdKV); ok { + if utcMilli, err := strconv.ParseInt(createdStr, 10, 64); err == nil { + created = time.UnixMilli(utcMilli) + } + } + + var owner user.ID + if ownerStr, ok := treeNode.Get(ownerKV); ok { + _ = owner.DecodeString(ownerStr) + } + + version.DeleteMarker = &data.DeleteMarkerInfo{ + FilePath: filePath, + Created: created, + Owner: owner, + } + } + return version } func (c *TreeClient) GetSettingsNode(ctx context.Context, cnrID *cid.ID) (*data.BucketSettings, error) { @@ -240,11 +274,11 @@ func (c *TreeClient) DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid. return nil, nil } -func (c *TreeClient) GetVersions(ctx context.Context, cnrID *cid.ID, filepath string) ([]*layer.NodeVersion, error) { +func (c *TreeClient) GetVersions(ctx context.Context, cnrID *cid.ID, filepath string) ([]*data.NodeVersion, error) { return c.getVersions(ctx, cnrID, versionTree, filepath, false) } -func (c *TreeClient) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*layer.NodeVersion, error) { +func (c *TreeClient) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { meta := []string{oidKV, isUnversionedKV, isDeleteMarkerKV} path := pathFromName(objectName) @@ -266,20 +300,14 @@ func (c *TreeClient) GetLatestVersionsByPrefix(ctx context.Context, cnrID *cid.I tailPrefix := path[len(path)-1] if len(path) > 1 { - meta := []string{fileNameKV} - - nodes, err := c.getNodes(ctx, cnrID, versionTree, fileNameKV, path[:len(path)-1], meta, true) + var err error + rootID, err = c.getPrefixNodeID(ctx, cnrID, path[:len(path)-1]) if err != nil { + if errors.Is(err, layer.ErrNodeNotFound) { + return nil, nil + } return nil, err } - if len(nodes) == 0 { - return nil, nil - } - if len(nodes) != 1 { - return nil, layer.ErrNodeNotFound - } - - rootID = nodes[0].NodeId } subTree, err := c.getSubTree(ctx, cnrID, versionTree, rootID, 1) @@ -289,18 +317,46 @@ func (c *TreeClient) GetLatestVersionsByPrefix(ctx context.Context, cnrID *cid.I var result []oid.ID for _, node := range subTree { - if node.GetNodeId() != 0 && hasPrefix(node, tailPrefix) { - latestNodes, err := c.getSubTreeLatestVersions(ctx, cnrID, node.GetNodeId()) + if node.GetNodeId() != rootID && hasPrefix(node, tailPrefix) { + latestNodes, err := c.getSubTreeVersions(ctx, cnrID, node.GetNodeId(), true) if err != nil { return nil, err } - result = append(result, latestNodes...) + + for _, latest := range latestNodes { + result = append(result, latest.OID) + } } } return result, nil } +func (c *TreeClient) getPrefixNodeID(ctx context.Context, cnrID *cid.ID, prefixPath []string) (uint64, error) { + meta := []string{fileNameKV, oidKV} + + nodes, err := c.getNodes(ctx, cnrID, versionTree, fileNameKV, prefixPath, meta, false) + if err != nil { + return 0, err + } + + var intermediateNodes []uint64 + for _, node := range nodes { + if !hasOID(node) { + intermediateNodes = append(intermediateNodes, node.GetNodeId()) + } + } + + if len(intermediateNodes) == 0 { + return 0, layer.ErrNodeNotFound + } + if len(intermediateNodes) > 1 { + return 0, fmt.Errorf("found more than one intermediate nodes") + } + + return intermediateNodes[0], nil +} + func hasPrefix(node *tree.GetSubTreeResponse_Body, prefix string) bool { for _, kv := range node.GetMeta() { if kv.GetKey() == fileNameKV { @@ -311,7 +367,17 @@ func hasPrefix(node *tree.GetSubTreeResponse_Body, prefix string) bool { return false } -func (c *TreeClient) getSubTreeLatestVersions(ctx context.Context, cnrID *cid.ID, nodeID uint64) ([]oid.ID, error) { +func hasOID(node *tree.GetNodeByPathResponse_Info) bool { + for _, kv := range node.GetMeta() { + if kv.GetKey() == oidKV { + return true + } + } + + return false +} + +func (c *TreeClient) getSubTreeVersions(ctx context.Context, cnrID *cid.ID, nodeID uint64, latestOnly bool) ([]*data.NodeVersion, error) { subTree, err := c.getSubTree(ctx, cnrID, versionTree, nodeID, maxGetSubTreeDepth) if err != nil { return nil, err @@ -319,10 +385,10 @@ func (c *TreeClient) getSubTreeLatestVersions(ctx context.Context, cnrID *cid.ID var emptyOID oid.ID - latestVersions := make(map[string]*TreeNode, len(subTree)) + versions := make(map[string][]*data.NodeVersion, len(subTree)) for _, node := range subTree { treeNode, err := newTreeNode(node) - if err != nil || treeNode.ObjID.Equals(emptyOID) { // invalid OID attribute + if err != nil || treeNode.ObjID.Equals(emptyOID) { // invalid or empty OID attribute continue } fileName, ok := treeNode.Get(fileNameKV) @@ -331,18 +397,24 @@ func (c *TreeClient) getSubTreeLatestVersions(ctx context.Context, cnrID *cid.ID } key := formLatestNodeKey(node.GetParentId(), fileName) - latest, ok := latestVersions[key] - if !ok || latest.TimeStamp <= treeNode.TimeStamp { // todo also compare oid - latestVersions[key] = treeNode + versionNodes, ok := versions[key] + if !ok { + versionNodes = []*data.NodeVersion{newNodeVersionFromTreeNode(treeNode)} + } else if !latestOnly { + versionNodes = append(versionNodes, newNodeVersionFromTreeNode(treeNode)) + } else if versionNodes[0].Timestamp <= treeNode.TimeStamp { + versionNodes[0] = newNodeVersionFromTreeNode(treeNode) } + + versions[key] = versionNodes } - result := make([]oid.ID, 0, len(latestVersions)) - for _, treeNode := range latestVersions { - if _, ok := treeNode.Get(isDeleteMarkerKV); ok { + result := make([]*data.NodeVersion, 0, len(versions)) // consider use len(subTree) + for _, version := range versions { + if latestOnly && version[0].DeleteMarker != nil { continue } - result = append(result, treeNode.ObjID) + result = append(result, version...) } return result, nil @@ -352,7 +424,42 @@ func formLatestNodeKey(parentID uint64, fileName string) string { return strconv.FormatUint(parentID, 10) + fileName } -func (c *TreeClient) GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*layer.BaseNodeVersion, error) { +func (c *TreeClient) GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]*data.NodeVersion, error) { + var rootID uint64 + path := strings.Split(prefix, separator) + tailPrefix := path[len(path)-1] + + if len(path) > 1 { + var err error + rootID, err = c.getPrefixNodeID(ctx, cnrID, path[:len(path)-1]) + if err != nil { + if errors.Is(err, layer.ErrNodeNotFound) { + return nil, nil + } + return nil, err + } + } + + subTree, err := c.getSubTree(ctx, cnrID, versionTree, rootID, 1) + if err != nil { + return nil, err + } + + var result []*data.NodeVersion + for _, node := range subTree { + if node.GetNodeId() != rootID && hasPrefix(node, tailPrefix) { + versions, err := c.getSubTreeVersions(ctx, cnrID, node.GetNodeId(), false) + if err != nil { + return nil, err + } + result = append(result, versions...) + } + } + + return result, nil +} + +func (c *TreeClient) GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) { meta := []string{oidKV} path := pathFromName(objectName) @@ -363,7 +470,7 @@ func (c *TreeClient) GetSystemVersion(ctx context.Context, cnrID *cid.ID, object return &node.BaseNodeVersion, nil } -func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID, attrPath string, path, meta []string) (*layer.NodeVersion, error) { +func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID, attrPath string, path, meta []string) (*data.NodeVersion, error) { nodes, err := c.getNodes(ctx, cnrID, treeID, attrPath, path, meta, true) if err != nil { if strings.Contains(err.Error(), "not found") { @@ -379,11 +486,11 @@ func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID return newNodeVersion(nodes[0]) } -func (c *TreeClient) GetUnversioned(ctx context.Context, cnrID *cid.ID, filepath string) (*layer.NodeVersion, error) { +func (c *TreeClient) GetUnversioned(ctx context.Context, cnrID *cid.ID, filepath string) (*data.NodeVersion, error) { return c.getUnversioned(ctx, cnrID, versionTree, filepath) } -func (c *TreeClient) getUnversioned(ctx context.Context, cnrID *cid.ID, treeID, filepath string) (*layer.NodeVersion, error) { +func (c *TreeClient) getUnversioned(ctx context.Context, cnrID *cid.ID, treeID, filepath string) (*data.NodeVersion, error) { nodes, err := c.getVersions(ctx, cnrID, treeID, filepath, true) if err != nil { return nil, err @@ -400,12 +507,12 @@ func (c *TreeClient) getUnversioned(ctx context.Context, cnrID *cid.ID, treeID, return nodes[0], nil } -func (c *TreeClient) AddVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *layer.NodeVersion) error { +func (c *TreeClient) AddVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *data.NodeVersion) error { return c.addVersion(ctx, cnrID, versionTree, fileNameKV, filepath, version) } -func (c *TreeClient) AddSystemVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *layer.BaseNodeVersion) error { - newVersion := &layer.NodeVersion{ +func (c *TreeClient) AddSystemVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *data.BaseNodeVersion) error { + newVersion := &data.NodeVersion{ BaseNodeVersion: *version, IsUnversioned: true, } @@ -428,15 +535,18 @@ func (c *TreeClient) Close() error { return nil } -func (c *TreeClient) addVersion(ctx context.Context, cnrID *cid.ID, treeID, attrPath, filepath string, version *layer.NodeVersion) error { +func (c *TreeClient) addVersion(ctx context.Context, cnrID *cid.ID, treeID, attrPath, filepath string, version *data.NodeVersion) error { path := pathFromName(filepath) meta := map[string]string{ oidKV: version.OID.EncodeToString(), attrPath: path[len(path)-1], } - if version.IsDeleteMarker { + if version.DeleteMarker != nil { meta[isDeleteMarkerKV] = "true" + meta[filePathKV] = version.DeleteMarker.FilePath + meta[ownerKV] = version.DeleteMarker.Owner.EncodeToString() + meta[createdKV] = strconv.FormatInt(version.DeleteMarker.Created.UTC().UnixMilli(), 10) } if version.IsUnversioned { @@ -460,7 +570,7 @@ func (c *TreeClient) addVersion(ctx context.Context, cnrID *cid.ID, treeID, attr return c.addNodeByPath(ctx, cnrID, treeID, path[:len(path)-1], meta) } -func (c *TreeClient) getVersions(ctx context.Context, cnrID *cid.ID, treeID, filepath string, onlyUnversioned bool) ([]*layer.NodeVersion, error) { +func (c *TreeClient) getVersions(ctx context.Context, cnrID *cid.ID, treeID, filepath string, onlyUnversioned bool) ([]*data.NodeVersion, error) { keysToReturn := []string{oidKV, isUnversionedKV, isDeleteMarkerKV} path := pathFromName(filepath) nodes, err := c.getNodes(ctx, cnrID, treeID, fileNameKV, path, keysToReturn, false) @@ -471,7 +581,7 @@ func (c *TreeClient) getVersions(ctx context.Context, cnrID *cid.ID, treeID, fil return nil, fmt.Errorf("couldn't get nodes: %w", err) } - result := make([]*layer.NodeVersion, 0, len(nodes)) + result := make([]*data.NodeVersion, 0, len(nodes)) for _, node := range nodes { nodeVersion, err := newNodeVersion(node) if err != nil { diff --git a/internal/neofstest/tree/tree_mock.go b/internal/neofstest/tree/tree_mock.go index ca48479ec..a32a75629 100644 --- a/internal/neofstest/tree/tree_mock.go +++ b/internal/neofstest/tree/tree_mock.go @@ -2,25 +2,28 @@ package tree import ( "context" + "errors" "sort" + "strings" "github.com/nspcc-dev/neofs-s3-gw/api/data" - "github.com/nspcc-dev/neofs-s3-gw/api/layer" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" ) type TreeServiceMock struct { settings map[string]*data.BucketSettings - versions map[string]map[string][]*layer.NodeVersion - system map[string]map[string]*layer.BaseNodeVersion + versions map[string]map[string][]*data.NodeVersion + system map[string]map[string]*data.BaseNodeVersion } +var ErrNodeNotFound = errors.New("not found") + func NewTreeService() *TreeServiceMock { return &TreeServiceMock{ settings: make(map[string]*data.BucketSettings), - versions: make(map[string]map[string][]*layer.NodeVersion), - system: make(map[string]map[string]*layer.BaseNodeVersion), + versions: make(map[string]map[string][]*data.NodeVersion), + system: make(map[string]map[string]*data.BaseNodeVersion), } } @@ -32,7 +35,7 @@ func (t *TreeServiceMock) PutSettingsNode(_ context.Context, id *cid.ID, setting func (t *TreeServiceMock) GetSettingsNode(_ context.Context, id *cid.ID) (*data.BucketSettings, error) { settings, ok := t.settings[id.EncodeToString()] if !ok { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } return settings, nil @@ -58,19 +61,29 @@ func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) ( panic("implement me") } -func (t *TreeServiceMock) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]*layer.NodeVersion, error) { - panic("implement me") -} - -func (t *TreeServiceMock) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*layer.NodeVersion, error) { +func (t *TreeServiceMock) GetVersions(_ context.Context, cnrID *cid.ID, objectName string) ([]*data.NodeVersion, error) { cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()] if !ok { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } versions, ok := cnrVersionsMap[objectName] if !ok { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound + } + + return versions, nil +} + +func (t *TreeServiceMock) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { + cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()] + if !ok { + return nil, ErrNodeNotFound + } + + versions, ok := cnrVersionsMap[objectName] + if !ok { + return nil, ErrNodeNotFound } sort.Slice(versions, func(i, j int) bool { @@ -81,21 +94,42 @@ func (t *TreeServiceMock) GetLatestVersion(ctx context.Context, cnrID *cid.ID, o return versions[len(versions)-1], nil } - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } -func (t *TreeServiceMock) GetLatestVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]oid.ID, error) { - panic("implement me") -} - -func (t *TreeServiceMock) GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*layer.NodeVersion, error) { - panic("implement me") -} - -func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *layer.NodeVersion) error { +func (t *TreeServiceMock) GetLatestVersionsByPrefix(_ context.Context, cnrID *cid.ID, prefix string) ([]oid.ID, error) { cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()] if !ok { - t.versions[cnrID.EncodeToString()] = map[string][]*layer.NodeVersion{ + return nil, ErrNodeNotFound + } + + var result []oid.ID + + for key, versions := range cnrVersionsMap { + if !strings.HasPrefix(key, prefix) { + continue + } + + sort.Slice(versions, func(i, j int) bool { + return versions[i].ID < versions[j].ID + }) + + if len(versions) != 0 { + result = append(result, versions[len(versions)-1].OID) + } + } + + return result, nil +} + +func (t *TreeServiceMock) GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) { + panic("implement me") +} + +func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *data.NodeVersion) error { + cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()] + if !ok { + t.versions[cnrID.EncodeToString()] = map[string][]*data.NodeVersion{ objectName: {newVersion}, } return nil @@ -103,7 +137,7 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectNam versions, ok := cnrVersionsMap[objectName] if !ok { - cnrVersionsMap[objectName] = []*layer.NodeVersion{newVersion} + cnrVersionsMap[objectName] = []*data.NodeVersion{newVersion} return nil } @@ -115,7 +149,19 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectNam newVersion.ID = versions[len(versions)-1].ID + 1 } - cnrVersionsMap[objectName] = append(versions, newVersion) + result := versions + + if newVersion.IsUnversioned { + result = make([]*data.NodeVersion, 0, len(versions)) + for _, node := range versions { + if !node.IsUnversioned { + result = append(result, node) + } + + } + } + + cnrVersionsMap[objectName] = append(result, newVersion) return nil } @@ -124,10 +170,10 @@ func (t *TreeServiceMock) RemoveVersion(ctx context.Context, cnrID *cid.ID, node panic("implement me") } -func (t *TreeServiceMock) AddSystemVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *layer.BaseNodeVersion) error { +func (t *TreeServiceMock) AddSystemVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *data.BaseNodeVersion) error { cnrSystemMap, ok := t.system[cnrID.EncodeToString()] if !ok { - t.system[cnrID.EncodeToString()] = map[string]*layer.BaseNodeVersion{ + t.system[cnrID.EncodeToString()] = map[string]*data.BaseNodeVersion{ objectName: newVersion, } return nil @@ -138,15 +184,15 @@ func (t *TreeServiceMock) AddSystemVersion(_ context.Context, cnrID *cid.ID, obj return nil } -func (t *TreeServiceMock) GetSystemVersion(_ context.Context, cnrID *cid.ID, objectName string) (*layer.BaseNodeVersion, error) { +func (t *TreeServiceMock) GetSystemVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) { cnrSystemMap, ok := t.system[cnrID.EncodeToString()] if !ok { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } sysVersion, ok := cnrSystemMap[objectName] if !ok { - return nil, layer.ErrNodeNotFound + return nil, ErrNodeNotFound } return sysVersion, nil @@ -155,3 +201,7 @@ func (t *TreeServiceMock) GetSystemVersion(_ context.Context, cnrID *cid.ID, obj func (t *TreeServiceMock) RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error { panic("implement me") } + +func (t *TreeServiceMock) GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]*data.NodeVersion, error) { + panic("implement me") +}