EliChin/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#420] Using tree service to list object versions

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-05-20 18:02:00 +03:00 committed by Alex Vanin
parent 55c38e73e6
commit 9c74cca9af
11 changed files with 374 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
api/cache/objectslist.go vendored
View file

@ -6,6 +6,7 @@ import (
"time"
"github.com/bluele/gcache"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"go.uber.org/zap"
@ -78,6 +79,21 @@ func (l *ObjectsListCache) Get(key ObjectsListKey) []oid.ID {
return result
}
// GetVersions returns a list of ObjectInfo.
func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion {
entry, err := l.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.([]*data.NodeVersion)
if !ok {
return nil
}
return result
}
// Put puts a list of objects to cache.
func (l *ObjectsListCache) Put(key ObjectsListKey, oids []oid.ID) error {
if len(oids) == 0 {
@ -87,6 +103,15 @@ func (l *ObjectsListCache) Put(key ObjectsListKey, oids []oid.ID) error {
return l.cache.Set(key, oids)
}
// PutVersions puts a list of object versions to cache.
func (l *ObjectsListCache) PutVersions(key ObjectsListKey, versions []*data.NodeVersion) error {
if len(versions) == 0 {
return fmt.Errorf("list versions is empty, cid: %s, prefix: %s", key.cid, key.prefix)
}
return l.cache.Set(key, versions)
}
// CleanCacheEntriesContainingObject deletes entries containing specified object.
func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string, cnr cid.ID) {
cidStr := cnr.EncodeToString()

7
api/data/info.go
View file

@ -28,9 +28,10 @@ type (
// ObjectInfo holds S3 object data.
ObjectInfo struct {
ID oid.ID
CID cid.ID
IsDir bool
ID oid.ID
CID cid.ID
IsDir bool
IsDeleteMarker bool
Bucket string
Name string

37
api/data/tree.go Normal file
View file

@ -0,0 +1,37 @@
package data
import (
"time"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
)
// NodeVersion represent node from tree service.
type NodeVersion struct {
BaseNodeVersion
DeleteMarker *DeleteMarkerInfo
IsUnversioned bool
}
// DeleteMarkerInfo is used to save object info if node in the tree service is delete marker.
// We need this information because the "delete marker" object is no longer stored in NeoFS.
type DeleteMarkerInfo struct {
FilePath string
Created time.Time
Owner user.ID
}
// ExtendedObjectInfo contains additional node info to be able to sort versions by timestamp.
type ExtendedObjectInfo struct {
ObjectInfo *ObjectInfo
NodeVersion *NodeVersion
}
// BaseNodeVersion is minimal node info from tree service.
// Basically used for "system" object.
type BaseNodeVersion struct {
ID uint64
OID oid.ID
Timestamp uint64
}

17
api/layer/layer.go
View file

@ -8,6 +8,7 @@ import (
"io"
"net/url"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
@ -562,12 +563,16 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
}
obj.DeleteMarkVersion = unversionedObjectVersionID
newVersion := &NodeVersion{
BaseNodeVersion: BaseNodeVersion{
newVersion := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
OID: *randOID,
},
IsDeleteMarker: true,
IsUnversioned: true,
DeleteMarker: &data.DeleteMarkerInfo{
FilePath: obj.Name,
Created: time.Now(),
Owner: n.Owner(ctx),
},
IsUnversioned: true,
}
if len(obj.VersionID) == 0 && settings.VersioningEnabled {
@ -595,14 +600,14 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
return obj
}
func (n *layer) removeVersionIfFound(ctx context.Context, bkt *data.BucketInfo, versions []*NodeVersion, obj *VersionedObject) (string, error) {
func (n *layer) removeVersionIfFound(ctx context.Context, bkt *data.BucketInfo, versions []*data.NodeVersion, obj *VersionedObject) (string, error) {
for _, version := range versions {
if version.OID.EncodeToString() != obj.VersionID {
continue
}
var deleteMarkVersion string
if version.IsDeleteMarker {
if version.DeleteMarker != nil {
deleteMarkVersion = obj.VersionID
}

59
api/layer/object.go
View file

@ -171,7 +171,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Object
own := n.Owner(ctx)
versioningEnabled := n.isVersioningEnabled(ctx, p.BktInfo)
newVersion := &NodeVersion{IsUnversioned: !versioningEnabled}
newVersion := &data.NodeVersion{IsUnversioned: !versioningEnabled}
r := p.Reader
if r != nil {
@ -283,7 +283,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
return nil, err
}
if node.IsDeleteMarker {
if node.DeleteMarker != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrNoSuchKey)
}
@ -365,7 +365,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
return nil, fmt.Errorf("couldn't get versions: %w", err)
}
var foundVersion *NodeVersion
var foundVersion *data.NodeVersion
for _, version := range versions {
if version.OID.EncodeToString() == p.VersionID {
foundVersion = version
@ -554,41 +554,56 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, bkt *data.BucketIn
return objects, nil
}
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, prefix, delimiter string) (map[string]*objectVersions, error) {
func (n *layer) getAllObjectsVersions(ctx context.Context, bkt *data.BucketInfo, prefix, delimiter string) (map[string][]*data.ExtendedObjectInfo, error) {
var err error
cacheKey := cache.CreateObjectsListCacheKey(&bkt.CID, prefix, false)
ids := n.listsCache.Get(cacheKey)
nodeVersions := n.listsCache.GetVersions(cacheKey)
if ids == nil {
ids, err = n.objectSearch(ctx, &findParams{bkt: bkt, prefix: prefix})
if nodeVersions == nil {
nodeVersions, err = n.treeService.GetAllVersionsByPrefix(ctx, &bkt.CID, prefix)
if err != nil {
return nil, err
}
if err = n.listsCache.Put(cacheKey, ids); err != nil {
if err = n.listsCache.PutVersions(cacheKey, nodeVersions); err != nil {
n.log.Error("couldn't cache list of objects", zap.Error(err))
}
}
versions := make(map[string]*objectVersions, len(ids)/2)
versions := make(map[string][]*data.ExtendedObjectInfo, len(nodeVersions))
for i := 0; i < len(ids); i++ {
obj := n.objectFromObjectsCacheOrNeoFS(ctx, bkt, ids[i])
if obj == nil {
continue
}
if oi := objectInfoFromMeta(bkt, obj, prefix, delimiter); oi != nil {
if isSystem(oi) {
for _, nodeVersion := range nodeVersions {
oi := &data.ObjectInfo{}
if nodeVersion.DeleteMarker != nil { // delete marker does not match any object in NeoFS
oi.ID = nodeVersion.OID
oi.Name = nodeVersion.DeleteMarker.FilePath
oi.Owner = nodeVersion.DeleteMarker.Owner
oi.Created = nodeVersion.DeleteMarker.Created
oi.IsDeleteMarker = true
} else {
obj := n.objectFromObjectsCacheOrNeoFS(ctx, bkt, nodeVersion.OID)
if obj == nil {
continue
}
objVersions, ok := versions[oi.Name]
if !ok {
objVersions = newObjectVersions(oi.Name)
oi = objectInfoFromMeta(bkt, obj, prefix, delimiter)
if oi == nil {
continue
}
objVersions.appendVersion(oi)
versions[oi.Name] = objVersions
}
eoi := &data.ExtendedObjectInfo{
ObjectInfo: oi,
NodeVersion: nodeVersion,
}
objVersions, ok := versions[oi.Name]
if !ok {
objVersions = []*data.ExtendedObjectInfo{eoi}
} else if !oi.IsDir {
objVersions = append(objVersions, eoi)
}
versions[oi.Name] = objVersions
}
return versions, nil

2
api/layer/system_object.go
View file

@ -122,7 +122,7 @@ func (n *layer) putSystemObjectIntoNeoFS(ctx context.Context, p *PutSystemObject
return nil, err
}
newVersion := &BaseNodeVersion{OID: *id}
newVersion := &data.BaseNodeVersion{OID: *id}
if err = n.treeService.AddSystemVersion(ctx, &p.BktInfo.CID, p.ObjName, newVersion); err != nil {
return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err)
}

24
api/layer/tree_service.go
View file

@ -30,28 +30,18 @@ type TreeService interface {
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in NeoFS
DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.ID, error)
GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]*NodeVersion, error)
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*NodeVersion, error)
GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]*data.NodeVersion, error)
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
GetLatestVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]oid.ID, error)
GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*NodeVersion, error)
AddVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *NodeVersion) error
GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]*data.NodeVersion, error)
GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
AddVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *data.NodeVersion) error
RemoveVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error
AddSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *BaseNodeVersion) error
GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*BaseNodeVersion, error)
AddSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string, newVersion *data.BaseNodeVersion) error
GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error)
RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error
}
type NodeVersion struct {
BaseNodeVersion
IsDeleteMarker bool
IsUnversioned bool
}
type BaseNodeVersion struct {
ID uint64
OID oid.ID
}
// ErrNodeNotFound is returned from Tree service in case of not found error.
var ErrNodeNotFound = errors.New("not found")

12
api/layer/versioning.go
View file

@ -267,7 +267,6 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar
var (
allObjects = make([]*data.ObjectInfo, 0, p.MaxKeys)
res = &ListObjectVersionsInfo{}
reverse = true
)
versions, err := n.getAllObjectsVersions(ctx, p.BktInfo, p.Prefix, p.Delimiter)
@ -282,7 +281,14 @@ func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsPar
sort.Strings(sortedNames)
for _, name := range sortedNames {
allObjects = append(allObjects, versions[name].getFiltered(reverse)...)
sortedVersions := versions[name]
sort.Slice(sortedVersions, func(i, j int) bool {
return sortedVersions[j].NodeVersion.Timestamp < sortedVersions[i].NodeVersion.Timestamp // sort in reverse order
})
for _, version := range sortedVersions {
allObjects = append(allObjects, version.ObjectInfo)
}
}
for i, obj := range allObjects {
@ -325,7 +331,7 @@ func triageVersions(objVersions []*ObjectVersionInfo) ([]*ObjectVersionInfo, []*
var resDelMarkVersions []*ObjectVersionInfo
for _, version := range objVersions {
if version.Object.Headers[VersionsDeleteMarkAttr] == DelMarkFullObject {
if version.Object.IsDeleteMarker {
resDelMarkVersions = append(resDelMarkVersions, version)
} else {
resVersion = append(resVersion, version)

12
api/layer/versioning_test.go
View file

@ -11,6 +11,7 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/creds/accessbox"
treetest "github.com/nspcc-dev/neofs-s3-gw/internal/neofstest/tree"
bearertest "github.com/nspcc-dev/neofs-sdk-go/bearer/test"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
@ -164,8 +165,9 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
}
layerCfg := &Config{
Caches: config,
AnonKey: AnonymousKey{Key: key},
Caches: config,
AnonKey: AnonymousKey{Key: key},
TreeService: treetest.NewTreeService(),
}
return &testContext{
@ -196,9 +198,8 @@ func TestSimpleVersioning(t *testing.T) {
obj1Content2 := []byte("content obj1 v2")
obj1v2 := tc.putObject(obj1Content2)
objv2, buffer2 := tc.getObject(tc.obj, "", false)
_, buffer2 := tc.getObject(tc.obj, "", false)
require.Equal(t, obj1Content2, buffer2)
require.Contains(t, objv2.Headers[versionsAddAttr], obj1v1.ID.EncodeToString())
_, buffer1 := tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), false)
require.Equal(t, obj1Content1, buffer1)
@ -215,9 +216,8 @@ func TestSimpleNoVersioning(t *testing.T) {
obj1Content2 := []byte("content obj1 v2")
obj1v2 := tc.putObject(obj1Content2)
objv2, buffer2 := tc.getObject(tc.obj, "", false)
_, buffer2 := tc.getObject(tc.obj, "", false)
require.Equal(t, obj1Content2, buffer2)
require.Contains(t, objv2.Headers[versionsDelAttr], obj1v1.ID.EncodeToString())
tc.getObject(tc.obj, obj1v1.ID.EncodeToString(), true)
tc.checkListObjects(obj1v2.ID)

202
internal/neofs/tree.go
View file

@ -7,6 +7,7 @@ import (
"io"
"strconv"
"strings"
"time"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neofs-s3-gw/api"
@ -16,6 +17,7 @@ import (
"github.com/nspcc-dev/neofs-s3-gw/internal/neofs/services/tree"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
"github.com/nspcc-dev/neofs-sdk-go/user"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
@ -42,7 +44,12 @@ const (
fileNameKV = "FileName"
systemNameKV = "SystemName"
isUnversionedKV = "IsUnversioned"
isDeleteMarkerKV = "IdDeleteMarker"
// keys for delete marker nodes
isDeleteMarkerKV = "IdDeleteMarker"
filePathKV = "FilePath"
ownerKV = "Owner"
createdKV = "Created"
settingsFileName = "bucket-settings"
notifConfFileName = "bucket-notifications"
@ -111,23 +118,50 @@ func (n *TreeNode) Get(key string) (string, bool) {
return value, ok
}
func newNodeVersion(node NodeResponse) (*layer.NodeVersion, error) {
func newNodeVersion(node NodeResponse) (*data.NodeVersion, error) {
treeNode, err := newTreeNode(node)
if err != nil {
return nil, fmt.Errorf("invalid tree node: %w", err)
}
return newNodeVersionFromTreeNode(treeNode), nil
}
func newNodeVersionFromTreeNode(treeNode *TreeNode) *data.NodeVersion {
_, isUnversioned := treeNode.Get(isUnversionedKV)
_, isDeleteMarker := treeNode.Get(isDeleteMarkerKV)
return &layer.NodeVersion{
BaseNodeVersion: layer.BaseNodeVersion{
ID: treeNode.ID,
OID: treeNode.ObjID,
version := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
ID: treeNode.ID,
OID: treeNode.ObjID,
Timestamp: treeNode.TimeStamp,
},
IsUnversioned: isUnversioned,
IsDeleteMarker: isDeleteMarker,
}, nil
IsUnversioned: isUnversioned,
}
if isDeleteMarker {
filePath, _ := treeNode.Get(filePathKV)
var created time.Time
if createdStr, ok := treeNode.Get(createdKV); ok {
if utcMilli, err := strconv.ParseInt(createdStr, 10, 64); err == nil {
created = time.UnixMilli(utcMilli)
}
}
var owner user.ID
if ownerStr, ok := treeNode.Get(ownerKV); ok {
_ = owner.DecodeString(ownerStr)
}
version.DeleteMarker = &data.DeleteMarkerInfo{
FilePath: filePath,
Created: created,
Owner: owner,
}
}
return version
}
func (c *TreeClient) GetSettingsNode(ctx context.Context, cnrID *cid.ID) (*data.BucketSettings, error) {
@ -240,11 +274,11 @@ func (c *TreeClient) DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (*oid.
return nil, nil
}
func (c *TreeClient) GetVersions(ctx context.Context, cnrID *cid.ID, filepath string) ([]*layer.NodeVersion, error) {
func (c *TreeClient) GetVersions(ctx context.Context, cnrID *cid.ID, filepath string) ([]*data.NodeVersion, error) {
return c.getVersions(ctx, cnrID, versionTree, filepath, false)
}
func (c *TreeClient) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*layer.NodeVersion, error) {
func (c *TreeClient) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
meta := []string{oidKV, isUnversionedKV, isDeleteMarkerKV}
path := pathFromName(objectName)
@ -266,20 +300,14 @@ func (c *TreeClient) GetLatestVersionsByPrefix(ctx context.Context, cnrID *cid.I
tailPrefix := path[len(path)-1]
if len(path) > 1 {
meta := []string{fileNameKV}
nodes, err := c.getNodes(ctx, cnrID, versionTree, fileNameKV, path[:len(path)-1], meta, true)
var err error
rootID, err = c.getPrefixNodeID(ctx, cnrID, path[:len(path)-1])
if err != nil {
if errors.Is(err, layer.ErrNodeNotFound) {
return nil, nil
}
return nil, err
}
if len(nodes) == 0 {
return nil, nil
}
if len(nodes) != 1 {
return nil, layer.ErrNodeNotFound
}
rootID = nodes[0].NodeId
}
subTree, err := c.getSubTree(ctx, cnrID, versionTree, rootID, 1)
@ -289,18 +317,46 @@ func (c *TreeClient) GetLatestVersionsByPrefix(ctx context.Context, cnrID *cid.I
var result []oid.ID
for _, node := range subTree {
if node.GetNodeId() != 0 && hasPrefix(node, tailPrefix) {
latestNodes, err := c.getSubTreeLatestVersions(ctx, cnrID, node.GetNodeId())
if node.GetNodeId() != rootID && hasPrefix(node, tailPrefix) {
latestNodes, err := c.getSubTreeVersions(ctx, cnrID, node.GetNodeId(), true)
if err != nil {
return nil, err
}
result = append(result, latestNodes...)
for _, latest := range latestNodes {
result = append(result, latest.OID)
}
}
}
return result, nil
}
func (c *TreeClient) getPrefixNodeID(ctx context.Context, cnrID *cid.ID, prefixPath []string) (uint64, error) {
meta := []string{fileNameKV, oidKV}
nodes, err := c.getNodes(ctx, cnrID, versionTree, fileNameKV, prefixPath, meta, false)
if err != nil {
return 0, err
}
var intermediateNodes []uint64
for _, node := range nodes {
if !hasOID(node) {
intermediateNodes = append(intermediateNodes, node.GetNodeId())
}
}
if len(intermediateNodes) == 0 {
return 0, layer.ErrNodeNotFound
}
if len(intermediateNodes) > 1 {
return 0, fmt.Errorf("found more than one intermediate nodes")
}
return intermediateNodes[0], nil
}
func hasPrefix(node *tree.GetSubTreeResponse_Body, prefix string) bool {
for _, kv := range node.GetMeta() {
if kv.GetKey() == fileNameKV {
@ -311,7 +367,17 @@ func hasPrefix(node *tree.GetSubTreeResponse_Body, prefix string) bool {
return false
}
func (c *TreeClient) getSubTreeLatestVersions(ctx context.Context, cnrID *cid.ID, nodeID uint64) ([]oid.ID, error) {
func hasOID(node *tree.GetNodeByPathResponse_Info) bool {
for _, kv := range node.GetMeta() {
if kv.GetKey() == oidKV {
return true
}
}
return false
}
func (c *TreeClient) getSubTreeVersions(ctx context.Context, cnrID *cid.ID, nodeID uint64, latestOnly bool) ([]*data.NodeVersion, error) {
subTree, err := c.getSubTree(ctx, cnrID, versionTree, nodeID, maxGetSubTreeDepth)
if err != nil {
return nil, err
@ -319,10 +385,10 @@ func (c *TreeClient) getSubTreeLatestVersions(ctx context.Context, cnrID *cid.ID
var emptyOID oid.ID
latestVersions := make(map[string]*TreeNode, len(subTree))
versions := make(map[string][]*data.NodeVersion, len(subTree))
for _, node := range subTree {
treeNode, err := newTreeNode(node)
if err != nil || treeNode.ObjID.Equals(emptyOID) { // invalid OID attribute
if err != nil || treeNode.ObjID.Equals(emptyOID) { // invalid or empty OID attribute
continue
}
fileName, ok := treeNode.Get(fileNameKV)
@ -331,18 +397,24 @@ func (c *TreeClient) getSubTreeLatestVersions(ctx context.Context, cnrID *cid.ID
}
key := formLatestNodeKey(node.GetParentId(), fileName)
latest, ok := latestVersions[key]
if !ok || latest.TimeStamp <= treeNode.TimeStamp { // todo also compare oid
latestVersions[key] = treeNode
versionNodes, ok := versions[key]
if !ok {
versionNodes = []*data.NodeVersion{newNodeVersionFromTreeNode(treeNode)}
} else if !latestOnly {
versionNodes = append(versionNodes, newNodeVersionFromTreeNode(treeNode))
} else if versionNodes[0].Timestamp <= treeNode.TimeStamp {
versionNodes[0] = newNodeVersionFromTreeNode(treeNode)
}
versions[key] = versionNodes
}
result := make([]oid.ID, 0, len(latestVersions))
for _, treeNode := range latestVersions {
if _, ok := treeNode.Get(isDeleteMarkerKV); ok {
result := make([]*data.NodeVersion, 0, len(versions)) // consider use len(subTree)
for _, version := range versions {
if latestOnly && version[0].DeleteMarker != nil {
continue
}
result = append(result, treeNode.ObjID)
result = append(result, version...)
}
return result, nil
@ -352,7 +424,42 @@ func formLatestNodeKey(parentID uint64, fileName string) string {
return strconv.FormatUint(parentID, 10) + fileName
}
func (c *TreeClient) GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*layer.BaseNodeVersion, error) {
func (c *TreeClient) GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]*data.NodeVersion, error) {
var rootID uint64
path := strings.Split(prefix, separator)
tailPrefix := path[len(path)-1]
if len(path) > 1 {
var err error
rootID, err = c.getPrefixNodeID(ctx, cnrID, path[:len(path)-1])
if err != nil {
if errors.Is(err, layer.ErrNodeNotFound) {
return nil, nil
}
return nil, err
}
}
subTree, err := c.getSubTree(ctx, cnrID, versionTree, rootID, 1)
if err != nil {
return nil, err
}
var result []*data.NodeVersion
for _, node := range subTree {
if node.GetNodeId() != rootID && hasPrefix(node, tailPrefix) {
versions, err := c.getSubTreeVersions(ctx, cnrID, node.GetNodeId(), false)
if err != nil {
return nil, err
}
result = append(result, versions...)
}
}
return result, nil
}
func (c *TreeClient) GetSystemVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) {
meta := []string{oidKV}
path := pathFromName(objectName)
@ -363,7 +470,7 @@ func (c *TreeClient) GetSystemVersion(ctx context.Context, cnrID *cid.ID, object
return &node.BaseNodeVersion, nil
}
func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID, attrPath string, path, meta []string) (*layer.NodeVersion, error) {
func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID, attrPath string, path, meta []string) (*data.NodeVersion, error) {
nodes, err := c.getNodes(ctx, cnrID, treeID, attrPath, path, meta, true)
if err != nil {
if strings.Contains(err.Error(), "not found") {
@ -379,11 +486,11 @@ func (c *TreeClient) getLatestVersion(ctx context.Context, cnrID *cid.ID, treeID
return newNodeVersion(nodes[0])
}
func (c *TreeClient) GetUnversioned(ctx context.Context, cnrID *cid.ID, filepath string) (*layer.NodeVersion, error) {
func (c *TreeClient) GetUnversioned(ctx context.Context, cnrID *cid.ID, filepath string) (*data.NodeVersion, error) {
return c.getUnversioned(ctx, cnrID, versionTree, filepath)
}
func (c *TreeClient) getUnversioned(ctx context.Context, cnrID *cid.ID, treeID, filepath string) (*layer.NodeVersion, error) {
func (c *TreeClient) getUnversioned(ctx context.Context, cnrID *cid.ID, treeID, filepath string) (*data.NodeVersion, error) {
nodes, err := c.getVersions(ctx, cnrID, treeID, filepath, true)
if err != nil {
return nil, err
@ -400,12 +507,12 @@ func (c *TreeClient) getUnversioned(ctx context.Context, cnrID *cid.ID, treeID,
return nodes[0], nil
}
func (c *TreeClient) AddVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *layer.NodeVersion) error {
func (c *TreeClient) AddVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *data.NodeVersion) error {
return c.addVersion(ctx, cnrID, versionTree, fileNameKV, filepath, version)
}
func (c *TreeClient) AddSystemVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *layer.BaseNodeVersion) error {
newVersion := &layer.NodeVersion{
func (c *TreeClient) AddSystemVersion(ctx context.Context, cnrID *cid.ID, filepath string, version *data.BaseNodeVersion) error {
newVersion := &data.NodeVersion{
BaseNodeVersion: *version,
IsUnversioned: true,
}
@ -428,15 +535,18 @@ func (c *TreeClient) Close() error {
return nil
}
func (c *TreeClient) addVersion(ctx context.Context, cnrID *cid.ID, treeID, attrPath, filepath string, version *layer.NodeVersion) error {
func (c *TreeClient) addVersion(ctx context.Context, cnrID *cid.ID, treeID, attrPath, filepath string, version *data.NodeVersion) error {
path := pathFromName(filepath)
meta := map[string]string{
oidKV: version.OID.EncodeToString(),
attrPath: path[len(path)-1],
}
if version.IsDeleteMarker {
if version.DeleteMarker != nil {
meta[isDeleteMarkerKV] = "true"
meta[filePathKV] = version.DeleteMarker.FilePath
meta[ownerKV] = version.DeleteMarker.Owner.EncodeToString()
meta[createdKV] = strconv.FormatInt(version.DeleteMarker.Created.UTC().UnixMilli(), 10)
}
if version.IsUnversioned {
@ -460,7 +570,7 @@ func (c *TreeClient) addVersion(ctx context.Context, cnrID *cid.ID, treeID, attr
return c.addNodeByPath(ctx, cnrID, treeID, path[:len(path)-1], meta)
}
func (c *TreeClient) getVersions(ctx context.Context, cnrID *cid.ID, treeID, filepath string, onlyUnversioned bool) ([]*layer.NodeVersion, error) {
func (c *TreeClient) getVersions(ctx context.Context, cnrID *cid.ID, treeID, filepath string, onlyUnversioned bool) ([]*data.NodeVersion, error) {
keysToReturn := []string{oidKV, isUnversionedKV, isDeleteMarkerKV}
path := pathFromName(filepath)
nodes, err := c.getNodes(ctx, cnrID, treeID, fileNameKV, path, keysToReturn, false)
@ -471,7 +581,7 @@ func (c *TreeClient) getVersions(ctx context.Context, cnrID *cid.ID, treeID, fil
return nil, fmt.Errorf("couldn't get nodes: %w", err)
}
result := make([]*layer.NodeVersion, 0, len(nodes))
result := make([]*data.NodeVersion, 0, len(nodes))
for _, node := range nodes {
nodeVersion, err := newNodeVersion(node)
if err != nil {

112
internal/neofstest/tree/tree_mock.go
View file

@ -2,25 +2,28 @@ package tree
import (
"context"
"errors"
"sort"
"strings"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)
type TreeServiceMock struct {
settings map[string]*data.BucketSettings
versions map[string]map[string][]*layer.NodeVersion
system map[string]map[string]*layer.BaseNodeVersion
versions map[string]map[string][]*data.NodeVersion
system map[string]map[string]*data.BaseNodeVersion
}
var ErrNodeNotFound = errors.New("not found")
func NewTreeService() *TreeServiceMock {
return &TreeServiceMock{
settings: make(map[string]*data.BucketSettings),
versions: make(map[string]map[string][]*layer.NodeVersion),
system: make(map[string]map[string]*layer.BaseNodeVersion),
versions: make(map[string]map[string][]*data.NodeVersion),
system: make(map[string]map[string]*data.BaseNodeVersion),
}
}
@ -32,7 +35,7 @@ func (t *TreeServiceMock) PutSettingsNode(_ context.Context, id *cid.ID, setting
func (t *TreeServiceMock) GetSettingsNode(_ context.Context, id *cid.ID) (*data.BucketSettings, error) {
settings, ok := t.settings[id.EncodeToString()]
if !ok {
return nil, layer.ErrNodeNotFound
return nil, ErrNodeNotFound
}
return settings, nil
@ -58,19 +61,29 @@ func (t *TreeServiceMock) DeleteBucketCORS(ctx context.Context, cnrID *cid.ID) (
panic("implement me")
}
func (t *TreeServiceMock) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string) ([]*layer.NodeVersion, error) {
panic("implement me")
}
func (t *TreeServiceMock) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*layer.NodeVersion, error) {
func (t *TreeServiceMock) GetVersions(_ context.Context, cnrID *cid.ID, objectName string) ([]*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
if !ok {
return nil, layer.ErrNodeNotFound
return nil, ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, layer.ErrNodeNotFound
return nil, ErrNodeNotFound
}
return versions, nil
}
func (t *TreeServiceMock) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, ErrNodeNotFound
}
sort.Slice(versions, func(i, j int) bool {
@ -81,21 +94,42 @@ func (t *TreeServiceMock) GetLatestVersion(ctx context.Context, cnrID *cid.ID, o
return versions[len(versions)-1], nil
}
return nil, layer.ErrNodeNotFound
return nil, ErrNodeNotFound
}
func (t *TreeServiceMock) GetLatestVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]oid.ID, error) {
panic("implement me")
}
func (t *TreeServiceMock) GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*layer.NodeVersion, error) {
panic("implement me")
}
func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *layer.NodeVersion) error {
func (t *TreeServiceMock) GetLatestVersionsByPrefix(_ context.Context, cnrID *cid.ID, prefix string) ([]oid.ID, error) {
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
if !ok {
t.versions[cnrID.EncodeToString()] = map[string][]*layer.NodeVersion{
return nil, ErrNodeNotFound
}
var result []oid.ID
for key, versions := range cnrVersionsMap {
if !strings.HasPrefix(key, prefix) {
continue
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
result = append(result, versions[len(versions)-1].OID)
}
}
return result, nil
}
func (t *TreeServiceMock) GetUnversioned(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error) {
panic("implement me")
}
func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *data.NodeVersion) error {
cnrVersionsMap, ok := t.versions[cnrID.EncodeToString()]
if !ok {
t.versions[cnrID.EncodeToString()] = map[string][]*data.NodeVersion{
objectName: {newVersion},
}
return nil
@ -103,7 +137,7 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectNam
versions, ok := cnrVersionsMap[objectName]
if !ok {
cnrVersionsMap[objectName] = []*layer.NodeVersion{newVersion}
cnrVersionsMap[objectName] = []*data.NodeVersion{newVersion}
return nil
}
@ -115,7 +149,19 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, cnrID *cid.ID, objectNam
newVersion.ID = versions[len(versions)-1].ID + 1
}
cnrVersionsMap[objectName] = append(versions, newVersion)
result := versions
if newVersion.IsUnversioned {
result = make([]*data.NodeVersion, 0, len(versions))
for _, node := range versions {
if !node.IsUnversioned {
result = append(result, node)
}
}
}
cnrVersionsMap[objectName] = append(result, newVersion)
return nil
}
@ -124,10 +170,10 @@ func (t *TreeServiceMock) RemoveVersion(ctx context.Context, cnrID *cid.ID, node
panic("implement me")
}
func (t *TreeServiceMock) AddSystemVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *layer.BaseNodeVersion) error {
func (t *TreeServiceMock) AddSystemVersion(_ context.Context, cnrID *cid.ID, objectName string, newVersion *data.BaseNodeVersion) error {
cnrSystemMap, ok := t.system[cnrID.EncodeToString()]
if !ok {
t.system[cnrID.EncodeToString()] = map[string]*layer.BaseNodeVersion{
t.system[cnrID.EncodeToString()] = map[string]*data.BaseNodeVersion{
objectName: newVersion,
}
return nil
@ -138,15 +184,15 @@ func (t *TreeServiceMock) AddSystemVersion(_ context.Context, cnrID *cid.ID, obj
return nil
}
func (t *TreeServiceMock) GetSystemVersion(_ context.Context, cnrID *cid.ID, objectName string) (*layer.BaseNodeVersion, error) {
func (t *TreeServiceMock) GetSystemVersion(_ context.Context, cnrID *cid.ID, objectName string) (*data.BaseNodeVersion, error) {
cnrSystemMap, ok := t.system[cnrID.EncodeToString()]
if !ok {
return nil, layer.ErrNodeNotFound
return nil, ErrNodeNotFound
}
sysVersion, ok := cnrSystemMap[objectName]
if !ok {
return nil, layer.ErrNodeNotFound
return nil, ErrNodeNotFound
}
return sysVersion, nil
@ -155,3 +201,7 @@ func (t *TreeServiceMock) GetSystemVersion(_ context.Context, cnrID *cid.ID, obj
func (t *TreeServiceMock) RemoveSystemVersion(ctx context.Context, cnrID *cid.ID, nodeID uint64) error {
panic("implement me")
}
func (t *TreeServiceMock) GetAllVersionsByPrefix(ctx context.Context, cnrID *cid.ID, prefix string) ([]*data.NodeVersion, error) {
panic("implement me")
}