Compare commits

...

3 commits

Author SHA1 Message Date
7a1665c660 [#Issue] tmp
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-07-11 16:33:11 +03:00
30ae1c469e [#XX] tree: new pilorama
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-06-25 15:05:37 +03:00
b0e52a4b5c [#XX] tree: Remove tree interface for the sake of simplicity
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-06-25 15:03:41 +03:00
25 changed files with 711 additions and 1792 deletions

View file

@ -61,8 +61,6 @@ func (e ExtendedObjectInfo) Version() string {
// BaseNodeVersion is minimal node info from tree service.
// Basically used for "system" object.
type BaseNodeVersion struct {
ID uint64
ParenID uint64
OID oid.ID
Timestamp uint64
Size uint64
@ -100,9 +98,6 @@ type ObjectTaggingInfo struct {
// MultipartInfo is multipart upload information.
type MultipartInfo struct {
// ID is node id in tree service.
// It's ignored when creating a new multipart upload.
ID uint64
Key string
UploadID string
Owner user.ID
@ -139,8 +134,6 @@ func (p *PartInfo) GetETag(md5Enabled bool) string {
// LockInfo is lock information to create appropriate tree node.
type LockInfo struct {
id uint64
legalHoldOID oid.ID
setLegalHold bool
@ -150,12 +143,8 @@ type LockInfo struct {
isCompliance bool
}
func NewLockInfo(id uint64) *LockInfo {
return &LockInfo{id: id}
}
func (l LockInfo) ID() uint64 {
return l.id
func NewLockInfo() *LockInfo {
return &LockInfo{}
}
func (l *LockInfo) SetLegalHold(objID oid.ID) {

View file

@ -264,6 +264,19 @@ func createMultipartUploadBase(hc *handlerContext, bktName, objName string, encr
return w
}
func abortMultipartUpload(hc *handlerContext, bktName, objName, uploadID string) {
w := abortMultipartUploadBase(hc, bktName, objName, uploadID)
assertStatus(hc.t, w, http.StatusNoContent)
}
func abortMultipartUploadBase(hc *handlerContext, bktName, objName, uploadID string) *httptest.ResponseRecorder {
query := make(url.Values)
query.Set(uploadIDQuery, uploadID)
w, r := prepareTestFullRequest(hc, bktName, objName, query, nil)
hc.Handler().AbortMultipartUploadHandler(w, r)
return w
}
func completeMultipartUpload(hc *handlerContext, bktName, objName, uploadID string, partsETags []string) {
w := completeMultipartUploadBase(hc, bktName, objName, uploadID, partsETags)
assertStatus(hc.t, w, http.StatusOK)

View file

@ -197,6 +197,19 @@ func TestGetObject(t *testing.T) {
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, errors.ErrNoSuchKey)
}
func TestGetObjectPiloramaPrefix(t *testing.T) {
hc := prepareHandlerContextWithMinCache(t)
bktName, objName, objName2 := "bucket", "obj", "obj2"
createBucket(hc, bktName)
hdr := putObjectContent(hc, bktName, objName, "content1")
putObjectContent(hc, bktName, objName2, "content2")
w := headObjectBase(hc, bktName, objName, emptyVersion)
assertStatus(t, w, http.StatusOK)
require.Equal(t, hdr.Get(api.ETag), w.Header().Get(api.ETag))
}
func TestGetObjectEnabledMD5(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket", "obj"

View file

@ -98,7 +98,7 @@ func TestMultipartReUploadPart(t *testing.T) {
innerUploadInfo, err := hc.tree.GetMultipartUpload(hc.context, bktInfo, objName, uploadInfo.UploadID)
require.NoError(t, err)
treeParts, err := hc.tree.GetParts(hc.Context(), bktInfo, innerUploadInfo.ID)
treeParts, err := hc.tree.GetParts(hc.Context(), bktInfo, innerUploadInfo)
require.NoError(t, err)
require.Len(t, treeParts, len(list.Parts))
@ -264,6 +264,25 @@ func TestMultipartUploadSize(t *testing.T) {
})
}
func TestMultipartUploadNewPilorama(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket-for-test-list-parts", "object-multipart"
bktInfo := createBucket(hc, bktName)
partSize := 1024
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
uploadPart(hc, bktName, objName, uploadInfo.UploadID, 2, partSize)
abortMultipartUpload(hc, bktName, objName, uploadInfo.UploadID)
require.Len(t, hc.tp.Objects(), 0)
parts, err := hc.tree.GetParts(hc.Context(), bktInfo.BktInfo, &data.MultipartInfo{Key: objName, UploadID: uploadInfo.UploadID})
require.NoError(t, err)
require.Len(t, parts, 0)
}
func TestListParts(t *testing.T) {
hc := prepareHandlerContext(t)

View file

@ -94,13 +94,10 @@ func TestListObjectsWithOldTreeNodes(t *testing.T) {
}
func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", 0, 0)
nodes, err := hc.treeMock.BurnedList(hc.Context(), bktInfo, "version", "")
require.NoError(hc.t, err)
for _, node := range nodes {
if node.GetNodeID() == 0 {
continue
}
meta := make(map[string]string, len(node.GetMeta()))
for _, m := range node.GetMeta() {
if m.GetKey() != "Created" && m.GetKey() != "Owner" {
@ -108,7 +105,7 @@ func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
}
}
err = hc.treeMock.MoveNode(hc.Context(), bktInfo, "version", node.GetNodeID(), node.GetParentID(), meta)
err = hc.treeMock.BurnedAdd(hc.Context(), bktInfo, "version", node.GetKey(), meta)
require.NoError(hc.t, err)
}
}

View file

@ -15,6 +15,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"go.opentelemetry.io/otel/trace"
@ -68,11 +69,12 @@ func transformToS3Error(err error) error {
}
if errors.Is(err, layer.ErrAccessDenied) ||
errors.Is(err, layer.ErrNodeAccessDenied) {
errors.Is(err, tree.ErrNodeAccessDenied) {
return s3errors.GetAPIError(s3errors.ErrAccessDenied)
}
if errors.Is(err, layer.ErrGatewayTimeout) {
if errors.Is(err, layer.ErrGatewayTimeout) ||
errors.Is(err, tree.ErrGatewayTimeout) {
return s3errors.GetAPIError(s3errors.ErrGatewayTimeout)
}

View file

@ -7,6 +7,7 @@ import (
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
"github.com/stretchr/testify/require"
)
@ -33,7 +34,7 @@ func TestTransformS3Errors(t *testing.T) {
},
{
name: "layer node access denied error to s3 access denied error",
err: layer.ErrNodeAccessDenied,
err: tree.ErrNodeAccessDenied,
expected: s3errors.ErrAccessDenied,
},
{

View file

@ -7,6 +7,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
)
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
@ -29,7 +30,7 @@ func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.Ob
tags, lockInfo, err = n.treeService.GetObjectTaggingAndLock(ctx, objVersion.BktInfo, nodeVersion)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, data.LockInfo{}, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
}
return nil, data.LockInfo{}, err

View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
"go.uber.org/zap"
)
@ -50,7 +51,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
}
objIDToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, objID)
objIDToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
objIDToDeleteNotFound := errorsStd.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objIDToDeleteNotFound {
return err
}
@ -71,7 +72,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error) {
cors, err := n.getCORS(ctx, bktInfo)
if err != nil {
if errorsStd.Is(err, ErrNodeNotFound) {
if errorsStd.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchCORSConfiguration), err.Error())
}
return nil, err
@ -82,7 +83,7 @@ func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*d
func (n *layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
objID, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
objIDNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
objIDNotFound := errorsStd.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objIDNotFound {
return err
}

View file

@ -19,6 +19,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -63,7 +64,7 @@ type (
resolver BucketResolver
ncontroller EventListener
cache *Cache
treeService TreeService
treeService *tree.Tree
features FeatureSettings
}
@ -73,7 +74,7 @@ type (
Cache *Cache
AnonKey AnonymousKey
Resolver BucketResolver
TreeService TreeService
TreeService *tree.Tree
Features FeatureSettings
}
@ -658,10 +659,10 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
}
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nodeVersion, obj); obj.Error != nil {
return n.handleObjectDeleteErrors(ctx, bkt, obj, nodeVersion.ID)
return n.handleObjectDeleteErrors(ctx, bkt, obj, nodeVersion)
}
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID)
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion)
n.cache.CleanListCacheEntriesContainingObject(obj.Name, bkt.CID)
return obj
}
@ -688,7 +689,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
if nullVersionToDelete != nil {
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nullVersionToDelete, obj); obj.Error != nil {
return n.handleObjectDeleteErrors(ctx, bkt, obj, nullVersionToDelete.ID)
return n.handleObjectDeleteErrors(ctx, bkt, obj, nullVersionToDelete)
}
}
}
@ -717,7 +718,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
IsUnversioned: settings.VersioningSuspended(),
}
if _, obj.Error = n.treeService.AddVersion(ctx, bkt, newVersion); obj.Error != nil {
if obj.Error = n.treeService.AddVersion(ctx, bkt, newVersion); obj.Error != nil {
return obj
}
@ -736,7 +737,7 @@ func (n *layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject)
return obj
}
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeID uint64) *VersionedObject {
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeVersion *data.NodeVersion) *VersionedObject {
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
return obj
}
@ -744,7 +745,7 @@ func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketIn
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeID)
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion)
if obj.Error == nil {
n.cache.DeleteObjectName(bkt.CID, bkt.Name, obj.Name)
}

View file

@ -142,7 +142,7 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
ListType: ListObjectsV2Type,
}
objects, next, err := n.getLatestObjectsVersions(ctx, prm)
objects, next, err := n.getLatestObjectsVersionsNew(ctx, prm)
if err != nil {
return nil, err
}
@ -225,6 +225,57 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVers
return
}
func (n *layer) getLatestObjectsVersionsNew(ctx context.Context, p commonLatestVersionsListingParams) (objects []*data.ExtendedNodeVersion, next *data.ExtendedNodeVersion, err error) {
if p.MaxKeys == 0 {
return nil, nil, nil
}
owner := n.BearerOwner(ctx)
cacheKey := cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, p.Bookmark)
session := n.cache.GetListSession(owner, cacheKey)
if session == nil {
session = &data.ListSession{NamesMap: make(map[string]struct{})}
session.Stream, err = n.treeService.InitVersionsByPrefixStream(ctx, p.BktInfo, p.Prefix, "", true)
if err != nil {
return nil, nil, err
}
} else {
session.Stream, err = n.treeService.InitVersionsByPrefixStream(ctx, p.BktInfo, p.Prefix, session.Next[0].Name(), true)
if err != nil {
return nil, nil, err
}
}
session.Context, session.Cancel = context.WithCancel(ctx)
generator, errorCh := nodesGeneratorStream(ctx, p.commonVersionsListingParams, session)
objOutCh, err := n.initWorkerPool(ctx, 2, p.commonVersionsListingParams, generator)
if err != nil {
return nil, nil, fmt.Errorf("failed to init worker pool: %w", err)
}
objects = make([]*data.ExtendedNodeVersion, 0, p.MaxKeys+1)
objects = append(objects, session.Next...)
for obj := range objOutCh {
objects = append(objects, obj)
}
if err = <-errorCh; err != nil {
return nil, nil, fmt.Errorf("failed to get next object from stream: %w", err)
}
sort.Slice(objects, func(i, j int) bool { return objects[i].NodeVersion.FilePath < objects[j].NodeVersion.FilePath })
if len(objects) > p.MaxKeys {
next = objects[p.MaxKeys]
session.Stream = nil
n.putListLatestVersionsSession(ctx, p, session, objects)
objects = objects[:p.MaxKeys]
}
return
}
func (n *layer) getAllObjectsVersions(ctx context.Context, p commonVersionsListingParams) ([]*data.ExtendedNodeVersion, bool, error) {
if p.MaxKeys == 0 {
return nil, false, nil
@ -337,7 +388,7 @@ func (n *layer) initNewVersionsByPrefixSession(ctx context.Context, p commonVers
session.Context = middleware.SetBox(session.Context, &middleware.Box{AccessBox: bd})
}
session.Stream, err = n.treeService.InitVersionsByPrefixStream(session.Context, p.BktInfo, p.Prefix, latestOnly)
session.Stream, err = n.treeService.InitVersionsByPrefixStream(session.Context, p.BktInfo, p.Prefix, "", latestOnly)
if err != nil {
return nil, err
}

View file

@ -20,6 +20,7 @@ import (
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/minio/sio"
@ -188,7 +189,7 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, error) {
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return "", fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
}
return "", err
@ -291,8 +292,8 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
MD5: hex.EncodeToString(md5Hash),
}
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
oldPartIDNotFound := errors.Is(err, ErrNoNodeToRemove)
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, partInfo)
oldPartIDNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !oldPartIDNotFound {
return nil, err
}
@ -322,7 +323,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error) {
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
}
return nil, err
@ -625,13 +626,13 @@ func (n *layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.MultipartInfo, map[int]*data.PartInfo, error) {
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Bkt, p.Key, p.UploadID)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
}
return nil, nil, err
}
parts, err := n.treeService.GetParts(ctx, p.Bkt, multipartInfo.ID)
parts, err := n.treeService.GetParts(ctx, p.Bkt, multipartInfo)
if err != nil {
return nil, nil, err
}

View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
"go.uber.org/zap"
)
@ -40,7 +41,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
}
objIDToDelete, err := n.treeService.PutNotificationConfigurationNode(ctx, p.BktInfo, objID)
objIDToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
objIDToDeleteNotFound := errorsStd.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objIDToDeleteNotFound {
return err
}
@ -65,7 +66,7 @@ func (n *layer) GetBucketNotificationConfiguration(ctx context.Context, bktInfo
}
objID, err := n.treeService.GetNotificationConfigurationNode(ctx, bktInfo)
objIDNotFound := errorsStd.Is(err, ErrNodeNotFound)
objIDNotFound := errorsStd.Is(err, tree.ErrNodeNotFound)
if err != nil && !objIDNotFound {
return nil, err
}

View file

@ -21,6 +21,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -315,7 +316,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
newVersion.MD5 = hex.EncodeToString(md5Hash)
}
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
if err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err)
}
@ -371,7 +372,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
node, err := n.treeService.GetLatestVersion(ctx, bkt, objectName)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchKey), err.Error())
}
return nil, err
@ -407,7 +408,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
if p.VersionID == data.UnversionedObjectVersionID {
foundVersion, err = n.treeService.GetUnversioned(ctx, bkt, p.Object)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion), err.Error())
}
return nil, err

View file

@ -12,6 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
@ -38,8 +39,8 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
}
}
lockInfo, err := n.treeService.GetLock(ctx, p.ObjVersion.BktInfo, versionNode.ID)
if err != nil && !errorsStd.Is(err, ErrNodeNotFound) {
lockInfo, err := n.treeService.GetLock(ctx, p.ObjVersion.BktInfo, versionNode)
if err != nil && !errorsStd.Is(err, tree.ErrNodeNotFound) {
return err
}
@ -91,7 +92,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
}
}
if err = n.treeService.PutLock(ctx, p.ObjVersion.BktInfo, versionNode.ID, lockInfo); err != nil {
if err = n.treeService.PutLock(ctx, p.ObjVersion.BktInfo, versionNode, lockInfo); err != nil {
return fmt.Errorf("couldn't put lock into tree: %w", err)
}
@ -140,8 +141,8 @@ func (n *layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion)
return nil, err
}
lockInfo, err := n.treeService.GetLock(ctx, objVersion.BktInfo, versionNode.ID)
if err != nil && !errorsStd.Is(err, ErrNodeNotFound) {
lockInfo, err := n.treeService.GetLock(ctx, objVersion.BktInfo, versionNode)
if err != nil && !errorsStd.Is(err, tree.ErrNodeNotFound) {
return nil, err
}
if lockInfo == nil {
@ -160,7 +161,7 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSCo
}
objID, err := n.treeService.GetBucketCORS(ctx, bkt)
objIDNotFound := errorsStd.Is(err, ErrNodeNotFound)
objIDNotFound := errorsStd.Is(err, tree.ErrNodeNotFound)
if err != nil && !objIDNotFound {
return nil, err
}
@ -198,7 +199,7 @@ func (n *layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
settings, err := n.treeService.GetSettingsNode(ctx, bktInfo)
if err != nil {
if !errorsStd.Is(err, ErrNodeNotFound) {
if !errorsStd.Is(err, tree.ErrNodeNotFound) {
return nil, err
}
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}

View file

@ -8,6 +8,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
@ -39,7 +40,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingPa
tags, err := n.treeService.GetObjectTagging(ctx, p.ObjectVersion.BktInfo, nodeVersion)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return "", nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
}
return "", nil, err
@ -62,7 +63,7 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingPa
err = n.treeService.PutObjectTagging(ctx, p.ObjectVersion.BktInfo, nodeVersion, p.TagSet)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
}
return nil, err
@ -81,7 +82,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion)
err = n.treeService.DeleteObjectTagging(ctx, p.BktInfo, version)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
}
return nil, err
@ -102,7 +103,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo)
}
tags, err := n.treeService.GetBucketTagging(ctx, bktInfo)
if err != nil && !errors.Is(err, ErrNodeNotFound) {
if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
return nil, err
}
@ -161,7 +162,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersi
if err == nil && version.IsDeleteMarker && !objVersion.NoErrorOnDeleteMarker {
return nil, fmt.Errorf("%w: found version is delete marker", s3errors.GetAPIError(s3errors.ErrNoSuchKey))
} else if errors.Is(err, ErrNodeNotFound) {
} else if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
}

View file

@ -1,448 +0,0 @@
package layer
import (
"context"
"fmt"
"io"
"sort"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
type VersionsByPrefixStreamMock struct {
result []*data.NodeVersion
offset int
}
func (s *VersionsByPrefixStreamMock) Next(context.Context) (*data.NodeVersion, error) {
if s.offset > len(s.result)-1 {
return nil, io.EOF
}
res := s.result[s.offset]
s.offset++
return res, nil
}
type TreeServiceMock struct {
settings map[string]*data.BucketSettings
versions map[string]map[string][]*data.NodeVersion
system map[string]map[string]*data.BaseNodeVersion
locks map[string]map[uint64]*data.LockInfo
tags map[string]map[uint64]map[string]string
multiparts map[string]map[string][]*data.MultipartInfo
parts map[string]map[int]*data.PartInfo
}
func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
// TODO implement object tagging
lock, err := t.GetLock(ctx, bktInfo, objVersion.ID)
return nil, lock, err
}
func (t *TreeServiceMock) GetObjectTagging(_ context.Context, bktInfo *data.BucketInfo, nodeVersion *data.NodeVersion) (map[string]string, error) {
cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()]
if !ok {
return nil, nil
}
return cnrTagsMap[nodeVersion.ID], nil
}
func (t *TreeServiceMock) PutObjectTagging(_ context.Context, bktInfo *data.BucketInfo, nodeVersion *data.NodeVersion, tagSet map[string]string) error {
cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()]
if !ok {
t.tags[bktInfo.CID.EncodeToString()] = map[uint64]map[string]string{
nodeVersion.ID: tagSet,
}
return nil
}
cnrTagsMap[nodeVersion.ID] = tagSet
return nil
}
func (t *TreeServiceMock) DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error {
return t.PutObjectTagging(ctx, bktInfo, objVersion, nil)
}
func (t *TreeServiceMock) GetBucketTagging(context.Context, *data.BucketInfo) (map[string]string, error) {
// TODO implement me
panic("implement me")
}
func (t *TreeServiceMock) PutBucketTagging(context.Context, *data.BucketInfo, map[string]string) error {
// TODO implement me
panic("implement me")
}
func (t *TreeServiceMock) DeleteBucketTagging(context.Context, *data.BucketInfo) error {
// TODO implement me
panic("implement me")
}
func NewTreeService() *TreeServiceMock {
return &TreeServiceMock{
settings: make(map[string]*data.BucketSettings),
versions: make(map[string]map[string][]*data.NodeVersion),
system: make(map[string]map[string]*data.BaseNodeVersion),
locks: make(map[string]map[uint64]*data.LockInfo),
tags: make(map[string]map[uint64]map[string]string),
multiparts: make(map[string]map[string][]*data.MultipartInfo),
parts: make(map[string]map[int]*data.PartInfo),
}
}
func (t *TreeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
t.settings[bktInfo.CID.EncodeToString()] = settings
return nil
}
func (t *TreeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
settings, ok := t.settings[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
return settings, nil
}
func (t *TreeServiceMock) GetNotificationConfigurationNode(context.Context, *data.BucketInfo) (oid.ID, error) {
panic("implement me")
}
func (t *TreeServiceMock) PutNotificationConfigurationNode(context.Context, *data.BucketInfo, oid.ID) (oid.ID, error) {
panic("implement me")
}
func (t *TreeServiceMock) GetBucketCORS(_ context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
return oid.ID{}, nil
}
node, ok := systemMap["cors"]
if !ok {
return oid.ID{}, nil
}
return node.OID, nil
}
func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
systemMap = make(map[string]*data.BaseNodeVersion)
}
systemMap["cors"] = &data.BaseNodeVersion{
OID: objID,
}
t.system[bktInfo.CID.EncodeToString()] = systemMap
return oid.ID{}, ErrNoNodeToRemove
}
func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) (oid.ID, error) {
panic("implement me")
}
func (t *TreeServiceMock) GetVersions(_ context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, ErrNodeNotFound
}
return versions, nil
}
func (t *TreeServiceMock) GetLatestVersion(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, ErrNodeNotFound
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
return versions[len(versions)-1], nil
}
return nil, ErrNodeNotFound
}
func (t *TreeServiceMock) InitVersionsByPrefixStream(_ context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) (data.VersionsStream, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
var result []*data.NodeVersion
for key, versions := range cnrVersionsMap {
if !strings.HasPrefix(key, prefix) {
continue
}
if !latestOnly {
result = append(result, versions...)
continue
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
result = append(result, versions[len(versions)-1])
}
}
return &VersionsByPrefixStreamMock{
result: result,
}, nil
}
func (t *TreeServiceMock) GetUnversioned(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
versions, ok := cnrVersionsMap[objectName]
if !ok {
return nil, ErrNodeNotFound
}
for _, version := range versions {
if version.IsUnversioned {
return version, nil
}
}
return nil, ErrNodeNotFound
}
func (t *TreeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{
newVersion.FilePath: {newVersion},
}
return newVersion.ID, nil
}
versions, ok := cnrVersionsMap[newVersion.FilePath]
if !ok {
cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion}
return newVersion.ID, nil
}
sort.Slice(versions, func(i, j int) bool {
return versions[i].ID < versions[j].ID
})
if len(versions) != 0 {
newVersion.ID = versions[len(versions)-1].ID + 1
newVersion.Timestamp = versions[len(versions)-1].Timestamp + 1
}
result := versions
if newVersion.IsUnversioned {
result = make([]*data.NodeVersion, 0, len(versions))
for _, node := range versions {
if !node.IsUnversioned {
result = append(result, node)
}
}
}
cnrVersionsMap[newVersion.FilePath] = append(result, newVersion)
return newVersion.ID, nil
}
func (t *TreeServiceMock) RemoveVersion(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) error {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return ErrNodeNotFound
}
for key, versions := range cnrVersionsMap {
for i, node := range versions {
if node.ID == nodeID {
cnrVersionsMap[key] = append(versions[:i], versions[i+1:]...)
return nil
}
}
}
return ErrNodeNotFound
}
func (t *TreeServiceMock) GetAllVersionsByPrefix(_ context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, nil
}
var result []*data.NodeVersion
for objName, versions := range cnrVersionsMap {
if strings.HasPrefix(objName, prefix) {
result = append(result, versions...)
}
}
return result, nil
}
func (t *TreeServiceMock) CreateMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error {
cnrMultipartsMap, ok := t.multiparts[bktInfo.CID.EncodeToString()]
if !ok {
t.multiparts[bktInfo.CID.EncodeToString()] = map[string][]*data.MultipartInfo{
info.Key: {info},
}
return nil
}
multiparts := cnrMultipartsMap[info.Key]
if len(multiparts) != 0 {
info.ID = multiparts[len(multiparts)-1].ID + 1
}
cnrMultipartsMap[info.Key] = append(multiparts, info)
return nil
}
func (t *TreeServiceMock) GetMultipartUploadsByPrefix(context.Context, *data.BucketInfo, string) ([]*data.MultipartInfo, error) {
panic("implement me")
}
func (t *TreeServiceMock) GetMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error) {
cnrMultipartsMap, ok := t.multiparts[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
multiparts := cnrMultipartsMap[objectName]
for _, multipart := range multiparts {
if multipart.UploadID == uploadID {
return multipart, nil
}
}
return nil, ErrNodeNotFound
}
func (t *TreeServiceMock) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error) {
multipartInfo, err := t.GetMultipartUpload(ctx, bktInfo, info.Key, info.UploadID)
if err != nil {
return oid.ID{}, err
}
if multipartInfo.ID != multipartNodeID {
return oid.ID{}, fmt.Errorf("invalid multipart info id")
}
partsMap, ok := t.parts[info.UploadID]
if !ok {
partsMap = make(map[int]*data.PartInfo)
}
partsMap[info.Number] = info
t.parts[info.UploadID] = partsMap
return oid.ID{}, nil
}
func (t *TreeServiceMock) GetParts(_ context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error) {
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
var foundMultipart *data.MultipartInfo
LOOP:
for _, multiparts := range cnrMultipartsMap {
for _, multipart := range multiparts {
if multipart.ID == multipartNodeID {
foundMultipart = multipart
break LOOP
}
}
}
if foundMultipart == nil {
return nil, ErrNodeNotFound
}
partsMap := t.parts[foundMultipart.UploadID]
result := make([]*data.PartInfo, 0, len(partsMap))
for _, part := range partsMap {
result = append(result, part)
}
return result, nil
}
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
var uploadID string
LOOP:
for key, multiparts := range cnrMultipartsMap {
for i, multipart := range multiparts {
if multipart.ID == multipartInfo.ID {
uploadID = multipart.UploadID
cnrMultipartsMap[key] = append(multiparts[:i], multiparts[i+1:]...)
break LOOP
}
}
}
if uploadID == "" {
return ErrNodeNotFound
}
delete(t.parts, uploadID)
return nil
}
func (t *TreeServiceMock) PutLock(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error {
cnrLockMap, ok := t.locks[bktInfo.CID.EncodeToString()]
if !ok {
t.locks[bktInfo.CID.EncodeToString()] = map[uint64]*data.LockInfo{
nodeID: lock,
}
return nil
}
cnrLockMap[nodeID] = lock
return nil
}
func (t *TreeServiceMock) GetLock(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) {
cnrLockMap, ok := t.locks[bktInfo.CID.EncodeToString()]
if !ok {
return nil, nil
}
return cnrLockMap[nodeID], nil
}

View file

@ -1,92 +0,0 @@
package layer
import (
"context"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
// PutSettingsNode update or create new settings node in tree service.
PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error
// GetSettingsNode retrieves the settings node from the tree service and form data.BucketSettings.
//
// If tree node is not found returns ErrNodeNotFound error.
GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
// GetNotificationConfigurationNode gets an object id that corresponds to object with bucket CORS.
//
// If tree node is not found returns ErrNodeNotFound error.
GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
// PutNotificationConfigurationNode puts a node to a system tree
// and returns objectID of a previous notif config which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
// GetBucketCORS gets an object id that corresponds to object with bucket CORS.
//
// If object id is not found returns ErrNodeNotFound error.
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error)
PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error
DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error
DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error
GetVersions(ctx context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error)
GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error)
InitVersionsByPrefixStream(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) (data.VersionsStream, error)
GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error)
AddVersion(ctx context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error)
RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) error
PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error
GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error)
CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error
DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error
GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error)
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
// AddPart puts a node to a system tree as a child of appropriate multipart upload
// and returns objectID of a previous part which must be deleted in FrostFS.
//
// If object id to remove is not found returns ErrNoNodeToRemove error.
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
// Compound methods for optimizations
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error)
}
var (
// ErrNodeNotFound is returned from Tree service in case of not found error.
ErrNodeNotFound = errors.New("not found")
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
ErrNodeAccessDenied = errors.New("access denied")
// ErrNoNodeToRemove is returned from Tree service in case of the lack of node with OID to remove.
ErrNoNodeToRemove = errors.New("no node to remove")
)

View file

@ -9,6 +9,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -167,10 +168,13 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
var owner user.ID
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
memTree, err := tree.NewTreeServiceClientMemory()
require.NoError(t, err)
layerCfg := &Config{
Cache: NewCache(config),
AnonKey: AnonymousKey{Key: key},
TreeService: NewTreeService(),
TreeService: tree.NewTree(memTree, logger),
Features: &FeatureSettingsMock{},
GateOwner: owner,
}

2
go.mod
View file

@ -37,6 +37,8 @@ require (
google.golang.org/protobuf v1.33.0
)
replace git.frostfs.info/TrueCloudLab/frostfs-sdk-go => git.frostfs.info/dkirillov/frostfs-sdk-go v0.0.0-20240621142828-0ee3fcc18ffd
require (
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect

4
go.sum
View file

@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f h1:vBLC1OSGMSn7lRJv/p1of0veifuBdZdztVrF9Vn+UFk=
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU=
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a h1:Bk1fB4cQASPKgAVGCdlBOEp5ohZfDxqK6fZM8eP+Emo=
@ -56,6 +54,8 @@ git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjq
git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8=
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4=
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw=
git.frostfs.info/dkirillov/frostfs-sdk-go v0.0.0-20240621142828-0ee3fcc18ffd h1:tLkZ8XzgDbElZvYHJSAR2FTx2IY0F3Y/FIYWRY3X4rY=
git.frostfs.info/dkirillov/frostfs-sdk-go v0.0.0-20240621142828-0ee3fcc18ffd/go.mod h1:e7H9nNFpx1Tj3R20Zoxy0Vo6Srlb6zV5L7ZQXqg9rn4=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=

View file

@ -15,30 +15,6 @@ import (
grpcService "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree/service"
)
type GetNodeByPathResponseInfoWrapper struct {
response *grpcService.GetNodeByPathResponse_Info
}
func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 {
return n.response.GetNodeId()
}
func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 {
return n.response.GetParentId()
}
func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 {
return n.response.GetTimestamp()
}
func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.response.Meta))
for i, value := range n.response.Meta {
res[i] = value
}
return res
}
type GetSubTreeResponseBodyWrapper struct {
response *grpcService.GetSubTreeResponse_Body
}
@ -63,6 +39,87 @@ func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta {
return res
}
type BurnedListResponseBodyWrapper struct {
response *grpcService.BurnedListResponse_Body
}
func (n BurnedListResponseBodyWrapper) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.response.Meta))
for i, value := range n.response.Meta {
res[i] = value
}
return res
}
func (n BurnedListResponseBodyWrapper) GetKey() string {
return n.response.GetKey()
}
func (n BurnedListResponseBodyWrapper) GetTimestamp() uint64 {
return n.response.GetTimestamp()
}
type BurnedGetResponseBodyWrapper struct {
response *grpcService.BurnedGetResponse_Body
key string
}
func (n BurnedGetResponseBodyWrapper) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.response.Meta))
for i, value := range n.response.Meta {
res[i] = value
}
return res
}
func (n BurnedGetResponseBodyWrapper) GetKey() string {
return n.key
}
func (n BurnedGetResponseBodyWrapper) GetTimestamp() uint64 {
return n.response.GetTimestamp()
}
type BurnedListByPrefixResponseBodyInfoWrapper struct {
response *grpcService.BurnedListByPrefixResponse_Body_Info
}
func (n BurnedListByPrefixResponseBodyInfoWrapper) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.response.Meta))
for i, value := range n.response.Meta {
res[i] = value
}
return res
}
func (n BurnedListByPrefixResponseBodyInfoWrapper) GetKey() string {
return n.response.GetKey()
}
func (n BurnedListByPrefixResponseBodyInfoWrapper) GetTimestamp() uint64 {
return n.response.GetTimestamp()
}
type BurnedGetLatestByPrefixResponseBodyWrapper struct {
response *grpcService.BurnedGetLatestByPrefixResponse_Body
}
func (n BurnedGetLatestByPrefixResponseBodyWrapper) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.response.Meta))
for i, value := range n.response.Meta {
res[i] = value
}
return res
}
func (n BurnedGetLatestByPrefixResponseBodyWrapper) GetKey() string {
return n.response.GetKey()
}
func (n BurnedGetLatestByPrefixResponseBodyWrapper) GetTimestamp() uint64 {
return n.response.GetTimestamp()
}
type PoolWrapper struct {
p *treepool.Pool
}
@ -71,77 +128,110 @@ func NewPoolWrapper(p *treepool.Pool) *PoolWrapper {
return &PoolWrapper{p: p}
}
func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) {
poolPrm := treepool.GetNodesParams{
CID: prm.BktInfo.CID,
TreeID: prm.TreeID,
Path: prm.Path,
Meta: prm.Meta,
PathAttribute: tree.FileNameKey,
LatestOnly: prm.LatestOnly,
AllAttrs: prm.AllAttrs,
BearerToken: getBearer(ctx, prm.BktInfo),
}
func (w *PoolWrapper) BurnedAdd(ctx context.Context, bktInfo *data.BucketInfo, treeID string, key string, meta map[string]string) error {
_, err := w.p.BurnedAdd(ctx, treepool.BurnedAddParams{
CID: bktInfo.CID,
TreeID: treeID,
Key: key,
Meta: meta,
BearerToken: getBearer(ctx, bktInfo),
})
return handleError(err)
}
nodes, err := w.p.GetNodes(ctx, poolPrm)
func (w *PoolWrapper) BurnedGet(ctx context.Context, bktInfo *data.BucketInfo, treeID string, key string) (tree.NodeResponse, error) {
resp, err := w.p.BurnedGet(ctx, treepool.BurnedGetParams{
CID: bktInfo.CID,
TreeID: treeID,
Key: key,
BearerToken: getBearer(ctx, bktInfo),
})
return BurnedGetResponseBodyWrapper{response: resp, key: key}, handleError(err)
}
func (w *PoolWrapper) BurnedGetLatestByPrefix(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefix string) (tree.NodeResponse, error) {
resp, err := w.p.BurnedGetLatestByPrefix(ctx, treepool.BurnedGetLatestByPrefixParams{
CID: bktInfo.CID,
TreeID: treeID,
Prefix: prefix,
BearerToken: getBearer(ctx, bktInfo),
})
return BurnedGetLatestByPrefixResponseBodyWrapper{resp}, handleError(err)
}
func (w *PoolWrapper) BurnedRemove(ctx context.Context, bktInfo *data.BucketInfo, treeID string, key string) error {
return handleError(w.p.BurnedRemove(ctx, treepool.BurnedRemoveParams{
CID: bktInfo.CID,
TreeID: treeID,
Key: key,
BearerToken: getBearer(ctx, bktInfo),
}))
}
func (w *PoolWrapper) BurnedListByPrefix(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefix string) ([]tree.NodeResponse, error) {
resp, err := w.p.BurnedListByPrefix(ctx, treepool.BurnedListByPrefixParams{
CID: bktInfo.CID,
TreeID: treeID,
Prefix: prefix,
BearerToken: getBearer(ctx, bktInfo),
})
if err != nil {
return nil, handleError(err)
}
res := make([]tree.NodeResponse, len(nodes))
for i, info := range nodes {
res[i] = GetNodeByPathResponseInfoWrapper{info}
res := make([]tree.NodeResponse, len(resp.GetList()))
for i, info := range resp.GetList() {
res[i] = BurnedListByPrefixResponseBodyInfoWrapper{response: info}
}
return res, nil
}
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]tree.NodeResponse, error) {
poolPrm := treepool.GetSubTreeParams{
func (w *PoolWrapper) BurnedList(ctx context.Context, bktInfo *data.BucketInfo, treeID, start string) ([]tree.NodeResponse, error) {
cli, err := w.p.BurnedList(ctx, treepool.BurnedListParams{
CID: bktInfo.CID,
TreeID: treeID,
RootID: rootID,
Depth: depth,
Start: start,
BearerToken: getBearer(ctx, bktInfo),
}
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
})
if err != nil {
return nil, handleError(err)
}
var subtree []tree.NodeResponse
var res []tree.NodeResponse
node, err := subTreeReader.Next()
node, err := cli.Next()
for err == nil {
subtree = append(subtree, GetSubTreeResponseBodyWrapper{node})
node, err = subTreeReader.Next()
res = append(res, BurnedListResponseBodyWrapper{node})
node, err = cli.Next()
}
if err != nil && err != io.EOF {
return nil, handleError(err)
}
return subtree, nil
return res, nil
}
type SubTreeStreamImpl struct {
r *treepool.SubTreeReader
buffer []*grpcService.GetSubTreeResponse_Body
const bufSize = 1000
type BurnedListStreamImpl struct {
r *treepool.BurnedListReader
buffer []*grpcService.BurnedListResponse_Body
eof bool
index int
ln int
}
const bufSize = 1000
func (s *SubTreeStreamImpl) Next() (tree.NodeResponse, error) {
func (s *BurnedListStreamImpl) Next() (tree.NodeResponse, error) {
if s.index != -1 {
node := s.buffer[s.index]
s.index++
if s.index >= s.ln {
s.index = -1
}
return GetSubTreeResponseBodyWrapper{response: node}, nil
return BurnedListResponseBodyWrapper{response: node}, nil
}
if s.eof {
return nil, io.EOF
@ -162,71 +252,24 @@ func (s *SubTreeStreamImpl) Next() (tree.NodeResponse, error) {
return s.Next()
}
func (w *PoolWrapper) GetSubTreeStream(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) (tree.SubTreeStream, error) {
poolPrm := treepool.GetSubTreeParams{
func (w *PoolWrapper) BurnedListStream(ctx context.Context, bktInfo *data.BucketInfo, treeID, start string) (tree.SubTreeStream, error) {
cli, err := w.p.BurnedList(ctx, treepool.BurnedListParams{
CID: bktInfo.CID,
TreeID: treeID,
RootID: rootID,
Depth: depth,
Start: start,
BearerToken: getBearer(ctx, bktInfo),
Order: treepool.AscendingOrder,
}
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
})
if err != nil {
return nil, handleError(err)
}
return &SubTreeStreamImpl{
r: subTreeReader,
buffer: make([]*grpcService.GetSubTreeResponse_Body, bufSize),
return &BurnedListStreamImpl{
r: cli,
buffer: make([]*grpcService.BurnedListResponse_Body, bufSize),
index: -1,
}, nil
}
func (w *PoolWrapper) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) {
nodeID, err := w.p.AddNode(ctx, treepool.AddNodeParams{
CID: bktInfo.CID,
TreeID: treeID,
Parent: parent,
Meta: meta,
BearerToken: getBearer(ctx, bktInfo),
})
return nodeID, handleError(err)
}
func (w *PoolWrapper) AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) {
nodeID, err := w.p.AddNodeByPath(ctx, treepool.AddNodeByPathParams{
CID: bktInfo.CID,
TreeID: treeID,
Path: path,
Meta: meta,
PathAttribute: tree.FileNameKey,
BearerToken: getBearer(ctx, bktInfo),
})
return nodeID, handleError(err)
}
func (w *PoolWrapper) MoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error {
return handleError(w.p.MoveNode(ctx, treepool.MoveNodeParams{
CID: bktInfo.CID,
TreeID: treeID,
NodeID: nodeID,
ParentID: parentID,
Meta: meta,
BearerToken: getBearer(ctx, bktInfo),
}))
}
func (w *PoolWrapper) RemoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error {
return handleError(w.p.RemoveNode(ctx, treepool.RemoveNodeParams{
CID: bktInfo.CID,
TreeID: treeID,
NodeID: nodeID,
BearerToken: getBearer(ctx, bktInfo),
}))
}
func getBearer(ctx context.Context, bktInfo *data.BucketInfo) []byte {
if bd, err := middleware.GetBoxData(ctx); err == nil {
if bd.Gate.BearerToken != nil {

File diff suppressed because it is too large Load diff

View file

@ -2,13 +2,13 @@ package tree
import (
"context"
"fmt"
"io"
"sort"
"strconv"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"golang.org/x/exp/slices"
)
type nodeMeta struct {
@ -26,17 +26,8 @@ func (m nodeMeta) GetValue() []byte {
type nodeResponse struct {
meta []nodeMeta
nodeID uint64
parentID uint64
timestamp uint64
}
func (n nodeResponse) GetNodeID() uint64 {
return n.nodeID
}
func (n nodeResponse) GetParentID() uint64 {
return n.parentID
key string
}
func (n nodeResponse) GetTimestamp() uint64 {
@ -48,16 +39,12 @@ func (n nodeResponse) GetMeta() []Meta {
for i, value := range n.meta {
res[i] = value
}
return res
return append(res, nodeMeta{key: "timestamp", value: []byte(strconv.FormatUint(n.timestamp, 10))})
}
func (n nodeResponse) getValue(key string) string {
for _, value := range n.meta {
if value.key == key {
return string(value.value)
}
}
return ""
func (n nodeResponse) GetKey() string {
return n.key
}
type ServiceClientMemory struct {
@ -66,118 +53,10 @@ type ServiceClientMemory struct {
type containerInfo struct {
bkt *data.BucketInfo
trees map[string]memoryTree
trees map[string]map[string]*nodeResponse
}
type memoryTree struct {
idCounter uint64
treeData *treeNodeMemory
}
type treeNodeMemory struct {
data nodeResponse
parent *treeNodeMemory
children []*treeNodeMemory
}
func (t *treeNodeMemory) getNode(nodeID uint64) *treeNodeMemory {
if t.data.nodeID == nodeID {
return t
}
for _, child := range t.children {
if node := child.getNode(nodeID); node != nil {
return node
}
}
return nil
}
func (t *memoryTree) getNodesByPath(path []string) []nodeResponse {
if len(path) == 0 {
return nil
}
var res []nodeResponse
for _, child := range t.treeData.children {
res = child.listNodesByPath(res, path)
}
return res
}
func (t *treeNodeMemory) listNodesByPath(res []nodeResponse, path []string) []nodeResponse {
if len(path) == 0 || t.data.getValue(FileNameKey) != path[0] {
return res
}
if len(path) == 1 {
return append(res, t.data)
}
for _, ch := range t.children {
res = ch.listNodesByPath(res, path[1:])
}
return res
}
func (t *memoryTree) createPathIfNotExist(parent *treeNodeMemory, path []string) *treeNodeMemory {
if len(path) == 0 {
return parent
}
var node *treeNodeMemory
for _, child := range parent.children {
if len(child.data.meta) == 1 && child.data.getValue(FileNameKey) == path[0] {
node = child
break
}
}
if node == nil {
node = &treeNodeMemory{
data: nodeResponse{
meta: []nodeMeta{{key: FileNameKey, value: []byte(path[0])}},
nodeID: t.idCounter,
parentID: parent.data.nodeID,
timestamp: uint64(time.Now().UnixMicro()),
},
parent: parent,
}
t.idCounter++
parent.children = append(parent.children, node)
}
return t.createPathIfNotExist(node, path[1:])
}
func (t *treeNodeMemory) removeChild(nodeID uint64) {
ind := -1
for i, ch := range t.children {
if ch.data.nodeID == nodeID {
ind = i
break
}
}
if ind != -1 {
t.children = append(t.children[:ind], t.children[ind+1:]...)
}
}
func (t *treeNodeMemory) listNodes(res []NodeResponse, depth uint32) []NodeResponse {
res = append(res, t.data)
if depth == 0 {
return res
}
for _, ch := range t.children {
res = ch.listNodes(res, depth-1)
}
return res
}
var _ ServiceClient = (*ServiceClientMemory)(nil)
func NewTreeServiceClientMemory() (*ServiceClientMemory, error) {
return &ServiceClientMemory{
@ -185,72 +64,121 @@ func NewTreeServiceClientMemory() (*ServiceClientMemory, error) {
}, nil
}
type nodeResponseWrapper struct {
nodeResponse
allAttr bool
attrs []string
func newContainerInfo(bktInfo *data.BucketInfo, treeID string) containerInfo {
return containerInfo{
bkt: bktInfo,
trees: map[string]map[string]*nodeResponse{
treeID: {},
},
}
}
func (n nodeResponseWrapper) GetMeta() []Meta {
res := make([]Meta, 0, len(n.meta))
for _, value := range n.meta {
if n.allAttr || slices.Contains(n.attrs, value.key) {
res = append(res, value)
}
}
return res
}
func (c *ServiceClientMemory) GetNodes(_ context.Context, p *GetNodesParams) ([]NodeResponse, error) {
cnr, ok := c.containers[p.BktInfo.CID.EncodeToString()]
if !ok {
return nil, nil
}
tr, ok := cnr.trees[p.TreeID]
if !ok {
return nil, nil
}
res := tr.getNodesByPath(p.Path)
sort.Slice(res, func(i, j int) bool {
return res[i].timestamp < res[j].timestamp
})
if p.LatestOnly && len(res) != 0 {
res = res[len(res)-1:]
}
res2 := make([]NodeResponse, len(res))
for i, n := range res {
res2[i] = nodeResponseWrapper{
nodeResponse: n,
allAttr: p.AllAttrs,
attrs: p.Meta,
}
}
return res2, nil
}
func (c *ServiceClientMemory) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error) {
func (c *ServiceClientMemory) BurnedAdd(_ context.Context, bktInfo *data.BucketInfo, treeID string, key string, meta map[string]string) error {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
return nil, nil
cnr = newContainerInfo(bktInfo, treeID)
c.containers[bktInfo.CID.EncodeToString()] = cnr
}
tr, ok := cnr.trees[treeID]
if !ok {
tr = make(map[string]*nodeResponse)
}
tr[key] = &nodeResponse{
meta: metaToNodeMeta(meta),
key: key,
timestamp: uint64(time.Now().UnixMicro()),
}
cnr.trees[treeID] = tr
return nil
}
func (c *ServiceClientMemory) BurnedGet(_ context.Context, bktInfo *data.BucketInfo, treeID string, key string) (NodeResponse, error) {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
node := tr.treeData.getNode(rootID)
if node == nil {
node, ok := cnr.trees[treeID][key]
if !ok {
return nil, ErrNodeNotFound
}
// we depth-1 in case of uint32 and 0 as mark to get all subtree leads to overflow and depth is getting quite big to walk all tree levels
return node.listNodes(nil, depth-1), nil
return node, nil
}
func (c *ServiceClientMemory) BurnedGetLatestByPrefix(_ context.Context, bktInfo *data.BucketInfo, treeID string, prefix string) (NodeResponse, error) {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
var res *nodeResponse
for k, val := range cnr.trees[treeID] {
if strings.HasPrefix(k, prefix) {
if res == nil || res.timestamp < val.timestamp {
res = val
}
}
}
if res == nil {
return nil, ErrNodeNotFound
}
return res, nil
}
func (c *ServiceClientMemory) BurnedRemove(_ context.Context, bktInfo *data.BucketInfo, treeID string, key string) error {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
return ErrNodeNotFound
}
tr, ok := cnr.trees[treeID]
if !ok {
return ErrNodeNotFound
}
delete(tr, key)
return nil
}
func (c *ServiceClientMemory) BurnedListByPrefix(_ context.Context, bktInfo *data.BucketInfo, treeID string, prefix string) ([]NodeResponse, error) {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}
var res []NodeResponse
for k, val := range cnr.trees[treeID] {
if strings.HasPrefix(k, prefix) {
res = append(res, val)
}
}
sort.Slice(res, func(i, j int) bool {
return res[i].GetKey() < res[j].GetKey()
})
return res, nil
}
func (c *ServiceClientMemory) BurnedList(ctx context.Context, bktInfo *data.BucketInfo, treeID, start string) ([]NodeResponse, error) {
res, err := c.BurnedListByPrefix(ctx, bktInfo, treeID, "")
if err != nil {
return nil, err
}
for i := range res {
if res[i].GetKey() > start {
return res[i:], nil
}
}
return nil, nil
}
type SubTreeStreamMemoryImpl struct {
@ -270,205 +198,20 @@ func (s *SubTreeStreamMemoryImpl) Next() (NodeResponse, error) {
return s.res[s.offset-1], nil
}
func (c *ServiceClientMemory) GetSubTreeStream(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) (SubTreeStream, error) {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
return &SubTreeStreamMemoryImpl{err: ErrNodeNotFound}, nil
func (c *ServiceClientMemory) BurnedListStream(ctx context.Context, bktInfo *data.BucketInfo, treeID, start string) (SubTreeStream, error) {
res, err := c.BurnedListByPrefix(ctx, bktInfo, treeID, "")
if err != nil {
return nil, err
}
tr, ok := cnr.trees[treeID]
if !ok {
return nil, ErrNodeNotFound
for i, item := range res {
if start <= item.GetKey() {
res = res[i:]
break
}
}
node := tr.treeData.getNode(rootID)
if node == nil {
return nil, ErrNodeNotFound
}
sortNode(tr.treeData)
return &SubTreeStreamMemoryImpl{
res: node.listNodes(nil, depth-1),
offset: 0,
}, nil
}
func newContainerInfo(bktInfo *data.BucketInfo, treeID string) containerInfo {
return containerInfo{
bkt: bktInfo,
trees: map[string]memoryTree{
treeID: {
idCounter: 1,
treeData: &treeNodeMemory{
data: nodeResponse{
timestamp: uint64(time.Now().UnixMicro()),
},
},
},
},
}
}
func newMemoryTree() memoryTree {
return memoryTree{
idCounter: 1,
treeData: &treeNodeMemory{
data: nodeResponse{
timestamp: uint64(time.Now().UnixMicro()),
},
},
}
}
func (c *ServiceClientMemory) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) {
return c.AddNodeBase(ctx, bktInfo, treeID, parent, meta, true)
}
func (c *ServiceClientMemory) AddNodeBase(_ context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string, needSort bool) (uint64, error) {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
cnr = newContainerInfo(bktInfo, treeID)
c.containers[bktInfo.CID.EncodeToString()] = cnr
}
tr, ok := cnr.trees[treeID]
if !ok {
tr = newMemoryTree()
cnr.trees[treeID] = tr
}
parentNode := tr.treeData.getNode(parent)
if parentNode == nil {
return 0, ErrNodeNotFound
}
newID := tr.idCounter
tr.idCounter++
tn := &treeNodeMemory{
data: nodeResponse{
meta: metaToNodeMeta(meta),
nodeID: newID,
parentID: parent,
timestamp: uint64(time.Now().UnixMicro()),
},
parent: parentNode,
}
parentNode.children = append(parentNode.children, tn)
if needSort {
sortNodes(parentNode.children)
}
cnr.trees[treeID] = tr
return newID, nil
}
func (c *ServiceClientMemory) AddNodeByPath(_ context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
cnr = newContainerInfo(bktInfo, treeID)
c.containers[bktInfo.CID.EncodeToString()] = cnr
}
tr, ok := cnr.trees[treeID]
if !ok {
tr = newMemoryTree()
cnr.trees[treeID] = tr
}
parentNode := tr.createPathIfNotExist(tr.treeData, path)
if parentNode == nil {
return 0, fmt.Errorf("create path '%s'", path)
}
newID := tr.idCounter
tr.idCounter++
tn := &treeNodeMemory{
data: nodeResponse{
meta: metaToNodeMeta(meta),
nodeID: newID,
parentID: parentNode.data.nodeID,
timestamp: uint64(time.Now().UnixMicro()),
},
parent: parentNode,
}
parentNode.children = append(parentNode.children, tn)
cnr.trees[treeID] = tr
return newID, nil
}
func (c *ServiceClientMemory) MoveNode(_ context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
return ErrNodeNotFound
}
tr, ok := cnr.trees[treeID]
if !ok {
return ErrNodeNotFound
}
node := tr.treeData.getNode(nodeID)
if node == nil {
return ErrNodeNotFound
}
newParent := tr.treeData.getNode(parentID)
if newParent == nil {
return ErrNodeNotFound
}
node.data.meta = metaToNodeMeta(meta)
node.data.parentID = parentID
newParent.children = append(newParent.children, node)
node.parent.removeChild(nodeID)
return nil
}
func sortNode(node *treeNodeMemory) {
if node == nil {
return
}
sortNodes(node.children)
for _, child := range node.children {
sortNode(child)
}
}
func sortNodes(list []*treeNodeMemory) {
sort.Slice(list, func(i, j int) bool {
return list[i].data.getValue(FileNameKey) < list[j].data.getValue(FileNameKey)
})
}
func (c *ServiceClientMemory) RemoveNode(_ context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error {
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
if !ok {
return ErrNodeNotFound
}
tr, ok := cnr.trees[treeID]
if !ok {
return ErrNodeNotFound
}
node := tr.treeData.getNode(nodeID)
if node == nil {
return ErrNodeNotFound
}
node.parent.removeChild(nodeID)
return nil
return &SubTreeStreamMemoryImpl{res: res}, nil
}
func metaToNodeMeta(m map[string]string) []nodeMeta {

View file

@ -163,12 +163,11 @@ func TestTreeServiceAddVersion(t *testing.T) {
IsUnversioned: true,
}
nodeID, err := treeService.AddVersion(ctx, bktInfo, version)
err = treeService.AddVersion(ctx, bktInfo, version)
require.NoError(t, err)
storedNode, err := treeService.GetUnversioned(ctx, bktInfo, "path/to/version")
require.NoError(t, err)
require.Equal(t, nodeID, storedNode.ID)
require.Equal(t, version.BaseNodeVersion.Size, storedNode.Size)
require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag)
require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag)
@ -183,10 +182,10 @@ func TestTreeServiceAddVersion(t *testing.T) {
func TestGetLatestNode(t *testing.T) {
for _, tc := range []struct {
name string
nodes []NodeResponse
expectedNodeID uint64
error bool
name string
nodes []NodeResponse
exceptedTimestamp uint64
error bool
}{
{
name: "empty",
@ -197,8 +196,6 @@ func TestGetLatestNode(t *testing.T) {
name: "one node of the object version",
nodes: []NodeResponse{
nodeResponse{
nodeID: 1,
parentID: 0,
timestamp: 1,
meta: []nodeMeta{
{
@ -208,20 +205,16 @@ func TestGetLatestNode(t *testing.T) {
},
},
},
expectedNodeID: 1,
exceptedTimestamp: 1,
},
{
name: "one node of the object version and one node of the secondary object",
nodes: []NodeResponse{
nodeResponse{
nodeID: 2,
parentID: 0,
timestamp: 3,
meta: []nodeMeta{},
},
nodeResponse{
nodeID: 1,
parentID: 0,
timestamp: 1,
meta: []nodeMeta{
{
@ -231,20 +224,16 @@ func TestGetLatestNode(t *testing.T) {
},
},
},
expectedNodeID: 1,
exceptedTimestamp: 1,
},
{
name: "all nodes represent a secondary object",
nodes: []NodeResponse{
nodeResponse{
nodeID: 2,
parentID: 0,
timestamp: 3,
meta: []nodeMeta{},
},
nodeResponse{
nodeID: 4,
parentID: 0,
timestamp: 5,
meta: []nodeMeta{},
},
@ -255,8 +244,6 @@ func TestGetLatestNode(t *testing.T) {
name: "several nodes of different types and with different timestamp",
nodes: []NodeResponse{
nodeResponse{
nodeID: 1,
parentID: 0,
timestamp: 1,
meta: []nodeMeta{
{
@ -266,14 +253,10 @@ func TestGetLatestNode(t *testing.T) {
},
},
nodeResponse{
nodeID: 3,
parentID: 0,
timestamp: 3,
meta: []nodeMeta{},
},
nodeResponse{
nodeID: 4,
parentID: 0,
timestamp: 4,
meta: []nodeMeta{
{
@ -283,13 +266,11 @@ func TestGetLatestNode(t *testing.T) {
},
},
nodeResponse{
nodeID: 6,
parentID: 0,
timestamp: 6,
meta: []nodeMeta{},
},
},
expectedNodeID: 4,
exceptedTimestamp: 4,
},
} {
t.Run(tc.name, func(t *testing.T) {
@ -298,9 +279,13 @@ func TestGetLatestNode(t *testing.T) {
require.Error(t, err)
return
}
require.NoError(t, err)
tNode, err := newTreeNode(actualNode)
require.NoError(t, err)
require.NoError(t, err)
require.Equal(t, tc.expectedNodeID, actualNode.GetNodeID())
require.EqualValues(t, tc.exceptedTimestamp, tNode.TimeStamp)
})
}
}