Compare commits
3 commits
master
...
poc/new-pi
Author | SHA1 | Date | |
---|---|---|---|
7a1665c660 | |||
30ae1c469e | |||
b0e52a4b5c |
25 changed files with 711 additions and 1792 deletions
|
@ -61,8 +61,6 @@ func (e ExtendedObjectInfo) Version() string {
|
|||
// BaseNodeVersion is minimal node info from tree service.
|
||||
// Basically used for "system" object.
|
||||
type BaseNodeVersion struct {
|
||||
ID uint64
|
||||
ParenID uint64
|
||||
OID oid.ID
|
||||
Timestamp uint64
|
||||
Size uint64
|
||||
|
@ -100,9 +98,6 @@ type ObjectTaggingInfo struct {
|
|||
|
||||
// MultipartInfo is multipart upload information.
|
||||
type MultipartInfo struct {
|
||||
// ID is node id in tree service.
|
||||
// It's ignored when creating a new multipart upload.
|
||||
ID uint64
|
||||
Key string
|
||||
UploadID string
|
||||
Owner user.ID
|
||||
|
@ -139,8 +134,6 @@ func (p *PartInfo) GetETag(md5Enabled bool) string {
|
|||
|
||||
// LockInfo is lock information to create appropriate tree node.
|
||||
type LockInfo struct {
|
||||
id uint64
|
||||
|
||||
legalHoldOID oid.ID
|
||||
setLegalHold bool
|
||||
|
||||
|
@ -150,12 +143,8 @@ type LockInfo struct {
|
|||
isCompliance bool
|
||||
}
|
||||
|
||||
func NewLockInfo(id uint64) *LockInfo {
|
||||
return &LockInfo{id: id}
|
||||
}
|
||||
|
||||
func (l LockInfo) ID() uint64 {
|
||||
return l.id
|
||||
func NewLockInfo() *LockInfo {
|
||||
return &LockInfo{}
|
||||
}
|
||||
|
||||
func (l *LockInfo) SetLegalHold(objID oid.ID) {
|
||||
|
|
|
@ -264,6 +264,19 @@ func createMultipartUploadBase(hc *handlerContext, bktName, objName string, encr
|
|||
return w
|
||||
}
|
||||
|
||||
func abortMultipartUpload(hc *handlerContext, bktName, objName, uploadID string) {
|
||||
w := abortMultipartUploadBase(hc, bktName, objName, uploadID)
|
||||
assertStatus(hc.t, w, http.StatusNoContent)
|
||||
}
|
||||
|
||||
func abortMultipartUploadBase(hc *handlerContext, bktName, objName, uploadID string) *httptest.ResponseRecorder {
|
||||
query := make(url.Values)
|
||||
query.Set(uploadIDQuery, uploadID)
|
||||
w, r := prepareTestFullRequest(hc, bktName, objName, query, nil)
|
||||
hc.Handler().AbortMultipartUploadHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func completeMultipartUpload(hc *handlerContext, bktName, objName, uploadID string, partsETags []string) {
|
||||
w := completeMultipartUploadBase(hc, bktName, objName, uploadID, partsETags)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
|
|
|
@ -197,6 +197,19 @@ func TestGetObject(t *testing.T) {
|
|||
getObjectAssertS3Error(hc, bktName, objName, emptyVersion, errors.ErrNoSuchKey)
|
||||
}
|
||||
|
||||
func TestGetObjectPiloramaPrefix(t *testing.T) {
|
||||
hc := prepareHandlerContextWithMinCache(t)
|
||||
bktName, objName, objName2 := "bucket", "obj", "obj2"
|
||||
createBucket(hc, bktName)
|
||||
|
||||
hdr := putObjectContent(hc, bktName, objName, "content1")
|
||||
putObjectContent(hc, bktName, objName2, "content2")
|
||||
|
||||
w := headObjectBase(hc, bktName, objName, emptyVersion)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
require.Equal(t, hdr.Get(api.ETag), w.Header().Get(api.ETag))
|
||||
}
|
||||
|
||||
func TestGetObjectEnabledMD5(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName := "bucket", "obj"
|
||||
|
|
|
@ -98,7 +98,7 @@ func TestMultipartReUploadPart(t *testing.T) {
|
|||
|
||||
innerUploadInfo, err := hc.tree.GetMultipartUpload(hc.context, bktInfo, objName, uploadInfo.UploadID)
|
||||
require.NoError(t, err)
|
||||
treeParts, err := hc.tree.GetParts(hc.Context(), bktInfo, innerUploadInfo.ID)
|
||||
treeParts, err := hc.tree.GetParts(hc.Context(), bktInfo, innerUploadInfo)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, treeParts, len(list.Parts))
|
||||
|
||||
|
@ -264,6 +264,25 @@ func TestMultipartUploadSize(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestMultipartUploadNewPilorama(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-test-list-parts", "object-multipart"
|
||||
bktInfo := createBucket(hc, bktName)
|
||||
|
||||
partSize := 1024
|
||||
uploadInfo := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
||||
uploadPart(hc, bktName, objName, uploadInfo.UploadID, 1, partSize)
|
||||
uploadPart(hc, bktName, objName, uploadInfo.UploadID, 2, partSize)
|
||||
|
||||
abortMultipartUpload(hc, bktName, objName, uploadInfo.UploadID)
|
||||
|
||||
require.Len(t, hc.tp.Objects(), 0)
|
||||
parts, err := hc.tree.GetParts(hc.Context(), bktInfo.BktInfo, &data.MultipartInfo{Key: objName, UploadID: uploadInfo.UploadID})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parts, 0)
|
||||
}
|
||||
|
||||
func TestListParts(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
|
|
|
@ -94,13 +94,10 @@ func TestListObjectsWithOldTreeNodes(t *testing.T) {
|
|||
}
|
||||
|
||||
func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
|
||||
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", 0, 0)
|
||||
nodes, err := hc.treeMock.BurnedList(hc.Context(), bktInfo, "version", "")
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
for _, node := range nodes {
|
||||
if node.GetNodeID() == 0 {
|
||||
continue
|
||||
}
|
||||
meta := make(map[string]string, len(node.GetMeta()))
|
||||
for _, m := range node.GetMeta() {
|
||||
if m.GetKey() != "Created" && m.GetKey() != "Owner" {
|
||||
|
@ -108,7 +105,7 @@ func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
|
|||
}
|
||||
}
|
||||
|
||||
err = hc.treeMock.MoveNode(hc.Context(), bktInfo, "version", node.GetNodeID(), node.GetParentID(), meta)
|
||||
err = hc.treeMock.BurnedAdd(hc.Context(), bktInfo, "version", node.GetKey(), meta)
|
||||
require.NoError(hc.t, err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -15,6 +15,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
@ -68,11 +69,12 @@ func transformToS3Error(err error) error {
|
|||
}
|
||||
|
||||
if errors.Is(err, layer.ErrAccessDenied) ||
|
||||
errors.Is(err, layer.ErrNodeAccessDenied) {
|
||||
errors.Is(err, tree.ErrNodeAccessDenied) {
|
||||
return s3errors.GetAPIError(s3errors.ErrAccessDenied)
|
||||
}
|
||||
|
||||
if errors.Is(err, layer.ErrGatewayTimeout) {
|
||||
if errors.Is(err, layer.ErrGatewayTimeout) ||
|
||||
errors.Is(err, tree.ErrGatewayTimeout) {
|
||||
return s3errors.GetAPIError(s3errors.ErrGatewayTimeout)
|
||||
}
|
||||
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -33,7 +34,7 @@ func TestTransformS3Errors(t *testing.T) {
|
|||
},
|
||||
{
|
||||
name: "layer node access denied error to s3 access denied error",
|
||||
err: layer.ErrNodeAccessDenied,
|
||||
err: tree.ErrNodeAccessDenied,
|
||||
expected: s3errors.ErrAccessDenied,
|
||||
},
|
||||
{
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
)
|
||||
|
||||
func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, data.LockInfo, error) {
|
||||
|
@ -29,7 +30,7 @@ func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *data.Ob
|
|||
|
||||
tags, lockInfo, err = n.treeService.GetObjectTaggingAndLock(ctx, objVersion.BktInfo, nodeVersion)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, data.LockInfo{}, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
||||
}
|
||||
return nil, data.LockInfo{}, err
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -50,7 +51,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
|||
}
|
||||
|
||||
objIDToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, objID)
|
||||
objIDToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
||||
objIDToDeleteNotFound := errorsStd.Is(err, tree.ErrNoNodeToRemove)
|
||||
if err != nil && !objIDToDeleteNotFound {
|
||||
return err
|
||||
}
|
||||
|
@ -71,7 +72,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
|||
func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*data.CORSConfiguration, error) {
|
||||
cors, err := n.getCORS(ctx, bktInfo)
|
||||
if err != nil {
|
||||
if errorsStd.Is(err, ErrNodeNotFound) {
|
||||
if errorsStd.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrNoSuchCORSConfiguration), err.Error())
|
||||
}
|
||||
return nil, err
|
||||
|
@ -82,7 +83,7 @@ func (n *layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*d
|
|||
|
||||
func (n *layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||
objID, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
|
||||
objIDNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
||||
objIDNotFound := errorsStd.Is(err, tree.ErrNoNodeToRemove)
|
||||
if err != nil && !objIDNotFound {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -19,6 +19,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
|
@ -63,7 +64,7 @@ type (
|
|||
resolver BucketResolver
|
||||
ncontroller EventListener
|
||||
cache *Cache
|
||||
treeService TreeService
|
||||
treeService *tree.Tree
|
||||
features FeatureSettings
|
||||
}
|
||||
|
||||
|
@ -73,7 +74,7 @@ type (
|
|||
Cache *Cache
|
||||
AnonKey AnonymousKey
|
||||
Resolver BucketResolver
|
||||
TreeService TreeService
|
||||
TreeService *tree.Tree
|
||||
Features FeatureSettings
|
||||
}
|
||||
|
||||
|
@ -658,10 +659,10 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
}
|
||||
|
||||
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nodeVersion, obj); obj.Error != nil {
|
||||
return n.handleObjectDeleteErrors(ctx, bkt, obj, nodeVersion.ID)
|
||||
return n.handleObjectDeleteErrors(ctx, bkt, obj, nodeVersion)
|
||||
}
|
||||
|
||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID)
|
||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion)
|
||||
n.cache.CleanListCacheEntriesContainingObject(obj.Name, bkt.CID)
|
||||
return obj
|
||||
}
|
||||
|
@ -688,7 +689,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
|
||||
if nullVersionToDelete != nil {
|
||||
if obj.DeleteMarkVersion, obj.Error = n.removeOldVersion(ctx, bkt, nullVersionToDelete, obj); obj.Error != nil {
|
||||
return n.handleObjectDeleteErrors(ctx, bkt, obj, nullVersionToDelete.ID)
|
||||
return n.handleObjectDeleteErrors(ctx, bkt, obj, nullVersionToDelete)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -717,7 +718,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
|||
IsUnversioned: settings.VersioningSuspended(),
|
||||
}
|
||||
|
||||
if _, obj.Error = n.treeService.AddVersion(ctx, bkt, newVersion); obj.Error != nil {
|
||||
if obj.Error = n.treeService.AddVersion(ctx, bkt, newVersion); obj.Error != nil {
|
||||
return obj
|
||||
}
|
||||
|
||||
|
@ -736,7 +737,7 @@ func (n *layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject)
|
|||
return obj
|
||||
}
|
||||
|
||||
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeID uint64) *VersionedObject {
|
||||
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeVersion *data.NodeVersion) *VersionedObject {
|
||||
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
|
||||
return obj
|
||||
}
|
||||
|
@ -744,7 +745,7 @@ func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketIn
|
|||
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
|
||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
|
||||
|
||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeID)
|
||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion)
|
||||
if obj.Error == nil {
|
||||
n.cache.DeleteObjectName(bkt.CID, bkt.Name, obj.Name)
|
||||
}
|
||||
|
|
|
@ -142,7 +142,7 @@ func (n *layer) ListObjectsV2(ctx context.Context, p *ListObjectsParamsV2) (*Lis
|
|||
ListType: ListObjectsV2Type,
|
||||
}
|
||||
|
||||
objects, next, err := n.getLatestObjectsVersions(ctx, prm)
|
||||
objects, next, err := n.getLatestObjectsVersionsNew(ctx, prm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -225,6 +225,57 @@ func (n *layer) getLatestObjectsVersions(ctx context.Context, p commonLatestVers
|
|||
return
|
||||
}
|
||||
|
||||
func (n *layer) getLatestObjectsVersionsNew(ctx context.Context, p commonLatestVersionsListingParams) (objects []*data.ExtendedNodeVersion, next *data.ExtendedNodeVersion, err error) {
|
||||
if p.MaxKeys == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
owner := n.BearerOwner(ctx)
|
||||
cacheKey := cache.CreateListSessionCacheKey(p.BktInfo.CID, p.Prefix, p.Bookmark)
|
||||
session := n.cache.GetListSession(owner, cacheKey)
|
||||
if session == nil {
|
||||
session = &data.ListSession{NamesMap: make(map[string]struct{})}
|
||||
session.Stream, err = n.treeService.InitVersionsByPrefixStream(ctx, p.BktInfo, p.Prefix, "", true)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
} else {
|
||||
session.Stream, err = n.treeService.InitVersionsByPrefixStream(ctx, p.BktInfo, p.Prefix, session.Next[0].Name(), true)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
session.Context, session.Cancel = context.WithCancel(ctx)
|
||||
|
||||
generator, errorCh := nodesGeneratorStream(ctx, p.commonVersionsListingParams, session)
|
||||
objOutCh, err := n.initWorkerPool(ctx, 2, p.commonVersionsListingParams, generator)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to init worker pool: %w", err)
|
||||
}
|
||||
|
||||
objects = make([]*data.ExtendedNodeVersion, 0, p.MaxKeys+1)
|
||||
objects = append(objects, session.Next...)
|
||||
for obj := range objOutCh {
|
||||
objects = append(objects, obj)
|
||||
}
|
||||
|
||||
if err = <-errorCh; err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to get next object from stream: %w", err)
|
||||
}
|
||||
|
||||
sort.Slice(objects, func(i, j int) bool { return objects[i].NodeVersion.FilePath < objects[j].NodeVersion.FilePath })
|
||||
|
||||
if len(objects) > p.MaxKeys {
|
||||
next = objects[p.MaxKeys]
|
||||
session.Stream = nil
|
||||
n.putListLatestVersionsSession(ctx, p, session, objects)
|
||||
objects = objects[:p.MaxKeys]
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (n *layer) getAllObjectsVersions(ctx context.Context, p commonVersionsListingParams) ([]*data.ExtendedNodeVersion, bool, error) {
|
||||
if p.MaxKeys == 0 {
|
||||
return nil, false, nil
|
||||
|
@ -337,7 +388,7 @@ func (n *layer) initNewVersionsByPrefixSession(ctx context.Context, p commonVers
|
|||
session.Context = middleware.SetBox(session.Context, &middleware.Box{AccessBox: bd})
|
||||
}
|
||||
|
||||
session.Stream, err = n.treeService.InitVersionsByPrefixStream(session.Context, p.BktInfo, p.Prefix, latestOnly)
|
||||
session.Stream, err = n.treeService.InitVersionsByPrefixStream(session.Context, p.BktInfo, p.Prefix, "", latestOnly)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -20,6 +20,7 @@ import (
|
|||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/minio/sio"
|
||||
|
@ -188,7 +189,7 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
|||
func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, error) {
|
||||
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return "", fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
|
||||
}
|
||||
return "", err
|
||||
|
@ -291,8 +292,8 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
MD5: hex.EncodeToString(md5Hash),
|
||||
}
|
||||
|
||||
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
|
||||
oldPartIDNotFound := errors.Is(err, ErrNoNodeToRemove)
|
||||
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, partInfo)
|
||||
oldPartIDNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
|
||||
if err != nil && !oldPartIDNotFound {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -322,7 +323,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.ObjectInfo, error) {
|
||||
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Info.Bkt, p.Info.Key, p.Info.UploadID)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
|
||||
}
|
||||
return nil, err
|
||||
|
@ -625,13 +626,13 @@ func (n *layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
|||
func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.MultipartInfo, map[int]*data.PartInfo, error) {
|
||||
multipartInfo, err := n.treeService.GetMultipartUpload(ctx, p.Bkt, p.Key, p.UploadID)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchUpload), err.Error())
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
parts, err := n.treeService.GetParts(ctx, p.Bkt, multipartInfo.ID)
|
||||
parts, err := n.treeService.GetParts(ctx, p.Bkt, multipartInfo)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -40,7 +41,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
|
|||
}
|
||||
|
||||
objIDToDelete, err := n.treeService.PutNotificationConfigurationNode(ctx, p.BktInfo, objID)
|
||||
objIDToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
||||
objIDToDeleteNotFound := errorsStd.Is(err, tree.ErrNoNodeToRemove)
|
||||
if err != nil && !objIDToDeleteNotFound {
|
||||
return err
|
||||
}
|
||||
|
@ -65,7 +66,7 @@ func (n *layer) GetBucketNotificationConfiguration(ctx context.Context, bktInfo
|
|||
}
|
||||
|
||||
objID, err := n.treeService.GetNotificationConfigurationNode(ctx, bktInfo)
|
||||
objIDNotFound := errorsStd.Is(err, ErrNodeNotFound)
|
||||
objIDNotFound := errorsStd.Is(err, tree.ErrNodeNotFound)
|
||||
if err != nil && !objIDNotFound {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
|
@ -315,7 +316,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
|||
newVersion.MD5 = hex.EncodeToString(md5Hash)
|
||||
}
|
||||
|
||||
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
||||
if err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
||||
return nil, fmt.Errorf("couldn't add new verion to tree service: %w", err)
|
||||
}
|
||||
|
||||
|
@ -371,7 +372,7 @@ func (n *layer) headLastVersionIfNotDeleted(ctx context.Context, bkt *data.Bucke
|
|||
|
||||
node, err := n.treeService.GetLatestVersion(ctx, bkt, objectName)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchKey), err.Error())
|
||||
}
|
||||
return nil, err
|
||||
|
@ -407,7 +408,7 @@ func (n *layer) headVersion(ctx context.Context, bkt *data.BucketInfo, p *HeadOb
|
|||
if p.VersionID == data.UnversionedObjectVersionID {
|
||||
foundVersion, err = n.treeService.GetUnversioned(ctx, bkt, p.Object)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, fmt.Errorf("%w: %s", apiErrors.GetAPIError(apiErrors.ErrNoSuchVersion), err.Error())
|
||||
}
|
||||
return nil, err
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
|
@ -38,8 +39,8 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
|||
}
|
||||
}
|
||||
|
||||
lockInfo, err := n.treeService.GetLock(ctx, p.ObjVersion.BktInfo, versionNode.ID)
|
||||
if err != nil && !errorsStd.Is(err, ErrNodeNotFound) {
|
||||
lockInfo, err := n.treeService.GetLock(ctx, p.ObjVersion.BktInfo, versionNode)
|
||||
if err != nil && !errorsStd.Is(err, tree.ErrNodeNotFound) {
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -91,7 +92,7 @@ func (n *layer) PutLockInfo(ctx context.Context, p *PutLockInfoParams) (err erro
|
|||
}
|
||||
}
|
||||
|
||||
if err = n.treeService.PutLock(ctx, p.ObjVersion.BktInfo, versionNode.ID, lockInfo); err != nil {
|
||||
if err = n.treeService.PutLock(ctx, p.ObjVersion.BktInfo, versionNode, lockInfo); err != nil {
|
||||
return fmt.Errorf("couldn't put lock into tree: %w", err)
|
||||
}
|
||||
|
||||
|
@ -140,8 +141,8 @@ func (n *layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion)
|
|||
return nil, err
|
||||
}
|
||||
|
||||
lockInfo, err := n.treeService.GetLock(ctx, objVersion.BktInfo, versionNode.ID)
|
||||
if err != nil && !errorsStd.Is(err, ErrNodeNotFound) {
|
||||
lockInfo, err := n.treeService.GetLock(ctx, objVersion.BktInfo, versionNode)
|
||||
if err != nil && !errorsStd.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
if lockInfo == nil {
|
||||
|
@ -160,7 +161,7 @@ func (n *layer) getCORS(ctx context.Context, bkt *data.BucketInfo) (*data.CORSCo
|
|||
}
|
||||
|
||||
objID, err := n.treeService.GetBucketCORS(ctx, bkt)
|
||||
objIDNotFound := errorsStd.Is(err, ErrNodeNotFound)
|
||||
objIDNotFound := errorsStd.Is(err, tree.ErrNodeNotFound)
|
||||
if err != nil && !objIDNotFound {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -198,7 +199,7 @@ func (n *layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
|
|||
|
||||
settings, err := n.treeService.GetSettingsNode(ctx, bktInfo)
|
||||
if err != nil {
|
||||
if !errorsStd.Is(err, ErrNodeNotFound) {
|
||||
if !errorsStd.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}
|
||||
|
|
|
@ -8,6 +8,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
|
@ -39,7 +40,7 @@ func (n *layer) GetObjectTagging(ctx context.Context, p *data.GetObjectTaggingPa
|
|||
|
||||
tags, err := n.treeService.GetObjectTagging(ctx, p.ObjectVersion.BktInfo, nodeVersion)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return "", nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
||||
}
|
||||
return "", nil, err
|
||||
|
@ -62,7 +63,7 @@ func (n *layer) PutObjectTagging(ctx context.Context, p *data.PutObjectTaggingPa
|
|||
|
||||
err = n.treeService.PutObjectTagging(ctx, p.ObjectVersion.BktInfo, nodeVersion, p.TagSet)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
||||
}
|
||||
return nil, err
|
||||
|
@ -81,7 +82,7 @@ func (n *layer) DeleteObjectTagging(ctx context.Context, p *data.ObjectVersion)
|
|||
|
||||
err = n.treeService.DeleteObjectTagging(ctx, p.BktInfo, version)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNodeNotFound) {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
||||
}
|
||||
return nil, err
|
||||
|
@ -102,7 +103,7 @@ func (n *layer) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo)
|
|||
}
|
||||
|
||||
tags, err := n.treeService.GetBucketTagging(ctx, bktInfo)
|
||||
if err != nil && !errors.Is(err, ErrNodeNotFound) {
|
||||
if err != nil && !errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
@ -161,7 +162,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersi
|
|||
|
||||
if err == nil && version.IsDeleteMarker && !objVersion.NoErrorOnDeleteMarker {
|
||||
return nil, fmt.Errorf("%w: found version is delete marker", s3errors.GetAPIError(s3errors.ErrNoSuchKey))
|
||||
} else if errors.Is(err, ErrNodeNotFound) {
|
||||
} else if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
return nil, fmt.Errorf("%w: %s", s3errors.GetAPIError(s3errors.ErrNoSuchKey), err.Error())
|
||||
}
|
||||
|
||||
|
|
|
@ -1,448 +0,0 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
type VersionsByPrefixStreamMock struct {
|
||||
result []*data.NodeVersion
|
||||
offset int
|
||||
}
|
||||
|
||||
func (s *VersionsByPrefixStreamMock) Next(context.Context) (*data.NodeVersion, error) {
|
||||
if s.offset > len(s.result)-1 {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
res := s.result[s.offset]
|
||||
s.offset++
|
||||
return res, nil
|
||||
}
|
||||
|
||||
type TreeServiceMock struct {
|
||||
settings map[string]*data.BucketSettings
|
||||
versions map[string]map[string][]*data.NodeVersion
|
||||
system map[string]map[string]*data.BaseNodeVersion
|
||||
locks map[string]map[uint64]*data.LockInfo
|
||||
tags map[string]map[uint64]map[string]string
|
||||
multiparts map[string]map[string][]*data.MultipartInfo
|
||||
parts map[string]map[int]*data.PartInfo
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
|
||||
// TODO implement object tagging
|
||||
lock, err := t.GetLock(ctx, bktInfo, objVersion.ID)
|
||||
return nil, lock, err
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetObjectTagging(_ context.Context, bktInfo *data.BucketInfo, nodeVersion *data.NodeVersion) (map[string]string, error) {
|
||||
cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return cnrTagsMap[nodeVersion.ID], nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) PutObjectTagging(_ context.Context, bktInfo *data.BucketInfo, nodeVersion *data.NodeVersion, tagSet map[string]string) error {
|
||||
cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
t.tags[bktInfo.CID.EncodeToString()] = map[uint64]map[string]string{
|
||||
nodeVersion.ID: tagSet,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
cnrTagsMap[nodeVersion.ID] = tagSet
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error {
|
||||
return t.PutObjectTagging(ctx, bktInfo, objVersion, nil)
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetBucketTagging(context.Context, *data.BucketInfo) (map[string]string, error) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) PutBucketTagging(context.Context, *data.BucketInfo, map[string]string) error {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) DeleteBucketTagging(context.Context, *data.BucketInfo) error {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func NewTreeService() *TreeServiceMock {
|
||||
return &TreeServiceMock{
|
||||
settings: make(map[string]*data.BucketSettings),
|
||||
versions: make(map[string]map[string][]*data.NodeVersion),
|
||||
system: make(map[string]map[string]*data.BaseNodeVersion),
|
||||
locks: make(map[string]map[uint64]*data.LockInfo),
|
||||
tags: make(map[string]map[uint64]map[string]string),
|
||||
multiparts: make(map[string]map[string][]*data.MultipartInfo),
|
||||
parts: make(map[string]map[int]*data.PartInfo),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) PutSettingsNode(_ context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
|
||||
t.settings[bktInfo.CID.EncodeToString()] = settings
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
|
||||
settings, ok := t.settings[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
return settings, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetNotificationConfigurationNode(context.Context, *data.BucketInfo) (oid.ID, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) PutNotificationConfigurationNode(context.Context, *data.BucketInfo, oid.ID) (oid.ID, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetBucketCORS(_ context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
|
||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return oid.ID{}, nil
|
||||
}
|
||||
|
||||
node, ok := systemMap["cors"]
|
||||
if !ok {
|
||||
return oid.ID{}, nil
|
||||
}
|
||||
|
||||
return node.OID, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
|
||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
systemMap = make(map[string]*data.BaseNodeVersion)
|
||||
}
|
||||
|
||||
systemMap["cors"] = &data.BaseNodeVersion{
|
||||
OID: objID,
|
||||
}
|
||||
|
||||
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
||||
|
||||
return oid.ID{}, ErrNoNodeToRemove
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) (oid.ID, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetVersions(_ context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error) {
|
||||
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
versions, ok := cnrVersionsMap[objectName]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
return versions, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetLatestVersion(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
|
||||
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
versions, ok := cnrVersionsMap[objectName]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
return versions[i].ID < versions[j].ID
|
||||
})
|
||||
|
||||
if len(versions) != 0 {
|
||||
return versions[len(versions)-1], nil
|
||||
}
|
||||
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) InitVersionsByPrefixStream(_ context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) (data.VersionsStream, error) {
|
||||
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
var result []*data.NodeVersion
|
||||
|
||||
for key, versions := range cnrVersionsMap {
|
||||
if !strings.HasPrefix(key, prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
if !latestOnly {
|
||||
result = append(result, versions...)
|
||||
continue
|
||||
}
|
||||
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
return versions[i].ID < versions[j].ID
|
||||
})
|
||||
|
||||
if len(versions) != 0 {
|
||||
result = append(result, versions[len(versions)-1])
|
||||
}
|
||||
}
|
||||
|
||||
return &VersionsByPrefixStreamMock{
|
||||
result: result,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetUnversioned(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
|
||||
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
versions, ok := cnrVersionsMap[objectName]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
for _, version := range versions {
|
||||
if version.IsUnversioned {
|
||||
return version, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) {
|
||||
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
t.versions[bktInfo.CID.EncodeToString()] = map[string][]*data.NodeVersion{
|
||||
newVersion.FilePath: {newVersion},
|
||||
}
|
||||
return newVersion.ID, nil
|
||||
}
|
||||
|
||||
versions, ok := cnrVersionsMap[newVersion.FilePath]
|
||||
if !ok {
|
||||
cnrVersionsMap[newVersion.FilePath] = []*data.NodeVersion{newVersion}
|
||||
return newVersion.ID, nil
|
||||
}
|
||||
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
return versions[i].ID < versions[j].ID
|
||||
})
|
||||
|
||||
if len(versions) != 0 {
|
||||
newVersion.ID = versions[len(versions)-1].ID + 1
|
||||
newVersion.Timestamp = versions[len(versions)-1].Timestamp + 1
|
||||
}
|
||||
|
||||
result := versions
|
||||
|
||||
if newVersion.IsUnversioned {
|
||||
result = make([]*data.NodeVersion, 0, len(versions))
|
||||
for _, node := range versions {
|
||||
if !node.IsUnversioned {
|
||||
result = append(result, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cnrVersionsMap[newVersion.FilePath] = append(result, newVersion)
|
||||
|
||||
return newVersion.ID, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) RemoveVersion(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) error {
|
||||
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
for key, versions := range cnrVersionsMap {
|
||||
for i, node := range versions {
|
||||
if node.ID == nodeID {
|
||||
cnrVersionsMap[key] = append(versions[:i], versions[i+1:]...)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetAllVersionsByPrefix(_ context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
|
||||
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var result []*data.NodeVersion
|
||||
for objName, versions := range cnrVersionsMap {
|
||||
if strings.HasPrefix(objName, prefix) {
|
||||
result = append(result, versions...)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) CreateMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error {
|
||||
cnrMultipartsMap, ok := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
t.multiparts[bktInfo.CID.EncodeToString()] = map[string][]*data.MultipartInfo{
|
||||
info.Key: {info},
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
multiparts := cnrMultipartsMap[info.Key]
|
||||
if len(multiparts) != 0 {
|
||||
info.ID = multiparts[len(multiparts)-1].ID + 1
|
||||
}
|
||||
cnrMultipartsMap[info.Key] = append(multiparts, info)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetMultipartUploadsByPrefix(context.Context, *data.BucketInfo, string) ([]*data.MultipartInfo, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error) {
|
||||
cnrMultipartsMap, ok := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
multiparts := cnrMultipartsMap[objectName]
|
||||
for _, multipart := range multiparts {
|
||||
if multipart.UploadID == uploadID {
|
||||
return multipart, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error) {
|
||||
multipartInfo, err := t.GetMultipartUpload(ctx, bktInfo, info.Key, info.UploadID)
|
||||
if err != nil {
|
||||
return oid.ID{}, err
|
||||
}
|
||||
|
||||
if multipartInfo.ID != multipartNodeID {
|
||||
return oid.ID{}, fmt.Errorf("invalid multipart info id")
|
||||
}
|
||||
|
||||
partsMap, ok := t.parts[info.UploadID]
|
||||
if !ok {
|
||||
partsMap = make(map[int]*data.PartInfo)
|
||||
}
|
||||
|
||||
partsMap[info.Number] = info
|
||||
|
||||
t.parts[info.UploadID] = partsMap
|
||||
return oid.ID{}, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetParts(_ context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error) {
|
||||
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||
|
||||
var foundMultipart *data.MultipartInfo
|
||||
|
||||
LOOP:
|
||||
for _, multiparts := range cnrMultipartsMap {
|
||||
for _, multipart := range multiparts {
|
||||
if multipart.ID == multipartNodeID {
|
||||
foundMultipart = multipart
|
||||
break LOOP
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if foundMultipart == nil {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
partsMap := t.parts[foundMultipart.UploadID]
|
||||
result := make([]*data.PartInfo, 0, len(partsMap))
|
||||
for _, part := range partsMap {
|
||||
result = append(result, part)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||
|
||||
var uploadID string
|
||||
|
||||
LOOP:
|
||||
for key, multiparts := range cnrMultipartsMap {
|
||||
for i, multipart := range multiparts {
|
||||
if multipart.ID == multipartInfo.ID {
|
||||
uploadID = multipart.UploadID
|
||||
cnrMultipartsMap[key] = append(multiparts[:i], multiparts[i+1:]...)
|
||||
break LOOP
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if uploadID == "" {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
delete(t.parts, uploadID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) PutLock(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error {
|
||||
cnrLockMap, ok := t.locks[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
t.locks[bktInfo.CID.EncodeToString()] = map[uint64]*data.LockInfo{
|
||||
nodeID: lock,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
cnrLockMap[nodeID] = lock
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetLock(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) {
|
||||
cnrLockMap, ok := t.locks[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return cnrLockMap[nodeID], nil
|
||||
}
|
|
@ -1,92 +0,0 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
// TreeService provide interface to interact with tree service using s3 data models.
|
||||
type TreeService interface {
|
||||
// PutSettingsNode update or create new settings node in tree service.
|
||||
PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error
|
||||
|
||||
// GetSettingsNode retrieves the settings node from the tree service and form data.BucketSettings.
|
||||
//
|
||||
// If tree node is not found returns ErrNodeNotFound error.
|
||||
GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
|
||||
|
||||
// GetNotificationConfigurationNode gets an object id that corresponds to object with bucket CORS.
|
||||
//
|
||||
// If tree node is not found returns ErrNodeNotFound error.
|
||||
GetNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||
|
||||
// PutNotificationConfigurationNode puts a node to a system tree
|
||||
// and returns objectID of a previous notif config which must be deleted in FrostFS.
|
||||
//
|
||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||
PutNotificationConfigurationNode(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
||||
|
||||
// GetBucketCORS gets an object id that corresponds to object with bucket CORS.
|
||||
//
|
||||
// If object id is not found returns ErrNodeNotFound error.
|
||||
GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||
|
||||
// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
|
||||
//
|
||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||
PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
|
||||
|
||||
// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
|
||||
//
|
||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||
DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
|
||||
|
||||
GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error)
|
||||
PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error
|
||||
DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) error
|
||||
|
||||
GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error)
|
||||
PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error
|
||||
DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error
|
||||
|
||||
GetVersions(ctx context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error)
|
||||
GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error)
|
||||
InitVersionsByPrefixStream(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) (data.VersionsStream, error)
|
||||
GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error)
|
||||
AddVersion(ctx context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error)
|
||||
RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) error
|
||||
|
||||
PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error
|
||||
GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error)
|
||||
|
||||
CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error
|
||||
DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) error
|
||||
GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error)
|
||||
GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, objectName, uploadID string) (*data.MultipartInfo, error)
|
||||
|
||||
// AddPart puts a node to a system tree as a child of appropriate multipart upload
|
||||
// and returns objectID of a previous part which must be deleted in FrostFS.
|
||||
//
|
||||
// If object id to remove is not found returns ErrNoNodeToRemove error.
|
||||
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
||||
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
|
||||
|
||||
// Compound methods for optimizations
|
||||
|
||||
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
|
||||
GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error)
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrNodeNotFound is returned from Tree service in case of not found error.
|
||||
ErrNodeNotFound = errors.New("not found")
|
||||
|
||||
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
|
||||
ErrNodeAccessDenied = errors.New("access denied")
|
||||
|
||||
// ErrNoNodeToRemove is returned from Tree service in case of the lack of node with OID to remove.
|
||||
ErrNoNodeToRemove = errors.New("no node to remove")
|
||||
)
|
|
@ -9,6 +9,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
|
||||
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
@ -167,10 +168,13 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
|||
var owner user.ID
|
||||
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
|
||||
|
||||
memTree, err := tree.NewTreeServiceClientMemory()
|
||||
require.NoError(t, err)
|
||||
|
||||
layerCfg := &Config{
|
||||
Cache: NewCache(config),
|
||||
AnonKey: AnonymousKey{Key: key},
|
||||
TreeService: NewTreeService(),
|
||||
TreeService: tree.NewTree(memTree, logger),
|
||||
Features: &FeatureSettingsMock{},
|
||||
GateOwner: owner,
|
||||
}
|
||||
|
|
2
go.mod
2
go.mod
|
@ -37,6 +37,8 @@ require (
|
|||
google.golang.org/protobuf v1.33.0
|
||||
)
|
||||
|
||||
replace git.frostfs.info/TrueCloudLab/frostfs-sdk-go => git.frostfs.info/dkirillov/frostfs-sdk-go v0.0.0-20240621142828-0ee3fcc18ffd
|
||||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect
|
||||
|
|
4
go.sum
4
go.sum
|
@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
|
|||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f h1:vBLC1OSGMSn7lRJv/p1of0veifuBdZdztVrF9Vn+UFk=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU=
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
|
||||
git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a h1:Bk1fB4cQASPKgAVGCdlBOEp5ohZfDxqK6fZM8eP+Emo=
|
||||
|
@ -56,6 +54,8 @@ git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjq
|
|||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8=
|
||||
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4=
|
||||
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw=
|
||||
git.frostfs.info/dkirillov/frostfs-sdk-go v0.0.0-20240621142828-0ee3fcc18ffd h1:tLkZ8XzgDbElZvYHJSAR2FTx2IY0F3Y/FIYWRY3X4rY=
|
||||
git.frostfs.info/dkirillov/frostfs-sdk-go v0.0.0-20240621142828-0ee3fcc18ffd/go.mod h1:e7H9nNFpx1Tj3R20Zoxy0Vo6Srlb6zV5L7ZQXqg9rn4=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
|
|
|
@ -15,30 +15,6 @@ import (
|
|||
grpcService "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree/service"
|
||||
)
|
||||
|
||||
type GetNodeByPathResponseInfoWrapper struct {
|
||||
response *grpcService.GetNodeByPathResponse_Info
|
||||
}
|
||||
|
||||
func (n GetNodeByPathResponseInfoWrapper) GetNodeID() uint64 {
|
||||
return n.response.GetNodeId()
|
||||
}
|
||||
|
||||
func (n GetNodeByPathResponseInfoWrapper) GetParentID() uint64 {
|
||||
return n.response.GetParentId()
|
||||
}
|
||||
|
||||
func (n GetNodeByPathResponseInfoWrapper) GetTimestamp() uint64 {
|
||||
return n.response.GetTimestamp()
|
||||
}
|
||||
|
||||
func (n GetNodeByPathResponseInfoWrapper) GetMeta() []tree.Meta {
|
||||
res := make([]tree.Meta, len(n.response.Meta))
|
||||
for i, value := range n.response.Meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
type GetSubTreeResponseBodyWrapper struct {
|
||||
response *grpcService.GetSubTreeResponse_Body
|
||||
}
|
||||
|
@ -63,6 +39,87 @@ func (n GetSubTreeResponseBodyWrapper) GetMeta() []tree.Meta {
|
|||
return res
|
||||
}
|
||||
|
||||
type BurnedListResponseBodyWrapper struct {
|
||||
response *grpcService.BurnedListResponse_Body
|
||||
}
|
||||
|
||||
func (n BurnedListResponseBodyWrapper) GetMeta() []tree.Meta {
|
||||
res := make([]tree.Meta, len(n.response.Meta))
|
||||
for i, value := range n.response.Meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (n BurnedListResponseBodyWrapper) GetKey() string {
|
||||
return n.response.GetKey()
|
||||
}
|
||||
|
||||
func (n BurnedListResponseBodyWrapper) GetTimestamp() uint64 {
|
||||
return n.response.GetTimestamp()
|
||||
}
|
||||
|
||||
type BurnedGetResponseBodyWrapper struct {
|
||||
response *grpcService.BurnedGetResponse_Body
|
||||
key string
|
||||
}
|
||||
|
||||
func (n BurnedGetResponseBodyWrapper) GetMeta() []tree.Meta {
|
||||
res := make([]tree.Meta, len(n.response.Meta))
|
||||
for i, value := range n.response.Meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (n BurnedGetResponseBodyWrapper) GetKey() string {
|
||||
return n.key
|
||||
}
|
||||
|
||||
func (n BurnedGetResponseBodyWrapper) GetTimestamp() uint64 {
|
||||
return n.response.GetTimestamp()
|
||||
}
|
||||
|
||||
type BurnedListByPrefixResponseBodyInfoWrapper struct {
|
||||
response *grpcService.BurnedListByPrefixResponse_Body_Info
|
||||
}
|
||||
|
||||
func (n BurnedListByPrefixResponseBodyInfoWrapper) GetMeta() []tree.Meta {
|
||||
res := make([]tree.Meta, len(n.response.Meta))
|
||||
for i, value := range n.response.Meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (n BurnedListByPrefixResponseBodyInfoWrapper) GetKey() string {
|
||||
return n.response.GetKey()
|
||||
}
|
||||
|
||||
func (n BurnedListByPrefixResponseBodyInfoWrapper) GetTimestamp() uint64 {
|
||||
return n.response.GetTimestamp()
|
||||
}
|
||||
|
||||
type BurnedGetLatestByPrefixResponseBodyWrapper struct {
|
||||
response *grpcService.BurnedGetLatestByPrefixResponse_Body
|
||||
}
|
||||
|
||||
func (n BurnedGetLatestByPrefixResponseBodyWrapper) GetMeta() []tree.Meta {
|
||||
res := make([]tree.Meta, len(n.response.Meta))
|
||||
for i, value := range n.response.Meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (n BurnedGetLatestByPrefixResponseBodyWrapper) GetKey() string {
|
||||
return n.response.GetKey()
|
||||
}
|
||||
|
||||
func (n BurnedGetLatestByPrefixResponseBodyWrapper) GetTimestamp() uint64 {
|
||||
return n.response.GetTimestamp()
|
||||
}
|
||||
|
||||
type PoolWrapper struct {
|
||||
p *treepool.Pool
|
||||
}
|
||||
|
@ -71,77 +128,110 @@ func NewPoolWrapper(p *treepool.Pool) *PoolWrapper {
|
|||
return &PoolWrapper{p: p}
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([]tree.NodeResponse, error) {
|
||||
poolPrm := treepool.GetNodesParams{
|
||||
CID: prm.BktInfo.CID,
|
||||
TreeID: prm.TreeID,
|
||||
Path: prm.Path,
|
||||
Meta: prm.Meta,
|
||||
PathAttribute: tree.FileNameKey,
|
||||
LatestOnly: prm.LatestOnly,
|
||||
AllAttrs: prm.AllAttrs,
|
||||
BearerToken: getBearer(ctx, prm.BktInfo),
|
||||
func (w *PoolWrapper) BurnedAdd(ctx context.Context, bktInfo *data.BucketInfo, treeID string, key string, meta map[string]string) error {
|
||||
_, err := w.p.BurnedAdd(ctx, treepool.BurnedAddParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
Key: key,
|
||||
Meta: meta,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
})
|
||||
return handleError(err)
|
||||
}
|
||||
|
||||
nodes, err := w.p.GetNodes(ctx, poolPrm)
|
||||
func (w *PoolWrapper) BurnedGet(ctx context.Context, bktInfo *data.BucketInfo, treeID string, key string) (tree.NodeResponse, error) {
|
||||
resp, err := w.p.BurnedGet(ctx, treepool.BurnedGetParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
Key: key,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
})
|
||||
|
||||
return BurnedGetResponseBodyWrapper{response: resp, key: key}, handleError(err)
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) BurnedGetLatestByPrefix(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefix string) (tree.NodeResponse, error) {
|
||||
resp, err := w.p.BurnedGetLatestByPrefix(ctx, treepool.BurnedGetLatestByPrefixParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
Prefix: prefix,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
})
|
||||
|
||||
return BurnedGetLatestByPrefixResponseBodyWrapper{resp}, handleError(err)
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) BurnedRemove(ctx context.Context, bktInfo *data.BucketInfo, treeID string, key string) error {
|
||||
return handleError(w.p.BurnedRemove(ctx, treepool.BurnedRemoveParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
Key: key,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
}))
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) BurnedListByPrefix(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefix string) ([]tree.NodeResponse, error) {
|
||||
resp, err := w.p.BurnedListByPrefix(ctx, treepool.BurnedListByPrefixParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
Prefix: prefix,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
res := make([]tree.NodeResponse, len(nodes))
|
||||
for i, info := range nodes {
|
||||
res[i] = GetNodeByPathResponseInfoWrapper{info}
|
||||
res := make([]tree.NodeResponse, len(resp.GetList()))
|
||||
for i, info := range resp.GetList() {
|
||||
res[i] = BurnedListByPrefixResponseBodyInfoWrapper{response: info}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]tree.NodeResponse, error) {
|
||||
poolPrm := treepool.GetSubTreeParams{
|
||||
func (w *PoolWrapper) BurnedList(ctx context.Context, bktInfo *data.BucketInfo, treeID, start string) ([]tree.NodeResponse, error) {
|
||||
cli, err := w.p.BurnedList(ctx, treepool.BurnedListParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
RootID: rootID,
|
||||
Depth: depth,
|
||||
Start: start,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
}
|
||||
|
||||
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
var subtree []tree.NodeResponse
|
||||
var res []tree.NodeResponse
|
||||
|
||||
node, err := subTreeReader.Next()
|
||||
node, err := cli.Next()
|
||||
for err == nil {
|
||||
subtree = append(subtree, GetSubTreeResponseBodyWrapper{node})
|
||||
node, err = subTreeReader.Next()
|
||||
res = append(res, BurnedListResponseBodyWrapper{node})
|
||||
node, err = cli.Next()
|
||||
}
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
return subtree, nil
|
||||
return res, nil
|
||||
}
|
||||
|
||||
type SubTreeStreamImpl struct {
|
||||
r *treepool.SubTreeReader
|
||||
buffer []*grpcService.GetSubTreeResponse_Body
|
||||
const bufSize = 1000
|
||||
|
||||
type BurnedListStreamImpl struct {
|
||||
r *treepool.BurnedListReader
|
||||
buffer []*grpcService.BurnedListResponse_Body
|
||||
eof bool
|
||||
index int
|
||||
ln int
|
||||
}
|
||||
|
||||
const bufSize = 1000
|
||||
|
||||
func (s *SubTreeStreamImpl) Next() (tree.NodeResponse, error) {
|
||||
func (s *BurnedListStreamImpl) Next() (tree.NodeResponse, error) {
|
||||
if s.index != -1 {
|
||||
node := s.buffer[s.index]
|
||||
s.index++
|
||||
if s.index >= s.ln {
|
||||
s.index = -1
|
||||
}
|
||||
return GetSubTreeResponseBodyWrapper{response: node}, nil
|
||||
return BurnedListResponseBodyWrapper{response: node}, nil
|
||||
}
|
||||
if s.eof {
|
||||
return nil, io.EOF
|
||||
|
@ -162,71 +252,24 @@ func (s *SubTreeStreamImpl) Next() (tree.NodeResponse, error) {
|
|||
return s.Next()
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) GetSubTreeStream(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) (tree.SubTreeStream, error) {
|
||||
poolPrm := treepool.GetSubTreeParams{
|
||||
func (w *PoolWrapper) BurnedListStream(ctx context.Context, bktInfo *data.BucketInfo, treeID, start string) (tree.SubTreeStream, error) {
|
||||
cli, err := w.p.BurnedList(ctx, treepool.BurnedListParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
RootID: rootID,
|
||||
Depth: depth,
|
||||
Start: start,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
Order: treepool.AscendingOrder,
|
||||
}
|
||||
|
||||
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, handleError(err)
|
||||
}
|
||||
|
||||
return &SubTreeStreamImpl{
|
||||
r: subTreeReader,
|
||||
buffer: make([]*grpcService.GetSubTreeResponse_Body, bufSize),
|
||||
return &BurnedListStreamImpl{
|
||||
r: cli,
|
||||
buffer: make([]*grpcService.BurnedListResponse_Body, bufSize),
|
||||
index: -1,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) {
|
||||
nodeID, err := w.p.AddNode(ctx, treepool.AddNodeParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
Parent: parent,
|
||||
Meta: meta,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
})
|
||||
return nodeID, handleError(err)
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) {
|
||||
nodeID, err := w.p.AddNodeByPath(ctx, treepool.AddNodeByPathParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
Path: path,
|
||||
Meta: meta,
|
||||
PathAttribute: tree.FileNameKey,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
})
|
||||
return nodeID, handleError(err)
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) MoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error {
|
||||
return handleError(w.p.MoveNode(ctx, treepool.MoveNodeParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
NodeID: nodeID,
|
||||
ParentID: parentID,
|
||||
Meta: meta,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
}))
|
||||
}
|
||||
|
||||
func (w *PoolWrapper) RemoveNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error {
|
||||
return handleError(w.p.RemoveNode(ctx, treepool.RemoveNodeParams{
|
||||
CID: bktInfo.CID,
|
||||
TreeID: treeID,
|
||||
NodeID: nodeID,
|
||||
BearerToken: getBearer(ctx, bktInfo),
|
||||
}))
|
||||
}
|
||||
|
||||
func getBearer(ctx context.Context, bktInfo *data.BucketInfo) []byte {
|
||||
if bd, err := middleware.GetBoxData(ctx); err == nil {
|
||||
if bd.Gate.BearerToken != nil {
|
||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -2,13 +2,13 @@ package tree
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
type nodeMeta struct {
|
||||
|
@ -26,17 +26,8 @@ func (m nodeMeta) GetValue() []byte {
|
|||
|
||||
type nodeResponse struct {
|
||||
meta []nodeMeta
|
||||
nodeID uint64
|
||||
parentID uint64
|
||||
timestamp uint64
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetNodeID() uint64 {
|
||||
return n.nodeID
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetParentID() uint64 {
|
||||
return n.parentID
|
||||
key string
|
||||
}
|
||||
|
||||
func (n nodeResponse) GetTimestamp() uint64 {
|
||||
|
@ -48,16 +39,12 @@ func (n nodeResponse) GetMeta() []Meta {
|
|||
for i, value := range n.meta {
|
||||
res[i] = value
|
||||
}
|
||||
return res
|
||||
|
||||
return append(res, nodeMeta{key: "timestamp", value: []byte(strconv.FormatUint(n.timestamp, 10))})
|
||||
}
|
||||
|
||||
func (n nodeResponse) getValue(key string) string {
|
||||
for _, value := range n.meta {
|
||||
if value.key == key {
|
||||
return string(value.value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
func (n nodeResponse) GetKey() string {
|
||||
return n.key
|
||||
}
|
||||
|
||||
type ServiceClientMemory struct {
|
||||
|
@ -66,118 +53,10 @@ type ServiceClientMemory struct {
|
|||
|
||||
type containerInfo struct {
|
||||
bkt *data.BucketInfo
|
||||
trees map[string]memoryTree
|
||||
trees map[string]map[string]*nodeResponse
|
||||
}
|
||||
|
||||
type memoryTree struct {
|
||||
idCounter uint64
|
||||
treeData *treeNodeMemory
|
||||
}
|
||||
|
||||
type treeNodeMemory struct {
|
||||
data nodeResponse
|
||||
parent *treeNodeMemory
|
||||
children []*treeNodeMemory
|
||||
}
|
||||
|
||||
func (t *treeNodeMemory) getNode(nodeID uint64) *treeNodeMemory {
|
||||
if t.data.nodeID == nodeID {
|
||||
return t
|
||||
}
|
||||
|
||||
for _, child := range t.children {
|
||||
if node := child.getNode(nodeID); node != nil {
|
||||
return node
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *memoryTree) getNodesByPath(path []string) []nodeResponse {
|
||||
if len(path) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var res []nodeResponse
|
||||
for _, child := range t.treeData.children {
|
||||
res = child.listNodesByPath(res, path)
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (t *treeNodeMemory) listNodesByPath(res []nodeResponse, path []string) []nodeResponse {
|
||||
if len(path) == 0 || t.data.getValue(FileNameKey) != path[0] {
|
||||
return res
|
||||
}
|
||||
|
||||
if len(path) == 1 {
|
||||
return append(res, t.data)
|
||||
}
|
||||
|
||||
for _, ch := range t.children {
|
||||
res = ch.listNodesByPath(res, path[1:])
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (t *memoryTree) createPathIfNotExist(parent *treeNodeMemory, path []string) *treeNodeMemory {
|
||||
if len(path) == 0 {
|
||||
return parent
|
||||
}
|
||||
|
||||
var node *treeNodeMemory
|
||||
for _, child := range parent.children {
|
||||
if len(child.data.meta) == 1 && child.data.getValue(FileNameKey) == path[0] {
|
||||
node = child
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if node == nil {
|
||||
node = &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
meta: []nodeMeta{{key: FileNameKey, value: []byte(path[0])}},
|
||||
nodeID: t.idCounter,
|
||||
parentID: parent.data.nodeID,
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
parent: parent,
|
||||
}
|
||||
t.idCounter++
|
||||
parent.children = append(parent.children, node)
|
||||
}
|
||||
|
||||
return t.createPathIfNotExist(node, path[1:])
|
||||
}
|
||||
|
||||
func (t *treeNodeMemory) removeChild(nodeID uint64) {
|
||||
ind := -1
|
||||
for i, ch := range t.children {
|
||||
if ch.data.nodeID == nodeID {
|
||||
ind = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if ind != -1 {
|
||||
t.children = append(t.children[:ind], t.children[ind+1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *treeNodeMemory) listNodes(res []NodeResponse, depth uint32) []NodeResponse {
|
||||
res = append(res, t.data)
|
||||
|
||||
if depth == 0 {
|
||||
return res
|
||||
}
|
||||
|
||||
for _, ch := range t.children {
|
||||
res = ch.listNodes(res, depth-1)
|
||||
}
|
||||
return res
|
||||
}
|
||||
var _ ServiceClient = (*ServiceClientMemory)(nil)
|
||||
|
||||
func NewTreeServiceClientMemory() (*ServiceClientMemory, error) {
|
||||
return &ServiceClientMemory{
|
||||
|
@ -185,72 +64,121 @@ func NewTreeServiceClientMemory() (*ServiceClientMemory, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
type nodeResponseWrapper struct {
|
||||
nodeResponse
|
||||
allAttr bool
|
||||
attrs []string
|
||||
}
|
||||
|
||||
func (n nodeResponseWrapper) GetMeta() []Meta {
|
||||
res := make([]Meta, 0, len(n.meta))
|
||||
for _, value := range n.meta {
|
||||
if n.allAttr || slices.Contains(n.attrs, value.key) {
|
||||
res = append(res, value)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) GetNodes(_ context.Context, p *GetNodesParams) ([]NodeResponse, error) {
|
||||
cnr, ok := c.containers[p.BktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[p.TreeID]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
res := tr.getNodesByPath(p.Path)
|
||||
sort.Slice(res, func(i, j int) bool {
|
||||
return res[i].timestamp < res[j].timestamp
|
||||
})
|
||||
|
||||
if p.LatestOnly && len(res) != 0 {
|
||||
res = res[len(res)-1:]
|
||||
}
|
||||
|
||||
res2 := make([]NodeResponse, len(res))
|
||||
for i, n := range res {
|
||||
res2[i] = nodeResponseWrapper{
|
||||
nodeResponse: n,
|
||||
allAttr: p.AllAttrs,
|
||||
attrs: p.Meta,
|
||||
func newContainerInfo(bktInfo *data.BucketInfo, treeID string) containerInfo {
|
||||
return containerInfo{
|
||||
bkt: bktInfo,
|
||||
trees: map[string]map[string]*nodeResponse{
|
||||
treeID: {},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return res2, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) ([]NodeResponse, error) {
|
||||
func (c *ServiceClientMemory) BurnedAdd(_ context.Context, bktInfo *data.BucketInfo, treeID string, key string, meta map[string]string) error {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
cnr = newContainerInfo(bktInfo, treeID)
|
||||
c.containers[bktInfo.CID.EncodeToString()] = cnr
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
tr = make(map[string]*nodeResponse)
|
||||
}
|
||||
|
||||
tr[key] = &nodeResponse{
|
||||
meta: metaToNodeMeta(meta),
|
||||
key: key,
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
}
|
||||
|
||||
cnr.trees[treeID] = tr
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) BurnedGet(_ context.Context, bktInfo *data.BucketInfo, treeID string, key string) (NodeResponse, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
node := tr.treeData.getNode(rootID)
|
||||
if node == nil {
|
||||
node, ok := cnr.trees[treeID][key]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
// we depth-1 in case of uint32 and 0 as mark to get all subtree leads to overflow and depth is getting quite big to walk all tree levels
|
||||
return node.listNodes(nil, depth-1), nil
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) BurnedGetLatestByPrefix(_ context.Context, bktInfo *data.BucketInfo, treeID string, prefix string) (NodeResponse, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
var res *nodeResponse
|
||||
for k, val := range cnr.trees[treeID] {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
if res == nil || res.timestamp < val.timestamp {
|
||||
res = val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if res == nil {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) BurnedRemove(_ context.Context, bktInfo *data.BucketInfo, treeID string, key string) error {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
delete(tr, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) BurnedListByPrefix(_ context.Context, bktInfo *data.BucketInfo, treeID string, prefix string) ([]NodeResponse, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
var res []NodeResponse
|
||||
for k, val := range cnr.trees[treeID] {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
res = append(res, val)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(res, func(i, j int) bool {
|
||||
return res[i].GetKey() < res[j].GetKey()
|
||||
})
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) BurnedList(ctx context.Context, bktInfo *data.BucketInfo, treeID, start string) ([]NodeResponse, error) {
|
||||
res, err := c.BurnedListByPrefix(ctx, bktInfo, treeID, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i := range res {
|
||||
if res[i].GetKey() > start {
|
||||
return res[i:], nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type SubTreeStreamMemoryImpl struct {
|
||||
|
@ -270,205 +198,20 @@ func (s *SubTreeStreamMemoryImpl) Next() (NodeResponse, error) {
|
|||
return s.res[s.offset-1], nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) GetSubTreeStream(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID uint64, depth uint32) (SubTreeStream, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return &SubTreeStreamMemoryImpl{err: ErrNodeNotFound}, nil
|
||||
func (c *ServiceClientMemory) BurnedListStream(ctx context.Context, bktInfo *data.BucketInfo, treeID, start string) (SubTreeStream, error) {
|
||||
res, err := c.BurnedListByPrefix(ctx, bktInfo, treeID, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
node := tr.treeData.getNode(rootID)
|
||||
if node == nil {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
sortNode(tr.treeData)
|
||||
|
||||
return &SubTreeStreamMemoryImpl{
|
||||
res: node.listNodes(nil, depth-1),
|
||||
offset: 0,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newContainerInfo(bktInfo *data.BucketInfo, treeID string) containerInfo {
|
||||
return containerInfo{
|
||||
bkt: bktInfo,
|
||||
trees: map[string]memoryTree{
|
||||
treeID: {
|
||||
idCounter: 1,
|
||||
treeData: &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
for i, item := range res {
|
||||
if start <= item.GetKey() {
|
||||
res = res[i:]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
func newMemoryTree() memoryTree {
|
||||
return memoryTree{
|
||||
idCounter: 1,
|
||||
treeData: &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error) {
|
||||
return c.AddNodeBase(ctx, bktInfo, treeID, parent, meta, true)
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) AddNodeBase(_ context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string, needSort bool) (uint64, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
cnr = newContainerInfo(bktInfo, treeID)
|
||||
c.containers[bktInfo.CID.EncodeToString()] = cnr
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
tr = newMemoryTree()
|
||||
cnr.trees[treeID] = tr
|
||||
}
|
||||
|
||||
parentNode := tr.treeData.getNode(parent)
|
||||
if parentNode == nil {
|
||||
return 0, ErrNodeNotFound
|
||||
}
|
||||
|
||||
newID := tr.idCounter
|
||||
tr.idCounter++
|
||||
|
||||
tn := &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
meta: metaToNodeMeta(meta),
|
||||
nodeID: newID,
|
||||
parentID: parent,
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
parent: parentNode,
|
||||
}
|
||||
|
||||
parentNode.children = append(parentNode.children, tn)
|
||||
if needSort {
|
||||
sortNodes(parentNode.children)
|
||||
}
|
||||
cnr.trees[treeID] = tr
|
||||
|
||||
return newID, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) AddNodeByPath(_ context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error) {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
cnr = newContainerInfo(bktInfo, treeID)
|
||||
c.containers[bktInfo.CID.EncodeToString()] = cnr
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
tr = newMemoryTree()
|
||||
cnr.trees[treeID] = tr
|
||||
}
|
||||
|
||||
parentNode := tr.createPathIfNotExist(tr.treeData, path)
|
||||
if parentNode == nil {
|
||||
return 0, fmt.Errorf("create path '%s'", path)
|
||||
}
|
||||
|
||||
newID := tr.idCounter
|
||||
tr.idCounter++
|
||||
|
||||
tn := &treeNodeMemory{
|
||||
data: nodeResponse{
|
||||
meta: metaToNodeMeta(meta),
|
||||
nodeID: newID,
|
||||
parentID: parentNode.data.nodeID,
|
||||
timestamp: uint64(time.Now().UnixMicro()),
|
||||
},
|
||||
parent: parentNode,
|
||||
}
|
||||
|
||||
parentNode.children = append(parentNode.children, tn)
|
||||
cnr.trees[treeID] = tr
|
||||
|
||||
return newID, nil
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) MoveNode(_ context.Context, bktInfo *data.BucketInfo, treeID string, nodeID, parentID uint64, meta map[string]string) error {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
node := tr.treeData.getNode(nodeID)
|
||||
if node == nil {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
newParent := tr.treeData.getNode(parentID)
|
||||
if newParent == nil {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
node.data.meta = metaToNodeMeta(meta)
|
||||
node.data.parentID = parentID
|
||||
|
||||
newParent.children = append(newParent.children, node)
|
||||
node.parent.removeChild(nodeID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortNode(node *treeNodeMemory) {
|
||||
if node == nil {
|
||||
return
|
||||
}
|
||||
|
||||
sortNodes(node.children)
|
||||
|
||||
for _, child := range node.children {
|
||||
sortNode(child)
|
||||
}
|
||||
}
|
||||
|
||||
func sortNodes(list []*treeNodeMemory) {
|
||||
sort.Slice(list, func(i, j int) bool {
|
||||
return list[i].data.getValue(FileNameKey) < list[j].data.getValue(FileNameKey)
|
||||
})
|
||||
}
|
||||
|
||||
func (c *ServiceClientMemory) RemoveNode(_ context.Context, bktInfo *data.BucketInfo, treeID string, nodeID uint64) error {
|
||||
cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
tr, ok := cnr.trees[treeID]
|
||||
if !ok {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
node := tr.treeData.getNode(nodeID)
|
||||
if node == nil {
|
||||
return ErrNodeNotFound
|
||||
}
|
||||
|
||||
node.parent.removeChild(nodeID)
|
||||
|
||||
return nil
|
||||
return &SubTreeStreamMemoryImpl{res: res}, nil
|
||||
}
|
||||
|
||||
func metaToNodeMeta(m map[string]string) []nodeMeta {
|
||||
|
|
|
@ -163,12 +163,11 @@ func TestTreeServiceAddVersion(t *testing.T) {
|
|||
IsUnversioned: true,
|
||||
}
|
||||
|
||||
nodeID, err := treeService.AddVersion(ctx, bktInfo, version)
|
||||
err = treeService.AddVersion(ctx, bktInfo, version)
|
||||
require.NoError(t, err)
|
||||
|
||||
storedNode, err := treeService.GetUnversioned(ctx, bktInfo, "path/to/version")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, nodeID, storedNode.ID)
|
||||
require.Equal(t, version.BaseNodeVersion.Size, storedNode.Size)
|
||||
require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag)
|
||||
require.Equal(t, version.BaseNodeVersion.ETag, storedNode.ETag)
|
||||
|
@ -185,7 +184,7 @@ func TestGetLatestNode(t *testing.T) {
|
|||
for _, tc := range []struct {
|
||||
name string
|
||||
nodes []NodeResponse
|
||||
expectedNodeID uint64
|
||||
exceptedTimestamp uint64
|
||||
error bool
|
||||
}{
|
||||
{
|
||||
|
@ -197,8 +196,6 @@ func TestGetLatestNode(t *testing.T) {
|
|||
name: "one node of the object version",
|
||||
nodes: []NodeResponse{
|
||||
nodeResponse{
|
||||
nodeID: 1,
|
||||
parentID: 0,
|
||||
timestamp: 1,
|
||||
meta: []nodeMeta{
|
||||
{
|
||||
|
@ -208,20 +205,16 @@ func TestGetLatestNode(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
expectedNodeID: 1,
|
||||
exceptedTimestamp: 1,
|
||||
},
|
||||
{
|
||||
name: "one node of the object version and one node of the secondary object",
|
||||
nodes: []NodeResponse{
|
||||
nodeResponse{
|
||||
nodeID: 2,
|
||||
parentID: 0,
|
||||
timestamp: 3,
|
||||
meta: []nodeMeta{},
|
||||
},
|
||||
nodeResponse{
|
||||
nodeID: 1,
|
||||
parentID: 0,
|
||||
timestamp: 1,
|
||||
meta: []nodeMeta{
|
||||
{
|
||||
|
@ -231,20 +224,16 @@ func TestGetLatestNode(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
expectedNodeID: 1,
|
||||
exceptedTimestamp: 1,
|
||||
},
|
||||
{
|
||||
name: "all nodes represent a secondary object",
|
||||
nodes: []NodeResponse{
|
||||
nodeResponse{
|
||||
nodeID: 2,
|
||||
parentID: 0,
|
||||
timestamp: 3,
|
||||
meta: []nodeMeta{},
|
||||
},
|
||||
nodeResponse{
|
||||
nodeID: 4,
|
||||
parentID: 0,
|
||||
timestamp: 5,
|
||||
meta: []nodeMeta{},
|
||||
},
|
||||
|
@ -255,8 +244,6 @@ func TestGetLatestNode(t *testing.T) {
|
|||
name: "several nodes of different types and with different timestamp",
|
||||
nodes: []NodeResponse{
|
||||
nodeResponse{
|
||||
nodeID: 1,
|
||||
parentID: 0,
|
||||
timestamp: 1,
|
||||
meta: []nodeMeta{
|
||||
{
|
||||
|
@ -266,14 +253,10 @@ func TestGetLatestNode(t *testing.T) {
|
|||
},
|
||||
},
|
||||
nodeResponse{
|
||||
nodeID: 3,
|
||||
parentID: 0,
|
||||
timestamp: 3,
|
||||
meta: []nodeMeta{},
|
||||
},
|
||||
nodeResponse{
|
||||
nodeID: 4,
|
||||
parentID: 0,
|
||||
timestamp: 4,
|
||||
meta: []nodeMeta{
|
||||
{
|
||||
|
@ -283,13 +266,11 @@ func TestGetLatestNode(t *testing.T) {
|
|||
},
|
||||
},
|
||||
nodeResponse{
|
||||
nodeID: 6,
|
||||
parentID: 0,
|
||||
timestamp: 6,
|
||||
meta: []nodeMeta{},
|
||||
},
|
||||
},
|
||||
expectedNodeID: 4,
|
||||
exceptedTimestamp: 4,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
@ -298,9 +279,13 @@ func TestGetLatestNode(t *testing.T) {
|
|||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
tNode, err := newTreeNode(actualNode)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expectedNodeID, actualNode.GetNodeID())
|
||||
require.EqualValues(t, tc.exceptedTimestamp, tNode.TimeStamp)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue