realloc/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#604] Add MFADelete tests with reworked mfa.Storage implementation

Browse source 
Signed-off-by: Alex Vanin <a.vanin@yadro.com>
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
This commit is contained in:
Alexey Vanin 2025-03-19 15:50:49 +03:00 • committed by Pavel Pogodaev
parent 0fc56cbfce
commit 7d6e20fdad
17 changed files with 440 additions and 233 deletions
Download patch file Download diff file Expand all files Collapse all files

1
api/cache/cache_test.go vendored
View file

@ -175,7 +175,6 @@ func TestSettingsCacheType(t *testing.T) {
key := "key"
settings := &data.BucketSettings{Versioning: data.Versioning{
VersioningStatus: data.VersioningEnabled,
MFADeleteStatus: data.MFADeleteEnabled,
}}
err := cache.PutSettings(key, settings)

4
api/data/info.go
View file

@ -163,6 +163,10 @@ func (b BucketSettings) MFADeleteEnabled() bool {
return b.Versioning.MFADeleteStatus == MFADeleteEnabled
}
func (b BucketSettings) MFADeleteDisabled() bool {
return b.Versioning.MFADeleteStatus == MFADeleteDisabled
}
func Quote(val string) string {
return "\"" + val + "\""
}

28
api/errors/errors.go
View file

@ -143,6 +143,10 @@ const (
// Add new error codes here.
ErrNotSupported
ErrMFAAuthNeeded
ErrCannotPutLifecycleConfiguration
ErrInvalidMFAHeader
ErrMFAAuthIsNotSupported
ErrVersioningNotSpecified
// SSE-S3 related API errors.
ErrInvalidEncryptionMethod
@ -1802,6 +1806,30 @@ var errorCodes = errorCodeMap{
Description: "AllowedHeader can not have more than one wildcard.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrCannotPutLifecycleConfiguration: {
ErrCode: ErrCannotPutLifecycleConfiguration,
Code: "ErrCannotPutLifecycleConfiguration",
Description: "Cannot put lifecycle configuration on a bucket that has MFA enabled",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMFAAuthIsNotSupported: {
ErrCode: ErrMFAAuthIsNotSupported,
Code: "ErrMFAAuthIsNotSupported",
Description: "MFA Authentication is not supported on a bucket with lifecycle configuration",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidMFAHeader: {
ErrCode: ErrInvalidMFAHeader,
Code: "ErrInvalidMFAHeader",
Description: "Invalid x-amz-mfa header",
HTTPStatusCode: http.StatusBadRequest,
},
ErrVersioningNotSpecified: {
ErrCode: ErrVersioningNotSpecified,
Code: "IllegalVersioningConfigurationException",
Description: "The Versioning element must be specified",
HTTPStatusCode: http.StatusBadRequest,
},
// Add your error structure here.
}

4
api/handler/copy_test.go
View file

@ -64,11 +64,11 @@ func TestCopyToItself(t *testing.T) {
copyObject(tc, bktName, objName, objName, CopyMeta{}, http.StatusBadRequest)
copyObject(tc, bktName, objName, objName, copyMeta, http.StatusOK)
putBucketVersioning(t, tc, bktName, true, "")
putBucketVersioning(t, tc, bktName, "Enabled")
copyObject(tc, bktName, objName, objName, CopyMeta{}, http.StatusOK)
copyObject(tc, bktName, objName, objName, copyMeta, http.StatusOK)
putBucketVersioning(t, tc, bktName, false, "")
putBucketVersioning(t, tc, bktName, "Suspended")
copyObject(tc, bktName, objName, objName, CopyMeta{}, http.StatusOK)
copyObject(tc, bktName, objName, objName, copyMeta, http.StatusOK)
}

4
api/handler/delete.go
View file

@ -100,7 +100,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
validate := totp.Validate(token, device.Key.Secret())
if !validate {
h.logAndSendError(ctx, w, "could not validate token", reqInfo, fmt.Errorf("mfa Authentication must be used for this request"))
h.logAndSendError(ctx, w, "could not validate token", reqInfo, errors.GetAPIError(errors.ErrMFAAuthNeeded))
return
}
}
@ -222,7 +222,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
validate := totp.Validate(token, device.Key.Secret())
if !validate {
h.logAndSendError(ctx, w, "could not validate token", reqInfo, fmt.Errorf("mfa Authentication must be used for this request"))
h.logAndSendError(ctx, w, "could not validate token", reqInfo, errors.GetAPIError(errors.ErrMFAAuthNeeded))
return
}
}

178
api/handler/delete_test.go
View file

@ -10,12 +10,15 @@ import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-mfa/mfa"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/pquerna/otp"
"github.com/pquerna/otp/totp"
"github.com/stretchr/testify/require"
)
@ -106,6 +109,27 @@ func TestForceDeleteBucket(t *testing.T) {
deleteBucketForce(t, hc, bktName, http.StatusNoContent, "true")
}
func TestForceDeleteBucketWithMFADelete(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName, deviceName := "bucket-for-removal", "object-to-delete", "device"
bktInfo := createTestBucket(hc, bktName)
key := createMFADevice(hc, bktName, deviceName)
putBucketVersioningMFADelete(hc, bktName, "Suspended", "Enabled", generateMFAHeader(key, deviceName))
putObject(hc, bktName, objName)
nodeVersion, err := hc.tree.GetUnversioned(hc.context, bktInfo, objName)
require.NoError(t, err)
var addr oid.Address
addr.SetContainer(bktInfo.CID)
addr.SetObject(nodeVersion.OID)
hc.owner = bktInfo.Owner
// force delete bucket fails when MFA Delete enabled
deleteBucketForce(t, hc, bktName, http.StatusConflict, "true")
}
func TestDeleteMultipleObjectCheckUniqueness(t *testing.T) {
hc := prepareHandlerContext(t)
@ -124,7 +148,7 @@ func TestDeleteObjectsError(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo := createTestBucket(hc, bktName)
putBucketVersioning(t, hc, bktName, true, "")
putBucketVersioning(t, hc, bktName, "Enabled")
putObject(hc, bktName, objName)
@ -302,7 +326,7 @@ func TestDeleteMarkerSuspended(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo, _ := createVersionedBucketAndObject(t, tc, bktName, objName)
putBucketVersioning(t, tc, bktName, false, "")
putBucketVersioning(t, tc, bktName, "Suspended")
t.Run("not create new delete marker if last version is delete marker", func(t *testing.T) {
deleteMarkerVersion, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
@ -350,7 +374,7 @@ func TestDeleteObjectCombined(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo, objInfo := createBucketAndObject(tc, bktName, objName)
putBucketVersioning(t, tc, bktName, true, "")
putBucketVersioning(t, tc, bktName, "Enabled")
checkFound(t, tc, bktName, objName, emptyVersion)
deleteObject(t, tc, bktName, objName, emptyVersion)
@ -367,13 +391,13 @@ func TestDeleteObjectSuspended(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
bktInfo, objInfo := createBucketAndObject(tc, bktName, objName)
putBucketVersioning(t, tc, bktName, true, "")
putBucketVersioning(t, tc, bktName, "Enabled")
checkFound(t, tc, bktName, objName, emptyVersion)
deleteObject(t, tc, bktName, objName, emptyVersion)
checkNotFound(t, tc, bktName, objName, emptyVersion)
putBucketVersioning(t, tc, bktName, false, "")
putBucketVersioning(t, tc, bktName, "Suspended")
deleteObject(t, tc, bktName, objName, emptyVersion)
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
@ -386,7 +410,7 @@ func TestDeleteMarkers(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
createTestBucket(tc, bktName)
putBucketVersioning(t, tc, bktName, true, "")
putBucketVersioning(t, tc, bktName, "Enabled")
checkNotFound(t, tc, bktName, objName, emptyVersion)
deleteObject(t, tc, bktName, objName, emptyVersion)
@ -405,7 +429,7 @@ func TestGetHeadDeleteMarker(t *testing.T) {
bktName, objName := "bucket-for-removal", "object-to-delete"
createTestBucket(hc, bktName)
putBucketVersioning(t, hc, bktName, true, "")
putBucketVersioning(t, hc, bktName, "Enabled")
putObject(hc, bktName, objName)
@ -473,13 +497,47 @@ func TestDeleteBucketByNotOwner(t *testing.T) {
}
func TestDeleteObjectMFAEnabled(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-name"
deviceName := "mfa"
serialNumber := "arn:aws:iam:::mfa/" + deviceName
token := "123456"
_ = createVersionedBucketMFAEnabled(hc, bktName, serialNumber+" "+token)
objName := "object-name"
deviceName := "device"
t.Run("versioned bucket", func(t *testing.T) {
hc := prepareHandlerContext(t)
bktInfo := createTestBucket(hc, bktName)
key := createMFADevice(hc, bktName, deviceName)
putBucketVersioningMFADelete(hc, bktName, "Enabled", "Enabled", generateMFAHeader(key, deviceName))
objInfo := createTestObject(hc, bktInfo, objName, encryption.Params{})
ver := objInfo.VersionID()
// delete object without MFA with error
deleteObjectErr(t, hc, bktName, objName, ver, apierr.GetAPIError(apierr.ErrMFAAuthNeeded))
// delete object with invalid MFA with error
mfaHeader := generateMFAHeader(key, deviceName)
deleteObjectWithMFAErr(hc, bktName, objName, ver, mfaHeader+"1", apierr.GetAPIError(apierr.ErrMFAAuthNeeded))
// delete object with MFA successfully
deleteObjectWithMFA(hc, bktName, objName, ver, generateMFAHeader(key, deviceName))
// disable MFA and delete object successfully
objInfo = createTestObject(hc, bktInfo, objName, encryption.Params{})
ver = objInfo.VersionID()
putBucketVersioningMFADelete(hc, bktName, "Enabled", "", generateMFAHeader(key, deviceName))
deleteObjectWithMFA(hc, bktName, objName, ver, generateMFAHeader(key, deviceName))
})
t.Run("versioned bucket without verId", func(t *testing.T) {
hc := prepareHandlerContext(t)
bktInfo := createTestBucket(hc, bktName)
key := createMFADevice(hc, bktName, deviceName)
putBucketVersioningMFADelete(hc, bktName, "Enabled", "Enabled", generateMFAHeader(key, deviceName))
objInfo := createTestObject(hc, bktInfo, objName, encryption.Params{})
deleteObject(t, hc, bktName, objInfo.Name, "")
})
}
func TestRemovalOnReplace(t *testing.T) {
@ -547,34 +605,67 @@ func createVersionedBucketAndObject(_ *testing.T, tc *handlerContext, bktName, o
func createVersionedBucket(hc *handlerContext, bktName string) *data.BucketInfo {
bktInfo := createTestBucket(hc, bktName)
putBucketVersioning(hc.t, hc, bktName, true, "")
putBucketVersioning(hc.t, hc, bktName, "Enabled")
return bktInfo
}
func createVersionedBucketMFAEnabled(hc *handlerContext, bktName, mfa string) *data.BucketInfo {
bktInfo := createTestBucket(hc, bktName)
putBucketVersioning(hc.t, hc, bktName, true, mfa)
func createMFADevice(hc *handlerContext, bktName, device string) *otp.Key {
otpKey, err := totp.Generate(totp.GenerateOpts{
Issuer: bktName,
AccountName: bktName,
})
require.NoError(hc.t, err)
return bktInfo
err = hc.h.mfa.CreateMFADevice(hc.context, mfa.SecretDevice{
Device: *mfa.NewDevice("", device, "/"),
Key: otpKey,
})
require.NoError(hc.t, err)
return otpKey
}
func putBucketVersioning(t *testing.T, tc *handlerContext, bktName string, enabled bool, mfa string) {
cfg := &VersioningConfiguration{Status: "Suspended"}
if enabled {
func generateMFAHeader(key *otp.Key, device string) string {
code, _ := totp.GenerateCode(key.Secret(), time.Now().UTC()) // error should never happen with otp.Key
return "arn:aws:iam:::mfa/" + device + " " + code
}
func putBucketVersioning(t *testing.T, hc *handlerContext, bktName string, status string) {
w := putBucketVersioningBase(hc, bktName, status, "", "")
assertStatus(t, w, http.StatusOK)
}
func putBucketVersioningMFADelete(hc *handlerContext, bktName string, versioning string, mfa string, mfaHeader string) {
w := putBucketVersioningBase(hc, bktName, versioning, mfa, mfaHeader)
assertStatus(hc.t, w, http.StatusOK)
}
func putBucketVersioningMFADeleteErr(hc *handlerContext, bktName string, versioning string, mfa string, mfaHeader string, err apierr.Error) {
w := putBucketVersioningBase(hc, bktName, versioning, mfa, mfaHeader)
assertS3Error(hc.t, w, err)
}
func putBucketVersioningBase(tc *handlerContext, bktName string, versioning string, mfa string, mfaHeader string) *httptest.ResponseRecorder {
cfg := &VersioningConfiguration{}
switch versioning {
case "Suspended":
cfg.Status = "Suspended"
case "Enabled":
cfg.Status = "Enabled"
}
if len(mfa) > 0 {
switch mfa {
case "Enabled":
cfg.MfaDelete = "Enabled"
case "Disabled":
cfg.MfaDelete = "Disabled"
}
w, r := prepareTestRequest(tc, bktName, "", cfg)
if len(mfa) > 0 {
r.Header.Set(api.AmzMFA, mfa)
if len(mfaHeader) > 0 {
r.Header.Set(api.AmzMFA, mfaHeader)
}
tc.Handler().PutBucketVersioningHandler(w, r)
assertStatus(t, w, http.StatusOK)
return w
}
func getBucketVersioning(hc *handlerContext, bktName string) *VersioningConfiguration {
@ -598,6 +689,37 @@ func deleteObject(t *testing.T, tc *handlerContext, bktName, objName, version st
return w.Header().Get(api.AmzVersionID), w.Header().Get(api.AmzDeleteMarker) != ""
}
func deleteObjectWithMFA(tc *handlerContext, bktName, objName, version, mfa string) (string, bool) {
query := make(url.Values)
query.Add(api.QueryVersionID, version)
w, r := prepareTestFullRequest(tc, bktName, objName, query, nil)
r.Header.Set(api.AmzMFA, mfa)
tc.Handler().DeleteObjectHandler(w, r)
assertStatus(tc.t, w, http.StatusNoContent)
return w.Header().Get(api.AmzVersionID), w.Header().Get(api.AmzDeleteMarker) != ""
}
func deleteObjectErr(t *testing.T, tc *handlerContext, bktName, objName, version string, err apierr.Error) {
query := make(url.Values)
query.Add(api.QueryVersionID, version)
w, r := prepareTestFullRequest(tc, bktName, objName, query, nil)
tc.Handler().DeleteObjectHandler(w, r)
assertS3Error(t, w, err)
}
func deleteObjectWithMFAErr(tc *handlerContext, bktName, objName, version, mfa string, err apierr.Error) {
query := make(url.Values)
query.Add(api.QueryVersionID, version)
w, r := prepareTestFullRequest(tc, bktName, objName, query, nil)
r.Header.Set(api.AmzMFA, mfa)
tc.Handler().DeleteObjectHandler(w, r)
assertS3Error(tc.t, w, err)
}
func deleteObjects(t *testing.T, tc *handlerContext, bktName string, objVersions [][2]string) *DeleteObjectsResponse {
w := deleteObjectsBase(tc, bktName, objVersions)
@ -696,6 +818,6 @@ func createSuspendedBucket(t *testing.T, tc *handlerContext, bktName string) *da
createTestBucket(tc, bktName)
bktInfo, err := tc.Layer().GetBucketInfo(tc.Context(), bktName)
require.NoError(t, err)
putBucketVersioning(t, tc, bktName, false, "")
putBucketVersioning(t, tc, bktName, "Suspended")
return bktInfo
}

107
api/handler/handlers_test.go
View file

@ -25,6 +25,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
intmfa "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/mfa"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -180,6 +181,7 @@ type handlerConfig struct {
cacheCfg *layer.CachesConfig
withoutCORS bool
withoutLifecycle bool
withoutMFA bool
}
func prepareHandlerContext(t *testing.T) *handlerContext {
@ -198,6 +200,7 @@ func prepareWithoutContainersHandlerContext(t *testing.T, cors, lifecycle bool)
cacheCfg: layer.DefaultCachesConfigs(log),
withoutCORS: cors,
withoutLifecycle: lifecycle,
withoutMFA: true,
}, log)
require.NoError(t, err)
return &handlerContext{
@ -289,7 +292,14 @@ func prepareHandlerContextBase(config *handlerConfig, log *zap.Logger) (*handler
cfg: cfg,
ape: newAPEMock(),
frostfsid: newFrostfsIDMock(),
mfa: newMFAMock(),
}
if !config.withoutMFA {
// this code creates one more container, so it may affect tests
h.mfa, err = newMFAMock(log, layerCfg.GateKey, tp, memCli)
if err != nil {
return nil, err
}
}
accessBox, err := newTestAccessBox(key)
@ -483,70 +493,45 @@ func newFrostfsIDMock() *frostfsidMock {
return &frostfsidMock{data: map[string]*keys.PublicKey{}}
}
func newMFAMock() *mfa.Manager {
cfg := mfa.Config{
Storage: newStorageMock(),
Unlocker: nil,
Container: cid.ID{},
Logger: nil,
type unlocker struct {
k *keys.PrivateKey
}
func (u unlocker) PrivateKey() *keys.PrivateKey {
return u.k
}
func (u unlocker) PublicKeys() []*keys.PublicKey {
return []*keys.PublicKey{
u.k.PublicKey(),
}
}
func newMFAMock(log *zap.Logger, key *keys.PrivateKey, p frostfs.FrostFS, t tree.ServiceClient) (*mfa.Manager, error) {
bktName := "mfa"
res, err := p.CreateContainer(context.Background(), frostfs.PrmContainerCreate{
Name: bktName,
Policy: getPlacementPolicy(),
})
if err != nil {
return nil, err
}
manager, _ := mfa.NewManager(cfg)
f := intmfa.NewMFAFrostFS(intmfa.FrostFSMFAConfig{
ObjStor: p,
TreeStor: t,
Key: key,
Logger: log,
})
return man
}
cfg := mfa.Config{
Storage: f,
Unlocker: unlocker{k: key},
Container: res.ContainerID,
Logger: log,
}
type man mfa.Manager
func (m man) GetMFADevice(ctx context.Context, ns, mfaName string) (*mfa.SecretDevice, error) {
// TODO implement me
panic("implement me")
}
type mfaOperations interface {
GetMFADevice(ctx context.Context, ns, mfaName string) (*mfa.SecretDevice, error)
}
type storage struct {
}
func newStorageMock() *storage {
return &storage{}
}
func (s *storage) CreateObject(_ context.Context, _ mfa.PrmObjectCreate) (oid.ID, error) {
// TODO implement me
panic("implement me")
}
func (s *storage) GetObject(_ context.Context, _ oid.Address) ([]byte, error) {
// TODO implement me
panic("implement me")
}
func (s *storage) DeleteObject(_ context.Context, _ oid.Address) error {
// TODO implement me
panic("implement me")
}
func (s *storage) SetTreeNode(_ context.Context, _ cid.ID, _ string, _ map[string]string) (*mfa.TreeMultiNode, error) {
// TODO implement me
panic("implement me")
}
func (s *storage) GetTreeNode(_ context.Context, _ cid.ID, _ string) (*mfa.TreeMultiNode, error) {
// TODO implement me
panic("implement me")
}
func (s *storage) DeleteTreeNode(_ context.Context, _ cid.ID, _ string) ([]*mfa.TreeNode, error) {
// TODO implement me
panic("implement me")
}
func (s *storage) GetTreeNodes(_ context.Context, _ cid.ID, _ string) ([]*mfa.TreeNode, error) {
// TODO implement me
panic("implement me")
return mfa.NewManager(cfg)
}
func (f *frostfsidMock) GetUserAddress(account, user string) (string, error) {

11
api/handler/lifecycle.go
View file

@ -94,6 +94,17 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
return
}
bktSettings, err := h.obj.GetBucketSettings(ctx, bktInfo)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket settings", reqInfo, err)
return
}
if bktSettings.MFADeleteEnabled() {
h.logAndSendError(ctx, w, "failed to add", reqInfo, apierr.GetAPIError(apierr.ErrCannotPutLifecycleConfiguration))
return
}
networkInfo, err := h.obj.GetNetworkInfo(ctx)
if err != nil {
h.logAndSendError(ctx, w, "could not get network info", reqInfo, err)

36
api/handler/lifecycle_test.go
View file

@ -521,6 +521,42 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) {
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
}
func TestPutMFADeleteWithLifecycleBucket(t *testing.T) {
hc := prepareHandlerContext(t)
lifecycle := &data.LifecycleConfiguration{
Rules: []data.LifecycleRule{
{
Status: data.LifecycleStatusEnabled,
Expiration: &data.LifecycleExpiration{
Days: ptr(21),
},
},
},
}
// S3 Lifecycle configuration on multi-factor authentication (MFA)-enabled buckets isn't supported.
t.Run("lifecycle in MFA Delete bucket", func(_ *testing.T) {
bktName := "mfa-delete-bucket"
deviceName := "device"
createBucket(hc, bktName)
key := createMFADevice(hc, bktName, deviceName)
putBucketVersioningMFADelete(hc, bktName, "Enabled", "Enabled", generateMFAHeader(key, deviceName))
putBucketLifecycleConfigurationErr(hc, bktName, lifecycle, nil, apierr.GetAPIError(apierr.ErrCannotPutLifecycleConfiguration))
})
// You cannot use MFA delete with lifecycle configurations.
t.Run("MFA Delete in lifecycle bucket", func(_ *testing.T) {
bktName := "lifecycle-bucket"
deviceName := "device2"
createBucket(hc, bktName)
key := createMFADevice(hc, bktName, deviceName)
putBucketLifecycleConfiguration(hc, bktName, lifecycle, nil, false)
putBucketVersioningMFADeleteErr(hc, bktName, "Enabled", "Enabled", generateMFAHeader(key, deviceName), apierr.GetAPIError(apierr.ErrMFAAuthIsNotSupported))
})
}
func TestPutBucketLifecycleCopiesNumbers(t *testing.T) {
t.Run("with lifecycle container", func(t *testing.T) {
hc := prepareHandlerContext(t)

4
api/handler/multipart_upload_test.go
View file

@ -69,7 +69,7 @@ func TestDeleteMultipartAllParts(t *testing.T) {
// versions bucket
createTestBucket(hc, bktName2)
putBucketVersioning(t, hc, bktName2, true, "")
putBucketVersioning(t, hc, bktName2, "Enabled")
multipartUpload(hc, bktName2, objName, nil, objLen, partSize)
_, hdr := getObject(hc, bktName2, objName)
versionID := hdr.Get("X-Amz-Version-Id")
@ -107,7 +107,7 @@ func TestSpecialMultipartName(t *testing.T) {
bktName, objName := "bucket", "bucket-settings"
createTestBucket(hc, bktName)
putBucketVersioning(t, hc, bktName, true, "")
putBucketVersioning(t, hc, bktName, "Enabled")
createMultipartUpload(hc, bktName, objName, nil)
res := getBucketVersioning(hc, bktName)

8
api/handler/object_list_test.go
View file

@ -64,7 +64,7 @@ func TestListObjectNullVersions(t *testing.T) {
createTestBucket(hc, bktName)
putObjectContent(hc, bktName, objName, "content")
putBucketVersioning(t, hc, bktName, true, "")
putBucketVersioning(t, hc, bktName, "Enabled")
putObjectContent(hc, bktName, objName, "content2")
result := listVersions(t, hc, bktName)
@ -226,7 +226,7 @@ func TestListObjectsLatestVersions(t *testing.T) {
bktName := "bucket-versioning-enabled"
createTestBucket(hc, bktName)
putBucketVersioning(t, hc, bktName, true, "")
putBucketVersioning(t, hc, bktName, "Enabled")
objName1, objName2 := "object1", "object2"
objContent1, objContent2 := "content1", "content2"
@ -762,7 +762,7 @@ func TestMintVersioningListObjectVersionsVersionIDContinuation(t *testing.T) {
bktName, objName := "mint-bucket-for-listing-versions", "objName"
createTestBucket(hc, bktName)
putBucketVersioning(t, hc, bktName, true, "")
putBucketVersioning(t, hc, bktName, "Enabled")
length := 10
objects := make([]string, length)
@ -795,7 +795,7 @@ func TestListObjectVersionsEncoding(t *testing.T) {
bktName := "bucket-for-listing-versions-encoding"
bktInfo := createTestBucket(hc, bktName)
putBucketVersioning(t, hc, bktName, true, "")
putBucketVersioning(t, hc, bktName, "Enabled")
objects := []string{"foo()/bar", "foo()/bar/xyzzy", "auux ab/thud", "asdf+b"}
for _, objName := range objects {

3
api/handler/util.go
View file

@ -170,6 +170,9 @@ func parseRange(s string) (*layer.RangeParams, error) {
}
func nameFromArn(arn string) string {
if len(arn) == 0 {
return ""
}
pts := strings.Split(arn, "/")
return pts[len(pts)-1]
}

22
api/handler/versioning.go
View file

@ -24,6 +24,12 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
h.logAndSendError(ctx, w, "couldn't decode versioning configuration", reqInfo, errors.GetAPIError(errors.ErrIllegalVersioningConfigurationException))
return
}
newMfa := len(configuration.MfaDelete) > 0
newStatus := len(configuration.Status) > 0
if !newStatus {
h.logAndSendError(ctx, w, "failed to put versioning", reqInfo, errors.GetAPIError(errors.ErrVersioningNotSpecified))
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
@ -37,12 +43,21 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
return
}
newMfa := len(configuration.MfaDelete) > 0
lifecycleCfg, err := h.obj.GetBucketLifecycleConfiguration(ctx, bktInfo)
if err != nil && !errors.IsS3Error(err, errors.ErrNoSuchLifecycleConfiguration) {
h.logAndSendError(ctx, w, "couldn't get lifecycle config", reqInfo, err)
return
}
if lifecycleCfg != nil && newMfa {
h.logAndSendError(ctx, w, "couldn't put versioning", reqInfo, errors.GetAPIError(errors.ErrMFAAuthIsNotSupported))
return
}
if settings.MFADeleteEnabled() || newMfa {
serialNumber, token, err = h.getMFAHeader(r)
if err != nil {
h.logAndSendError(ctx, w, "invalid x-amz-mfa header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
h.logAndSendError(ctx, w, "invalid x-amz-mfa header", reqInfo, errors.GetAPIError(errors.ErrInvalidMFAHeader))
return
}
device, err := h.mfa.GetMFADevice(ctx, reqInfo.Namespace, nameFromArn(serialNumber))
@ -68,6 +83,7 @@ func (h *handler) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Requ
newSettings.Versioning.MFASerialNumber = serialNumber
case data.MFADeleteDisabled:
newSettings.Versioning.MFADeleteStatus = data.MFADeleteDisabled
newSettings.Versioning.MFASerialNumber = ""
default:
h.logAndSendError(ctx, w, "failed to get mfa configuration", reqInfo, nil)
return
@ -127,6 +143,8 @@ func formVersioningConfiguration(settings *data.BucketSettings) *VersioningConfi
}
if settings.MFADeleteEnabled() {
res.MfaDelete = data.MFADeleteEnabled
} else if settings.MFADeleteDisabled() {
res.MfaDelete = data.MFADeleteDisabled
}
return res

45
api/handler/versioning_test.go Normal file
View file

@ -0,0 +1,45 @@
package handler
import (
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)
func TestVersioningWithMFADelete(t *testing.T) {
bktName := "bucket-name"
deviceName := "device"
hc := prepareHandlerContext(t)
createBucket(hc, bktName)
key := createMFADevice(hc, bktName, deviceName)
putBucketVersioningMFADeleteErr(hc, bktName, "", "Enabled", generateMFAHeader(key, deviceName), errors.GetAPIError(errors.ErrVersioningNotSpecified))
putBucketVersioningMFADeleteErr(hc, bktName, "", "", "", errors.GetAPIError(errors.ErrVersioningNotSpecified))
// set MFA Delete status
putBucketVersioningMFADelete(hc, bktName, "Suspended", "Enabled", generateMFAHeader(key, deviceName))
require.Equal(t, "Enabled", getBucketVersioning(hc, bktName).MfaDelete)
require.Equal(t, "Suspended", getBucketVersioning(hc, bktName).Status)
// try to change versioning without MFA Header
putBucketVersioningMFADeleteErr(hc, bktName, "Enabled", "Enabled", "", errors.GetAPIError(errors.ErrInvalidMFAHeader))
require.Equal(t, "Enabled", getBucketVersioning(hc, bktName).MfaDelete)
require.Equal(t, "Suspended", getBucketVersioning(hc, bktName).Status)
// change versioning with MFA successfully
putBucketVersioningMFADelete(hc, bktName, "Enabled", "Enabled", generateMFAHeader(key, deviceName))
require.Equal(t, "Enabled", getBucketVersioning(hc, bktName).MfaDelete)
require.Equal(t, "Enabled", getBucketVersioning(hc, bktName).Status)
// try to disable MFA without MFA Header
putBucketVersioningMFADeleteErr(hc, bktName, "Enabled", "Disabled", "", errors.GetAPIError(errors.ErrInvalidMFAHeader))
require.Equal(t, "Enabled", getBucketVersioning(hc, bktName).MfaDelete)
require.Equal(t, "Enabled", getBucketVersioning(hc, bktName).Status)
// try to disable MFA successfully
putBucketVersioningMFADelete(hc, bktName, "Enabled", "Disabled", generateMFAHeader(key, deviceName))
require.Equal(t, "Disabled", getBucketVersioning(hc, bktName).MfaDelete)
require.Equal(t, "Enabled", getBucketVersioning(hc, bktName).Status)
}

3
api/layer/versioning_test.go
View file

@ -302,7 +302,6 @@ func TestVersioningDeleteObject(t *testing.T) {
tc := prepareContext(t)
settings := &data.BucketSettings{Versioning: data.Versioning{
VersioningStatus: data.VersioningEnabled,
MFADeleteStatus: data.MFADeleteDisabled,
}}
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo,
@ -327,7 +326,6 @@ func TestGetUnversioned(t *testing.T) {
settings := &data.BucketSettings{Versioning: data.Versioning{
VersioningStatus: data.VersioningUnversioned,
MFADeleteStatus: data.MFADeleteDisabled,
}}
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo,
@ -344,7 +342,6 @@ func TestVersioningDeleteSpecificObjectVersion(t *testing.T) {
tc := prepareContext(t)
settings := &data.BucketSettings{Versioning: data.Versioning{
VersioningStatus: data.VersioningEnabled,
MFADeleteStatus: data.MFADeleteDisabled,
}}
err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{
BktInfo: tc.bktInfo,

35
cmd/s3-gw/app.go
View file

@ -35,6 +35,7 @@ import (
containerClient "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/container"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/frostfsid"
ffidcontract "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/frostfsid/contract"
intmfa "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/mfa"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/policy"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/policy/contract"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services"
@ -80,8 +81,6 @@ type (
frostfsid *frostfsid.FrostFSID
mfaManager *mfa.Manager
policyStorage *policy.Storage
servers []Server
@ -258,7 +257,6 @@ func (a *App) init(ctx context.Context) {
a.initFrostfsID(rpcCli)
a.initPolicyStorage(rpcCli)
a.initAPI(ctx, rpcCli)
a.initMfaManager(ctx)
a.initMetrics()
a.initServers(ctx)
a.initTracing(ctx)
@ -722,7 +720,8 @@ func (s *appSettings) LifecycleCopiesNumbers() []uint32 {
func (a *App) initAPI(ctx context.Context, rpcCli *rpcclient.Client) {
a.initLayer(ctx, rpcCli)
a.initHandler()
a.initHandler(ctx)
}
func (a *App) initMetrics() {
@ -1271,7 +1270,7 @@ func getFrostfsIDCacheConfig(v *viper.Viper, l *zap.Logger) *cache.Config {
return cacheCfg
}
func (a *App) initMfaManager(ctx context.Context) {
func (a *App) initMfaManager(ctx context.Context) *mfa.Manager {
var err error
var mfaCnrInfo *data.BucketInfo
@ -1280,33 +1279,31 @@ func (a *App) initMfaManager(ctx context.Context) {
if err != nil {
a.log.Fatal(logs.CouldNotFetchMFAContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
}
} else {
a.log.Fatal(logs.CouldNotFetchMFAContainerInfo, logs.TagField(logs.TagApp))
}
mfaConfig, err := a.fetchMFAConfig(mfaCnrInfo.CID)
if err != nil {
a.log.Fatal(logs.CouldNotInitMFAClient, zap.Error(err), logs.TagField(logs.TagApp))
}
mfaConfig := a.fetchMFAConfig(mfaCnrInfo.CID)
manager, err := mfa.NewManager(mfaConfig)
if err != nil {
a.log.Fatal(logs.CouldNotInitMFAClient, zap.Error(err), logs.TagField(logs.TagApp))
}
a.mfaManager = manager
return manager
}
func (a *App) initHandler() {
func (a *App) initHandler(ctx context.Context) {
var err error
a.api, err = handler.New(a.log, a.obj, a.settings, a.policyStorage, a.frostfsid, a.mfaManager)
manager := a.initMfaManager(ctx)
a.api, err = handler.New(a.log, a.obj, a.settings, a.policyStorage, a.frostfsid, manager)
if err != nil {
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err), logs.TagField(logs.TagApp))
}
}
func (a *App) fetchMFAConfig(id cid.ID) (mfa.Config, error) {
mfaFrostFS := frostfs.NewMFAFrostFS(frostfs.MFAFrostFSConfig{
Pool: a.pool,
TreePool: a.treePool,
func (a *App) fetchMFAConfig(id cid.ID) mfa.Config {
mfaFrostFS := intmfa.NewMFAFrostFS(intmfa.FrostFSMFAConfig{
ObjStor: frostfs.NewFrostFS(a.pool, a.key),
TreeStor: services.NewPoolWrapper(a.treePool),
Key: a.key,
Logger: a.log,
})
@ -1318,7 +1315,7 @@ func (a *App) fetchMFAConfig(id cid.ID) (mfa.Config, error) {
Logger: a.log,
}
return config, nil
return config
}
func (a *App) getServer(address string) Server {

172
internal/frostfs/mfa.go → internal/frostfs/mfa/mfa.go
View file

@ -1,4 +1,4 @@
package frostfs
package mfa
import (
"bytes"
@ -9,37 +9,41 @@ import (
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-mfa/mfa"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"go.uber.org/zap"
)
// MFAFrostFS is a mediator which implements mfa.Storage through pool.Pool and treepool.Pool.
type MFAFrostFS struct {
frostFS *FrostFS
treePool *treepool.Pool
// FrostFSMFA is a mediator which implements mfa.Storage through pool.Pool and treepool.Pool.
type FrostFSMFA struct {
objStor frostfs.FrostFS
treeStor tree.ServiceClient
log *zap.Logger
key *keys.PrivateKey
}
func (m *MFAFrostFS) PrivateKey() *keys.PrivateKey {
var (
_ mfa.Storage = (*FrostFSMFA)(nil)
_ mfa.KeyStore = (*FrostFSMFA)(nil)
)
func (m *FrostFSMFA) PrivateKey() *keys.PrivateKey {
return m.key
}
func (m *MFAFrostFS) PublicKeys() []*keys.PublicKey {
// TODO implement me
func (m *FrostFSMFA) PublicKeys() []*keys.PublicKey {
panic("Readonly MFA manager should not call PublicKeys()")
}
func (m *MFAFrostFS) CreateObject(ctx context.Context, create mfa.PrmObjectCreate) (oid.ID, error) {
object, err := m.frostFS.CreateObject(ctx, frostfs.PrmObjectCreate{
func (m *FrostFSMFA) CreateObject(ctx context.Context, create mfa.PrmObjectCreate) (oid.ID, error) {
object, err := m.objStor.CreateObject(ctx, frostfs.PrmObjectCreate{
Container: create.Container,
Payload: bytes.NewReader(create.Payload),
Filepath: create.FilePath,
@ -47,21 +51,21 @@ func (m *MFAFrostFS) CreateObject(ctx context.Context, create mfa.PrmObjectCreat
WithoutHomomorphicHash: true,
})
if err != nil {
return [32]byte{}, err
return oid.ID{}, err
}
return object.ObjectID, nil
}
func (m *MFAFrostFS) DeleteObject(ctx context.Context, address oid.Address) error {
func (m *FrostFSMFA) DeleteObject(ctx context.Context, address oid.Address) error {
prm := frostfs.PrmObjectDelete{
Container: address.Container(),
Object: address.Object(),
}
return m.frostFS.DeleteObject(ctx, prm)
return m.objStor.DeleteObject(ctx, prm)
}
func (m *MFAFrostFS) SetTreeNode(ctx context.Context, cnrID cid.ID, name string, meta map[string]string) (*mfa.TreeMultiNode, error) {
func (m *FrostFSMFA) SetTreeNode(ctx context.Context, cnrID cid.ID, name string, meta map[string]string) (*mfa.TreeMultiNode, error) {
if len(name) == 0 {
return nil, errors.New("tree node name must not be empty")
}
@ -76,15 +80,7 @@ func (m *MFAFrostFS) SetTreeNode(ctx context.Context, cnrID cid.ID, name string,
}
if isErrNotFound {
prmAdd := treepool.AddNodeByPathParams{
CID: cnrID,
TreeID: mfaTreeName,
Path: path[:len(path)-1],
Meta: meta,
PathAttribute: fileNameKey,
}
if _, err = m.treePool.AddNodeByPath(ctx, prmAdd); err != nil {
if _, err = m.treeStor.AddNodeByPath(ctx, &data.BucketInfo{CID: cnrID}, mfaTreeName, path[:len(path)-1], meta); err != nil {
return nil, fmt.Errorf("add node by path: %w", err)
}
@ -92,15 +88,7 @@ func (m *MFAFrostFS) SetTreeNode(ctx context.Context, cnrID cid.ID, name string,
}
node := multiNode.Latest()
prmMove := treepool.MoveNodeParams{
CID: cnrID,
TreeID: mfaTreeName,
NodeID: node.ID,
ParentID: node.ParentID,
Meta: meta,
}
if err = m.treePool.MoveNode(ctx, prmMove); err != nil {
if err = m.treeStor.MoveNode(ctx, &data.BucketInfo{CID: cnrID}, mfaTreeName, node.ID, node.ParentID, meta); err != nil {
return nil, fmt.Errorf("move node: %w", err)
}
@ -112,7 +100,7 @@ func (m *MFAFrostFS) SetTreeNode(ctx context.Context, cnrID cid.ID, name string,
return mfaMultiNode, nil
}
func (m *MFAFrostFS) GetTreeNode(ctx context.Context, cnrID cid.ID, name string) (*mfa.TreeMultiNode, error) {
func (m *FrostFSMFA) GetTreeNode(ctx context.Context, cnrID cid.ID, name string) (*mfa.TreeMultiNode, error) {
multiNode, err := m.getTreeNode(ctx, cnrID, pathFromName(name))
if err != nil {
return nil, fmt.Errorf("couldn't get node: %w", err)
@ -121,7 +109,7 @@ func (m *MFAFrostFS) GetTreeNode(ctx context.Context, cnrID cid.ID, name string)
return multiNode.ToMFAMultiNode(), nil
}
func (m *MFAFrostFS) DeleteTreeNode(ctx context.Context, cnrID cid.ID, name string) ([]*mfa.TreeNode, error) {
func (m *FrostFSMFA) DeleteTreeNode(ctx context.Context, cnrID cid.ID, name string) ([]*mfa.TreeNode, error) {
multiNode, err := m.getTreeNode(ctx, cnrID, pathFromName(name))
if err != nil {
return nil, fmt.Errorf("couldn't get node: %w", err)
@ -139,7 +127,7 @@ func (m *MFAFrostFS) DeleteTreeNode(ctx context.Context, cnrID cid.ID, name stri
return res, nil
}
func (m *MFAFrostFS) GetTreeNodes(ctx context.Context, cnrID cid.ID, prefix string) ([]*mfa.TreeNode, error) {
func (m *FrostFSMFA) GetTreeNodes(ctx context.Context, cnrID cid.ID, prefix string) ([]*mfa.TreeNode, error) {
rootID := []uint64{0}
if len(prefix) != 0 {
var err error
@ -152,18 +140,7 @@ func (m *MFAFrostFS) GetTreeNodes(ctx context.Context, cnrID cid.ID, prefix stri
}
}
prm := treepool.GetSubTreeParams{
CID: cnrID,
TreeID: mfaTreeName,
RootID: rootID,
}
subTreeCli, err := m.treePool.GetSubTree(ctx, prm) // todo use streaming https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/issues/561
if err != nil {
return nil, err
}
allNodes, err := subTreeCli.ReadAll()
allNodes, err := m.treeStor.GetSubTree(ctx, &data.BucketInfo{CID: cnrID}, mfaTreeName, rootID, 0, false)
if err != nil {
return nil, err
}
@ -184,8 +161,8 @@ func (m *MFAFrostFS) GetTreeNodes(ctx context.Context, cnrID cid.ID, prefix stri
return res, nil
}
func filterUnique(allNodes []*apitree.GetSubTreeResponseBody) map[string]*apitree.GetSubTreeResponseBody {
res := make(map[string]*apitree.GetSubTreeResponseBody, len(allNodes))
func filterUnique(allNodes []tree.NodeResponse) map[string]tree.NodeResponse {
res := make(map[string]tree.NodeResponse, len(allNodes))
for _, node := range allNodes {
var name string
for _, kv := range node.GetMeta() {
@ -215,9 +192,9 @@ func getMaxTimestamp(timestamps []uint64) uint64 {
return maxTimestamp
}
type MFAFrostFSConfig struct {
Pool *pool.Pool
TreePool *treepool.Pool
type FrostFSMFAConfig struct {
ObjStor frostfs.FrostFS
TreeStor tree.ServiceClient
Key *keys.PrivateKey
Logger *zap.Logger
}
@ -240,17 +217,17 @@ const (
)
// NewMFAFrostFS creates new MFAFrostFS using provided pool.Pool.
func NewMFAFrostFS(cfg MFAFrostFSConfig) *MFAFrostFS {
return &MFAFrostFS{
frostFS: NewFrostFS(cfg.Pool, cfg.Key),
treePool: cfg.TreePool,
func NewMFAFrostFS(cfg FrostFSMFAConfig) *FrostFSMFA {
return &FrostFSMFA{
objStor: cfg.ObjStor,
treeStor: cfg.TreeStor,
log: cfg.Logger,
key: cfg.Key,
}
}
func (m *MFAFrostFS) GetObject(ctx context.Context, addr oid.Address) ([]byte, error) {
res, err := m.frostFS.GetObject(ctx, frostfs.PrmObjectGet{
func (m *FrostFSMFA) GetObject(ctx context.Context, addr oid.Address) ([]byte, error) {
res, err := m.objStor.GetObject(ctx, frostfs.PrmObjectGet{
Container: addr.Container(),
Object: addr.Object(),
})
@ -267,21 +244,19 @@ func (m *MFAFrostFS) GetObject(ctx context.Context, addr oid.Address) ([]byte, e
return io.ReadAll(res.Payload)
}
func (m *MFAFrostFS) getTreeNode(ctx context.Context, cnrID cid.ID, path []string) (*multiSystemNode, error) {
prmGetNodes := treepool.GetNodesParams{
CID: cnrID,
func (m *FrostFSMFA) getTreeNode(ctx context.Context, cnrID cid.ID, path []string) (*multiSystemNode, error) {
prmGetNodes := &tree.GetNodesParams{
BktInfo: &data.BucketInfo{CID: cnrID},
TreeID: mfaTreeName,
Path: path,
PathAttribute: fileNameKey,
LatestOnly: true,
AllAttrs: true,
}
nodes, err := m.treePool.GetNodes(ctx, prmGetNodes)
nodes, err := m.treeStor.GetNodes(ctx, prmGetNodes)
if err != nil {
if errors.Is(err, treepool.ErrNodeNotFound) {
return nil, fmt.Errorf("%s: %s", "mfa.ErrTreeNodeNotFound", err.Error())
// return nil, fmt.Errorf("%w: %s", mfa.ErrTreeNodeNotFound, err.Error())
if errors.Is(err, tree.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", mfa.ErrTreeNodeNotFound, err.Error())
}
return nil, fmt.Errorf("get nodes: %w", err)
}
@ -296,7 +271,7 @@ func (m *MFAFrostFS) getTreeNode(ctx context.Context, cnrID cid.ID, path []strin
return newMultiNode(nodes)
}
func (m *MFAFrostFS) cleanOldNodes(ctx context.Context, nodes []*treeNode, cnrID cid.ID) []*treeNode {
func (m *FrostFSMFA) cleanOldNodes(ctx context.Context, nodes []*treeNode, cnrID cid.ID) []*treeNode {
res := make([]*treeNode, 0, len(nodes))
for _, node := range nodes {
@ -310,14 +285,8 @@ func (m *MFAFrostFS) cleanOldNodes(ctx context.Context, nodes []*treeNode, cnrID
return res
}
func (m *MFAFrostFS) removeTreeNode(ctx context.Context, cnrID cid.ID, nodeID uint64) error {
prmRemoveNode := treepool.RemoveNodeParams{
CID: cnrID,
TreeID: mfaTreeName,
NodeID: nodeID,
}
err := m.treePool.RemoveNode(ctx, prmRemoveNode)
func (m *FrostFSMFA) removeTreeNode(ctx context.Context, cnrID cid.ID, nodeID uint64) error {
err := m.treeStor.RemoveNode(ctx, &data.BucketInfo{CID: cnrID}, mfaTreeName, nodeID)
if err != nil {
if errors.Is(err, treepool.ErrNodeNotFound) {
return fmt.Errorf("%w: %s", mfa.ErrTreeNodeNotFound, err.Error())
@ -328,16 +297,16 @@ func (m *MFAFrostFS) removeTreeNode(ctx context.Context, cnrID cid.ID, nodeID ui
return nil
}
func (m *MFAFrostFS) getPrefixNodeID(ctx context.Context, cnrID cid.ID, prefixPath []string) ([]uint64, error) {
p := treepool.GetNodesParams{
CID: cnrID,
func (m *FrostFSMFA) getPrefixNodeID(ctx context.Context, cnrID cid.ID, prefixPath []string) ([]uint64, error) {
p := &tree.GetNodesParams{
BktInfo: &data.BucketInfo{CID: cnrID},
TreeID: mfaTreeName,
Path: prefixPath,
LatestOnly: false,
AllAttrs: true,
}
nodes, err := m.treePool.GetNodes(ctx, p)
nodes, err := m.treeStor.GetNodes(ctx, p)
if err != nil {
if errors.Is(err, treepool.ErrNodeNotFound) {
return nil, fmt.Errorf("%w: %s", mfa.ErrTreeNodeNotFound, err.Error())
@ -348,7 +317,7 @@ func (m *MFAFrostFS) getPrefixNodeID(ctx context.Context, cnrID cid.ID, prefixPa
var intermediateNodes []uint64
for _, node := range nodes {
if isIntermediate(node.GetMeta()) {
intermediateNodes = append(intermediateNodes, node.GetNodeID())
intermediateNodes = append(intermediateNodes, node.GetNodeID()...)
}
}
@ -359,7 +328,7 @@ func (m *MFAFrostFS) getPrefixNodeID(ctx context.Context, cnrID cid.ID, prefixPa
return intermediateNodes, nil
}
func isIntermediate(meta []*apitree.KeyValue) bool {
func isIntermediate(meta []tree.Meta) bool {
if len(meta) != 1 {
return false
}
@ -367,9 +336,8 @@ func isIntermediate(meta []*apitree.KeyValue) bool {
return meta[0].GetKey() == fileNameKey
}
func newMultiNode(nodes []*apitree.GetNodeByPathResponseInfo) (*multiSystemNode, error) {
func newMultiNode(nodes []tree.NodeResponse) (*multiSystemNode, error) {
var (
err error
index int
maxTimestamp uint64
)
@ -381,13 +349,22 @@ func newMultiNode(nodes []*apitree.GetNodeByPathResponseInfo) (*multiSystemNode,
treeNodes := make([]*treeNode, len(nodes))
for i, node := range nodes {
if treeNodes[i] = newTreeNode(node); err != nil {
return nil, fmt.Errorf("parse tree node response: %w", err)
if len(node.GetTimestamp()) == 0 || len(node.GetNodeID()) == 0 || len(node.GetParentID()) == 0 {
// this should never happen when GetNodes returns response.
return nil, fmt.Errorf("parse tree node response: %v", node)
}
if maxTimestamp < node.GetTimestamp() {
treeNodes[i] = &treeNode{
ID: node.GetNodeID()[0],
ParentID: node.GetParentID()[0],
TimeStamp: node.GetTimestamp()[0],
Meta: make(map[string]string, len(node.GetMeta())),
}
for _, kv := range node.GetMeta() {
treeNodes[i].Meta[kv.GetKey()] = string(kv.GetValue())
}
if maxTimestamp < node.GetTimestamp()[0] {
index = i
maxTimestamp = node.GetTimestamp()
maxTimestamp = node.GetTimestamp()[0]
}
}
@ -411,21 +388,6 @@ func (m *multiSystemNode) ToMFAMultiNode() *mfa.TreeMultiNode {
return res
}
func newTreeNode(nodeInfo *apitree.GetNodeByPathResponseInfo) *treeNode {
tNode := &treeNode{
ID: nodeInfo.GetNodeID(),
ParentID: nodeInfo.GetParentID(),
TimeStamp: nodeInfo.GetTimestamp(),
Meta: make(map[string]string, len(nodeInfo.GetMeta())),
}
for _, kv := range nodeInfo.GetMeta() {
tNode.Meta[kv.GetKey()] = string(kv.GetValue())
}
return tNode
}
func (m *multiSystemNode) Old() []*treeNode {
return m.nodes[1:]
}