forked from TrueCloudLab/frostfs-s3-gw
[#42] Support expiration lifecycle
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
This commit is contained in:
parent
3dc989d7fe
commit
a594deaa37
22 changed files with 1206 additions and 59 deletions
20
api/cache/system.go
vendored
20
api/cache/system.go
vendored
|
@ -88,6 +88,22 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
|||
return result
|
||||
}
|
||||
|
||||
func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfiguration {
|
||||
entry, err := o.cache.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, ok := entry.(*data.LifecycleConfiguration)
|
||||
if !ok {
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
||||
entry, err := o.cache.Get(key)
|
||||
if err != nil {
|
||||
|
@ -133,6 +149,10 @@ func (o *SystemCache) PutCORS(key string, obj *data.CORSConfiguration) error {
|
|||
return o.cache.Set(key, obj)
|
||||
}
|
||||
|
||||
func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleConfiguration) error {
|
||||
return o.cache.Set(key, obj)
|
||||
}
|
||||
|
||||
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
|
||||
return o.cache.Set(key, settings)
|
||||
}
|
||||
|
|
|
@ -14,6 +14,7 @@ import (
|
|||
const (
|
||||
bktSettingsObject = ".s3-settings"
|
||||
bktCORSConfigurationObject = ".s3-cors"
|
||||
bktLifecycleConfigurationObject = ".s3-lifecycle"
|
||||
|
||||
VersioningUnversioned = "Unversioned"
|
||||
VersioningEnabled = "Enabled"
|
||||
|
@ -91,6 +92,10 @@ func (b *BucketInfo) CORSObjectName() string {
|
|||
return b.CID.EncodeToString() + bktCORSConfigurationObject
|
||||
}
|
||||
|
||||
func (b *BucketInfo) LifecycleConfigurationObjectName() string {
|
||||
return b.CID.EncodeToString() + bktLifecycleConfigurationObject
|
||||
}
|
||||
|
||||
// VersionID returns object version from ObjectInfo.
|
||||
func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() }
|
||||
|
||||
|
|
54
api/data/lifecycle.go
Normal file
54
api/data/lifecycle.go
Normal file
|
@ -0,0 +1,54 @@
|
|||
package data
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
const (
|
||||
LifecycleStatusEnabled = "Enabled"
|
||||
LifecycleStatusDisabled = "Disabled"
|
||||
)
|
||||
|
||||
type (
|
||||
LifecycleConfiguration struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LifecycleConfiguration" json:"-"`
|
||||
Rules []LifecycleRule `xml:"Rule"`
|
||||
}
|
||||
|
||||
LifecycleRule struct {
|
||||
Status string `xml:"Status,omitempty"`
|
||||
AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
|
||||
Expiration *LifecycleExpiration `xml:"Expiration,omitempty"`
|
||||
Filter *LifecycleRuleFilter `xml:"Filter,omitempty"`
|
||||
ID string `xml:"ID,omitempty"`
|
||||
NonCurrentVersionExpiration *NonCurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
|
||||
}
|
||||
|
||||
AbortIncompleteMultipartUpload struct {
|
||||
DaysAfterInitiation *int `xml:"DaysAfterInitiation,omitempty"`
|
||||
}
|
||||
|
||||
LifecycleExpiration struct {
|
||||
Date string `xml:"Date,omitempty"`
|
||||
Days *int `xml:"Days,omitempty"`
|
||||
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
|
||||
}
|
||||
|
||||
LifecycleRuleFilter struct {
|
||||
And *LifecycleRuleAndOperator `xml:"And,omitempty"`
|
||||
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
||||
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
|
||||
Prefix string `xml:"Prefix,omitempty"`
|
||||
Tag *Tag `xml:"Tag,omitempty"`
|
||||
}
|
||||
|
||||
LifecycleRuleAndOperator struct {
|
||||
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
||||
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
|
||||
Prefix string `xml:"Prefix,omitempty"`
|
||||
Tags []Tag `xml:"Tag"`
|
||||
}
|
||||
|
||||
NonCurrentVersionExpiration struct {
|
||||
NewerNonCurrentVersions *int `xml:"NewerNoncurrentVersions,omitempty"`
|
||||
NonCurrentDays *int `xml:"NoncurrentDays,omitempty"`
|
||||
}
|
||||
)
|
235
api/handler/lifecycle.go
Normal file
235
api/handler/lifecycle.go
Normal file
|
@ -0,0 +1,235 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRules = 1000
|
||||
maxRuleIDLen = 255
|
||||
maxNewerNoncurrentVersions = 100
|
||||
)
|
||||
|
||||
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := h.obj.GetBucketLifecycleConfiguration(ctx, bktInfo)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket lifecycle configuration", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, cfg); err != nil {
|
||||
h.logAndSendError(w, "could not encode GetBucketLifecycle response", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
tee := io.TeeReader(r.Body, &buf)
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
// Content-Md5 is required and should be set
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
|
||||
if _, ok := r.Header[api.ContentMD5]; !ok {
|
||||
h.logAndSendError(w, "missing Content-MD5", reqInfo, apiErr.GetAPIError(apiErr.ErrMissingContentMD5))
|
||||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := new(data.LifecycleConfiguration)
|
||||
if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil {
|
||||
h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
if err = checkLifecycleConfiguration(cfg); err != nil {
|
||||
h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
params := &layer.PutBucketLifecycleParams{
|
||||
BktInfo: bktInfo,
|
||||
LifecycleCfg: cfg,
|
||||
LifecycleReader: &buf,
|
||||
MD5Hash: r.Header.Get(api.ContentMD5),
|
||||
}
|
||||
|
||||
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = h.obj.PutBucketLifecycleConfiguration(ctx, params); err != nil {
|
||||
h.logAndSendError(w, "could not put bucket lifecycle configuration", reqInfo, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
reqInfo := middleware.GetReqInfo(ctx)
|
||||
|
||||
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = h.obj.DeleteBucketLifecycleConfiguration(ctx, bktInfo); err != nil {
|
||||
h.logAndSendError(w, "could not delete bucket lifecycle configuration", reqInfo, err)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
|
||||
if len(cfg.Rules) > maxRules {
|
||||
return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
|
||||
}
|
||||
|
||||
ids := make(map[string]struct{}, len(cfg.Rules))
|
||||
for _, rule := range cfg.Rules {
|
||||
if _, ok := ids[rule.ID]; ok && rule.ID != "" {
|
||||
return fmt.Errorf("duplicate 'ID': %s", rule.ID)
|
||||
}
|
||||
ids[rule.ID] = struct{}{}
|
||||
|
||||
if len(rule.ID) > maxRuleIDLen {
|
||||
return fmt.Errorf("'ID' value cannot be longer than %d characters", maxRuleIDLen)
|
||||
}
|
||||
|
||||
if rule.Status != data.LifecycleStatusEnabled && rule.Status != data.LifecycleStatusDisabled {
|
||||
return fmt.Errorf("invalid lifecycle status: %s", rule.Status)
|
||||
}
|
||||
|
||||
if rule.AbortIncompleteMultipartUpload == nil && rule.Expiration == nil && rule.NonCurrentVersionExpiration == nil {
|
||||
return fmt.Errorf("at least one action needs to be specified in a rule")
|
||||
}
|
||||
|
||||
if rule.AbortIncompleteMultipartUpload != nil {
|
||||
if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != nil &&
|
||||
*rule.AbortIncompleteMultipartUpload.DaysAfterInitiation <= 0 {
|
||||
return fmt.Errorf("days after initiation must be a positive integer: %d", *rule.AbortIncompleteMultipartUpload.DaysAfterInitiation)
|
||||
}
|
||||
|
||||
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
|
||||
return fmt.Errorf("abort incomplete multipart upload cannot be specified with tags")
|
||||
}
|
||||
}
|
||||
|
||||
if rule.Expiration != nil {
|
||||
if rule.Expiration.ExpiredObjectDeleteMarker != nil {
|
||||
if rule.Expiration.Days != nil || rule.Expiration.Date != "" {
|
||||
return fmt.Errorf("expired object delete marker cannot be specified with days or date")
|
||||
}
|
||||
|
||||
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
|
||||
return fmt.Errorf("expired object delete marker cannot be specified with tags")
|
||||
}
|
||||
}
|
||||
|
||||
if rule.Expiration.Days != nil && *rule.Expiration.Days <= 0 {
|
||||
return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days)
|
||||
}
|
||||
|
||||
if _, err := time.Parse("2006-01-02T15:04:05.000Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil {
|
||||
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
|
||||
}
|
||||
}
|
||||
|
||||
if rule.NonCurrentVersionExpiration != nil {
|
||||
if rule.NonCurrentVersionExpiration.NewerNonCurrentVersions != nil &&
|
||||
(*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions > maxNewerNoncurrentVersions ||
|
||||
*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions <= 0) {
|
||||
return fmt.Errorf("invalid value of newer noncurrent versions: %d", *rule.NonCurrentVersionExpiration.NewerNonCurrentVersions)
|
||||
}
|
||||
|
||||
if rule.NonCurrentVersionExpiration.NonCurrentDays != nil && *rule.NonCurrentVersionExpiration.NonCurrentDays <= 0 {
|
||||
return fmt.Errorf("invalid value of noncurrent days: %d", *rule.NonCurrentVersionExpiration.NonCurrentDays)
|
||||
}
|
||||
}
|
||||
|
||||
if err := checkLifecycleRuleFilter(rule.Filter); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
|
||||
if filter == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var fields int
|
||||
|
||||
if filter.And != nil {
|
||||
fields++
|
||||
for _, tag := range filter.And.Tags {
|
||||
err := checkTag(tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if filter.And.ObjectSizeGreaterThan != nil && filter.And.ObjectSizeLessThan != nil &&
|
||||
*filter.And.ObjectSizeLessThan <= *filter.And.ObjectSizeGreaterThan {
|
||||
return fmt.Errorf("the maximum object size must be larger than the minimum object size")
|
||||
}
|
||||
}
|
||||
|
||||
if filter.ObjectSizeGreaterThan != nil {
|
||||
fields++
|
||||
}
|
||||
|
||||
if filter.ObjectSizeLessThan != nil {
|
||||
fields++
|
||||
}
|
||||
|
||||
if filter.Prefix != "" {
|
||||
fields++
|
||||
}
|
||||
|
||||
if filter.Tag != nil {
|
||||
fields++
|
||||
err := checkTag(*filter.Tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if fields > 1 {
|
||||
return fmt.Errorf("filter cannot have more than one field")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
457
api/handler/lifecycle_test.go
Normal file
457
api/handler/lifecycle_test.go
Normal file
|
@ -0,0 +1,457 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"github.com/mr-tron/base58"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
||||
hc := prepareHandlerContextWithMinCache(t)
|
||||
|
||||
bktName := "bucket-lifecycle"
|
||||
createBucket(hc, bktName)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body *data.LifecycleConfiguration
|
||||
error bool
|
||||
}{
|
||||
{
|
||||
name: "correct configuration",
|
||||
body: &data.LifecycleConfiguration{
|
||||
XMLName: xml.Name{
|
||||
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
|
||||
Local: "LifecycleConfiguration",
|
||||
},
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
Date: time.Now().Format("2006-01-02T15:04:05.000Z"),
|
||||
},
|
||||
Filter: &data.LifecycleRuleFilter{
|
||||
And: &data.LifecycleRuleAndOperator{
|
||||
Prefix: "prefix/",
|
||||
Tags: []data.Tag{{Key: "key", Value: "value"}, {Key: "tag", Value: ""}},
|
||||
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||
DaysAfterInitiation: ptr(14),
|
||||
},
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
ExpiredObjectDeleteMarker: ptr(true),
|
||||
},
|
||||
Filter: &data.LifecycleRuleFilter{
|
||||
ObjectSizeLessThan: ptr(uint64(100)),
|
||||
},
|
||||
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||
NewerNonCurrentVersions: ptr(1),
|
||||
NonCurrentDays: ptr(21),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "too many rules",
|
||||
body: func() *data.LifecycleConfiguration {
|
||||
lifecycle := new(data.LifecycleConfiguration)
|
||||
for i := 0; i <= maxRules; i++ {
|
||||
lifecycle.Rules = append(lifecycle.Rules, data.LifecycleRule{
|
||||
ID: "Rule" + strconv.Itoa(i),
|
||||
})
|
||||
}
|
||||
return lifecycle
|
||||
}(),
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "duplicate rule ID",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
ID: "Rule",
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "Rule",
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "too long rule ID",
|
||||
body: func() *data.LifecycleConfiguration {
|
||||
id := make([]byte, maxRuleIDLen+1)
|
||||
_, err := io.ReadFull(rand.Reader, id)
|
||||
require.NoError(t, err)
|
||||
return &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
ID: base58.Encode(id)[:maxRuleIDLen+1],
|
||||
},
|
||||
},
|
||||
}
|
||||
}(),
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "invalid status",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: "invalid",
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "no actions",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Filter: &data.LifecycleRuleFilter{
|
||||
Prefix: "prefix/",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "invalid days after initiation",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||
DaysAfterInitiation: ptr(0),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "invalid expired object delete marker declaration",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
ExpiredObjectDeleteMarker: ptr(false),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "invalid expiration days",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(0),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "invalid expiration date",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Date: "invalid",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "newer noncurrent versions is too small",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||
NewerNonCurrentVersions: ptr(0),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "newer noncurrent versions is too large",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||
NewerNonCurrentVersions: ptr(101),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "invalid noncurrent days",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||
NonCurrentDays: ptr(0),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "more than one filter field",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
Filter: &data.LifecycleRuleFilter{
|
||||
Prefix: "prefix/",
|
||||
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "invalid tag in filter",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
Filter: &data.LifecycleRuleFilter{
|
||||
Tag: &data.Tag{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "abort incomplete multipart upload with tag",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||
DaysAfterInitiation: ptr(14),
|
||||
},
|
||||
Filter: &data.LifecycleRuleFilter{
|
||||
Tag: &data.Tag{Key: "key", Value: "value"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "expired object delete marker with tag",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
ExpiredObjectDeleteMarker: ptr(true),
|
||||
},
|
||||
Filter: &data.LifecycleRuleFilter{
|
||||
And: &data.LifecycleRuleAndOperator{
|
||||
Tags: []data.Tag{{Key: "key", Value: "value"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "invalid size range",
|
||||
body: &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
Filter: &data.LifecycleRuleFilter{
|
||||
And: &data.LifecycleRuleAndOperator{
|
||||
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||
ObjectSizeLessThan: ptr(uint64(100)),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
error: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if tc.error {
|
||||
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apiErrors.GetAPIError(apiErrors.ErrMalformedXML))
|
||||
return
|
||||
}
|
||||
|
||||
putBucketLifecycleConfiguration(hc, bktName, tc.body)
|
||||
|
||||
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
||||
require.Equal(t, *tc.body, *cfg)
|
||||
|
||||
deleteBucketLifecycleConfiguration(hc, bktName)
|
||||
getBucketLifecycleConfigurationErr(hc, bktName, apiErrors.GetAPIError(apiErrors.ErrNoSuchLifecycleConfiguration))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-lifecycle-md5"
|
||||
createBucket(hc, bktName)
|
||||
|
||||
lifecycle := &data.LifecycleConfiguration{
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, "", lifecycle)
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMissingContentMD5))
|
||||
|
||||
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||
r.Header.Set(api.ContentMD5, "")
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest))
|
||||
|
||||
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||
r.Header.Set(api.ContentMD5, "some-hash")
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest))
|
||||
}
|
||||
|
||||
func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-lifecycle-invalid-xml"
|
||||
createBucket(hc, bktName)
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{})
|
||||
r.Header.Set(api.ContentMD5, "")
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMalformedXML))
|
||||
}
|
||||
|
||||
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) {
|
||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apiErrors.Error) {
|
||||
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||
assertS3Error(hc.t, w, err)
|
||||
}
|
||||
|
||||
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, "", cfg)
|
||||
|
||||
rawBody, err := xml.Marshal(cfg)
|
||||
require.NoError(hc.t, err)
|
||||
|
||||
hash := md5.New()
|
||||
hash.Write(rawBody)
|
||||
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
||||
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func getBucketLifecycleConfiguration(hc *handlerContext, bktName string) *data.LifecycleConfiguration {
|
||||
w := getBucketLifecycleConfigurationBase(hc, bktName)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
res := &data.LifecycleConfiguration{}
|
||||
parseTestResponse(hc.t, w, res)
|
||||
return res
|
||||
}
|
||||
|
||||
func getBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, err apiErrors.Error) {
|
||||
w := getBucketLifecycleConfigurationBase(hc, bktName)
|
||||
assertS3Error(hc.t, w, err)
|
||||
}
|
||||
|
||||
func getBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
hc.Handler().GetBucketLifecycleHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func deleteBucketLifecycleConfiguration(hc *handlerContext, bktName string) {
|
||||
w := deleteBucketLifecycleConfigurationBase(hc, bktName)
|
||||
assertStatus(hc.t, w, http.StatusNoContent)
|
||||
}
|
||||
|
||||
func deleteBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
||||
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||
hc.Handler().DeleteBucketLifecycleHandler(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
|
@ -7,10 +7,6 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
)
|
||||
|
||||
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||
}
|
||||
|
||||
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||
}
|
||||
|
|
|
@ -11,10 +11,6 @@ func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Requ
|
|||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
@ -51,10 +47,6 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request)
|
|||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
||||
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||
}
|
||||
|
|
|
@ -257,3 +257,29 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
|||
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
|
||||
c.systemCache.Delete(bktInfo.CORSObjectName())
|
||||
}
|
||||
|
||||
func (c *Cache) GetLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo) *data.LifecycleConfiguration {
|
||||
key := bkt.LifecycleConfigurationObjectName()
|
||||
|
||||
if !c.accessCache.Get(owner, key) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return c.systemCache.GetLifecycleConfiguration(key)
|
||||
}
|
||||
|
||||
func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.LifecycleConfiguration) {
|
||||
key := bkt.LifecycleConfigurationObjectName()
|
||||
|
||||
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) {
|
||||
c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName())
|
||||
}
|
||||
|
|
|
@ -56,6 +56,7 @@ type (
|
|||
features FeatureSettings
|
||||
gateKey *keys.PrivateKey
|
||||
corsCnrInfo *data.BucketInfo
|
||||
lifecycleCnrInfo *data.BucketInfo
|
||||
}
|
||||
|
||||
Config struct {
|
||||
|
@ -68,6 +69,7 @@ type (
|
|||
Features FeatureSettings
|
||||
GateKey *keys.PrivateKey
|
||||
CORSCnrInfo *data.BucketInfo
|
||||
LifecycleCnrInfo *data.BucketInfo
|
||||
}
|
||||
|
||||
// AnonymousKey contains data for anonymous requests.
|
||||
|
@ -243,6 +245,7 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer {
|
|||
features: config.Features,
|
||||
gateKey: config.GateKey,
|
||||
corsCnrInfo: config.CORSCnrInfo,
|
||||
lifecycleCnrInfo: config.LifecycleCnrInfo,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -826,6 +829,11 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
|||
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
|
||||
}
|
||||
|
||||
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
|
||||
if treeErr != nil {
|
||||
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr))
|
||||
}
|
||||
|
||||
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete container: %w", err)
|
||||
|
@ -835,5 +843,9 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
|||
n.deleteCORSObject(ctx, p.BktInfo, corsObj)
|
||||
}
|
||||
|
||||
if treeErr == nil && !lifecycleObj.Container().Equals(p.BktInfo.CID) {
|
||||
n.deleteLifecycleObject(ctx, p.BktInfo, lifecycleObj)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
148
api/layer/lifecycle.go
Normal file
148
api/layer/lifecycle.go
Normal file
|
@ -0,0 +1,148 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type PutBucketLifecycleParams struct {
|
||||
BktInfo *data.BucketInfo
|
||||
LifecycleCfg *data.LifecycleConfiguration
|
||||
LifecycleReader io.Reader
|
||||
CopiesNumbers []uint32
|
||||
MD5Hash string
|
||||
}
|
||||
|
||||
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
|
||||
prm := PrmObjectCreate{
|
||||
Payload: p.LifecycleReader,
|
||||
Filepath: p.BktInfo.LifecycleConfigurationObjectName(),
|
||||
CreationTime: TimeNow(ctx),
|
||||
}
|
||||
|
||||
var lifecycleBkt *data.BucketInfo
|
||||
if n.lifecycleCnrInfo == nil {
|
||||
lifecycleBkt = p.BktInfo
|
||||
prm.CopiesNumber = p.CopiesNumbers
|
||||
} else {
|
||||
lifecycleBkt = n.lifecycleCnrInfo
|
||||
prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||
}
|
||||
|
||||
prm.Container = lifecycleBkt.CID
|
||||
|
||||
_, objID, _, md5, err := n.objectPutAndHash(ctx, prm, lifecycleBkt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("put lifecycle object: %w", err)
|
||||
}
|
||||
|
||||
hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash)
|
||||
if err != nil {
|
||||
return apiErr.GetAPIError(apiErr.ErrInvalidDigest)
|
||||
}
|
||||
|
||||
if !bytes.Equal(hashBytes, md5) {
|
||||
n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, objID))
|
||||
|
||||
return apiErr.GetAPIError(apiErr.ErrInvalidDigest)
|
||||
}
|
||||
|
||||
objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, objID))
|
||||
objsToDeleteNotFound := errors.Is(err, ErrNoNodeToRemove)
|
||||
if err != nil && !objsToDeleteNotFound {
|
||||
return err
|
||||
}
|
||||
|
||||
if !objsToDeleteNotFound {
|
||||
for _, addr := range objsToDelete {
|
||||
n.deleteLifecycleObject(ctx, p.BktInfo, addr)
|
||||
}
|
||||
}
|
||||
|
||||
n.cache.PutLifecycleConfiguration(n.BearerOwner(ctx), p.BktInfo, p.LifecycleCfg)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteLifecycleObject removes object and logs in case of error.
|
||||
func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
|
||||
var prmAuth PrmAuth
|
||||
lifecycleBkt := bktInfo
|
||||
if !addr.Container().Equals(bktInfo.CID) {
|
||||
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
|
||||
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||
}
|
||||
|
||||
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
|
||||
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
|
||||
zap.String("oid", addr.Object().EncodeToString()))
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.LifecycleConfiguration, error) {
|
||||
owner := n.BearerOwner(ctx)
|
||||
if cfg := n.cache.GetLifecycleConfiguration(owner, bktInfo); cfg != nil {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
addr, err := n.treeService.GetBucketLifecycleConfiguration(ctx, bktInfo)
|
||||
objNotFound := errors.Is(err, ErrNodeNotFound)
|
||||
if err != nil && !objNotFound {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if objNotFound {
|
||||
return nil, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), err.Error())
|
||||
}
|
||||
|
||||
var prmAuth PrmAuth
|
||||
lifecycleBkt := bktInfo
|
||||
if !addr.Container().Equals(bktInfo.CID) {
|
||||
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
|
||||
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||
}
|
||||
|
||||
obj, err := n.objectGetWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get lifecycle object: %w", err)
|
||||
}
|
||||
|
||||
lifecycleCfg := &data.LifecycleConfiguration{}
|
||||
|
||||
if err = xml.NewDecoder(obj.Payload).Decode(&lifecycleCfg); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal lifecycle configuration: %w", err)
|
||||
}
|
||||
|
||||
n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg)
|
||||
|
||||
return lifecycleCfg, nil
|
||||
}
|
||||
|
||||
func (n *Layer) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||
objs, err := n.treeService.DeleteBucketLifecycleConfiguration(ctx, bktInfo)
|
||||
objsNotFound := errors.Is(err, ErrNoNodeToRemove)
|
||||
if err != nil && !objsNotFound {
|
||||
return err
|
||||
}
|
||||
if !objsNotFound {
|
||||
for _, addr := range objs {
|
||||
n.deleteLifecycleObject(ctx, bktInfo, addr)
|
||||
}
|
||||
}
|
||||
|
||||
n.cache.DeleteLifecycleConfiguration(bktInfo)
|
||||
|
||||
return nil
|
||||
}
|
65
api/layer/lifecycle_test.go
Normal file
65
api/layer/lifecycle_test.go
Normal file
|
@ -0,0 +1,65 @@
|
|||
package layer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBucketLifecycle(t *testing.T) {
|
||||
tc := prepareContext(t)
|
||||
|
||||
lifecycle := &data.LifecycleConfiguration{
|
||||
XMLName: xml.Name{
|
||||
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
|
||||
Local: "LifecycleConfiguration",
|
||||
},
|
||||
Rules: []data.LifecycleRule{
|
||||
{
|
||||
Status: data.LifecycleStatusEnabled,
|
||||
Expiration: &data.LifecycleExpiration{
|
||||
Days: ptr(21),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
lifecycleBytes, err := xml.Marshal(lifecycle)
|
||||
require.NoError(t, err)
|
||||
hash := md5.New()
|
||||
hash.Write(lifecycleBytes)
|
||||
|
||||
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||
require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err))
|
||||
|
||||
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{
|
||||
BktInfo: tc.bktInfo,
|
||||
LifecycleCfg: lifecycle,
|
||||
LifecycleReader: bytes.NewReader(lifecycleBytes),
|
||||
MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg, err := tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, *lifecycle, *cfg)
|
||||
|
||||
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||
require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err))
|
||||
}
|
||||
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
|
@ -395,6 +395,51 @@ LOOP:
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) PutBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
systemMap = make(map[string]*data.BaseNodeVersion)
|
||||
}
|
||||
|
||||
systemMap["lifecycle"] = &data.BaseNodeVersion{
|
||||
OID: addr.Object(),
|
||||
}
|
||||
|
||||
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
||||
|
||||
return nil, ErrNoNodeToRemove
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) GetBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return oid.Address{}, ErrNodeNotFound
|
||||
}
|
||||
|
||||
node, ok := systemMap["lifecycle"]
|
||||
if !ok {
|
||||
return oid.Address{}, ErrNodeNotFound
|
||||
}
|
||||
|
||||
return newAddress(bktInfo.CID, node.OID), nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) DeleteBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
||||
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||
if !ok {
|
||||
return nil, ErrNoNodeToRemove
|
||||
}
|
||||
|
||||
node, ok := systemMap["lifecycle"]
|
||||
if !ok {
|
||||
return nil, ErrNoNodeToRemove
|
||||
}
|
||||
|
||||
delete(systemMap, "lifecycle")
|
||||
|
||||
return []oid.Address{newAddress(bktInfo.CID, node.OID)}, nil
|
||||
}
|
||||
|
||||
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||
|
||||
|
|
|
@ -63,6 +63,10 @@ type TreeService interface {
|
|||
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
||||
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
|
||||
|
||||
PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error)
|
||||
GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
|
||||
DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
|
||||
|
||||
// Compound methods for optimizations
|
||||
|
||||
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
|
||||
|
|
|
@ -356,7 +356,7 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router {
|
|||
Handler(named(s3middleware.DeleteBucketPolicyOperation, h.DeleteBucketPolicyHandler))).
|
||||
Add(NewFilter().
|
||||
Queries(s3middleware.LifecycleQuery).
|
||||
Handler(named(s3middleware.PutBucketLifecycleOperation, h.PutBucketLifecycleHandler))).
|
||||
Handler(named(s3middleware.DeleteBucketLifecycleOperation, h.DeleteBucketLifecycleHandler))).
|
||||
Add(NewFilter().
|
||||
Queries(s3middleware.EncryptionQuery).
|
||||
Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))).
|
||||
|
|
|
@ -183,6 +183,14 @@ func (a *App) initLayer(ctx context.Context) {
|
|||
}
|
||||
}
|
||||
|
||||
var lifecycleCnrInfo *data.BucketInfo
|
||||
if a.cfg.IsSet(cfgContainersLifecycle) {
|
||||
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
|
||||
if err != nil {
|
||||
a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
layerCfg := &layer.Config{
|
||||
Cache: layer.NewCache(getCacheOptions(a.cfg, a.log)),
|
||||
AnonKey: layer.AnonymousKey{
|
||||
|
@ -194,6 +202,7 @@ func (a *App) initLayer(ctx context.Context) {
|
|||
Features: a.settings,
|
||||
GateKey: a.key,
|
||||
CORSCnrInfo: corsCnrInfo,
|
||||
LifecycleCnrInfo: lifecycleCnrInfo,
|
||||
}
|
||||
|
||||
// prepare object layer
|
||||
|
|
|
@ -178,6 +178,7 @@ const ( // Settings.
|
|||
|
||||
// Containers.
|
||||
cfgContainersCORS = "containers.cors"
|
||||
cfgContainersLifecycle = "containers.lifecycle"
|
||||
|
||||
// Command line args.
|
||||
cmdHelp = "help"
|
||||
|
|
|
@ -218,3 +218,4 @@ S3_GW_RETRY_STRATEGY=exponential
|
|||
|
||||
# Containers properties
|
||||
S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||
S3_GW_CONTAINERS_LIFECYCLE=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||
|
|
|
@ -256,3 +256,4 @@ retry:
|
|||
# Containers properties
|
||||
containers:
|
||||
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||
|
|
|
@ -716,8 +716,10 @@ Section for well-known containers to store s3-related data and settings.
|
|||
```yaml
|
||||
containers:
|
||||
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|-----------|----------|---------------|---------------|--------------------------------------------------------------------------------------|
|
||||
|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------|
|
||||
| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. |
|
||||
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
|
||||
|
|
2
go.mod
2
go.mod
|
@ -15,6 +15,7 @@ require (
|
|||
github.com/go-chi/chi/v5 v5.0.8
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/minio/sio v0.3.0
|
||||
github.com/mr-tron/base58 v1.2.0
|
||||
github.com/nspcc-dev/neo-go v0.106.2
|
||||
github.com/panjf2000/ants/v2 v2.5.0
|
||||
github.com/prometheus/client_golang v1.19.0
|
||||
|
@ -64,7 +65,6 @@ require (
|
|||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
|
||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect
|
||||
github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
|
||||
|
|
|
@ -154,4 +154,9 @@ const (
|
|||
FailedToParsePartInfo = "failed to parse part info"
|
||||
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
|
||||
CloseCredsObjectPayload = "close creds object payload"
|
||||
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object"
|
||||
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration"
|
||||
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
|
||||
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
|
||||
GetBucketLifecycle = "get bucket lifecycle"
|
||||
)
|
||||
|
|
|
@ -117,6 +117,7 @@ const (
|
|||
settingsFileName = "bucket-settings"
|
||||
corsFilename = "bucket-cors"
|
||||
bucketTaggingFilename = "bucket-tagging"
|
||||
bucketLifecycleFilename = "bucket-lifecycle"
|
||||
|
||||
// versionTree -- ID of a tree with object versions.
|
||||
versionTree = "version"
|
||||
|
@ -1447,6 +1448,74 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
||||
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||
if err != nil && !isErrNotFound {
|
||||
return nil, fmt.Errorf("couldn't get node: %w", err)
|
||||
}
|
||||
|
||||
meta := make(map[string]string)
|
||||
meta[FileNameKey] = bucketLifecycleFilename
|
||||
meta[oidKV] = addr.Object().EncodeToString()
|
||||
meta[cidKV] = addr.Container().EncodeToString()
|
||||
|
||||
if isErrNotFound {
|
||||
if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, layer.ErrNoNodeToRemove
|
||||
}
|
||||
|
||||
latest := multiNode.Latest()
|
||||
ind := latest.GetLatestNodeIndex()
|
||||
if latest.IsSplit() {
|
||||
c.reqLogger(ctx).Error(logs.BucketLifecycleNodeHasMultipleIDs)
|
||||
}
|
||||
|
||||
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
||||
return nil, fmt.Errorf("move lifecycle node: %w", err)
|
||||
}
|
||||
|
||||
objToDelete := make([]oid.Address, 1, len(multiNode.nodes))
|
||||
objToDelete[0], err = getTreeNodeAddress(latest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse object addr of latest lifecycle node in tree: %w", err)
|
||||
}
|
||||
|
||||
objToDelete = append(objToDelete, c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)...)
|
||||
|
||||
return objToDelete, nil
|
||||
}
|
||||
|
||||
func (c *Tree) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
||||
node, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||
if err != nil {
|
||||
return oid.Address{}, fmt.Errorf("get lifecycle node: %w", err)
|
||||
}
|
||||
|
||||
return getTreeNodeAddress(node.Latest())
|
||||
}
|
||||
|
||||
func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
||||
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||
if err != nil && !isErrNotFound {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if isErrNotFound {
|
||||
return nil, layer.ErrNoNodeToRemove
|
||||
}
|
||||
|
||||
objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo)
|
||||
if len(objToDelete) != len(multiNode.nodes) {
|
||||
return nil, fmt.Errorf("clean old lifecycle nodes: %w", err)
|
||||
}
|
||||
|
||||
return objToDelete, nil
|
||||
}
|
||||
|
||||
func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||
err := c.service.RemoveNode(ctx, bktInfo, systemTree, multipartInfo.ID)
|
||||
if err != nil {
|
||||
|
|
Loading…
Reference in a new issue