From 08f9cfb39215ed7e8108934a37310c5bcd5c343c Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 12 Aug 2024 14:22:56 +0300 Subject: [PATCH] [#42] Support expiration lifecycle Signed-off-by: Marina Biryukova --- api/cache/system.go | 20 ++ api/data/info.go | 9 +- api/data/lifecycle.go | 54 ++++ api/handler/lifecycle.go | 235 +++++++++++++++++ api/handler/lifecycle_test.go | 457 ++++++++++++++++++++++++++++++++++ api/handler/not_support.go | 4 - api/handler/unimplemented.go | 8 - api/layer/cache.go | 26 ++ api/layer/layer.go | 70 +++--- api/layer/lifecycle.go | 148 +++++++++++ api/layer/lifecycle_test.go | 65 +++++ api/layer/tree_mock.go | 45 ++++ api/layer/tree_service.go | 4 + api/router.go | 2 +- cmd/s3-gw/app.go | 21 +- cmd/s3-gw/app_settings.go | 3 +- config/config.env | 1 + config/config.yaml | 1 + docs/configuration.md | 10 +- go.mod | 2 +- internal/logs/logs.go | 5 + pkg/service/tree/tree.go | 77 +++++- 22 files changed, 1207 insertions(+), 60 deletions(-) create mode 100644 api/data/lifecycle.go create mode 100644 api/handler/lifecycle.go create mode 100644 api/handler/lifecycle_test.go create mode 100644 api/layer/lifecycle.go create mode 100644 api/layer/lifecycle_test.go diff --git a/api/cache/system.go b/api/cache/system.go index c0f51a5..0395f58 100644 --- a/api/cache/system.go +++ b/api/cache/system.go @@ -88,6 +88,22 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration { return result } +func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfiguration { + entry, err := o.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.(*data.LifecycleConfiguration) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result))) + return nil + } + + return result +} + func (o *SystemCache) GetSettings(key string) *data.BucketSettings { entry, err := o.cache.Get(key) if err != nil { @@ -133,6 +149,10 @@ func (o *SystemCache) PutCORS(key string, obj *data.CORSConfiguration) error { return o.cache.Set(key, obj) } +func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleConfiguration) error { + return o.cache.Set(key, obj) +} + func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error { return o.cache.Set(key, settings) } diff --git a/api/data/info.go b/api/data/info.go index 686349b..daac9a3 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -12,8 +12,9 @@ import ( ) const ( - bktSettingsObject = ".s3-settings" - bktCORSConfigurationObject = ".s3-cors" + bktSettingsObject = ".s3-settings" + bktCORSConfigurationObject = ".s3-cors" + bktLifecycleConfigurationObject = ".s3-lifecycle" VersioningUnversioned = "Unversioned" VersioningEnabled = "Enabled" @@ -91,6 +92,10 @@ func (b *BucketInfo) CORSObjectName() string { return b.CID.EncodeToString() + bktCORSConfigurationObject } +func (b *BucketInfo) LifecycleConfigurationObjectName() string { + return b.CID.EncodeToString() + bktLifecycleConfigurationObject +} + // VersionID returns object version from ObjectInfo. func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() } diff --git a/api/data/lifecycle.go b/api/data/lifecycle.go new file mode 100644 index 0000000..cb3c7cb --- /dev/null +++ b/api/data/lifecycle.go @@ -0,0 +1,54 @@ +package data + +import "encoding/xml" + +const ( + LifecycleStatusEnabled = "Enabled" + LifecycleStatusDisabled = "Disabled" +) + +type ( + LifecycleConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LifecycleConfiguration" json:"-"` + Rules []LifecycleRule `xml:"Rule"` + } + + LifecycleRule struct { + Status string `xml:"Status,omitempty"` + AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"` + Expiration *LifecycleExpiration `xml:"Expiration,omitempty"` + Filter *LifecycleRuleFilter `xml:"Filter,omitempty"` + ID string `xml:"ID,omitempty"` + NonCurrentVersionExpiration *NonCurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"` + } + + AbortIncompleteMultipartUpload struct { + DaysAfterInitiation *int `xml:"DaysAfterInitiation,omitempty"` + } + + LifecycleExpiration struct { + Date string `xml:"Date,omitempty"` + Days *int `xml:"Days,omitempty"` + ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"` + } + + LifecycleRuleFilter struct { + And *LifecycleRuleAndOperator `xml:"And,omitempty"` + ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"` + ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"` + Prefix string `xml:"Prefix,omitempty"` + Tag *Tag `xml:"Tag,omitempty"` + } + + LifecycleRuleAndOperator struct { + ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"` + ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"` + Prefix string `xml:"Prefix,omitempty"` + Tags []Tag `xml:"Tag"` + } + + NonCurrentVersionExpiration struct { + NewerNonCurrentVersions *int `xml:"NewerNoncurrentVersions,omitempty"` + NonCurrentDays *int `xml:"NoncurrentDays,omitempty"` + } +) diff --git a/api/handler/lifecycle.go b/api/handler/lifecycle.go new file mode 100644 index 0000000..60bdce0 --- /dev/null +++ b/api/handler/lifecycle.go @@ -0,0 +1,235 @@ +package handler + +import ( + "bytes" + "fmt" + "io" + "net/http" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" +) + +const ( + maxRules = 1000 + maxRuleIDLen = 255 + maxNewerNoncurrentVersions = 100 +) + +func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + reqInfo := middleware.GetReqInfo(ctx) + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + + cfg, err := h.obj.GetBucketLifecycleConfiguration(ctx, bktInfo) + if err != nil { + h.logAndSendError(w, "could not get bucket lifecycle configuration", reqInfo, err) + return + } + + if err = middleware.EncodeToResponse(w, cfg); err != nil { + h.logAndSendError(w, "could not encode GetBucketLifecycle response", reqInfo, err) + return + } +} + +func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { + var buf bytes.Buffer + + tee := io.TeeReader(r.Body, &buf) + ctx := r.Context() + reqInfo := middleware.GetReqInfo(ctx) + + // Content-Md5 is required and should be set + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html + if _, ok := r.Header[api.ContentMD5]; !ok { + h.logAndSendError(w, "missing Content-MD5", reqInfo, apiErr.GetAPIError(apiErr.ErrMissingContentMD5)) + return + } + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + + cfg := new(data.LifecycleConfiguration) + if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil { + h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error())) + return + } + + if err = checkLifecycleConfiguration(cfg); err != nil { + h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error())) + return + } + + params := &layer.PutBucketLifecycleParams{ + BktInfo: bktInfo, + LifecycleCfg: cfg, + LifecycleReader: &buf, + MD5Hash: r.Header.Get(api.ContentMD5), + } + + params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint) + if err != nil { + h.logAndSendError(w, "invalid copies number", reqInfo, err) + return + } + + if err = h.obj.PutBucketLifecycleConfiguration(ctx, params); err != nil { + h.logAndSendError(w, "could not put bucket lifecycle configuration", reqInfo, err) + return + } +} + +func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + reqInfo := middleware.GetReqInfo(ctx) + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(w, "could not get bucket info", reqInfo, err) + return + } + + if err = h.obj.DeleteBucketLifecycleConfiguration(ctx, bktInfo); err != nil { + h.logAndSendError(w, "could not delete bucket lifecycle configuration", reqInfo, err) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error { + if len(cfg.Rules) > maxRules { + return fmt.Errorf("number of rules cannot be greater than %d", maxRules) + } + + ids := make(map[string]struct{}, len(cfg.Rules)) + for _, rule := range cfg.Rules { + if _, ok := ids[rule.ID]; ok && rule.ID != "" { + return fmt.Errorf("duplicate 'ID': %s", rule.ID) + } + ids[rule.ID] = struct{}{} + + if len(rule.ID) > maxRuleIDLen { + return fmt.Errorf("'ID' value cannot be longer than %d characters", maxRuleIDLen) + } + + if rule.Status != data.LifecycleStatusEnabled && rule.Status != data.LifecycleStatusDisabled { + return fmt.Errorf("invalid lifecycle status: %s", rule.Status) + } + + if rule.AbortIncompleteMultipartUpload == nil && rule.Expiration == nil && rule.NonCurrentVersionExpiration == nil { + return fmt.Errorf("at least one action needs to be specified in a rule") + } + + if rule.AbortIncompleteMultipartUpload != nil { + if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != nil && + *rule.AbortIncompleteMultipartUpload.DaysAfterInitiation <= 0 { + return fmt.Errorf("days after initiation must be a positive integer: %d", *rule.AbortIncompleteMultipartUpload.DaysAfterInitiation) + } + + if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) { + return fmt.Errorf("abort incomplete multipart upload cannot be specified with tags") + } + } + + if rule.Expiration != nil { + if rule.Expiration.ExpiredObjectDeleteMarker != nil { + if rule.Expiration.Days != nil || rule.Expiration.Date != "" { + return fmt.Errorf("expired object delete marker cannot be specified with days or date") + } + + if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) { + return fmt.Errorf("expired object delete marker cannot be specified with tags") + } + } + + if rule.Expiration.Days != nil && *rule.Expiration.Days <= 0 { + return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days) + } + + if _, err := time.Parse("2006-01-02T15:04:05.000Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil { + return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date) + } + } + + if rule.NonCurrentVersionExpiration != nil { + if rule.NonCurrentVersionExpiration.NewerNonCurrentVersions != nil && + (*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions > maxNewerNoncurrentVersions || + *rule.NonCurrentVersionExpiration.NewerNonCurrentVersions <= 0) { + return fmt.Errorf("invalid value of newer noncurrent versions: %d", *rule.NonCurrentVersionExpiration.NewerNonCurrentVersions) + } + + if rule.NonCurrentVersionExpiration.NonCurrentDays != nil && *rule.NonCurrentVersionExpiration.NonCurrentDays <= 0 { + return fmt.Errorf("invalid value of noncurrent days: %d", *rule.NonCurrentVersionExpiration.NonCurrentDays) + } + } + + if err := checkLifecycleRuleFilter(rule.Filter); err != nil { + return err + } + } + + return nil +} + +func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error { + if filter == nil { + return nil + } + + var fields int + + if filter.And != nil { + fields++ + for _, tag := range filter.And.Tags { + err := checkTag(tag) + if err != nil { + return err + } + } + + if filter.And.ObjectSizeGreaterThan != nil && filter.And.ObjectSizeLessThan != nil && + *filter.And.ObjectSizeLessThan <= *filter.And.ObjectSizeGreaterThan { + return fmt.Errorf("the maximum object size must be larger than the minimum object size") + } + } + + if filter.ObjectSizeGreaterThan != nil { + fields++ + } + + if filter.ObjectSizeLessThan != nil { + fields++ + } + + if filter.Prefix != "" { + fields++ + } + + if filter.Tag != nil { + fields++ + err := checkTag(*filter.Tag) + if err != nil { + return err + } + } + + if fields > 1 { + return fmt.Errorf("filter cannot have more than one field") + } + + return nil +} diff --git a/api/handler/lifecycle_test.go b/api/handler/lifecycle_test.go new file mode 100644 index 0000000..9f48089 --- /dev/null +++ b/api/handler/lifecycle_test.go @@ -0,0 +1,457 @@ +package handler + +import ( + "crypto/md5" + "crypto/rand" + "encoding/base64" + "encoding/xml" + "io" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "github.com/mr-tron/base58" + "github.com/stretchr/testify/require" +) + +func TestPutBucketLifecycleConfiguration(t *testing.T) { + hc := prepareHandlerContextWithMinCache(t) + + bktName := "bucket-lifecycle" + createBucket(hc, bktName) + + for _, tc := range []struct { + name string + body *data.LifecycleConfiguration + error bool + }{ + { + name: "correct configuration", + body: &data.LifecycleConfiguration{ + XMLName: xml.Name{ + Space: `http://s3.amazonaws.com/doc/2006-03-01/`, + Local: "LifecycleConfiguration", + }, + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + Date: time.Now().Format("2006-01-02T15:04:05.000Z"), + }, + Filter: &data.LifecycleRuleFilter{ + And: &data.LifecycleRuleAndOperator{ + Prefix: "prefix/", + Tags: []data.Tag{{Key: "key", Value: "value"}, {Key: "tag", Value: ""}}, + ObjectSizeGreaterThan: ptr(uint64(100)), + }, + }, + }, + { + Status: data.LifecycleStatusEnabled, + AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{ + DaysAfterInitiation: ptr(14), + }, + Expiration: &data.LifecycleExpiration{ + ExpiredObjectDeleteMarker: ptr(true), + }, + Filter: &data.LifecycleRuleFilter{ + ObjectSizeLessThan: ptr(uint64(100)), + }, + NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{ + NewerNonCurrentVersions: ptr(1), + NonCurrentDays: ptr(21), + }, + }, + }, + }, + }, + { + name: "too many rules", + body: func() *data.LifecycleConfiguration { + lifecycle := new(data.LifecycleConfiguration) + for i := 0; i <= maxRules; i++ { + lifecycle.Rules = append(lifecycle.Rules, data.LifecycleRule{ + ID: "Rule" + strconv.Itoa(i), + }) + } + return lifecycle + }(), + error: true, + }, + { + name: "duplicate rule ID", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + ID: "Rule", + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + }, + }, + { + ID: "Rule", + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + }, + }, + }, + }, + error: true, + }, + { + name: "too long rule ID", + body: func() *data.LifecycleConfiguration { + id := make([]byte, maxRuleIDLen+1) + _, err := io.ReadFull(rand.Reader, id) + require.NoError(t, err) + return &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + ID: base58.Encode(id)[:maxRuleIDLen+1], + }, + }, + } + }(), + error: true, + }, + { + name: "invalid status", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: "invalid", + }, + }, + }, + error: true, + }, + { + name: "no actions", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Filter: &data.LifecycleRuleFilter{ + Prefix: "prefix/", + }, + }, + }, + }, + error: true, + }, + { + name: "invalid days after initiation", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{ + DaysAfterInitiation: ptr(0), + }, + }, + }, + }, + error: true, + }, + { + name: "invalid expired object delete marker declaration", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + ExpiredObjectDeleteMarker: ptr(false), + }, + }, + }, + }, + error: true, + }, + { + name: "invalid expiration days", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(0), + }, + }, + }, + }, + error: true, + }, + { + name: "invalid expiration date", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Date: "invalid", + }, + }, + }, + }, + error: true, + }, + { + name: "newer noncurrent versions is too small", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{ + NewerNonCurrentVersions: ptr(0), + }, + }, + }, + }, + error: true, + }, + { + name: "newer noncurrent versions is too large", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{ + NewerNonCurrentVersions: ptr(101), + }, + }, + }, + }, + error: true, + }, + { + name: "invalid noncurrent days", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{ + NonCurrentDays: ptr(0), + }, + }, + }, + }, + error: true, + }, + { + name: "more than one filter field", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + }, + Filter: &data.LifecycleRuleFilter{ + Prefix: "prefix/", + ObjectSizeGreaterThan: ptr(uint64(100)), + }, + }, + }, + }, + error: true, + }, + { + name: "invalid tag in filter", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + }, + Filter: &data.LifecycleRuleFilter{ + Tag: &data.Tag{}, + }, + }, + }, + }, + error: true, + }, + { + name: "abort incomplete multipart upload with tag", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{ + DaysAfterInitiation: ptr(14), + }, + Filter: &data.LifecycleRuleFilter{ + Tag: &data.Tag{Key: "key", Value: "value"}, + }, + }, + }, + }, + error: true, + }, + { + name: "expired object delete marker with tag", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + ExpiredObjectDeleteMarker: ptr(true), + }, + Filter: &data.LifecycleRuleFilter{ + And: &data.LifecycleRuleAndOperator{ + Tags: []data.Tag{{Key: "key", Value: "value"}}, + }, + }, + }, + }, + }, + error: true, + }, + { + name: "invalid size range", + body: &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + }, + Filter: &data.LifecycleRuleFilter{ + And: &data.LifecycleRuleAndOperator{ + ObjectSizeGreaterThan: ptr(uint64(100)), + ObjectSizeLessThan: ptr(uint64(100)), + }, + }, + }, + }, + }, + error: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.error { + putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)) + return + } + + putBucketLifecycleConfiguration(hc, bktName, tc.body) + + cfg := getBucketLifecycleConfiguration(hc, bktName) + require.Equal(t, *tc.body, *cfg) + + deleteBucketLifecycleConfiguration(hc, bktName) + getBucketLifecycleConfigurationErr(hc, bktName, apiErrors.GetAPIError(apiErrors.ErrNoSuchLifecycleConfiguration)) + }) + } +} + +func TestPutBucketLifecycleInvalidMD5(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName := "bucket-lifecycle-md5" + createBucket(hc, bktName) + + lifecycle := &data.LifecycleConfiguration{ + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + }, + }, + }, + } + + w, r := prepareTestRequest(hc, bktName, "", lifecycle) + hc.Handler().PutBucketLifecycleHandler(w, r) + assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMissingContentMD5)) + + w, r = prepareTestRequest(hc, bktName, "", lifecycle) + r.Header.Set(api.ContentMD5, "") + hc.Handler().PutBucketLifecycleHandler(w, r) + assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)) + + w, r = prepareTestRequest(hc, bktName, "", lifecycle) + r.Header.Set(api.ContentMD5, "some-hash") + hc.Handler().PutBucketLifecycleHandler(w, r) + assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)) +} + +func TestPutBucketLifecycleInvalidXML(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName := "bucket-lifecycle-invalid-xml" + createBucket(hc, bktName) + + w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{}) + r.Header.Set(api.ContentMD5, "") + hc.Handler().PutBucketLifecycleHandler(w, r) + assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)) +} + +func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) { + w := putBucketLifecycleConfigurationBase(hc, bktName, cfg) + assertStatus(hc.t, w, http.StatusOK) +} + +func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apiErrors.Error) { + w := putBucketLifecycleConfigurationBase(hc, bktName, cfg) + assertS3Error(hc.t, w, err) +} + +func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder { + w, r := prepareTestRequest(hc, bktName, "", cfg) + + rawBody, err := xml.Marshal(cfg) + require.NoError(hc.t, err) + + hash := md5.New() + hash.Write(rawBody) + r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil))) + hc.Handler().PutBucketLifecycleHandler(w, r) + return w +} + +func getBucketLifecycleConfiguration(hc *handlerContext, bktName string) *data.LifecycleConfiguration { + w := getBucketLifecycleConfigurationBase(hc, bktName) + assertStatus(hc.t, w, http.StatusOK) + res := &data.LifecycleConfiguration{} + parseTestResponse(hc.t, w, res) + return res +} + +func getBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, err apiErrors.Error) { + w := getBucketLifecycleConfigurationBase(hc, bktName) + assertS3Error(hc.t, w, err) +} + +func getBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder { + w, r := prepareTestRequest(hc, bktName, "", nil) + hc.Handler().GetBucketLifecycleHandler(w, r) + return w +} + +func deleteBucketLifecycleConfiguration(hc *handlerContext, bktName string) { + w := deleteBucketLifecycleConfigurationBase(hc, bktName) + assertStatus(hc.t, w, http.StatusNoContent) +} + +func deleteBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder { + w, r := prepareTestRequest(hc, bktName, "", nil) + hc.Handler().DeleteBucketLifecycleHandler(w, r) + return w +} + +func ptr[T any](t T) *T { + return &t +} diff --git a/api/handler/not_support.go b/api/handler/not_support.go index 423e840..85c8317 100644 --- a/api/handler/not_support.go +++ b/api/handler/not_support.go @@ -7,10 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" ) -func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported)) -} - func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported)) } diff --git a/api/handler/unimplemented.go b/api/handler/unimplemented.go index 0c90789..b06e2c6 100644 --- a/api/handler/unimplemented.go +++ b/api/handler/unimplemented.go @@ -11,10 +11,6 @@ func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Requ h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } -func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } @@ -51,10 +47,6 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } -func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } diff --git a/api/layer/cache.go b/api/layer/cache.go index 71520f2..ca9a592 100644 --- a/api/layer/cache.go +++ b/api/layer/cache.go @@ -257,3 +257,29 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) { c.systemCache.Delete(bktInfo.CORSObjectName()) } + +func (c *Cache) GetLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo) *data.LifecycleConfiguration { + key := bkt.LifecycleConfigurationObjectName() + + if !c.accessCache.Get(owner, key) { + return nil + } + + return c.systemCache.GetLifecycleConfiguration(key) +} + +func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.LifecycleConfiguration) { + key := bkt.LifecycleConfigurationObjectName() + + if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil { + c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err)) + } + + if err := c.accessCache.Put(owner, key); err != nil { + c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err)) + } +} + +func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) { + c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName()) +} diff --git a/api/layer/layer.go b/api/layer/layer.go index eddcda2..a4b6a7c 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -46,28 +46,30 @@ type ( } Layer struct { - frostFS FrostFS - gateOwner user.ID - log *zap.Logger - anonKey AnonymousKey - resolver BucketResolver - cache *Cache - treeService TreeService - features FeatureSettings - gateKey *keys.PrivateKey - corsCnrInfo *data.BucketInfo + frostFS FrostFS + gateOwner user.ID + log *zap.Logger + anonKey AnonymousKey + resolver BucketResolver + cache *Cache + treeService TreeService + features FeatureSettings + gateKey *keys.PrivateKey + corsCnrInfo *data.BucketInfo + lifecycleCnrInfo *data.BucketInfo } Config struct { - GateOwner user.ID - ChainAddress string - Cache *Cache - AnonKey AnonymousKey - Resolver BucketResolver - TreeService TreeService - Features FeatureSettings - GateKey *keys.PrivateKey - CORSCnrInfo *data.BucketInfo + GateOwner user.ID + ChainAddress string + Cache *Cache + AnonKey AnonymousKey + Resolver BucketResolver + TreeService TreeService + Features FeatureSettings + GateKey *keys.PrivateKey + CORSCnrInfo *data.BucketInfo + LifecycleCnrInfo *data.BucketInfo } // AnonymousKey contains data for anonymous requests. @@ -233,16 +235,17 @@ func (p HeadObjectParams) Versioned() bool { // and establishes gRPC connection with the node. func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer { return &Layer{ - frostFS: frostFS, - log: log, - gateOwner: config.GateOwner, - anonKey: config.AnonKey, - resolver: config.Resolver, - cache: config.Cache, - treeService: config.TreeService, - features: config.Features, - gateKey: config.GateKey, - corsCnrInfo: config.CORSCnrInfo, + frostFS: frostFS, + log: log, + gateOwner: config.GateOwner, + anonKey: config.AnonKey, + resolver: config.Resolver, + cache: config.Cache, + treeService: config.TreeService, + features: config.Features, + gateKey: config.GateKey, + corsCnrInfo: config.CORSCnrInfo, + lifecycleCnrInfo: config.LifecycleCnrInfo, } } @@ -826,6 +829,11 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err)) } + lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo) + if treeErr != nil { + n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr)) + } + err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken) if err != nil { return fmt.Errorf("delete container: %w", err) @@ -835,5 +843,9 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { n.deleteCORSObject(ctx, p.BktInfo, corsObj) } + if treeErr == nil && !lifecycleObj.Container().Equals(p.BktInfo.CID) { + n.deleteLifecycleObject(ctx, p.BktInfo, lifecycleObj) + } + return nil } diff --git a/api/layer/lifecycle.go b/api/layer/lifecycle.go new file mode 100644 index 0000000..69f07c6 --- /dev/null +++ b/api/layer/lifecycle.go @@ -0,0 +1,148 @@ +package layer + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/xml" + "errors" + "fmt" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" +) + +type PutBucketLifecycleParams struct { + BktInfo *data.BucketInfo + LifecycleCfg *data.LifecycleConfiguration + LifecycleReader io.Reader + CopiesNumbers []uint32 + MD5Hash string +} + +func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error { + prm := PrmObjectCreate{ + Payload: p.LifecycleReader, + Filepath: p.BktInfo.LifecycleConfigurationObjectName(), + CreationTime: TimeNow(ctx), + } + + var lifecycleBkt *data.BucketInfo + if n.lifecycleCnrInfo == nil { + lifecycleBkt = p.BktInfo + prm.CopiesNumber = p.CopiesNumbers + } else { + lifecycleBkt = n.lifecycleCnrInfo + prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey + } + + prm.Container = lifecycleBkt.CID + + _, objID, _, md5, err := n.objectPutAndHash(ctx, prm, lifecycleBkt) + if err != nil { + return fmt.Errorf("put lifecycle object: %w", err) + } + + hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash) + if err != nil { + return apiErr.GetAPIError(apiErr.ErrInvalidDigest) + } + + if !bytes.Equal(hashBytes, md5) { + n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, objID)) + + return apiErr.GetAPIError(apiErr.ErrInvalidDigest) + } + + objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, objID)) + objsToDeleteNotFound := errors.Is(err, ErrNoNodeToRemove) + if err != nil && !objsToDeleteNotFound { + return err + } + + if !objsToDeleteNotFound { + for _, addr := range objsToDelete { + n.deleteLifecycleObject(ctx, p.BktInfo, addr) + } + } + + n.cache.PutLifecycleConfiguration(n.BearerOwner(ctx), p.BktInfo, p.LifecycleCfg) + + return nil +} + +// deleteLifecycleObject removes object and logs in case of error. +func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) { + var prmAuth PrmAuth + lifecycleBkt := bktInfo + if !addr.Container().Equals(bktInfo.CID) { + lifecycleBkt = &data.BucketInfo{CID: addr.Container()} + prmAuth.PrivateKey = &n.gateKey.PrivateKey + } + + if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil { + n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err), + zap.String("cid", lifecycleBkt.CID.EncodeToString()), + zap.String("oid", addr.Object().EncodeToString())) + } +} + +func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.LifecycleConfiguration, error) { + owner := n.BearerOwner(ctx) + if cfg := n.cache.GetLifecycleConfiguration(owner, bktInfo); cfg != nil { + return cfg, nil + } + + addr, err := n.treeService.GetBucketLifecycleConfiguration(ctx, bktInfo) + objNotFound := errors.Is(err, ErrNodeNotFound) + if err != nil && !objNotFound { + return nil, err + } + + if objNotFound { + return nil, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), err.Error()) + } + + var prmAuth PrmAuth + lifecycleBkt := bktInfo + if !addr.Container().Equals(bktInfo.CID) { + lifecycleBkt = &data.BucketInfo{CID: addr.Container()} + prmAuth.PrivateKey = &n.gateKey.PrivateKey + } + + obj, err := n.objectGetWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth) + if err != nil { + return nil, fmt.Errorf("get lifecycle object: %w", err) + } + + lifecycleCfg := &data.LifecycleConfiguration{} + + if err = xml.NewDecoder(obj.Payload).Decode(&lifecycleCfg); err != nil { + return nil, fmt.Errorf("unmarshal lifecycle configuration: %w", err) + } + + n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg) + + return lifecycleCfg, nil +} + +func (n *Layer) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) error { + objs, err := n.treeService.DeleteBucketLifecycleConfiguration(ctx, bktInfo) + objsNotFound := errors.Is(err, ErrNoNodeToRemove) + if err != nil && !objsNotFound { + return err + } + if !objsNotFound { + for _, addr := range objs { + n.deleteLifecycleObject(ctx, bktInfo, addr) + } + } + + n.cache.DeleteLifecycleConfiguration(bktInfo) + + return nil +} diff --git a/api/layer/lifecycle_test.go b/api/layer/lifecycle_test.go new file mode 100644 index 0000000..ec89897 --- /dev/null +++ b/api/layer/lifecycle_test.go @@ -0,0 +1,65 @@ +package layer + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/xml" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors" + "github.com/stretchr/testify/require" +) + +func TestBucketLifecycle(t *testing.T) { + tc := prepareContext(t) + + lifecycle := &data.LifecycleConfiguration{ + XMLName: xml.Name{ + Space: `http://s3.amazonaws.com/doc/2006-03-01/`, + Local: "LifecycleConfiguration", + }, + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + }, + }, + }, + } + lifecycleBytes, err := xml.Marshal(lifecycle) + require.NoError(t, err) + hash := md5.New() + hash.Write(lifecycleBytes) + + _, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo) + require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err)) + + err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo) + require.NoError(t, err) + + err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{ + BktInfo: tc.bktInfo, + LifecycleCfg: lifecycle, + LifecycleReader: bytes.NewReader(lifecycleBytes), + MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)), + }) + require.NoError(t, err) + + cfg, err := tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo) + require.NoError(t, err) + require.Equal(t, *lifecycle, *cfg) + + err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo) + require.NoError(t, err) + + _, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo) + require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err)) +} + +func ptr[T any](t T) *T { + return &t +} diff --git a/api/layer/tree_mock.go b/api/layer/tree_mock.go index daac689..b3b56b2 100644 --- a/api/layer/tree_mock.go +++ b/api/layer/tree_mock.go @@ -395,6 +395,51 @@ LOOP: return result, nil } +func (t *TreeServiceMock) PutBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) { + systemMap, ok := t.system[bktInfo.CID.EncodeToString()] + if !ok { + systemMap = make(map[string]*data.BaseNodeVersion) + } + + systemMap["lifecycle"] = &data.BaseNodeVersion{ + OID: addr.Object(), + } + + t.system[bktInfo.CID.EncodeToString()] = systemMap + + return nil, ErrNoNodeToRemove +} + +func (t *TreeServiceMock) GetBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) { + systemMap, ok := t.system[bktInfo.CID.EncodeToString()] + if !ok { + return oid.Address{}, ErrNodeNotFound + } + + node, ok := systemMap["lifecycle"] + if !ok { + return oid.Address{}, ErrNodeNotFound + } + + return newAddress(bktInfo.CID, node.OID), nil +} + +func (t *TreeServiceMock) DeleteBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) { + systemMap, ok := t.system[bktInfo.CID.EncodeToString()] + if !ok { + return nil, ErrNoNodeToRemove + } + + node, ok := systemMap["lifecycle"] + if !ok { + return nil, ErrNoNodeToRemove + } + + delete(systemMap, "lifecycle") + + return []oid.Address{newAddress(bktInfo.CID, node.OID)}, nil +} + func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error { cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()] diff --git a/api/layer/tree_service.go b/api/layer/tree_service.go index 9f31051..5c74f64 100644 --- a/api/layer/tree_service.go +++ b/api/layer/tree_service.go @@ -63,6 +63,10 @@ type TreeService interface { AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error) + PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) + GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) + DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) + // Compound methods for optimizations // GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation. diff --git a/api/router.go b/api/router.go index 0f86e2e..8e97400 100644 --- a/api/router.go +++ b/api/router.go @@ -356,7 +356,7 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router { Handler(named(s3middleware.DeleteBucketPolicyOperation, h.DeleteBucketPolicyHandler))). Add(NewFilter(). Queries(s3middleware.LifecycleQuery). - Handler(named(s3middleware.PutBucketLifecycleOperation, h.PutBucketLifecycleHandler))). + Handler(named(s3middleware.DeleteBucketLifecycleOperation, h.DeleteBucketLifecycleHandler))). Add(NewFilter(). Queries(s3middleware.EncryptionQuery). Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))). diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index 0b818e0..4e754e1 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -183,17 +183,26 @@ func (a *App) initLayer(ctx context.Context) { } } + var lifecycleCnrInfo *data.BucketInfo + if a.cfg.IsSet(cfgContainersLifecycle) { + lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle) + if err != nil { + a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err)) + } + } + layerCfg := &layer.Config{ Cache: layer.NewCache(getCacheOptions(a.cfg, a.log)), AnonKey: layer.AnonymousKey{ Key: randomKey, }, - GateOwner: gateOwner, - Resolver: a.bucketResolver, - TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log), - Features: a.settings, - GateKey: a.key, - CORSCnrInfo: corsCnrInfo, + GateOwner: gateOwner, + Resolver: a.bucketResolver, + TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log), + Features: a.settings, + GateKey: a.key, + CORSCnrInfo: corsCnrInfo, + LifecycleCnrInfo: lifecycleCnrInfo, } // prepare object layer diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index 3aacea4..631c703 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -177,7 +177,8 @@ const ( // Settings. cfgSourceIPHeader = "source_ip_header" // Containers. - cfgContainersCORS = "containers.cors" + cfgContainersCORS = "containers.cors" + cfgContainersLifecycle = "containers.lifecycle" // Command line args. cmdHelp = "help" diff --git a/config/config.env b/config/config.env index dd4438a..0ffb31e 100644 --- a/config/config.env +++ b/config/config.env @@ -218,3 +218,4 @@ S3_GW_RETRY_STRATEGY=exponential # Containers properties S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj +S3_GW_CONTAINERS_LIFECYCLE=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj diff --git a/config/config.yaml b/config/config.yaml index aae9e0b..9088ad2 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -256,3 +256,4 @@ retry: # Containers properties containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj diff --git a/docs/configuration.md b/docs/configuration.md index 23ba071..37d251f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -715,9 +715,11 @@ Section for well-known containers to store s3-related data and settings. ```yaml containers: - cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-----------|----------|---------------|---------------|--------------------------------------------------------------------------------------| -| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------| +| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. | +| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. | diff --git a/go.mod b/go.mod index 9984415..f94d5c6 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/go-chi/chi/v5 v5.0.8 github.com/google/uuid v1.6.0 github.com/minio/sio v0.3.0 + github.com/mr-tron/base58 v1.2.0 github.com/nspcc-dev/neo-go v0.106.2 github.com/panjf2000/ants/v2 v2.5.0 github.com/prometheus/client_golang v1.19.0 @@ -64,7 +65,6 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d875eb7..948da36 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -154,4 +154,9 @@ const ( FailedToParsePartInfo = "failed to parse part info" CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" CloseCredsObjectPayload = "close creds object payload" + CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object" + CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration" + CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info" + BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids" + GetBucketLifecycle = "get bucket lifecycle" ) diff --git a/pkg/service/tree/tree.go b/pkg/service/tree/tree.go index 259a473..234e359 100644 --- a/pkg/service/tree/tree.go +++ b/pkg/service/tree/tree.go @@ -114,9 +114,10 @@ const ( ownerKV = "Owner" createdKV = "Created" - settingsFileName = "bucket-settings" - corsFilename = "bucket-cors" - bucketTaggingFilename = "bucket-tagging" + settingsFileName = "bucket-settings" + corsFilename = "bucket-cors" + bucketTaggingFilename = "bucket-tagging" + bucketLifecycleFilename = "bucket-lifecycle" // versionTree -- ID of a tree with object versions. versionTree = "version" @@ -560,7 +561,7 @@ func (c *Tree) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ( objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo) if len(objToDelete) != len(multiNode.nodes) { - return nil, fmt.Errorf("clean old cors nodes: %w", err) + return nil, fmt.Errorf("failed to clean all old cors nodes") } return objToDelete, nil @@ -1447,6 +1448,74 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart return result, nil } +func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) { + multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename) + isErrNotFound := errors.Is(err, layer.ErrNodeNotFound) + if err != nil && !isErrNotFound { + return nil, fmt.Errorf("couldn't get node: %w", err) + } + + meta := make(map[string]string) + meta[FileNameKey] = bucketLifecycleFilename + meta[oidKV] = addr.Object().EncodeToString() + meta[cidKV] = addr.Container().EncodeToString() + + if isErrNotFound { + if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil { + return nil, err + } + return nil, layer.ErrNoNodeToRemove + } + + latest := multiNode.Latest() + ind := latest.GetLatestNodeIndex() + if latest.IsSplit() { + c.reqLogger(ctx).Error(logs.BucketLifecycleNodeHasMultipleIDs) + } + + if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil { + return nil, fmt.Errorf("move lifecycle node: %w", err) + } + + objToDelete := make([]oid.Address, 1, len(multiNode.nodes)) + objToDelete[0], err = getTreeNodeAddress(latest) + if err != nil { + return nil, fmt.Errorf("parse object addr of latest lifecycle node in tree: %w", err) + } + + objToDelete = append(objToDelete, c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)...) + + return objToDelete, nil +} + +func (c *Tree) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) { + node, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename) + if err != nil { + return oid.Address{}, fmt.Errorf("get lifecycle node: %w", err) + } + + return getTreeNodeAddress(node.Latest()) +} + +func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) { + multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename) + isErrNotFound := errors.Is(err, layer.ErrNodeNotFound) + if err != nil && !isErrNotFound { + return nil, err + } + + if isErrNotFound { + return nil, layer.ErrNoNodeToRemove + } + + objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo) + if len(objToDelete) != len(multiNode.nodes) { + return nil, fmt.Errorf("failed to clean all old lifecycle nodes") + } + + return objToDelete, nil +} + func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error { err := c.service.RemoveNode(ctx, bktInfo, systemTree, multipartInfo.ID) if err != nil {