[#42] Support expiration lifecycle
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
This commit is contained in:
parent
28723f4a68
commit
481520705a
22 changed files with 1207 additions and 60 deletions
20
api/cache/system.go
vendored
20
api/cache/system.go
vendored
|
@ -88,6 +88,22 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfiguration {
|
||||||
|
entry, err := o.cache.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, ok := entry.(*data.LifecycleConfiguration)
|
||||||
|
if !ok {
|
||||||
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
||||||
entry, err := o.cache.Get(key)
|
entry, err := o.cache.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -133,6 +149,10 @@ func (o *SystemCache) PutCORS(key string, obj *data.CORSConfiguration) error {
|
||||||
return o.cache.Set(key, obj)
|
return o.cache.Set(key, obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleConfiguration) error {
|
||||||
|
return o.cache.Set(key, obj)
|
||||||
|
}
|
||||||
|
|
||||||
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
|
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
|
||||||
return o.cache.Set(key, settings)
|
return o.cache.Set(key, settings)
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,8 +12,9 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
bktSettingsObject = ".s3-settings"
|
bktSettingsObject = ".s3-settings"
|
||||||
bktCORSConfigurationObject = ".s3-cors"
|
bktCORSConfigurationObject = ".s3-cors"
|
||||||
|
bktLifecycleConfigurationObject = ".s3-lifecycle"
|
||||||
|
|
||||||
VersioningUnversioned = "Unversioned"
|
VersioningUnversioned = "Unversioned"
|
||||||
VersioningEnabled = "Enabled"
|
VersioningEnabled = "Enabled"
|
||||||
|
@ -91,6 +92,10 @@ func (b *BucketInfo) CORSObjectName() string {
|
||||||
return b.CID.EncodeToString() + bktCORSConfigurationObject
|
return b.CID.EncodeToString() + bktCORSConfigurationObject
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *BucketInfo) LifecycleConfigurationObjectName() string {
|
||||||
|
return b.CID.EncodeToString() + bktLifecycleConfigurationObject
|
||||||
|
}
|
||||||
|
|
||||||
// VersionID returns object version from ObjectInfo.
|
// VersionID returns object version from ObjectInfo.
|
||||||
func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() }
|
func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() }
|
||||||
|
|
||||||
|
|
54
api/data/lifecycle.go
Normal file
54
api/data/lifecycle.go
Normal file
|
@ -0,0 +1,54 @@
|
||||||
|
package data
|
||||||
|
|
||||||
|
import "encoding/xml"
|
||||||
|
|
||||||
|
const (
|
||||||
|
LifecycleStatusEnabled = "Enabled"
|
||||||
|
LifecycleStatusDisabled = "Disabled"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
LifecycleConfiguration struct {
|
||||||
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LifecycleConfiguration" json:"-"`
|
||||||
|
Rules []LifecycleRule `xml:"Rule"`
|
||||||
|
}
|
||||||
|
|
||||||
|
LifecycleRule struct {
|
||||||
|
Status string `xml:"Status,omitempty"`
|
||||||
|
AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
|
||||||
|
Expiration *LifecycleExpiration `xml:"Expiration,omitempty"`
|
||||||
|
Filter *LifecycleRuleFilter `xml:"Filter,omitempty"`
|
||||||
|
ID string `xml:"ID,omitempty"`
|
||||||
|
NonCurrentVersionExpiration *NonCurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
AbortIncompleteMultipartUpload struct {
|
||||||
|
DaysAfterInitiation *int `xml:"DaysAfterInitiation,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
LifecycleExpiration struct {
|
||||||
|
Date string `xml:"Date,omitempty"`
|
||||||
|
Days *int `xml:"Days,omitempty"`
|
||||||
|
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
LifecycleRuleFilter struct {
|
||||||
|
And *LifecycleRuleAndOperator `xml:"And,omitempty"`
|
||||||
|
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
||||||
|
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
|
||||||
|
Prefix string `xml:"Prefix,omitempty"`
|
||||||
|
Tag *Tag `xml:"Tag,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
LifecycleRuleAndOperator struct {
|
||||||
|
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
||||||
|
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
|
||||||
|
Prefix string `xml:"Prefix,omitempty"`
|
||||||
|
Tags []Tag `xml:"Tag"`
|
||||||
|
}
|
||||||
|
|
||||||
|
NonCurrentVersionExpiration struct {
|
||||||
|
NewerNonCurrentVersions *int `xml:"NewerNoncurrentVersions,omitempty"`
|
||||||
|
NonCurrentDays *int `xml:"NoncurrentDays,omitempty"`
|
||||||
|
}
|
||||||
|
)
|
235
api/handler/lifecycle.go
Normal file
235
api/handler/lifecycle.go
Normal file
|
@ -0,0 +1,235 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxRules = 1000
|
||||||
|
maxRuleIDLen = 255
|
||||||
|
maxNewerNoncurrentVersions = 100
|
||||||
|
)
|
||||||
|
|
||||||
|
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := h.obj.GetBucketLifecycleConfiguration(ctx, bktInfo)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket lifecycle configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = middleware.EncodeToResponse(w, cfg); err != nil {
|
||||||
|
h.logAndSendError(w, "could not encode GetBucketLifecycle response", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
tee := io.TeeReader(r.Body, &buf)
|
||||||
|
ctx := r.Context()
|
||||||
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
|
// Content-Md5 is required and should be set
|
||||||
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
|
||||||
|
if _, ok := r.Header[api.ContentMD5]; !ok {
|
||||||
|
h.logAndSendError(w, "missing Content-MD5", reqInfo, apiErr.GetAPIError(apiErr.ErrMissingContentMD5))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := new(data.LifecycleConfiguration)
|
||||||
|
if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil {
|
||||||
|
h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error()))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = checkLifecycleConfiguration(cfg); err != nil {
|
||||||
|
h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error()))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
params := &layer.PutBucketLifecycleParams{
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
LifecycleCfg: cfg,
|
||||||
|
LifecycleReader: &buf,
|
||||||
|
MD5Hash: r.Header.Get(api.ContentMD5),
|
||||||
|
}
|
||||||
|
|
||||||
|
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.PutBucketLifecycleConfiguration(ctx, params); err != nil {
|
||||||
|
h.logAndSendError(w, "could not put bucket lifecycle configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.DeleteBucketLifecycleConfiguration(ctx, bktInfo); err != nil {
|
||||||
|
h.logAndSendError(w, "could not delete bucket lifecycle configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
|
||||||
|
if len(cfg.Rules) > maxRules {
|
||||||
|
return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
|
||||||
|
}
|
||||||
|
|
||||||
|
ids := make(map[string]struct{}, len(cfg.Rules))
|
||||||
|
for _, rule := range cfg.Rules {
|
||||||
|
if _, ok := ids[rule.ID]; ok && rule.ID != "" {
|
||||||
|
return fmt.Errorf("duplicate 'ID': %s", rule.ID)
|
||||||
|
}
|
||||||
|
ids[rule.ID] = struct{}{}
|
||||||
|
|
||||||
|
if len(rule.ID) > maxRuleIDLen {
|
||||||
|
return fmt.Errorf("'ID' value cannot be longer than %d characters", maxRuleIDLen)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Status != data.LifecycleStatusEnabled && rule.Status != data.LifecycleStatusDisabled {
|
||||||
|
return fmt.Errorf("invalid lifecycle status: %s", rule.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.AbortIncompleteMultipartUpload == nil && rule.Expiration == nil && rule.NonCurrentVersionExpiration == nil {
|
||||||
|
return fmt.Errorf("at least one action needs to be specified in a rule")
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.AbortIncompleteMultipartUpload != nil {
|
||||||
|
if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != nil &&
|
||||||
|
*rule.AbortIncompleteMultipartUpload.DaysAfterInitiation <= 0 {
|
||||||
|
return fmt.Errorf("days after initiation must be a positive integer: %d", *rule.AbortIncompleteMultipartUpload.DaysAfterInitiation)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
|
||||||
|
return fmt.Errorf("abort incomplete multipart upload cannot be specified with tags")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Expiration != nil {
|
||||||
|
if rule.Expiration.ExpiredObjectDeleteMarker != nil {
|
||||||
|
if rule.Expiration.Days != nil || rule.Expiration.Date != "" {
|
||||||
|
return fmt.Errorf("expired object delete marker cannot be specified with days or date")
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
|
||||||
|
return fmt.Errorf("expired object delete marker cannot be specified with tags")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Expiration.Days != nil && *rule.Expiration.Days <= 0 {
|
||||||
|
return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := time.Parse("2006-01-02T15:04:05.000Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil {
|
||||||
|
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.NonCurrentVersionExpiration != nil {
|
||||||
|
if rule.NonCurrentVersionExpiration.NewerNonCurrentVersions != nil &&
|
||||||
|
(*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions > maxNewerNoncurrentVersions ||
|
||||||
|
*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions <= 0) {
|
||||||
|
return fmt.Errorf("invalid value of newer noncurrent versions: %d", *rule.NonCurrentVersionExpiration.NewerNonCurrentVersions)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.NonCurrentVersionExpiration.NonCurrentDays != nil && *rule.NonCurrentVersionExpiration.NonCurrentDays <= 0 {
|
||||||
|
return fmt.Errorf("invalid value of noncurrent days: %d", *rule.NonCurrentVersionExpiration.NonCurrentDays)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := checkLifecycleRuleFilter(rule.Filter); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
|
||||||
|
if filter == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var fields int
|
||||||
|
|
||||||
|
if filter.And != nil {
|
||||||
|
fields++
|
||||||
|
for _, tag := range filter.And.Tags {
|
||||||
|
err := checkTag(tag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.And.ObjectSizeGreaterThan != nil && filter.And.ObjectSizeLessThan != nil &&
|
||||||
|
*filter.And.ObjectSizeLessThan <= *filter.And.ObjectSizeGreaterThan {
|
||||||
|
return fmt.Errorf("the maximum object size must be larger than the minimum object size")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.ObjectSizeGreaterThan != nil {
|
||||||
|
fields++
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.ObjectSizeLessThan != nil {
|
||||||
|
fields++
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.Prefix != "" {
|
||||||
|
fields++
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.Tag != nil {
|
||||||
|
fields++
|
||||||
|
err := checkTag(*filter.Tag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields > 1 {
|
||||||
|
return fmt.Errorf("filter cannot have more than one field")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
457
api/handler/lifecycle_test.go
Normal file
457
api/handler/lifecycle_test.go
Normal file
|
@ -0,0 +1,457 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/md5"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"github.com/mr-tron/base58"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
||||||
|
hc := prepareHandlerContextWithMinCache(t)
|
||||||
|
|
||||||
|
bktName := "bucket-lifecycle"
|
||||||
|
createBucket(hc, bktName)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
body *data.LifecycleConfiguration
|
||||||
|
error bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "correct configuration",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
XMLName: xml.Name{
|
||||||
|
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
|
||||||
|
Local: "LifecycleConfiguration",
|
||||||
|
},
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
Date: time.Now().Format("2006-01-02T15:04:05.000Z"),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
And: &data.LifecycleRuleAndOperator{
|
||||||
|
Prefix: "prefix/",
|
||||||
|
Tags: []data.Tag{{Key: "key", Value: "value"}, {Key: "tag", Value: ""}},
|
||||||
|
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||||
|
DaysAfterInitiation: ptr(14),
|
||||||
|
},
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
ExpiredObjectDeleteMarker: ptr(true),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
ObjectSizeLessThan: ptr(uint64(100)),
|
||||||
|
},
|
||||||
|
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||||
|
NewerNonCurrentVersions: ptr(1),
|
||||||
|
NonCurrentDays: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "too many rules",
|
||||||
|
body: func() *data.LifecycleConfiguration {
|
||||||
|
lifecycle := new(data.LifecycleConfiguration)
|
||||||
|
for i := 0; i <= maxRules; i++ {
|
||||||
|
lifecycle.Rules = append(lifecycle.Rules, data.LifecycleRule{
|
||||||
|
ID: "Rule" + strconv.Itoa(i),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return lifecycle
|
||||||
|
}(),
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate rule ID",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
ID: "Rule",
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "Rule",
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "too long rule ID",
|
||||||
|
body: func() *data.LifecycleConfiguration {
|
||||||
|
id := make([]byte, maxRuleIDLen+1)
|
||||||
|
_, err := io.ReadFull(rand.Reader, id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
ID: base58.Encode(id)[:maxRuleIDLen+1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}(),
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid status",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: "invalid",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no actions",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
Prefix: "prefix/",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid days after initiation",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||||
|
DaysAfterInitiation: ptr(0),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid expired object delete marker declaration",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
ExpiredObjectDeleteMarker: ptr(false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid expiration days",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(0),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid expiration date",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Date: "invalid",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "newer noncurrent versions is too small",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||||
|
NewerNonCurrentVersions: ptr(0),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "newer noncurrent versions is too large",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||||
|
NewerNonCurrentVersions: ptr(101),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid noncurrent days",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||||
|
NonCurrentDays: ptr(0),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "more than one filter field",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
Prefix: "prefix/",
|
||||||
|
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid tag in filter",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
Tag: &data.Tag{},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "abort incomplete multipart upload with tag",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||||
|
DaysAfterInitiation: ptr(14),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
Tag: &data.Tag{Key: "key", Value: "value"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "expired object delete marker with tag",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
ExpiredObjectDeleteMarker: ptr(true),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
And: &data.LifecycleRuleAndOperator{
|
||||||
|
Tags: []data.Tag{{Key: "key", Value: "value"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid size range",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
And: &data.LifecycleRuleAndOperator{
|
||||||
|
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||||
|
ObjectSizeLessThan: ptr(uint64(100)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if tc.error {
|
||||||
|
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apiErrors.GetAPIError(apiErrors.ErrMalformedXML))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
putBucketLifecycleConfiguration(hc, bktName, tc.body)
|
||||||
|
|
||||||
|
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
||||||
|
require.Equal(t, *tc.body, *cfg)
|
||||||
|
|
||||||
|
deleteBucketLifecycleConfiguration(hc, bktName)
|
||||||
|
getBucketLifecycleConfigurationErr(hc, bktName, apiErrors.GetAPIError(apiErrors.ErrNoSuchLifecycleConfiguration))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-lifecycle-md5"
|
||||||
|
createBucket(hc, bktName)
|
||||||
|
|
||||||
|
lifecycle := &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMissingContentMD5))
|
||||||
|
|
||||||
|
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
r.Header.Set(api.ContentMD5, "")
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest))
|
||||||
|
|
||||||
|
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
r.Header.Set(api.ContentMD5, "some-hash")
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-lifecycle-invalid-xml"
|
||||||
|
createBucket(hc, bktName)
|
||||||
|
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{})
|
||||||
|
r.Header.Set(api.ContentMD5, "")
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMalformedXML))
|
||||||
|
}
|
||||||
|
|
||||||
|
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) {
|
||||||
|
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||||
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apiErrors.Error) {
|
||||||
|
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||||
|
assertS3Error(hc.t, w, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder {
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", cfg)
|
||||||
|
|
||||||
|
rawBody, err := xml.Marshal(cfg)
|
||||||
|
require.NoError(hc.t, err)
|
||||||
|
|
||||||
|
hash := md5.New()
|
||||||
|
hash.Write(rawBody)
|
||||||
|
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func getBucketLifecycleConfiguration(hc *handlerContext, bktName string) *data.LifecycleConfiguration {
|
||||||
|
w := getBucketLifecycleConfigurationBase(hc, bktName)
|
||||||
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
|
res := &data.LifecycleConfiguration{}
|
||||||
|
parseTestResponse(hc.t, w, res)
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func getBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, err apiErrors.Error) {
|
||||||
|
w := getBucketLifecycleConfigurationBase(hc, bktName)
|
||||||
|
assertS3Error(hc.t, w, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
|
hc.Handler().GetBucketLifecycleHandler(w, r)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteBucketLifecycleConfiguration(hc *handlerContext, bktName string) {
|
||||||
|
w := deleteBucketLifecycleConfigurationBase(hc, bktName)
|
||||||
|
assertStatus(hc.t, w, http.StatusNoContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
|
hc.Handler().DeleteBucketLifecycleHandler(w, r)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func ptr[T any](t T) *T {
|
||||||
|
return &t
|
||||||
|
}
|
|
@ -7,10 +7,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,10 +11,6 @@ func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Requ
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
@ -51,10 +47,6 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request)
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
|
@ -257,3 +257,29 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
||||||
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
|
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
|
||||||
c.systemCache.Delete(bktInfo.CORSObjectName())
|
c.systemCache.Delete(bktInfo.CORSObjectName())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo) *data.LifecycleConfiguration {
|
||||||
|
key := bkt.LifecycleConfigurationObjectName()
|
||||||
|
|
||||||
|
if !c.accessCache.Get(owner, key) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.systemCache.GetLifecycleConfiguration(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.LifecycleConfiguration) {
|
||||||
|
key := bkt.LifecycleConfigurationObjectName()
|
||||||
|
|
||||||
|
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
|
||||||
|
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) {
|
||||||
|
c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName())
|
||||||
|
}
|
||||||
|
|
|
@ -46,28 +46,30 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
Layer struct {
|
Layer struct {
|
||||||
frostFS FrostFS
|
frostFS FrostFS
|
||||||
gateOwner user.ID
|
gateOwner user.ID
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
anonKey AnonymousKey
|
anonKey AnonymousKey
|
||||||
resolver BucketResolver
|
resolver BucketResolver
|
||||||
cache *Cache
|
cache *Cache
|
||||||
treeService TreeService
|
treeService TreeService
|
||||||
features FeatureSettings
|
features FeatureSettings
|
||||||
gateKey *keys.PrivateKey
|
gateKey *keys.PrivateKey
|
||||||
corsCnrInfo *data.BucketInfo
|
corsCnrInfo *data.BucketInfo
|
||||||
|
lifecycleCnrInfo *data.BucketInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
Config struct {
|
Config struct {
|
||||||
GateOwner user.ID
|
GateOwner user.ID
|
||||||
ChainAddress string
|
ChainAddress string
|
||||||
Cache *Cache
|
Cache *Cache
|
||||||
AnonKey AnonymousKey
|
AnonKey AnonymousKey
|
||||||
Resolver BucketResolver
|
Resolver BucketResolver
|
||||||
TreeService TreeService
|
TreeService TreeService
|
||||||
Features FeatureSettings
|
Features FeatureSettings
|
||||||
GateKey *keys.PrivateKey
|
GateKey *keys.PrivateKey
|
||||||
CORSCnrInfo *data.BucketInfo
|
CORSCnrInfo *data.BucketInfo
|
||||||
|
LifecycleCnrInfo *data.BucketInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
// AnonymousKey contains data for anonymous requests.
|
// AnonymousKey contains data for anonymous requests.
|
||||||
|
@ -233,16 +235,17 @@ func (p HeadObjectParams) Versioned() bool {
|
||||||
// and establishes gRPC connection with the node.
|
// and establishes gRPC connection with the node.
|
||||||
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer {
|
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer {
|
||||||
return &Layer{
|
return &Layer{
|
||||||
frostFS: frostFS,
|
frostFS: frostFS,
|
||||||
log: log,
|
log: log,
|
||||||
gateOwner: config.GateOwner,
|
gateOwner: config.GateOwner,
|
||||||
anonKey: config.AnonKey,
|
anonKey: config.AnonKey,
|
||||||
resolver: config.Resolver,
|
resolver: config.Resolver,
|
||||||
cache: config.Cache,
|
cache: config.Cache,
|
||||||
treeService: config.TreeService,
|
treeService: config.TreeService,
|
||||||
features: config.Features,
|
features: config.Features,
|
||||||
gateKey: config.GateKey,
|
gateKey: config.GateKey,
|
||||||
corsCnrInfo: config.CORSCnrInfo,
|
corsCnrInfo: config.CORSCnrInfo,
|
||||||
|
lifecycleCnrInfo: config.LifecycleCnrInfo,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -826,6 +829,11 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
|
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
|
||||||
|
if treeErr != nil {
|
||||||
|
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr))
|
||||||
|
}
|
||||||
|
|
||||||
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("delete container: %w", err)
|
return fmt.Errorf("delete container: %w", err)
|
||||||
|
@ -835,5 +843,9 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
n.deleteCORSObject(ctx, p.BktInfo, corsObj)
|
n.deleteCORSObject(ctx, p.BktInfo, corsObj)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if treeErr == nil && !lifecycleObj.Container().Equals(p.BktInfo.CID) {
|
||||||
|
n.deleteLifecycleObject(ctx, p.BktInfo, lifecycleObj)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
148
api/layer/lifecycle.go
Normal file
148
api/layer/lifecycle.go
Normal file
|
@ -0,0 +1,148 @@
|
||||||
|
package layer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PutBucketLifecycleParams struct {
|
||||||
|
BktInfo *data.BucketInfo
|
||||||
|
LifecycleCfg *data.LifecycleConfiguration
|
||||||
|
LifecycleReader io.Reader
|
||||||
|
CopiesNumbers []uint32
|
||||||
|
MD5Hash string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
|
||||||
|
prm := PrmObjectCreate{
|
||||||
|
Payload: p.LifecycleReader,
|
||||||
|
Filepath: p.BktInfo.LifecycleConfigurationObjectName(),
|
||||||
|
CreationTime: TimeNow(ctx),
|
||||||
|
}
|
||||||
|
|
||||||
|
var lifecycleBkt *data.BucketInfo
|
||||||
|
if n.lifecycleCnrInfo == nil {
|
||||||
|
lifecycleBkt = p.BktInfo
|
||||||
|
prm.CopiesNumber = p.CopiesNumbers
|
||||||
|
} else {
|
||||||
|
lifecycleBkt = n.lifecycleCnrInfo
|
||||||
|
prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||||
|
}
|
||||||
|
|
||||||
|
prm.Container = lifecycleBkt.CID
|
||||||
|
|
||||||
|
_, objID, _, md5, err := n.objectPutAndHash(ctx, prm, lifecycleBkt)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("put lifecycle object: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash)
|
||||||
|
if err != nil {
|
||||||
|
return apiErr.GetAPIError(apiErr.ErrInvalidDigest)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(hashBytes, md5) {
|
||||||
|
n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, objID))
|
||||||
|
|
||||||
|
return apiErr.GetAPIError(apiErr.ErrInvalidDigest)
|
||||||
|
}
|
||||||
|
|
||||||
|
objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, objID))
|
||||||
|
objsToDeleteNotFound := errors.Is(err, ErrNoNodeToRemove)
|
||||||
|
if err != nil && !objsToDeleteNotFound {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !objsToDeleteNotFound {
|
||||||
|
for _, addr := range objsToDelete {
|
||||||
|
n.deleteLifecycleObject(ctx, p.BktInfo, addr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n.cache.PutLifecycleConfiguration(n.BearerOwner(ctx), p.BktInfo, p.LifecycleCfg)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deleteLifecycleObject removes object and logs in case of error.
|
||||||
|
func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
|
||||||
|
var prmAuth PrmAuth
|
||||||
|
lifecycleBkt := bktInfo
|
||||||
|
if !addr.Container().Equals(bktInfo.CID) {
|
||||||
|
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
|
||||||
|
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
|
||||||
|
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
|
||||||
|
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
|
||||||
|
zap.String("oid", addr.Object().EncodeToString()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.LifecycleConfiguration, error) {
|
||||||
|
owner := n.BearerOwner(ctx)
|
||||||
|
if cfg := n.cache.GetLifecycleConfiguration(owner, bktInfo); cfg != nil {
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, err := n.treeService.GetBucketLifecycleConfiguration(ctx, bktInfo)
|
||||||
|
objNotFound := errors.Is(err, ErrNodeNotFound)
|
||||||
|
if err != nil && !objNotFound {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if objNotFound {
|
||||||
|
return nil, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
var prmAuth PrmAuth
|
||||||
|
lifecycleBkt := bktInfo
|
||||||
|
if !addr.Container().Equals(bktInfo.CID) {
|
||||||
|
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
|
||||||
|
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||||
|
}
|
||||||
|
|
||||||
|
obj, err := n.objectGetWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("get lifecycle object: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
lifecycleCfg := &data.LifecycleConfiguration{}
|
||||||
|
|
||||||
|
if err = xml.NewDecoder(obj.Payload).Decode(&lifecycleCfg); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal lifecycle configuration: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg)
|
||||||
|
|
||||||
|
return lifecycleCfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Layer) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
|
objs, err := n.treeService.DeleteBucketLifecycleConfiguration(ctx, bktInfo)
|
||||||
|
objsNotFound := errors.Is(err, ErrNoNodeToRemove)
|
||||||
|
if err != nil && !objsNotFound {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !objsNotFound {
|
||||||
|
for _, addr := range objs {
|
||||||
|
n.deleteLifecycleObject(ctx, bktInfo, addr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n.cache.DeleteLifecycleConfiguration(bktInfo)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
65
api/layer/lifecycle_test.go
Normal file
65
api/layer/lifecycle_test.go
Normal file
|
@ -0,0 +1,65 @@
|
||||||
|
package layer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBucketLifecycle(t *testing.T) {
|
||||||
|
tc := prepareContext(t)
|
||||||
|
|
||||||
|
lifecycle := &data.LifecycleConfiguration{
|
||||||
|
XMLName: xml.Name{
|
||||||
|
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
|
||||||
|
Local: "LifecycleConfiguration",
|
||||||
|
},
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
lifecycleBytes, err := xml.Marshal(lifecycle)
|
||||||
|
require.NoError(t, err)
|
||||||
|
hash := md5.New()
|
||||||
|
hash.Write(lifecycleBytes)
|
||||||
|
|
||||||
|
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err))
|
||||||
|
|
||||||
|
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{
|
||||||
|
BktInfo: tc.bktInfo,
|
||||||
|
LifecycleCfg: lifecycle,
|
||||||
|
LifecycleReader: bytes.NewReader(lifecycleBytes),
|
||||||
|
MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
cfg, err := tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, *lifecycle, *cfg)
|
||||||
|
|
||||||
|
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ptr[T any](t T) *T {
|
||||||
|
return &t
|
||||||
|
}
|
|
@ -395,6 +395,51 @@ LOOP:
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *TreeServiceMock) PutBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
||||||
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
systemMap = make(map[string]*data.BaseNodeVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
systemMap["lifecycle"] = &data.BaseNodeVersion{
|
||||||
|
OID: addr.Object(),
|
||||||
|
}
|
||||||
|
|
||||||
|
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
||||||
|
|
||||||
|
return nil, ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *TreeServiceMock) GetBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
||||||
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return oid.Address{}, ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
node, ok := systemMap["lifecycle"]
|
||||||
|
if !ok {
|
||||||
|
return oid.Address{}, ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
return newAddress(bktInfo.CID, node.OID), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *TreeServiceMock) DeleteBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
||||||
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
node, ok := systemMap["lifecycle"]
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(systemMap, "lifecycle")
|
||||||
|
|
||||||
|
return []oid.Address{newAddress(bktInfo.CID, node.OID)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||||
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||||
|
|
||||||
|
|
|
@ -63,6 +63,10 @@ type TreeService interface {
|
||||||
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
||||||
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
|
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
|
||||||
|
|
||||||
|
PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error)
|
||||||
|
GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
|
||||||
|
DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
|
||||||
|
|
||||||
// Compound methods for optimizations
|
// Compound methods for optimizations
|
||||||
|
|
||||||
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
|
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
|
||||||
|
|
|
@ -356,7 +356,7 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router {
|
||||||
Handler(named(s3middleware.DeleteBucketPolicyOperation, h.DeleteBucketPolicyHandler))).
|
Handler(named(s3middleware.DeleteBucketPolicyOperation, h.DeleteBucketPolicyHandler))).
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.LifecycleQuery).
|
Queries(s3middleware.LifecycleQuery).
|
||||||
Handler(named(s3middleware.PutBucketLifecycleOperation, h.PutBucketLifecycleHandler))).
|
Handler(named(s3middleware.DeleteBucketLifecycleOperation, h.DeleteBucketLifecycleHandler))).
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.EncryptionQuery).
|
Queries(s3middleware.EncryptionQuery).
|
||||||
Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))).
|
Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))).
|
||||||
|
|
|
@ -183,17 +183,26 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var lifecycleCnrInfo *data.BucketInfo
|
||||||
|
if a.cfg.IsSet(cfgContainersLifecycle) {
|
||||||
|
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
|
||||||
|
if err != nil {
|
||||||
|
a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
layerCfg := &layer.Config{
|
layerCfg := &layer.Config{
|
||||||
Cache: layer.NewCache(getCacheOptions(a.cfg, a.log)),
|
Cache: layer.NewCache(getCacheOptions(a.cfg, a.log)),
|
||||||
AnonKey: layer.AnonymousKey{
|
AnonKey: layer.AnonymousKey{
|
||||||
Key: randomKey,
|
Key: randomKey,
|
||||||
},
|
},
|
||||||
GateOwner: gateOwner,
|
GateOwner: gateOwner,
|
||||||
Resolver: a.bucketResolver,
|
Resolver: a.bucketResolver,
|
||||||
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
||||||
Features: a.settings,
|
Features: a.settings,
|
||||||
GateKey: a.key,
|
GateKey: a.key,
|
||||||
CORSCnrInfo: corsCnrInfo,
|
CORSCnrInfo: corsCnrInfo,
|
||||||
|
LifecycleCnrInfo: lifecycleCnrInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepare object layer
|
// prepare object layer
|
||||||
|
|
|
@ -177,7 +177,8 @@ const ( // Settings.
|
||||||
cfgSourceIPHeader = "source_ip_header"
|
cfgSourceIPHeader = "source_ip_header"
|
||||||
|
|
||||||
// Containers.
|
// Containers.
|
||||||
cfgContainersCORS = "containers.cors"
|
cfgContainersCORS = "containers.cors"
|
||||||
|
cfgContainersLifecycle = "containers.lifecycle"
|
||||||
|
|
||||||
// Command line args.
|
// Command line args.
|
||||||
cmdHelp = "help"
|
cmdHelp = "help"
|
||||||
|
|
|
@ -218,3 +218,4 @@ S3_GW_RETRY_STRATEGY=exponential
|
||||||
|
|
||||||
# Containers properties
|
# Containers properties
|
||||||
S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
S3_GW_CONTAINERS_LIFECYCLE=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
|
|
@ -256,3 +256,4 @@ retry:
|
||||||
# Containers properties
|
# Containers properties
|
||||||
containers:
|
containers:
|
||||||
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
|
|
@ -715,9 +715,11 @@ Section for well-known containers to store s3-related data and settings.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
containers:
|
containers:
|
||||||
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
```
|
```
|
||||||
|
|
||||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|-----------|----------|---------------|---------------|--------------------------------------------------------------------------------------|
|
|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------|
|
||||||
| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. |
|
| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. |
|
||||||
|
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
|
||||||
|
|
2
go.mod
2
go.mod
|
@ -15,6 +15,7 @@ require (
|
||||||
github.com/go-chi/chi/v5 v5.0.8
|
github.com/go-chi/chi/v5 v5.0.8
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/minio/sio v0.3.0
|
github.com/minio/sio v0.3.0
|
||||||
|
github.com/mr-tron/base58 v1.2.0
|
||||||
github.com/nspcc-dev/neo-go v0.106.2
|
github.com/nspcc-dev/neo-go v0.106.2
|
||||||
github.com/panjf2000/ants/v2 v2.5.0
|
github.com/panjf2000/ants/v2 v2.5.0
|
||||||
github.com/prometheus/client_golang v1.19.0
|
github.com/prometheus/client_golang v1.19.0
|
||||||
|
@ -64,7 +65,6 @@ require (
|
||||||
github.com/magiconair/properties v1.8.7 // indirect
|
github.com/magiconair/properties v1.8.7 // indirect
|
||||||
github.com/mailru/easyjson v0.7.7 // indirect
|
github.com/mailru/easyjson v0.7.7 // indirect
|
||||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
|
||||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
|
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
|
||||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect
|
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect
|
||||||
github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
|
github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
|
||||||
|
|
|
@ -154,4 +154,9 @@ const (
|
||||||
FailedToParsePartInfo = "failed to parse part info"
|
FailedToParsePartInfo = "failed to parse part info"
|
||||||
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
|
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
|
||||||
CloseCredsObjectPayload = "close creds object payload"
|
CloseCredsObjectPayload = "close creds object payload"
|
||||||
|
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object"
|
||||||
|
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration"
|
||||||
|
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
|
||||||
|
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
|
||||||
|
GetBucketLifecycle = "get bucket lifecycle"
|
||||||
)
|
)
|
||||||
|
|
|
@ -114,9 +114,10 @@ const (
|
||||||
ownerKV = "Owner"
|
ownerKV = "Owner"
|
||||||
createdKV = "Created"
|
createdKV = "Created"
|
||||||
|
|
||||||
settingsFileName = "bucket-settings"
|
settingsFileName = "bucket-settings"
|
||||||
corsFilename = "bucket-cors"
|
corsFilename = "bucket-cors"
|
||||||
bucketTaggingFilename = "bucket-tagging"
|
bucketTaggingFilename = "bucket-tagging"
|
||||||
|
bucketLifecycleFilename = "bucket-lifecycle"
|
||||||
|
|
||||||
// versionTree -- ID of a tree with object versions.
|
// versionTree -- ID of a tree with object versions.
|
||||||
versionTree = "version"
|
versionTree = "version"
|
||||||
|
@ -560,7 +561,7 @@ func (c *Tree) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (
|
||||||
|
|
||||||
objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo)
|
objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo)
|
||||||
if len(objToDelete) != len(multiNode.nodes) {
|
if len(objToDelete) != len(multiNode.nodes) {
|
||||||
return nil, fmt.Errorf("clean old cors nodes: %w", err)
|
return nil, fmt.Errorf("failed to clean all old cors nodes")
|
||||||
}
|
}
|
||||||
|
|
||||||
return objToDelete, nil
|
return objToDelete, nil
|
||||||
|
@ -1447,6 +1448,74 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
||||||
|
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
|
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||||
|
if err != nil && !isErrNotFound {
|
||||||
|
return nil, fmt.Errorf("couldn't get node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
meta := make(map[string]string)
|
||||||
|
meta[FileNameKey] = bucketLifecycleFilename
|
||||||
|
meta[oidKV] = addr.Object().EncodeToString()
|
||||||
|
meta[cidKV] = addr.Container().EncodeToString()
|
||||||
|
|
||||||
|
if isErrNotFound {
|
||||||
|
if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, layer.ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
latest := multiNode.Latest()
|
||||||
|
ind := latest.GetLatestNodeIndex()
|
||||||
|
if latest.IsSplit() {
|
||||||
|
c.reqLogger(ctx).Error(logs.BucketLifecycleNodeHasMultipleIDs)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
||||||
|
return nil, fmt.Errorf("move lifecycle node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
objToDelete := make([]oid.Address, 1, len(multiNode.nodes))
|
||||||
|
objToDelete[0], err = getTreeNodeAddress(latest)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse object addr of latest lifecycle node in tree: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
objToDelete = append(objToDelete, c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)...)
|
||||||
|
|
||||||
|
return objToDelete, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Tree) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
||||||
|
node, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
|
if err != nil {
|
||||||
|
return oid.Address{}, fmt.Errorf("get lifecycle node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return getTreeNodeAddress(node.Latest())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
||||||
|
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
|
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||||
|
if err != nil && !isErrNotFound {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if isErrNotFound {
|
||||||
|
return nil, layer.ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo)
|
||||||
|
if len(objToDelete) != len(multiNode.nodes) {
|
||||||
|
return nil, fmt.Errorf("failed to clean all old lifecycle nodes")
|
||||||
|
}
|
||||||
|
|
||||||
|
return objToDelete, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||||
err := c.service.RemoveNode(ctx, bktInfo, systemTree, multipartInfo.ID)
|
err := c.service.RemoveNode(ctx, bktInfo, systemTree, multipartInfo.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
Loading…
Reference in a new issue