Compare commits
5 commits
master
...
feature/li
Author | SHA1 | Date | |
---|---|---|---|
924349c1c5 | |||
1e67e74a67 | |||
ab48d174c5 | |||
846ca63dc6 | |||
a594deaa37 |
35 changed files with 1451 additions and 155 deletions
20
api/cache/system.go
vendored
20
api/cache/system.go
vendored
|
@ -88,6 +88,22 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfiguration {
|
||||||
|
entry, err := o.cache.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, ok := entry.(*data.LifecycleConfiguration)
|
||||||
|
if !ok {
|
||||||
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
||||||
entry, err := o.cache.Get(key)
|
entry, err := o.cache.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -133,6 +149,10 @@ func (o *SystemCache) PutCORS(key string, obj *data.CORSConfiguration) error {
|
||||||
return o.cache.Set(key, obj)
|
return o.cache.Set(key, obj)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleConfiguration) error {
|
||||||
|
return o.cache.Set(key, obj)
|
||||||
|
}
|
||||||
|
|
||||||
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
|
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
|
||||||
return o.cache.Set(key, settings)
|
return o.cache.Set(key, settings)
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,8 +12,9 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
bktSettingsObject = ".s3-settings"
|
bktSettingsObject = ".s3-settings"
|
||||||
bktCORSConfigurationObject = ".s3-cors"
|
bktCORSConfigurationObject = ".s3-cors"
|
||||||
|
bktLifecycleConfigurationObject = ".s3-lifecycle"
|
||||||
|
|
||||||
VersioningUnversioned = "Unversioned"
|
VersioningUnversioned = "Unversioned"
|
||||||
VersioningEnabled = "Enabled"
|
VersioningEnabled = "Enabled"
|
||||||
|
@ -81,6 +82,15 @@ type (
|
||||||
VersionID string
|
VersionID string
|
||||||
NoErrorOnDeleteMarker bool
|
NoErrorOnDeleteMarker bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CreatedObjectInfo stores created object info.
|
||||||
|
CreatedObjectInfo struct {
|
||||||
|
ID oid.ID
|
||||||
|
Size uint64
|
||||||
|
HashSum []byte
|
||||||
|
MD5Sum []byte
|
||||||
|
CreationEpoch uint64
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// SettingsObjectName is a system name for a bucket settings file.
|
// SettingsObjectName is a system name for a bucket settings file.
|
||||||
|
@ -91,6 +101,10 @@ func (b *BucketInfo) CORSObjectName() string {
|
||||||
return b.CID.EncodeToString() + bktCORSConfigurationObject
|
return b.CID.EncodeToString() + bktCORSConfigurationObject
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *BucketInfo) LifecycleConfigurationObjectName() string {
|
||||||
|
return b.CID.EncodeToString() + bktLifecycleConfigurationObject
|
||||||
|
}
|
||||||
|
|
||||||
// VersionID returns object version from ObjectInfo.
|
// VersionID returns object version from ObjectInfo.
|
||||||
func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() }
|
func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() }
|
||||||
|
|
||||||
|
|
54
api/data/lifecycle.go
Normal file
54
api/data/lifecycle.go
Normal file
|
@ -0,0 +1,54 @@
|
||||||
|
package data
|
||||||
|
|
||||||
|
import "encoding/xml"
|
||||||
|
|
||||||
|
const (
|
||||||
|
LifecycleStatusEnabled = "Enabled"
|
||||||
|
LifecycleStatusDisabled = "Disabled"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
LifecycleConfiguration struct {
|
||||||
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LifecycleConfiguration" json:"-"`
|
||||||
|
Rules []LifecycleRule `xml:"Rule"`
|
||||||
|
}
|
||||||
|
|
||||||
|
LifecycleRule struct {
|
||||||
|
Status string `xml:"Status,omitempty"`
|
||||||
|
AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
|
||||||
|
Expiration *LifecycleExpiration `xml:"Expiration,omitempty"`
|
||||||
|
Filter *LifecycleRuleFilter `xml:"Filter,omitempty"`
|
||||||
|
ID string `xml:"ID,omitempty"`
|
||||||
|
NonCurrentVersionExpiration *NonCurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
AbortIncompleteMultipartUpload struct {
|
||||||
|
DaysAfterInitiation *int `xml:"DaysAfterInitiation,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
LifecycleExpiration struct {
|
||||||
|
Date string `xml:"Date,omitempty"`
|
||||||
|
Days *int `xml:"Days,omitempty"`
|
||||||
|
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
LifecycleRuleFilter struct {
|
||||||
|
And *LifecycleRuleAndOperator `xml:"And,omitempty"`
|
||||||
|
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
||||||
|
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
|
||||||
|
Prefix string `xml:"Prefix,omitempty"`
|
||||||
|
Tag *Tag `xml:"Tag,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
LifecycleRuleAndOperator struct {
|
||||||
|
ObjectSizeGreaterThan *uint64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
||||||
|
ObjectSizeLessThan *uint64 `xml:"ObjectSizeLessThan,omitempty"`
|
||||||
|
Prefix string `xml:"Prefix,omitempty"`
|
||||||
|
Tags []Tag `xml:"Tag"`
|
||||||
|
}
|
||||||
|
|
||||||
|
NonCurrentVersionExpiration struct {
|
||||||
|
NewerNonCurrentVersions *int `xml:"NewerNoncurrentVersions,omitempty"`
|
||||||
|
NonCurrentDays *int `xml:"NoncurrentDays,omitempty"`
|
||||||
|
}
|
||||||
|
)
|
|
@ -72,6 +72,7 @@ type BaseNodeVersion struct {
|
||||||
Created *time.Time
|
Created *time.Time
|
||||||
Owner *user.ID
|
Owner *user.ID
|
||||||
IsDeleteMarker bool
|
IsDeleteMarker bool
|
||||||
|
CreationEpoch uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *BaseNodeVersion) GetETag(md5Enabled bool) string {
|
func (v *BaseNodeVersion) GetETag(md5Enabled bool) string {
|
||||||
|
@ -110,6 +111,7 @@ type MultipartInfo struct {
|
||||||
Meta map[string]string
|
Meta map[string]string
|
||||||
CopiesNumbers []uint32
|
CopiesNumbers []uint32
|
||||||
Finished bool
|
Finished bool
|
||||||
|
CreationEpoch uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// PartInfo is upload information about part.
|
// PartInfo is upload information about part.
|
||||||
|
|
|
@ -81,10 +81,17 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
networkInfo, err := h.obj.GetNetworkInfo(ctx)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get network info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
p := &layer.DeleteObjectParams{
|
p := &layer.DeleteObjectParams{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
Objects: versionedObject,
|
Objects: versionedObject,
|
||||||
Settings: bktSettings,
|
Settings: bktSettings,
|
||||||
|
NetworkInfo: networkInfo,
|
||||||
}
|
}
|
||||||
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
||||||
deletedObject := deletedObjects[0]
|
deletedObject := deletedObjects[0]
|
||||||
|
@ -181,11 +188,18 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
networkInfo, err := h.obj.GetNetworkInfo(ctx)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get network info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
p := &layer.DeleteObjectParams{
|
p := &layer.DeleteObjectParams{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
Objects: toRemove,
|
Objects: toRemove,
|
||||||
Settings: bktSettings,
|
Settings: bktSettings,
|
||||||
IsMultiple: true,
|
NetworkInfo: networkInfo,
|
||||||
|
IsMultiple: true,
|
||||||
}
|
}
|
||||||
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
deletedObjects := h.obj.DeleteObjects(ctx, p)
|
||||||
|
|
||||||
|
|
|
@ -33,6 +33,7 @@ import (
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
"go.uber.org/zap/zaptest"
|
||||||
"golang.org/x/exp/slices"
|
"golang.org/x/exp/slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -141,18 +142,19 @@ func (c *configMock) Domains() []string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(zap.NewExample()))
|
log := zaptest.NewLogger(t)
|
||||||
|
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(log), log)
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
|
func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
|
||||||
return prepareHandlerContextBase(t, getMinCacheConfig(zap.NewExample()))
|
log := zaptest.NewLogger(t)
|
||||||
|
return prepareHandlerContextBase(t, getMinCacheConfig(log), log)
|
||||||
}
|
}
|
||||||
|
|
||||||
func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *handlerContext {
|
func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig, log *zap.Logger) *handlerContext {
|
||||||
key, err := keys.NewPrivateKey()
|
key, err := keys.NewPrivateKey()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
l := zap.NewExample()
|
|
||||||
tp := layer.NewTestFrostFS(key)
|
tp := layer.NewTestFrostFS(key)
|
||||||
|
|
||||||
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
testResolver := &resolver.Resolver{Name: "test_resolver"}
|
||||||
|
@ -166,7 +168,7 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
|
||||||
memCli, err := tree.NewTreeServiceClientMemory()
|
memCli, err := tree.NewTreeServiceClientMemory()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
treeMock := tree.NewTree(memCli, zap.NewExample())
|
treeMock := tree.NewTree(memCli, log)
|
||||||
|
|
||||||
features := &layer.FeatureSettingsMock{}
|
features := &layer.FeatureSettingsMock{}
|
||||||
|
|
||||||
|
@ -187,8 +189,8 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
|
||||||
defaultPolicy: pp,
|
defaultPolicy: pp,
|
||||||
}
|
}
|
||||||
h := &handler{
|
h := &handler{
|
||||||
log: l,
|
log: log,
|
||||||
obj: layer.NewLayer(l, tp, layerCfg),
|
obj: layer.NewLayer(log, tp, layerCfg),
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
ape: newAPEMock(),
|
ape: newAPEMock(),
|
||||||
frostfsid: newFrostfsIDMock(),
|
frostfsid: newFrostfsIDMock(),
|
||||||
|
|
235
api/handler/lifecycle.go
Normal file
235
api/handler/lifecycle.go
Normal file
|
@ -0,0 +1,235 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxRules = 1000
|
||||||
|
maxRuleIDLen = 255
|
||||||
|
maxNewerNoncurrentVersions = 100
|
||||||
|
)
|
||||||
|
|
||||||
|
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := h.obj.GetBucketLifecycleConfiguration(ctx, bktInfo)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket lifecycle configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = middleware.EncodeToResponse(w, cfg); err != nil {
|
||||||
|
h.logAndSendError(w, "could not encode GetBucketLifecycle response", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
tee := io.TeeReader(r.Body, &buf)
|
||||||
|
ctx := r.Context()
|
||||||
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
|
// Content-Md5 is required and should be set
|
||||||
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
|
||||||
|
if _, ok := r.Header[api.ContentMD5]; !ok {
|
||||||
|
h.logAndSendError(w, "missing Content-MD5", reqInfo, apiErr.GetAPIError(apiErr.ErrMissingContentMD5))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := new(data.LifecycleConfiguration)
|
||||||
|
if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil {
|
||||||
|
h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error()))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = checkLifecycleConfiguration(cfg); err != nil {
|
||||||
|
h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error()))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
params := &layer.PutBucketLifecycleParams{
|
||||||
|
BktInfo: bktInfo,
|
||||||
|
LifecycleCfg: cfg,
|
||||||
|
LifecycleReader: &buf,
|
||||||
|
MD5Hash: r.Header.Get(api.ContentMD5),
|
||||||
|
}
|
||||||
|
|
||||||
|
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "invalid copies number", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.PutBucketLifecycleConfiguration(ctx, params); err != nil {
|
||||||
|
h.logAndSendError(w, "could not put bucket lifecycle configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
reqInfo := middleware.GetReqInfo(ctx)
|
||||||
|
|
||||||
|
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
|
||||||
|
if err != nil {
|
||||||
|
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = h.obj.DeleteBucketLifecycleConfiguration(ctx, bktInfo); err != nil {
|
||||||
|
h.logAndSendError(w, "could not delete bucket lifecycle configuration", reqInfo, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
|
||||||
|
if len(cfg.Rules) > maxRules {
|
||||||
|
return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
|
||||||
|
}
|
||||||
|
|
||||||
|
ids := make(map[string]struct{}, len(cfg.Rules))
|
||||||
|
for _, rule := range cfg.Rules {
|
||||||
|
if _, ok := ids[rule.ID]; ok && rule.ID != "" {
|
||||||
|
return fmt.Errorf("duplicate 'ID': %s", rule.ID)
|
||||||
|
}
|
||||||
|
ids[rule.ID] = struct{}{}
|
||||||
|
|
||||||
|
if len(rule.ID) > maxRuleIDLen {
|
||||||
|
return fmt.Errorf("'ID' value cannot be longer than %d characters", maxRuleIDLen)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Status != data.LifecycleStatusEnabled && rule.Status != data.LifecycleStatusDisabled {
|
||||||
|
return fmt.Errorf("invalid lifecycle status: %s", rule.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.AbortIncompleteMultipartUpload == nil && rule.Expiration == nil && rule.NonCurrentVersionExpiration == nil {
|
||||||
|
return fmt.Errorf("at least one action needs to be specified in a rule")
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.AbortIncompleteMultipartUpload != nil {
|
||||||
|
if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != nil &&
|
||||||
|
*rule.AbortIncompleteMultipartUpload.DaysAfterInitiation <= 0 {
|
||||||
|
return fmt.Errorf("days after initiation must be a positive integer: %d", *rule.AbortIncompleteMultipartUpload.DaysAfterInitiation)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
|
||||||
|
return fmt.Errorf("abort incomplete multipart upload cannot be specified with tags")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Expiration != nil {
|
||||||
|
if rule.Expiration.ExpiredObjectDeleteMarker != nil {
|
||||||
|
if rule.Expiration.Days != nil || rule.Expiration.Date != "" {
|
||||||
|
return fmt.Errorf("expired object delete marker cannot be specified with days or date")
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Filter != nil && (rule.Filter.Tag != nil || (rule.Filter.And != nil && len(rule.Filter.And.Tags) > 0)) {
|
||||||
|
return fmt.Errorf("expired object delete marker cannot be specified with tags")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.Expiration.Days != nil && *rule.Expiration.Days <= 0 {
|
||||||
|
return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil {
|
||||||
|
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.NonCurrentVersionExpiration != nil {
|
||||||
|
if rule.NonCurrentVersionExpiration.NewerNonCurrentVersions != nil &&
|
||||||
|
(*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions > maxNewerNoncurrentVersions ||
|
||||||
|
*rule.NonCurrentVersionExpiration.NewerNonCurrentVersions <= 0) {
|
||||||
|
return fmt.Errorf("invalid value of newer noncurrent versions: %d", *rule.NonCurrentVersionExpiration.NewerNonCurrentVersions)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rule.NonCurrentVersionExpiration.NonCurrentDays != nil && *rule.NonCurrentVersionExpiration.NonCurrentDays <= 0 {
|
||||||
|
return fmt.Errorf("invalid value of noncurrent days: %d", *rule.NonCurrentVersionExpiration.NonCurrentDays)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := checkLifecycleRuleFilter(rule.Filter); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
|
||||||
|
if filter == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var fields int
|
||||||
|
|
||||||
|
if filter.And != nil {
|
||||||
|
fields++
|
||||||
|
for _, tag := range filter.And.Tags {
|
||||||
|
err := checkTag(tag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.And.ObjectSizeGreaterThan != nil && filter.And.ObjectSizeLessThan != nil &&
|
||||||
|
*filter.And.ObjectSizeLessThan <= *filter.And.ObjectSizeGreaterThan {
|
||||||
|
return fmt.Errorf("the maximum object size must be larger than the minimum object size")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.ObjectSizeGreaterThan != nil {
|
||||||
|
fields++
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.ObjectSizeLessThan != nil {
|
||||||
|
fields++
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.Prefix != "" {
|
||||||
|
fields++
|
||||||
|
}
|
||||||
|
|
||||||
|
if filter.Tag != nil {
|
||||||
|
fields++
|
||||||
|
err := checkTag(*filter.Tag)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields > 1 {
|
||||||
|
return fmt.Errorf("filter cannot have more than one field")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
457
api/handler/lifecycle_test.go
Normal file
457
api/handler/lifecycle_test.go
Normal file
|
@ -0,0 +1,457 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/md5"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"github.com/mr-tron/base58"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPutBucketLifecycleConfiguration(t *testing.T) {
|
||||||
|
hc := prepareHandlerContextWithMinCache(t)
|
||||||
|
|
||||||
|
bktName := "bucket-lifecycle"
|
||||||
|
createBucket(hc, bktName)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
body *data.LifecycleConfiguration
|
||||||
|
error bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "correct configuration",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
XMLName: xml.Name{
|
||||||
|
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
|
||||||
|
Local: "LifecycleConfiguration",
|
||||||
|
},
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
Date: time.Now().Format("2006-01-02T15:04:05.000Z"),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
And: &data.LifecycleRuleAndOperator{
|
||||||
|
Prefix: "prefix/",
|
||||||
|
Tags: []data.Tag{{Key: "key", Value: "value"}, {Key: "tag", Value: ""}},
|
||||||
|
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||||
|
DaysAfterInitiation: ptr(14),
|
||||||
|
},
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
ExpiredObjectDeleteMarker: ptr(true),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
ObjectSizeLessThan: ptr(uint64(100)),
|
||||||
|
},
|
||||||
|
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||||
|
NewerNonCurrentVersions: ptr(1),
|
||||||
|
NonCurrentDays: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "too many rules",
|
||||||
|
body: func() *data.LifecycleConfiguration {
|
||||||
|
lifecycle := new(data.LifecycleConfiguration)
|
||||||
|
for i := 0; i <= maxRules; i++ {
|
||||||
|
lifecycle.Rules = append(lifecycle.Rules, data.LifecycleRule{
|
||||||
|
ID: "Rule" + strconv.Itoa(i),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return lifecycle
|
||||||
|
}(),
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate rule ID",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
ID: "Rule",
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "Rule",
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "too long rule ID",
|
||||||
|
body: func() *data.LifecycleConfiguration {
|
||||||
|
id := make([]byte, maxRuleIDLen+1)
|
||||||
|
_, err := io.ReadFull(rand.Reader, id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
ID: base58.Encode(id)[:maxRuleIDLen+1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}(),
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid status",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: "invalid",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no actions",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
Prefix: "prefix/",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid days after initiation",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||||
|
DaysAfterInitiation: ptr(0),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid expired object delete marker declaration",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
ExpiredObjectDeleteMarker: ptr(false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid expiration days",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(0),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid expiration date",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Date: "invalid",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "newer noncurrent versions is too small",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||||
|
NewerNonCurrentVersions: ptr(0),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "newer noncurrent versions is too large",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||||
|
NewerNonCurrentVersions: ptr(101),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid noncurrent days",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
NonCurrentVersionExpiration: &data.NonCurrentVersionExpiration{
|
||||||
|
NonCurrentDays: ptr(0),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "more than one filter field",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
Prefix: "prefix/",
|
||||||
|
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid tag in filter",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
Tag: &data.Tag{},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "abort incomplete multipart upload with tag",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
AbortIncompleteMultipartUpload: &data.AbortIncompleteMultipartUpload{
|
||||||
|
DaysAfterInitiation: ptr(14),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
Tag: &data.Tag{Key: "key", Value: "value"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "expired object delete marker with tag",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
ExpiredObjectDeleteMarker: ptr(true),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
And: &data.LifecycleRuleAndOperator{
|
||||||
|
Tags: []data.Tag{{Key: "key", Value: "value"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid size range",
|
||||||
|
body: &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
Filter: &data.LifecycleRuleFilter{
|
||||||
|
And: &data.LifecycleRuleAndOperator{
|
||||||
|
ObjectSizeGreaterThan: ptr(uint64(100)),
|
||||||
|
ObjectSizeLessThan: ptr(uint64(100)),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
error: true,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if tc.error {
|
||||||
|
putBucketLifecycleConfigurationErr(hc, bktName, tc.body, apiErrors.GetAPIError(apiErrors.ErrMalformedXML))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
putBucketLifecycleConfiguration(hc, bktName, tc.body)
|
||||||
|
|
||||||
|
cfg := getBucketLifecycleConfiguration(hc, bktName)
|
||||||
|
require.Equal(t, *tc.body, *cfg)
|
||||||
|
|
||||||
|
deleteBucketLifecycleConfiguration(hc, bktName)
|
||||||
|
getBucketLifecycleConfigurationErr(hc, bktName, apiErrors.GetAPIError(apiErrors.ErrNoSuchLifecycleConfiguration))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-lifecycle-md5"
|
||||||
|
createBucket(hc, bktName)
|
||||||
|
|
||||||
|
lifecycle := &data.LifecycleConfiguration{
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMissingContentMD5))
|
||||||
|
|
||||||
|
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
r.Header.Set(api.ContentMD5, "")
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest))
|
||||||
|
|
||||||
|
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
|
||||||
|
r.Header.Set(api.ContentMD5, "some-hash")
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPutBucketLifecycleInvalidXML(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-lifecycle-invalid-xml"
|
||||||
|
createBucket(hc, bktName)
|
||||||
|
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{})
|
||||||
|
r.Header.Set(api.ContentMD5, "")
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
assertS3Error(hc.t, w, apiErrors.GetAPIError(apiErrors.ErrMalformedXML))
|
||||||
|
}
|
||||||
|
|
||||||
|
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) {
|
||||||
|
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||||
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
|
}
|
||||||
|
|
||||||
|
func putBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration, err apiErrors.Error) {
|
||||||
|
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
|
||||||
|
assertS3Error(hc.t, w, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func putBucketLifecycleConfigurationBase(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) *httptest.ResponseRecorder {
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", cfg)
|
||||||
|
|
||||||
|
rawBody, err := xml.Marshal(cfg)
|
||||||
|
require.NoError(hc.t, err)
|
||||||
|
|
||||||
|
hash := md5.New()
|
||||||
|
hash.Write(rawBody)
|
||||||
|
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(hash.Sum(nil)))
|
||||||
|
hc.Handler().PutBucketLifecycleHandler(w, r)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func getBucketLifecycleConfiguration(hc *handlerContext, bktName string) *data.LifecycleConfiguration {
|
||||||
|
w := getBucketLifecycleConfigurationBase(hc, bktName)
|
||||||
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
|
res := &data.LifecycleConfiguration{}
|
||||||
|
parseTestResponse(hc.t, w, res)
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func getBucketLifecycleConfigurationErr(hc *handlerContext, bktName string, err apiErrors.Error) {
|
||||||
|
w := getBucketLifecycleConfigurationBase(hc, bktName)
|
||||||
|
assertS3Error(hc.t, w, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
|
hc.Handler().GetBucketLifecycleHandler(w, r)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteBucketLifecycleConfiguration(hc *handlerContext, bktName string) {
|
||||||
|
w := deleteBucketLifecycleConfigurationBase(hc, bktName)
|
||||||
|
assertStatus(hc.t, w, http.StatusNoContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteBucketLifecycleConfigurationBase(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
|
||||||
|
w, r := prepareTestRequest(hc, bktName, "", nil)
|
||||||
|
hc.Handler().DeleteBucketLifecycleHandler(w, r)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func ptr[T any](t T) *T {
|
||||||
|
return &t
|
||||||
|
}
|
|
@ -7,10 +7,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
h.logAndSendError(w, "not supported", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotSupported))
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -15,8 +16,11 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.uber.org/zap"
|
||||||
"go.uber.org/zap/zaptest"
|
"go.uber.org/zap/zaptest"
|
||||||
|
"go.uber.org/zap/zaptest/observer"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseContinuationToken(t *testing.T) {
|
func TestParseContinuationToken(t *testing.T) {
|
||||||
|
@ -93,6 +97,27 @@ func TestListObjectsWithOldTreeNodes(t *testing.T) {
|
||||||
checkListVersionsOldNodes(hc, listVers.Version, objInfos)
|
checkListVersionsOldNodes(hc, listVers.Version, objInfos)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestListObjectsVersionsSkipLogTaggingNodesError(t *testing.T) {
|
||||||
|
loggerCore, observedLog := observer.New(zap.DebugLevel)
|
||||||
|
log := zap.New(loggerCore)
|
||||||
|
hc := prepareHandlerContextBase(t, layer.DefaultCachesConfigs(log), log)
|
||||||
|
|
||||||
|
bktName, objName := "bucket-versioning-enabled", "versions/object"
|
||||||
|
bktInfo := createTestBucket(hc, bktName)
|
||||||
|
|
||||||
|
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
||||||
|
createTestObject(hc, bktInfo, objName, encryption.Params{})
|
||||||
|
|
||||||
|
putObjectTagging(hc.t, hc, bktName, objName, map[string]string{"tag1": "val1"})
|
||||||
|
|
||||||
|
listObjectsVersions(hc, bktName, "", "", "", "", -1)
|
||||||
|
|
||||||
|
filtered := observedLog.Filter(func(entry observer.LoggedEntry) bool {
|
||||||
|
return strings.Contains(entry.Message, logs.ParseTreeNode)
|
||||||
|
})
|
||||||
|
require.Empty(t, filtered)
|
||||||
|
}
|
||||||
|
|
||||||
func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
|
func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
|
||||||
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", []uint64{0}, 0)
|
nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", []uint64{0}, 0)
|
||||||
require.NoError(hc.t, err)
|
require.NoError(hc.t, err)
|
||||||
|
@ -138,11 +163,12 @@ func checkListVersionsOldNodes(hc *handlerContext, list []ObjectVersionResponse,
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestListObjectsContextCanceled(t *testing.T) {
|
func TestListObjectsContextCanceled(t *testing.T) {
|
||||||
layerCfg := layer.DefaultCachesConfigs(zaptest.NewLogger(t))
|
log := zaptest.NewLogger(t)
|
||||||
|
layerCfg := layer.DefaultCachesConfigs(log)
|
||||||
layerCfg.SessionList.Lifetime = time.Hour
|
layerCfg.SessionList.Lifetime = time.Hour
|
||||||
layerCfg.SessionList.Size = 1
|
layerCfg.SessionList.Size = 1
|
||||||
|
|
||||||
hc := prepareHandlerContextBase(t, layerCfg)
|
hc := prepareHandlerContextBase(t, layerCfg, log)
|
||||||
|
|
||||||
bktName := "bucket-versioning-enabled"
|
bktName := "bucket-versioning-enabled"
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
bktInfo := createTestBucket(hc, bktName)
|
||||||
|
|
|
@ -11,10 +11,6 @@ func (h *handler) SelectObjectContentHandler(w http.ResponseWriter, r *http.Requ
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
@ -51,10 +47,6 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request)
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
h.logAndSendError(w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
|
||||||
}
|
}
|
||||||
|
|
|
@ -257,3 +257,29 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
||||||
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
|
func (c *Cache) DeleteCORS(bktInfo *data.BucketInfo) {
|
||||||
c.systemCache.Delete(bktInfo.CORSObjectName())
|
c.systemCache.Delete(bktInfo.CORSObjectName())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Cache) GetLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo) *data.LifecycleConfiguration {
|
||||||
|
key := bkt.LifecycleConfigurationObjectName()
|
||||||
|
|
||||||
|
if !c.accessCache.Get(owner, key) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.systemCache.GetLifecycleConfiguration(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.LifecycleConfiguration) {
|
||||||
|
key := bkt.LifecycleConfigurationObjectName()
|
||||||
|
|
||||||
|
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
|
||||||
|
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) {
|
||||||
|
c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName())
|
||||||
|
}
|
||||||
|
|
|
@ -55,12 +55,12 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
|
|
||||||
prm.Container = corsBkt.CID
|
prm.Container = corsBkt.CID
|
||||||
|
|
||||||
_, objID, _, _, err := n.objectPutAndHash(ctx, prm, corsBkt)
|
createdObj, err := n.objectPutAndHash(ctx, prm, corsBkt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("put cors object: %w", err)
|
return fmt.Errorf("put cors object: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
objsToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, newAddress(corsBkt.CID, objID))
|
objsToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, newAddress(corsBkt.CID, createdObj.ID))
|
||||||
objToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
objToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
|
||||||
if err != nil && !objToDeleteNotFound {
|
if err != nil && !objToDeleteNotFound {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -166,6 +166,12 @@ type PrmObjectCreate struct {
|
||||||
BufferMaxSize uint64
|
BufferMaxSize uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CreateObjectResult is a result parameter of FrostFS.CreateObject operation.
|
||||||
|
type CreateObjectResult struct {
|
||||||
|
ObjectID oid.ID
|
||||||
|
CreationEpoch uint64
|
||||||
|
}
|
||||||
|
|
||||||
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||||
type PrmObjectDelete struct {
|
type PrmObjectDelete struct {
|
||||||
// Authentication parameters.
|
// Authentication parameters.
|
||||||
|
@ -261,15 +267,15 @@ type FrostFS interface {
|
||||||
|
|
||||||
// CreateObject creates and saves a parameterized object in the FrostFS container.
|
// CreateObject creates and saves a parameterized object in the FrostFS container.
|
||||||
// It sets 'Timestamp' attribute to the current time.
|
// It sets 'Timestamp' attribute to the current time.
|
||||||
// It returns the ID of the saved object.
|
// It returns the ID and creation epoch of the saved object.
|
||||||
//
|
//
|
||||||
// Creation time should be written into the object (UTC).
|
// Creation time should be written into the object (UTC).
|
||||||
//
|
//
|
||||||
// It returns ErrAccessDenied on write access violation.
|
// It returns ErrAccessDenied on write access violation.
|
||||||
//
|
//
|
||||||
// It returns exactly one non-zero value. It returns any error encountered which
|
// It returns exactly one non-nil value. It returns any error encountered which
|
||||||
// prevented the container from being created.
|
// prevented the object from being created.
|
||||||
CreateObject(context.Context, PrmObjectCreate) (oid.ID, error)
|
CreateObject(context.Context, PrmObjectCreate) (*CreateObjectResult, error)
|
||||||
|
|
||||||
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
|
// DeleteObject marks the object to be removed from the FrostFS container by identifier.
|
||||||
// Successful return does not guarantee actual removal.
|
// Successful return does not guarantee actual removal.
|
||||||
|
@ -295,4 +301,7 @@ type FrostFS interface {
|
||||||
//
|
//
|
||||||
// It returns any error encountered which prevented computing epochs.
|
// It returns any error encountered which prevented computing epochs.
|
||||||
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
|
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
|
||||||
|
|
||||||
|
// NetworkInfo returns parameters of FrostFS network.
|
||||||
|
NetworkInfo(context.Context) (netmap.NetworkInfo, error)
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,6 +18,7 @@ import (
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
|
@ -255,10 +256,10 @@ func (t *TestFrostFS) RangeObject(ctx context.Context, prm PrmObjectRange) (io.R
|
||||||
return io.NopCloser(bytes.NewReader(payload)), nil
|
return io.NopCloser(bytes.NewReader(payload)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.ID, error) {
|
func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (*CreateObjectResult, error) {
|
||||||
b := make([]byte, 32)
|
b := make([]byte, 32)
|
||||||
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
if _, err := io.ReadFull(rand.Reader, b); err != nil {
|
||||||
return oid.ID{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
var id oid.ID
|
var id oid.ID
|
||||||
id.SetSHA256(sha256.Sum256(b))
|
id.SetSHA256(sha256.Sum256(b))
|
||||||
|
@ -266,7 +267,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
|
||||||
attrs := make([]object.Attribute, 0)
|
attrs := make([]object.Attribute, 0)
|
||||||
|
|
||||||
if err := t.objectPutErrors[prm.Filepath]; err != nil {
|
if err := t.objectPutErrors[prm.Filepath]; err != nil {
|
||||||
return oid.ID{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if prm.Filepath != "" {
|
if prm.Filepath != "" {
|
||||||
|
@ -311,7 +312,7 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
|
||||||
if prm.Payload != nil {
|
if prm.Payload != nil {
|
||||||
all, err := io.ReadAll(prm.Payload)
|
all, err := io.ReadAll(prm.Payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return oid.ID{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
obj.SetPayload(all)
|
obj.SetPayload(all)
|
||||||
obj.SetPayloadSize(uint64(len(all)))
|
obj.SetPayloadSize(uint64(len(all)))
|
||||||
|
@ -325,7 +326,10 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
|
||||||
|
|
||||||
addr := newAddress(cnrID, objID)
|
addr := newAddress(cnrID, objID)
|
||||||
t.objects[addr.EncodeToString()] = obj
|
t.objects[addr.EncodeToString()] = obj
|
||||||
return objID, nil
|
return &CreateObjectResult{
|
||||||
|
ObjectID: objID,
|
||||||
|
CreationEpoch: t.currentEpoch - 1,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
func (t *TestFrostFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error {
|
||||||
|
@ -404,6 +408,13 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) ([]o
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) {
|
||||||
|
ni := netmap.NetworkInfo{}
|
||||||
|
ni.SetCurrentEpoch(t.currentEpoch)
|
||||||
|
|
||||||
|
return ni, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID) bool {
|
func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID) bool {
|
||||||
cnr, ok := t.containers[cnrID.EncodeToString()]
|
cnr, ok := t.containers[cnrID.EncodeToString()]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|
|
@ -46,28 +46,30 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
Layer struct {
|
Layer struct {
|
||||||
frostFS FrostFS
|
frostFS FrostFS
|
||||||
gateOwner user.ID
|
gateOwner user.ID
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
anonKey AnonymousKey
|
anonKey AnonymousKey
|
||||||
resolver BucketResolver
|
resolver BucketResolver
|
||||||
cache *Cache
|
cache *Cache
|
||||||
treeService TreeService
|
treeService TreeService
|
||||||
features FeatureSettings
|
features FeatureSettings
|
||||||
gateKey *keys.PrivateKey
|
gateKey *keys.PrivateKey
|
||||||
corsCnrInfo *data.BucketInfo
|
corsCnrInfo *data.BucketInfo
|
||||||
|
lifecycleCnrInfo *data.BucketInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
Config struct {
|
Config struct {
|
||||||
GateOwner user.ID
|
GateOwner user.ID
|
||||||
ChainAddress string
|
ChainAddress string
|
||||||
Cache *Cache
|
Cache *Cache
|
||||||
AnonKey AnonymousKey
|
AnonKey AnonymousKey
|
||||||
Resolver BucketResolver
|
Resolver BucketResolver
|
||||||
TreeService TreeService
|
TreeService TreeService
|
||||||
Features FeatureSettings
|
Features FeatureSettings
|
||||||
GateKey *keys.PrivateKey
|
GateKey *keys.PrivateKey
|
||||||
CORSCnrInfo *data.BucketInfo
|
CORSCnrInfo *data.BucketInfo
|
||||||
|
LifecycleCnrInfo *data.BucketInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
// AnonymousKey contains data for anonymous requests.
|
// AnonymousKey contains data for anonymous requests.
|
||||||
|
@ -122,10 +124,11 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
DeleteObjectParams struct {
|
DeleteObjectParams struct {
|
||||||
BktInfo *data.BucketInfo
|
BktInfo *data.BucketInfo
|
||||||
Objects []*VersionedObject
|
Objects []*VersionedObject
|
||||||
Settings *data.BucketSettings
|
Settings *data.BucketSettings
|
||||||
IsMultiple bool
|
NetworkInfo netmap.NetworkInfo
|
||||||
|
IsMultiple bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// PutSettingsParams stores object copy request parameters.
|
// PutSettingsParams stores object copy request parameters.
|
||||||
|
@ -233,16 +236,17 @@ func (p HeadObjectParams) Versioned() bool {
|
||||||
// and establishes gRPC connection with the node.
|
// and establishes gRPC connection with the node.
|
||||||
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer {
|
func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) *Layer {
|
||||||
return &Layer{
|
return &Layer{
|
||||||
frostFS: frostFS,
|
frostFS: frostFS,
|
||||||
log: log,
|
log: log,
|
||||||
gateOwner: config.GateOwner,
|
gateOwner: config.GateOwner,
|
||||||
anonKey: config.AnonKey,
|
anonKey: config.AnonKey,
|
||||||
resolver: config.Resolver,
|
resolver: config.Resolver,
|
||||||
cache: config.Cache,
|
cache: config.Cache,
|
||||||
treeService: config.TreeService,
|
treeService: config.TreeService,
|
||||||
features: config.Features,
|
features: config.Features,
|
||||||
gateKey: config.GateKey,
|
gateKey: config.GateKey,
|
||||||
corsCnrInfo: config.CORSCnrInfo,
|
corsCnrInfo: config.CORSCnrInfo,
|
||||||
|
lifecycleCnrInfo: config.LifecycleCnrInfo,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -542,7 +546,8 @@ func getRandomOID() (oid.ID, error) {
|
||||||
return objID, nil
|
return objID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings *data.BucketSettings, obj *VersionedObject) *VersionedObject {
|
func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings *data.BucketSettings, obj *VersionedObject,
|
||||||
|
networkInfo netmap.NetworkInfo) *VersionedObject {
|
||||||
if len(obj.VersionID) != 0 || settings.Unversioned() {
|
if len(obj.VersionID) != 0 || settings.Unversioned() {
|
||||||
var nodeVersions []*data.NodeVersion
|
var nodeVersions []*data.NodeVersion
|
||||||
if nodeVersions, obj.Error = n.getNodeVersionsToDelete(ctx, bkt, obj); obj.Error != nil {
|
if nodeVersions, obj.Error = n.getNodeVersionsToDelete(ctx, bkt, obj); obj.Error != nil {
|
||||||
|
@ -624,6 +629,7 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
|
||||||
Created: &now,
|
Created: &now,
|
||||||
Owner: &n.gateOwner,
|
Owner: &n.gateOwner,
|
||||||
IsDeleteMarker: true,
|
IsDeleteMarker: true,
|
||||||
|
CreationEpoch: networkInfo.CurrentEpoch(),
|
||||||
},
|
},
|
||||||
IsUnversioned: settings.VersioningSuspended(),
|
IsUnversioned: settings.VersioningSuspended(),
|
||||||
}
|
}
|
||||||
|
@ -766,7 +772,7 @@ func (n *Layer) removeCombinedObject(ctx context.Context, bkt *data.BucketInfo,
|
||||||
// DeleteObjects from the storage.
|
// DeleteObjects from the storage.
|
||||||
func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject {
|
func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*VersionedObject {
|
||||||
for i, obj := range p.Objects {
|
for i, obj := range p.Objects {
|
||||||
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj)
|
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo)
|
||||||
if p.IsMultiple && p.Objects[i].Error != nil {
|
if p.IsMultiple && p.Objects[i].Error != nil {
|
||||||
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
|
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
|
||||||
}
|
}
|
||||||
|
@ -826,6 +832,11 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
|
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
|
||||||
|
if treeErr != nil {
|
||||||
|
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr))
|
||||||
|
}
|
||||||
|
|
||||||
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("delete container: %w", err)
|
return fmt.Errorf("delete container: %w", err)
|
||||||
|
@ -835,5 +846,18 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
|
||||||
n.deleteCORSObject(ctx, p.BktInfo, corsObj)
|
n.deleteCORSObject(ctx, p.BktInfo, corsObj)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if treeErr == nil && !lifecycleObj.Container().Equals(p.BktInfo.CID) {
|
||||||
|
n.deleteLifecycleObject(ctx, p.BktInfo, lifecycleObj)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (n *Layer) GetNetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
||||||
|
networkInfo, err := n.frostFS.NetworkInfo(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return networkInfo, fmt.Errorf("get network info: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return networkInfo, nil
|
||||||
|
}
|
||||||
|
|
148
api/layer/lifecycle.go
Normal file
148
api/layer/lifecycle.go
Normal file
|
@ -0,0 +1,148 @@
|
||||||
|
package layer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PutBucketLifecycleParams struct {
|
||||||
|
BktInfo *data.BucketInfo
|
||||||
|
LifecycleCfg *data.LifecycleConfiguration
|
||||||
|
LifecycleReader io.Reader
|
||||||
|
CopiesNumbers []uint32
|
||||||
|
MD5Hash string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
|
||||||
|
prm := PrmObjectCreate{
|
||||||
|
Payload: p.LifecycleReader,
|
||||||
|
Filepath: p.BktInfo.LifecycleConfigurationObjectName(),
|
||||||
|
CreationTime: TimeNow(ctx),
|
||||||
|
}
|
||||||
|
|
||||||
|
var lifecycleBkt *data.BucketInfo
|
||||||
|
if n.lifecycleCnrInfo == nil {
|
||||||
|
lifecycleBkt = p.BktInfo
|
||||||
|
prm.CopiesNumber = p.CopiesNumbers
|
||||||
|
} else {
|
||||||
|
lifecycleBkt = n.lifecycleCnrInfo
|
||||||
|
prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||||
|
}
|
||||||
|
|
||||||
|
prm.Container = lifecycleBkt.CID
|
||||||
|
|
||||||
|
createdObj, err := n.objectPutAndHash(ctx, prm, lifecycleBkt)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("put lifecycle object: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash)
|
||||||
|
if err != nil {
|
||||||
|
return apiErr.GetAPIError(apiErr.ErrInvalidDigest)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(hashBytes, createdObj.MD5Sum) {
|
||||||
|
n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
|
||||||
|
|
||||||
|
return apiErr.GetAPIError(apiErr.ErrInvalidDigest)
|
||||||
|
}
|
||||||
|
|
||||||
|
objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
|
||||||
|
objsToDeleteNotFound := errors.Is(err, ErrNoNodeToRemove)
|
||||||
|
if err != nil && !objsToDeleteNotFound {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !objsToDeleteNotFound {
|
||||||
|
for _, addr := range objsToDelete {
|
||||||
|
n.deleteLifecycleObject(ctx, p.BktInfo, addr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n.cache.PutLifecycleConfiguration(n.BearerOwner(ctx), p.BktInfo, p.LifecycleCfg)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deleteLifecycleObject removes object and logs in case of error.
|
||||||
|
func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
|
||||||
|
var prmAuth PrmAuth
|
||||||
|
lifecycleBkt := bktInfo
|
||||||
|
if !addr.Container().Equals(bktInfo.CID) {
|
||||||
|
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
|
||||||
|
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
|
||||||
|
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
|
||||||
|
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
|
||||||
|
zap.String("oid", addr.Object().EncodeToString()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (*data.LifecycleConfiguration, error) {
|
||||||
|
owner := n.BearerOwner(ctx)
|
||||||
|
if cfg := n.cache.GetLifecycleConfiguration(owner, bktInfo); cfg != nil {
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, err := n.treeService.GetBucketLifecycleConfiguration(ctx, bktInfo)
|
||||||
|
objNotFound := errors.Is(err, ErrNodeNotFound)
|
||||||
|
if err != nil && !objNotFound {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if objNotFound {
|
||||||
|
return nil, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
var prmAuth PrmAuth
|
||||||
|
lifecycleBkt := bktInfo
|
||||||
|
if !addr.Container().Equals(bktInfo.CID) {
|
||||||
|
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
|
||||||
|
prmAuth.PrivateKey = &n.gateKey.PrivateKey
|
||||||
|
}
|
||||||
|
|
||||||
|
obj, err := n.objectGetWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("get lifecycle object: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
lifecycleCfg := &data.LifecycleConfiguration{}
|
||||||
|
|
||||||
|
if err = xml.NewDecoder(obj.Payload).Decode(&lifecycleCfg); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal lifecycle configuration: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg)
|
||||||
|
|
||||||
|
return lifecycleCfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Layer) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||||
|
objs, err := n.treeService.DeleteBucketLifecycleConfiguration(ctx, bktInfo)
|
||||||
|
objsNotFound := errors.Is(err, ErrNoNodeToRemove)
|
||||||
|
if err != nil && !objsNotFound {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !objsNotFound {
|
||||||
|
for _, addr := range objs {
|
||||||
|
n.deleteLifecycleObject(ctx, bktInfo, addr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n.cache.DeleteLifecycleConfiguration(bktInfo)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
65
api/layer/lifecycle_test.go
Normal file
65
api/layer/lifecycle_test.go
Normal file
|
@ -0,0 +1,65 @@
|
||||||
|
package layer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
apiErr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
frostfsErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBucketLifecycle(t *testing.T) {
|
||||||
|
tc := prepareContext(t)
|
||||||
|
|
||||||
|
lifecycle := &data.LifecycleConfiguration{
|
||||||
|
XMLName: xml.Name{
|
||||||
|
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
|
||||||
|
Local: "LifecycleConfiguration",
|
||||||
|
},
|
||||||
|
Rules: []data.LifecycleRule{
|
||||||
|
{
|
||||||
|
Status: data.LifecycleStatusEnabled,
|
||||||
|
Expiration: &data.LifecycleExpiration{
|
||||||
|
Days: ptr(21),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
lifecycleBytes, err := xml.Marshal(lifecycle)
|
||||||
|
require.NoError(t, err)
|
||||||
|
hash := md5.New()
|
||||||
|
hash.Write(lifecycleBytes)
|
||||||
|
|
||||||
|
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err))
|
||||||
|
|
||||||
|
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{
|
||||||
|
BktInfo: tc.bktInfo,
|
||||||
|
LifecycleCfg: lifecycle,
|
||||||
|
LifecycleReader: bytes.NewReader(lifecycleBytes),
|
||||||
|
MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)),
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
cfg, err := tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, *lifecycle, *cfg)
|
||||||
|
|
||||||
|
err = tc.layer.DeleteBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo)
|
||||||
|
require.Equal(t, apiErr.GetAPIError(apiErr.ErrNoSuchLifecycleConfiguration), frostfsErrors.UnwrapErr(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ptr[T any](t T) *T {
|
||||||
|
return &t
|
||||||
|
}
|
|
@ -150,6 +150,11 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
||||||
metaSize += len(p.Data.TagSet)
|
metaSize += len(p.Data.TagSet)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
networkInfo, err := n.frostFS.NetworkInfo(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("get network info: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
info := &data.MultipartInfo{
|
info := &data.MultipartInfo{
|
||||||
Key: p.Info.Key,
|
Key: p.Info.Key,
|
||||||
UploadID: p.Info.UploadID,
|
UploadID: p.Info.UploadID,
|
||||||
|
@ -157,6 +162,7 @@ func (n *Layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
|
||||||
Created: TimeNow(ctx),
|
Created: TimeNow(ctx),
|
||||||
Meta: make(map[string]string, metaSize),
|
Meta: make(map[string]string, metaSize),
|
||||||
CopiesNumbers: p.CopiesNumbers,
|
CopiesNumbers: p.CopiesNumbers,
|
||||||
|
CreationEpoch: networkInfo.CurrentEpoch(),
|
||||||
}
|
}
|
||||||
|
|
||||||
for key, val := range p.Header {
|
for key, val := range p.Header {
|
||||||
|
@ -229,7 +235,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
prm.Attributes[0][0], prm.Attributes[0][1] = UploadIDAttributeName, p.Info.UploadID
|
prm.Attributes[0][0], prm.Attributes[0][1] = UploadIDAttributeName, p.Info.UploadID
|
||||||
prm.Attributes[1][0], prm.Attributes[1][1] = UploadPartNumberAttributeName, strconv.Itoa(p.PartNumber)
|
prm.Attributes[1][0], prm.Attributes[1][1] = UploadPartNumberAttributeName, strconv.Itoa(p.PartNumber)
|
||||||
|
|
||||||
size, id, hash, md5Hash, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
createdObj, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -238,21 +244,21 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
|
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
if hex.EncodeToString(hashBytes) != hex.EncodeToString(md5Hash) {
|
if hex.EncodeToString(hashBytes) != hex.EncodeToString(createdObj.MD5Sum) {
|
||||||
prm := PrmObjectDelete{
|
prm := PrmObjectDelete{
|
||||||
Object: id,
|
Object: createdObj.ID,
|
||||||
Container: bktInfo.CID,
|
Container: bktInfo.CID,
|
||||||
}
|
}
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
err = n.frostFS.DeleteObject(ctx, prm)
|
err = n.frostFS.DeleteObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
}
|
}
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
|
return nil, s3errors.GetAPIError(s3errors.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if p.Info.Encryption.Enabled() {
|
if p.Info.Encryption.Enabled() {
|
||||||
size = decSize
|
createdObj.Size = decSize
|
||||||
}
|
}
|
||||||
|
|
||||||
if !p.Info.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
|
if !p.Info.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
|
||||||
|
@ -260,10 +266,10 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
if !bytes.Equal(contentHashBytes, hash) {
|
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
||||||
err = n.objectDelete(ctx, bktInfo, id)
|
err = n.objectDelete(ctx, bktInfo, createdObj.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
}
|
}
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
|
@ -271,17 +277,17 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.UploadPart,
|
n.reqLogger(ctx).Debug(logs.UploadPart,
|
||||||
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
||||||
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
|
|
||||||
partInfo := &data.PartInfo{
|
partInfo := &data.PartInfo{
|
||||||
Key: p.Info.Key,
|
Key: p.Info.Key,
|
||||||
UploadID: p.Info.UploadID,
|
UploadID: p.Info.UploadID,
|
||||||
Number: p.PartNumber,
|
Number: p.PartNumber,
|
||||||
OID: id,
|
OID: createdObj.ID,
|
||||||
Size: size,
|
Size: createdObj.Size,
|
||||||
ETag: hex.EncodeToString(hash),
|
ETag: hex.EncodeToString(createdObj.HashSum),
|
||||||
Created: prm.CreationTime,
|
Created: prm.CreationTime,
|
||||||
MD5: hex.EncodeToString(md5Hash),
|
MD5: hex.EncodeToString(createdObj.MD5Sum),
|
||||||
}
|
}
|
||||||
|
|
||||||
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
|
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)
|
||||||
|
@ -298,7 +304,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
}
|
}
|
||||||
|
|
||||||
objInfo := &data.ObjectInfo{
|
objInfo := &data.ObjectInfo{
|
||||||
ID: id,
|
ID: createdObj.ID,
|
||||||
CID: bktInfo.CID,
|
CID: bktInfo.CID,
|
||||||
|
|
||||||
Owner: bktInfo.Owner,
|
Owner: bktInfo.Owner,
|
||||||
|
|
|
@ -271,7 +271,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
prm.Attributes = append(prm.Attributes, [2]string{k, v})
|
||||||
}
|
}
|
||||||
|
|
||||||
size, id, hash, md5Hash, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
createdObj, err := n.objectPutAndHash(ctx, prm, p.BktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -280,10 +280,10 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
if !bytes.Equal(headerMd5Hash, md5Hash) {
|
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
|
||||||
err = n.objectDelete(ctx, p.BktInfo, id)
|
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
}
|
}
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidDigest)
|
||||||
}
|
}
|
||||||
|
@ -294,25 +294,26 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
if !bytes.Equal(contentHashBytes, hash) {
|
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
|
||||||
err = n.objectDelete(ctx, p.BktInfo, id)
|
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
}
|
}
|
||||||
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
|
||||||
now := TimeNow(ctx)
|
now := TimeNow(ctx)
|
||||||
newVersion := &data.NodeVersion{
|
newVersion := &data.NodeVersion{
|
||||||
BaseNodeVersion: data.BaseNodeVersion{
|
BaseNodeVersion: data.BaseNodeVersion{
|
||||||
OID: id,
|
OID: createdObj.ID,
|
||||||
ETag: hex.EncodeToString(hash),
|
ETag: hex.EncodeToString(createdObj.HashSum),
|
||||||
FilePath: p.Object,
|
FilePath: p.Object,
|
||||||
Size: p.Size,
|
Size: p.Size,
|
||||||
Created: &now,
|
Created: &now,
|
||||||
Owner: &n.gateOwner,
|
Owner: &n.gateOwner,
|
||||||
|
CreationEpoch: createdObj.CreationEpoch,
|
||||||
},
|
},
|
||||||
IsUnversioned: !bktSettings.VersioningEnabled(),
|
IsUnversioned: !bktSettings.VersioningEnabled(),
|
||||||
IsCombined: p.Header[MultipartObjectSize] != "",
|
IsCombined: p.Header[MultipartObjectSize] != "",
|
||||||
|
@ -320,7 +321,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
if len(p.CompleteMD5Hash) > 0 {
|
if len(p.CompleteMD5Hash) > 0 {
|
||||||
newVersion.MD5 = p.CompleteMD5Hash
|
newVersion.MD5 = p.CompleteMD5Hash
|
||||||
} else {
|
} else {
|
||||||
newVersion.MD5 = hex.EncodeToString(md5Hash)
|
newVersion.MD5 = hex.EncodeToString(createdObj.MD5Sum)
|
||||||
}
|
}
|
||||||
|
|
||||||
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
if newVersion.ID, err = n.treeService.AddVersion(ctx, p.BktInfo, newVersion); err != nil {
|
||||||
|
@ -332,7 +333,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
ObjVersion: &data.ObjectVersion{
|
ObjVersion: &data.ObjectVersion{
|
||||||
BktInfo: p.BktInfo,
|
BktInfo: p.BktInfo,
|
||||||
ObjectName: p.Object,
|
ObjectName: p.Object,
|
||||||
VersionID: id.EncodeToString(),
|
VersionID: createdObj.ID.EncodeToString(),
|
||||||
},
|
},
|
||||||
NewLock: p.Lock,
|
NewLock: p.Lock,
|
||||||
CopiesNumbers: p.CopiesNumbers,
|
CopiesNumbers: p.CopiesNumbers,
|
||||||
|
@ -347,13 +348,13 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
n.cache.CleanListCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
n.cache.CleanListCacheEntriesContainingObject(p.Object, p.BktInfo.CID)
|
||||||
|
|
||||||
objInfo := &data.ObjectInfo{
|
objInfo := &data.ObjectInfo{
|
||||||
ID: id,
|
ID: createdObj.ID,
|
||||||
CID: p.BktInfo.CID,
|
CID: p.BktInfo.CID,
|
||||||
|
|
||||||
Owner: n.gateOwner,
|
Owner: n.gateOwner,
|
||||||
Bucket: p.BktInfo.Name,
|
Bucket: p.BktInfo.Name,
|
||||||
Name: p.Object,
|
Name: p.Object,
|
||||||
Size: size,
|
Size: createdObj.Size,
|
||||||
Created: prm.CreationTime,
|
Created: prm.CreationTime,
|
||||||
Headers: p.Header,
|
Headers: p.Header,
|
||||||
ContentType: p.Header[api.ContentType],
|
ContentType: p.Header[api.ContentType],
|
||||||
|
@ -491,8 +492,7 @@ func (n *Layer) objectDeleteBase(ctx context.Context, bktInfo *data.BucketInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
|
// objectPutAndHash prepare auth parameters and invoke frostfs.CreateObject.
|
||||||
// Returns object ID and payload sha256 hash.
|
func (n *Layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (*data.CreatedObjectInfo, error) {
|
||||||
func (n *Layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, []byte, error) {
|
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
prm.ClientCut = n.features.ClientCut()
|
prm.ClientCut = n.features.ClientCut()
|
||||||
prm.BufferMaxSize = n.features.BufferMaxSizeForPut()
|
prm.BufferMaxSize = n.features.BufferMaxSizeForPut()
|
||||||
|
@ -505,15 +505,21 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
|
||||||
hash.Write(buf)
|
hash.Write(buf)
|
||||||
md5Hash.Write(buf)
|
md5Hash.Write(buf)
|
||||||
})
|
})
|
||||||
id, err := n.frostFS.CreateObject(ctx, prm)
|
res, err := n.frostFS.CreateObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
||||||
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0, oid.ID{}, nil, nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return size, id, hash.Sum(nil), md5Hash.Sum(nil), nil
|
return &data.CreatedObjectInfo{
|
||||||
|
ID: res.ObjectID,
|
||||||
|
Size: size,
|
||||||
|
HashSum: hash.Sum(nil),
|
||||||
|
MD5Sum: md5Hash.Sum(nil),
|
||||||
|
CreationEpoch: res.CreationEpoch,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type logWrapper struct {
|
type logWrapper struct {
|
||||||
|
|
|
@ -44,7 +44,7 @@ func TestGoroutinesDontLeakInPutAndHash(t *testing.T) {
|
||||||
|
|
||||||
expErr := errors.New("some error")
|
expErr := errors.New("some error")
|
||||||
tc.testFrostFS.SetObjectPutError(tc.obj, expErr)
|
tc.testFrostFS.SetObjectPutError(tc.obj, expErr)
|
||||||
_, _, _, _, err = tc.layer.objectPutAndHash(tc.ctx, prm, tc.bktInfo)
|
_, err = tc.layer.objectPutAndHash(tc.ctx, prm, tc.bktInfo)
|
||||||
require.ErrorIs(t, err, expErr)
|
require.ErrorIs(t, err, expErr)
|
||||||
require.Empty(t, payload.Len(), "body must be read out otherwise goroutines can leak in wrapReader")
|
require.Empty(t, payload.Len(), "body must be read out otherwise goroutines can leak in wrapReader")
|
||||||
}
|
}
|
||||||
|
|
|
@ -126,8 +126,12 @@ func (n *Layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
|
||||||
return oid.ID{}, err
|
return oid.ID{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, id, _, _, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
createdObj, err := n.objectPutAndHash(ctx, prm, bktInfo)
|
||||||
return id, err
|
if err != nil {
|
||||||
|
return oid.ID{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return createdObj.ID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) {
|
func (n *Layer) GetLockInfo(ctx context.Context, objVersion *data.ObjectVersion) (*data.LockInfo, error) {
|
||||||
|
|
|
@ -395,6 +395,51 @@ LOOP:
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *TreeServiceMock) PutBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
||||||
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
systemMap = make(map[string]*data.BaseNodeVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
systemMap["lifecycle"] = &data.BaseNodeVersion{
|
||||||
|
OID: addr.Object(),
|
||||||
|
}
|
||||||
|
|
||||||
|
t.system[bktInfo.CID.EncodeToString()] = systemMap
|
||||||
|
|
||||||
|
return nil, ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *TreeServiceMock) GetBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
||||||
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return oid.Address{}, ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
node, ok := systemMap["lifecycle"]
|
||||||
|
if !ok {
|
||||||
|
return oid.Address{}, ErrNodeNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
return newAddress(bktInfo.CID, node.OID), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *TreeServiceMock) DeleteBucketLifecycleConfiguration(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
||||||
|
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
node, ok := systemMap["lifecycle"]
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(systemMap, "lifecycle")
|
||||||
|
|
||||||
|
return []oid.Address{newAddress(bktInfo.CID, node.OID)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
func (t *TreeServiceMock) DeleteMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||||
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
cnrMultipartsMap := t.multiparts[bktInfo.CID.EncodeToString()]
|
||||||
|
|
||||||
|
|
|
@ -63,6 +63,10 @@ type TreeService interface {
|
||||||
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error)
|
||||||
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
|
GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error)
|
||||||
|
|
||||||
|
PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error)
|
||||||
|
GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
|
||||||
|
DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
|
||||||
|
|
||||||
// Compound methods for optimizations
|
// Compound methods for optimizations
|
||||||
|
|
||||||
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
|
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
|
||||||
|
|
|
@ -356,7 +356,7 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router {
|
||||||
Handler(named(s3middleware.DeleteBucketPolicyOperation, h.DeleteBucketPolicyHandler))).
|
Handler(named(s3middleware.DeleteBucketPolicyOperation, h.DeleteBucketPolicyHandler))).
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.LifecycleQuery).
|
Queries(s3middleware.LifecycleQuery).
|
||||||
Handler(named(s3middleware.PutBucketLifecycleOperation, h.PutBucketLifecycleHandler))).
|
Handler(named(s3middleware.DeleteBucketLifecycleOperation, h.DeleteBucketLifecycleHandler))).
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.EncryptionQuery).
|
Queries(s3middleware.EncryptionQuery).
|
||||||
Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))).
|
Handler(named(s3middleware.DeleteBucketEncryptionOperation, h.DeleteBucketEncryptionHandler))).
|
||||||
|
|
|
@ -183,17 +183,26 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var lifecycleCnrInfo *data.BucketInfo
|
||||||
|
if a.cfg.IsSet(cfgContainersLifecycle) {
|
||||||
|
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
|
||||||
|
if err != nil {
|
||||||
|
a.log.Fatal(logs.CouldNotFetchLifecycleContainerInfo, zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
layerCfg := &layer.Config{
|
layerCfg := &layer.Config{
|
||||||
Cache: layer.NewCache(getCacheOptions(a.cfg, a.log)),
|
Cache: layer.NewCache(getCacheOptions(a.cfg, a.log)),
|
||||||
AnonKey: layer.AnonymousKey{
|
AnonKey: layer.AnonymousKey{
|
||||||
Key: randomKey,
|
Key: randomKey,
|
||||||
},
|
},
|
||||||
GateOwner: gateOwner,
|
GateOwner: gateOwner,
|
||||||
Resolver: a.bucketResolver,
|
Resolver: a.bucketResolver,
|
||||||
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
||||||
Features: a.settings,
|
Features: a.settings,
|
||||||
GateKey: a.key,
|
GateKey: a.key,
|
||||||
CORSCnrInfo: corsCnrInfo,
|
CORSCnrInfo: corsCnrInfo,
|
||||||
|
LifecycleCnrInfo: lifecycleCnrInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepare object layer
|
// prepare object layer
|
||||||
|
|
|
@ -177,7 +177,8 @@ const ( // Settings.
|
||||||
cfgSourceIPHeader = "source_ip_header"
|
cfgSourceIPHeader = "source_ip_header"
|
||||||
|
|
||||||
// Containers.
|
// Containers.
|
||||||
cfgContainersCORS = "containers.cors"
|
cfgContainersCORS = "containers.cors"
|
||||||
|
cfgContainersLifecycle = "containers.lifecycle"
|
||||||
|
|
||||||
// Command line args.
|
// Command line args.
|
||||||
cmdHelp = "help"
|
cmdHelp = "help"
|
||||||
|
|
|
@ -218,3 +218,4 @@ S3_GW_RETRY_STRATEGY=exponential
|
||||||
|
|
||||||
# Containers properties
|
# Containers properties
|
||||||
S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
S3_GW_CONTAINERS_LIFECYCLE=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
|
|
@ -256,3 +256,4 @@ retry:
|
||||||
# Containers properties
|
# Containers properties
|
||||||
containers:
|
containers:
|
||||||
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
|
|
@ -715,9 +715,11 @@ Section for well-known containers to store s3-related data and settings.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
containers:
|
containers:
|
||||||
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
|
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
|
||||||
```
|
```
|
||||||
|
|
||||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|-----------|----------|---------------|---------------|--------------------------------------------------------------------------------------|
|
|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------|
|
||||||
| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. |
|
| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. |
|
||||||
|
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
|
||||||
|
|
2
go.mod
2
go.mod
|
@ -15,6 +15,7 @@ require (
|
||||||
github.com/go-chi/chi/v5 v5.0.8
|
github.com/go-chi/chi/v5 v5.0.8
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/minio/sio v0.3.0
|
github.com/minio/sio v0.3.0
|
||||||
|
github.com/mr-tron/base58 v1.2.0
|
||||||
github.com/nspcc-dev/neo-go v0.106.2
|
github.com/nspcc-dev/neo-go v0.106.2
|
||||||
github.com/panjf2000/ants/v2 v2.5.0
|
github.com/panjf2000/ants/v2 v2.5.0
|
||||||
github.com/prometheus/client_golang v1.19.0
|
github.com/prometheus/client_golang v1.19.0
|
||||||
|
@ -64,7 +65,6 @@ require (
|
||||||
github.com/magiconair/properties v1.8.7 // indirect
|
github.com/magiconair/properties v1.8.7 // indirect
|
||||||
github.com/mailru/easyjson v0.7.7 // indirect
|
github.com/mailru/easyjson v0.7.7 // indirect
|
||||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
|
||||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
|
github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect
|
||||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect
|
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect
|
||||||
github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
|
github.com/nspcc-dev/rfc6979 v0.2.1 // indirect
|
||||||
|
|
|
@ -137,12 +137,17 @@ func (x *AuthmateFrostFS) CreateObject(ctx context.Context, prm tokens.PrmObject
|
||||||
attributes = append(attributes, [2]string{attr.Key(), attr.Value()})
|
attributes = append(attributes, [2]string{attr.Key(), attr.Value()})
|
||||||
}
|
}
|
||||||
|
|
||||||
return x.frostFS.CreateObject(ctx, layer.PrmObjectCreate{
|
res, err := x.frostFS.CreateObject(ctx, layer.PrmObjectCreate{
|
||||||
Container: prm.Container,
|
Container: prm.Container,
|
||||||
Filepath: prm.Filepath,
|
Filepath: prm.Filepath,
|
||||||
Attributes: attributes,
|
Attributes: attributes,
|
||||||
Payload: bytes.NewReader(prm.Payload),
|
Payload: bytes.NewReader(prm.Payload),
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return oid.ID{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.ObjectID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *AuthmateFrostFS) getCredVersions(ctx context.Context, addr oid.Address) (*crdt.ObjectVersions, error) {
|
func (x *AuthmateFrostFS) getCredVersions(ctx context.Context, addr oid.Address) (*crdt.ObjectVersions, error) {
|
||||||
|
|
|
@ -15,6 +15,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
|
@ -51,7 +52,7 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TimeToEpoch implements frostfs.FrostFS interface method.
|
// TimeToEpoch implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
|
func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
|
||||||
dur := futureTime.Sub(now)
|
dur := futureTime.Sub(now)
|
||||||
if dur < 0 {
|
if dur < 0 {
|
||||||
|
@ -87,7 +88,7 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u
|
||||||
return curr, epoch, nil
|
return curr, epoch, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Container implements frostfs.FrostFS interface method.
|
// Container implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) Container(ctx context.Context, layerPrm layer.PrmContainer) (*container.Container, error) {
|
func (x *FrostFS) Container(ctx context.Context, layerPrm layer.PrmContainer) (*container.Container, error) {
|
||||||
prm := pool.PrmContainerGet{
|
prm := pool.PrmContainerGet{
|
||||||
ContainerID: layerPrm.ContainerID,
|
ContainerID: layerPrm.ContainerID,
|
||||||
|
@ -102,7 +103,7 @@ func (x *FrostFS) Container(ctx context.Context, layerPrm layer.PrmContainer) (*
|
||||||
return &res, nil
|
return &res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateContainer implements frostfs.FrostFS interface method.
|
// CreateContainer implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) CreateContainer(ctx context.Context, prm layer.PrmContainerCreate) (*layer.ContainerCreateResult, error) {
|
func (x *FrostFS) CreateContainer(ctx context.Context, prm layer.PrmContainerCreate) (*layer.ContainerCreateResult, error) {
|
||||||
var cnr container.Container
|
var cnr container.Container
|
||||||
cnr.Init()
|
cnr.Init()
|
||||||
|
@ -150,7 +151,7 @@ func (x *FrostFS) CreateContainer(ctx context.Context, prm layer.PrmContainerCre
|
||||||
}, handleObjectError("save container via connection pool", err)
|
}, handleObjectError("save container via connection pool", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// UserContainers implements frostfs.FrostFS interface method.
|
// UserContainers implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) UserContainers(ctx context.Context, layerPrm layer.PrmUserContainers) ([]cid.ID, error) {
|
func (x *FrostFS) UserContainers(ctx context.Context, layerPrm layer.PrmUserContainers) ([]cid.ID, error) {
|
||||||
prm := pool.PrmContainerList{
|
prm := pool.PrmContainerList{
|
||||||
OwnerID: layerPrm.UserID,
|
OwnerID: layerPrm.UserID,
|
||||||
|
@ -161,7 +162,7 @@ func (x *FrostFS) UserContainers(ctx context.Context, layerPrm layer.PrmUserCont
|
||||||
return r, handleObjectError("list user containers via connection pool", err)
|
return r, handleObjectError("list user containers via connection pool", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteContainer implements frostfs.FrostFS interface method.
|
// DeleteContainer implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) DeleteContainer(ctx context.Context, id cid.ID, token *session.Container) error {
|
func (x *FrostFS) DeleteContainer(ctx context.Context, id cid.ID, token *session.Container) error {
|
||||||
prm := pool.PrmContainerDelete{ContainerID: id, Session: token, WaitParams: &x.await}
|
prm := pool.PrmContainerDelete{ContainerID: id, Session: token, WaitParams: &x.await}
|
||||||
|
|
||||||
|
@ -169,8 +170,8 @@ func (x *FrostFS) DeleteContainer(ctx context.Context, id cid.ID, token *session
|
||||||
return handleObjectError("delete container via connection pool", err)
|
return handleObjectError("delete container via connection pool", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateObject implements frostfs.FrostFS interface method.
|
// CreateObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (oid.ID, error) {
|
func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (*layer.CreateObjectResult, error) {
|
||||||
attrNum := len(prm.Attributes) + 1 // + creation time
|
attrNum := len(prm.Attributes) + 1 // + creation time
|
||||||
|
|
||||||
if prm.Filepath != "" {
|
if prm.Filepath != "" {
|
||||||
|
@ -239,10 +240,13 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
|
||||||
|
|
||||||
res, err := x.pool.PutObject(ctx, prmPut)
|
res, err := x.pool.PutObject(ctx, prmPut)
|
||||||
if err = handleObjectError("save object via connection pool", err); err != nil {
|
if err = handleObjectError("save object via connection pool", err); err != nil {
|
||||||
return oid.ID{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return res.ObjectID, nil
|
return &layer.CreateObjectResult{
|
||||||
|
ObjectID: res.ObjectID,
|
||||||
|
CreationEpoch: res.Epoch,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// wraps io.ReadCloser and transforms Read errors related to access violation
|
// wraps io.ReadCloser and transforms Read errors related to access violation
|
||||||
|
@ -259,7 +263,7 @@ func (x payloadReader) Read(p []byte) (int, error) {
|
||||||
return n, handleObjectError("read payload", err)
|
return n, handleObjectError("read payload", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// HeadObject implements frostfs.FrostFS interface method.
|
// HeadObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) HeadObject(ctx context.Context, prm layer.PrmObjectHead) (*object.Object, error) {
|
func (x *FrostFS) HeadObject(ctx context.Context, prm layer.PrmObjectHead) (*object.Object, error) {
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
|
@ -282,7 +286,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm layer.PrmObjectHead) (*obj
|
||||||
return &res, nil
|
return &res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetObject implements frostfs.FrostFS interface method.
|
// GetObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) GetObject(ctx context.Context, prm layer.PrmObjectGet) (*layer.Object, error) {
|
func (x *FrostFS) GetObject(ctx context.Context, prm layer.PrmObjectGet) (*layer.Object, error) {
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
|
@ -308,7 +312,7 @@ func (x *FrostFS) GetObject(ctx context.Context, prm layer.PrmObjectGet) (*layer
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RangeObject implements frostfs.FrostFS interface method.
|
// RangeObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) RangeObject(ctx context.Context, prm layer.PrmObjectRange) (io.ReadCloser, error) {
|
func (x *FrostFS) RangeObject(ctx context.Context, prm layer.PrmObjectRange) (io.ReadCloser, error) {
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
|
@ -333,7 +337,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm layer.PrmObjectRange) (io
|
||||||
return payloadReader{&res}, nil
|
return payloadReader{&res}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteObject implements frostfs.FrostFS interface method.
|
// DeleteObject implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) DeleteObject(ctx context.Context, prm layer.PrmObjectDelete) error {
|
func (x *FrostFS) DeleteObject(ctx context.Context, prm layer.PrmObjectDelete) error {
|
||||||
var addr oid.Address
|
var addr oid.Address
|
||||||
addr.SetContainer(prm.Container)
|
addr.SetContainer(prm.Container)
|
||||||
|
@ -352,7 +356,7 @@ func (x *FrostFS) DeleteObject(ctx context.Context, prm layer.PrmObjectDelete) e
|
||||||
return handleObjectError("mark object removal via connection pool", err)
|
return handleObjectError("mark object removal via connection pool", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SearchObjects implements frostfs.FrostFS interface method.
|
// SearchObjects implements layer.FrostFS interface method.
|
||||||
func (x *FrostFS) SearchObjects(ctx context.Context, prm layer.PrmObjectSearch) ([]oid.ID, error) {
|
func (x *FrostFS) SearchObjects(ctx context.Context, prm layer.PrmObjectSearch) ([]oid.ID, error) {
|
||||||
filters := object.NewSearchFilters()
|
filters := object.NewSearchFilters()
|
||||||
filters.AddRootFilter()
|
filters.AddRootFilter()
|
||||||
|
@ -389,6 +393,16 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm layer.PrmObjectSearch)
|
||||||
return buf, handleObjectError("read object list", err)
|
return buf, handleObjectError("read object list", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NetworkInfo implements layer.FrostFS interface method.
|
||||||
|
func (x *FrostFS) NetworkInfo(ctx context.Context) (netmap.NetworkInfo, error) {
|
||||||
|
ni, err := x.pool.NetworkInfo(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return ni, handleObjectError("get network info via connection pool", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ni, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ResolverFrostFS represents virtual connection to the FrostFS network.
|
// ResolverFrostFS represents virtual connection to the FrostFS network.
|
||||||
// It implements resolver.FrostFS.
|
// It implements resolver.FrostFS.
|
||||||
type ResolverFrostFS struct {
|
type ResolverFrostFS struct {
|
||||||
|
|
|
@ -154,4 +154,9 @@ const (
|
||||||
FailedToParsePartInfo = "failed to parse part info"
|
FailedToParsePartInfo = "failed to parse part info"
|
||||||
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
|
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
|
||||||
CloseCredsObjectPayload = "close creds object payload"
|
CloseCredsObjectPayload = "close creds object payload"
|
||||||
|
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object"
|
||||||
|
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration"
|
||||||
|
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
|
||||||
|
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
|
||||||
|
GetBucketLifecycle = "get bucket lifecycle"
|
||||||
)
|
)
|
||||||
|
|
|
@ -82,6 +82,8 @@ var (
|
||||||
|
|
||||||
// ErrGatewayTimeout is returned from ServiceClient service in case of timeout error.
|
// ErrGatewayTimeout is returned from ServiceClient service in case of timeout error.
|
||||||
ErrGatewayTimeout = layer.ErrGatewayTimeout
|
ErrGatewayTimeout = layer.ErrGatewayTimeout
|
||||||
|
|
||||||
|
errNodeDoesntContainFileName = fmt.Errorf("node doesn't contain FileName")
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -101,6 +103,7 @@ const (
|
||||||
etagKV = "ETag"
|
etagKV = "ETag"
|
||||||
md5KV = "MD5"
|
md5KV = "MD5"
|
||||||
finishedKV = "Finished"
|
finishedKV = "Finished"
|
||||||
|
creationEpochKV = "CreationEpoch"
|
||||||
|
|
||||||
// keys for lock.
|
// keys for lock.
|
||||||
isLockKV = "IsLock"
|
isLockKV = "IsLock"
|
||||||
|
@ -114,9 +117,10 @@ const (
|
||||||
ownerKV = "Owner"
|
ownerKV = "Owner"
|
||||||
createdKV = "Created"
|
createdKV = "Created"
|
||||||
|
|
||||||
settingsFileName = "bucket-settings"
|
settingsFileName = "bucket-settings"
|
||||||
corsFilename = "bucket-cors"
|
corsFilename = "bucket-cors"
|
||||||
bucketTaggingFilename = "bucket-tagging"
|
bucketTaggingFilename = "bucket-tagging"
|
||||||
|
bucketLifecycleFilename = "bucket-lifecycle"
|
||||||
|
|
||||||
// versionTree -- ID of a tree with object versions.
|
// versionTree -- ID of a tree with object versions.
|
||||||
versionTree = "version"
|
versionTree = "version"
|
||||||
|
@ -272,6 +276,14 @@ func newNodeVersionFromTreeNode(log *zap.Logger, filePath string, treeNode *tree
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if creationEpoch, ok := treeNode.Get(creationEpochKV); ok {
|
||||||
|
if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil {
|
||||||
|
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err))
|
||||||
|
} else {
|
||||||
|
version.CreationEpoch = epoch
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return version, nil
|
return version, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -353,6 +365,14 @@ func newMultipartInfoFromTreeNode(log *zap.Logger, filePath string, treeNode *tr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if creationEpoch, ok := treeNode.Get(creationEpochKV); ok {
|
||||||
|
if epoch, err := strconv.ParseUint(creationEpoch, 10, 64); err != nil {
|
||||||
|
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, creationEpoch), zap.Error(err))
|
||||||
|
} else {
|
||||||
|
multipartInfo.CreationEpoch = epoch
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return multipartInfo, nil
|
return multipartInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -388,6 +408,12 @@ func newMultipartInfo(log *zap.Logger, node NodeResponse) (*data.MultipartInfo,
|
||||||
} else {
|
} else {
|
||||||
multipartInfo.Finished = isFinished
|
multipartInfo.Finished = isFinished
|
||||||
}
|
}
|
||||||
|
case creationEpochKV:
|
||||||
|
if epoch, err := strconv.ParseUint(string(kv.GetValue()), 10, 64); err != nil {
|
||||||
|
log.Warn(logs.InvalidTreeKV, zap.String(creationEpochKV, string(kv.GetValue())), zap.Error(err))
|
||||||
|
} else {
|
||||||
|
multipartInfo.CreationEpoch = epoch
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
multipartInfo.Meta[kv.GetKey()] = string(kv.GetValue())
|
multipartInfo.Meta[kv.GetKey()] = string(kv.GetValue())
|
||||||
}
|
}
|
||||||
|
@ -758,7 +784,7 @@ func (c *Tree) GetVersions(ctx context.Context, bktInfo *data.BucketInfo, filepa
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
|
func (c *Tree) GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
|
||||||
meta := []string{oidKV, isCombinedKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV, md5KV}
|
meta := []string{oidKV, isCombinedKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV, md5KV, creationEpochKV}
|
||||||
path := pathFromName(objectName)
|
path := pathFromName(objectName)
|
||||||
|
|
||||||
p := &GetNodesParams{
|
p := &GetNodesParams{
|
||||||
|
@ -995,7 +1021,9 @@ func (s *VersionsByPrefixStreamImpl) getNodeVersionFromInnerStream() (*data.Node
|
||||||
func (s *VersionsByPrefixStreamImpl) parseNodeResponse(node NodeResponse) (res *data.NodeVersion, skip bool, err error) {
|
func (s *VersionsByPrefixStreamImpl) parseNodeResponse(node NodeResponse) (res *data.NodeVersion, skip bool, err error) {
|
||||||
trNode, fileName, err := parseTreeNode(node)
|
trNode, fileName, err := parseTreeNode(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.log.Debug(logs.ParseTreeNode, zap.Error(err))
|
if !errors.Is(err, errNodeDoesntContainFileName) {
|
||||||
|
s.log.Debug(logs.ParseTreeNode, zap.Error(err))
|
||||||
|
}
|
||||||
return nil, true, nil
|
return nil, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1206,7 +1234,7 @@ func parseTreeNode(node NodeResponse) (*treeNode, string, error) {
|
||||||
|
|
||||||
fileName, ok := tNode.FileName()
|
fileName, ok := tNode.FileName()
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, "", fmt.Errorf("doesn't contain FileName")
|
return nil, "", errNodeDoesntContainFileName
|
||||||
}
|
}
|
||||||
|
|
||||||
return tNode, fileName, nil
|
return tNode, fileName, nil
|
||||||
|
@ -1447,6 +1475,74 @@ func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipart
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Tree) PutBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
|
||||||
|
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
|
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||||
|
if err != nil && !isErrNotFound {
|
||||||
|
return nil, fmt.Errorf("couldn't get node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
meta := make(map[string]string)
|
||||||
|
meta[FileNameKey] = bucketLifecycleFilename
|
||||||
|
meta[oidKV] = addr.Object().EncodeToString()
|
||||||
|
meta[cidKV] = addr.Container().EncodeToString()
|
||||||
|
|
||||||
|
if isErrNotFound {
|
||||||
|
if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, layer.ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
latest := multiNode.Latest()
|
||||||
|
ind := latest.GetLatestNodeIndex()
|
||||||
|
if latest.IsSplit() {
|
||||||
|
c.reqLogger(ctx).Error(logs.BucketLifecycleNodeHasMultipleIDs)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
|
||||||
|
return nil, fmt.Errorf("move lifecycle node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
objToDelete := make([]oid.Address, 1, len(multiNode.nodes))
|
||||||
|
objToDelete[0], err = getTreeNodeAddress(latest)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse object addr of latest lifecycle node in tree: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
objToDelete = append(objToDelete, c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)...)
|
||||||
|
|
||||||
|
return objToDelete, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Tree) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
|
||||||
|
node, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
|
if err != nil {
|
||||||
|
return oid.Address{}, fmt.Errorf("get lifecycle node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return getTreeNodeAddress(node.Latest())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
|
||||||
|
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketLifecycleFilename)
|
||||||
|
isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
|
||||||
|
if err != nil && !isErrNotFound {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if isErrNotFound {
|
||||||
|
return nil, layer.ErrNoNodeToRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo)
|
||||||
|
if len(objToDelete) != len(multiNode.nodes) {
|
||||||
|
return nil, fmt.Errorf("clean old lifecycle nodes: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return objToDelete, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
|
||||||
err := c.service.RemoveNode(ctx, bktInfo, systemTree, multipartInfo.ID)
|
err := c.service.RemoveNode(ctx, bktInfo, systemTree, multipartInfo.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -1536,10 +1632,11 @@ func (c *Tree) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.Bucket
|
||||||
func (c *Tree) addVersion(ctx context.Context, bktInfo *data.BucketInfo, treeID string, version *data.NodeVersion) (uint64, error) {
|
func (c *Tree) addVersion(ctx context.Context, bktInfo *data.BucketInfo, treeID string, version *data.NodeVersion) (uint64, error) {
|
||||||
path := pathFromName(version.FilePath)
|
path := pathFromName(version.FilePath)
|
||||||
meta := map[string]string{
|
meta := map[string]string{
|
||||||
oidKV: version.OID.EncodeToString(),
|
oidKV: version.OID.EncodeToString(),
|
||||||
FileNameKey: path[len(path)-1],
|
FileNameKey: path[len(path)-1],
|
||||||
ownerKV: version.Owner.EncodeToString(),
|
ownerKV: version.Owner.EncodeToString(),
|
||||||
createdKV: strconv.FormatInt(version.Created.UTC().UnixMilli(), 10),
|
createdKV: strconv.FormatInt(version.Created.UTC().UnixMilli(), 10),
|
||||||
|
creationEpochKV: strconv.FormatUint(version.CreationEpoch, 10),
|
||||||
}
|
}
|
||||||
|
|
||||||
if version.Size > 0 {
|
if version.Size > 0 {
|
||||||
|
@ -1593,7 +1690,7 @@ func (c *Tree) clearOutdatedVersionInfo(ctx context.Context, bktInfo *data.Bucke
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) getVersions(ctx context.Context, bktInfo *data.BucketInfo, treeID, filepath string, onlyUnversioned bool) ([]*data.NodeVersion, error) {
|
func (c *Tree) getVersions(ctx context.Context, bktInfo *data.BucketInfo, treeID, filepath string, onlyUnversioned bool) ([]*data.NodeVersion, error) {
|
||||||
keysToReturn := []string{oidKV, isCombinedKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV, md5KV}
|
keysToReturn := []string{oidKV, isCombinedKV, isUnversionedKV, isDeleteMarkerKV, etagKV, sizeKV, md5KV, creationEpochKV}
|
||||||
path := pathFromName(filepath)
|
path := pathFromName(filepath)
|
||||||
p := &GetNodesParams{
|
p := &GetNodesParams{
|
||||||
BktInfo: bktInfo,
|
BktInfo: bktInfo,
|
||||||
|
@ -1651,6 +1748,7 @@ func metaFromMultipart(info *data.MultipartInfo, fileName string) map[string]str
|
||||||
if info.Finished {
|
if info.Finished {
|
||||||
info.Meta[finishedKV] = strconv.FormatBool(info.Finished)
|
info.Meta[finishedKV] = strconv.FormatBool(info.Finished)
|
||||||
}
|
}
|
||||||
|
info.Meta[creationEpochKV] = strconv.FormatUint(info.CreationEpoch, 10)
|
||||||
|
|
||||||
return info.Meta
|
return info.Meta
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue