TrueCloudLab/frostfs-s3-gw
18
3
Fork 20
Code Issues 29 Pull requests 8 Projects Releases 23 Packages Activity Actions

feature/lifecycle_convert_date #516

Merged
alexvanin merged 2 commits from mbiryukova/frostfs-s3-gw:feature/lifecycle_convert_date into master 2024-10-22 14:21:50 +00:00
Conversation 0 Commits 2 Files changed 9 +179 -63
9 changed files with 179 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
api/data/lifecycle.go
View file

@ -27,9 +27,10 @@ type (
}
LifecycleExpiration struct {
Date string `xml:"Date,omitempty"`
Days *int `xml:"Days,omitempty"`
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
Date string `xml:"Date,omitempty"`
Days *int `xml:"Days,omitempty"`
Epoch *uint64 `xml:"Epoch,omitempty"`
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
}
LifecycleRuleFilter struct {

69
api/handler/lifecycle.go
View file

@ -2,6 +2,9 @@ package handler
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"fmt"
"io"
"net/http"
@ -12,6 +15,8 @@ import (
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
)
const (
@ -56,9 +61,9 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
@ -68,16 +73,37 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
return
}
if err = checkLifecycleConfiguration(cfg); err != nil {
bodyMD5, err := getContentMD5(&buf)
if err != nil {
h.logAndSendError(w, "could not get content md5", reqInfo, err)
return
}
if !bytes.Equal(headerMD5, bodyMD5) {
h.logAndSendError(w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
networkInfo, err := h.obj.GetNetworkInfo(ctx)
if err != nil {
h.logAndSendError(w, "could not get network info", reqInfo, err)
return
}
if err = checkLifecycleConfiguration(ctx, cfg, &networkInfo); err != nil {
h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
params := &layer.PutBucketLifecycleParams{
BktInfo: bktInfo,
LifecycleCfg: cfg,
LifecycleReader: &buf,
MD5Hash: r.Header.Get(api.ContentMD5),
BktInfo: bktInfo,
LifecycleCfg: cfg,
}
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
@ -110,13 +136,15 @@ func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Re
w.WriteHeader(http.StatusNoContent)
}
func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
func checkLifecycleConfiguration(ctx context.Context, cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error {
now := layer.TimeNow(ctx)
if len(cfg.Rules) > maxRules {
return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
}
ids := make(map[string]struct{}, len(cfg.Rules))
for _, rule := range cfg.Rules {
for i, rule := range cfg.Rules {
if _, ok := ids[rule.ID]; ok && rule.ID != "" {
return fmt.Errorf("duplicate 'ID': %s", rule.ID)
}
@ -160,8 +188,18 @@ func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days)
}
if _, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil {
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
if rule.Expiration.Date != "" {
parsedTime, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date)
if err != nil {
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
}
epoch, err := util.TimeToEpoch(ni, now, parsedTime)
if err != nil {
return fmt.Errorf("convert time to epoch: %w", err)
}
cfg.Rules[i].Expiration.Epoch = &epoch
}
}
@ -233,3 +271,12 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
return nil
}
func getContentMD5(reader io.Reader) ([]byte, error) {
hash := md5.New()
_, err := io.Copy(hash, reader)
if err != nil {
return nil, err
}
return hash.Sum(nil), nil
}

11
api/handler/lifecycle_test.go
View file

@ -1,6 +1,7 @@
package handler
import (
"bytes"
"crypto/md5"
"crypto/rand"
"encoding/base64"
@ -393,8 +394,14 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) {
bktName := "bucket-lifecycle-invalid-xml"
createBucket(hc, bktName)
w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{})
r.Header.Set(api.ContentMD5, "")
cfg := &data.CORSConfiguration{}
body, err := xml.Marshal(cfg)
require.NoError(t, err)
contentMD5, err := getContentMD5(bytes.NewReader(body))
require.NoError(t, err)
w, r := prepareTestRequest(hc, bktName, "", cfg)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(contentMD5))
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
}

2
api/layer/frostfs_mock.go
View file

@ -413,6 +413,8 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm frostfs.PrmObjectSear
func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) {
ni := netmap.NetworkInfo{}
ni.SetCurrentEpoch(t.currentEpoch)
ni.SetEpochDuration(60)
ni.SetMsPerBlock(1000)
return ni, nil
}

34
api/layer/lifecycle.go
View file

@ -3,11 +3,9 @@ package layer
import (
"bytes"
"context"
"encoding/base64"
"encoding/xml"
"errors"
"fmt"
"io"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
@ -19,16 +17,19 @@ import (
)
type PutBucketLifecycleParams struct {
BktInfo *data.BucketInfo
LifecycleCfg *data.LifecycleConfiguration
LifecycleReader io.Reader
CopiesNumbers []uint32
MD5Hash string
BktInfo *data.BucketInfo
LifecycleCfg *data.LifecycleConfiguration
CopiesNumbers []uint32
}
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
cfgBytes, err := xml.Marshal(p.LifecycleCfg)
if err != nil {
return fmt.Errorf("marshal lifecycle configuration: %w", err)
}
prm := frostfs.PrmObjectCreate{
Payload: p.LifecycleReader,
Payload: bytes.NewReader(cfgBytes),
Filepath: p.BktInfo.LifecycleConfigurationObjectName(),
CreationTime: TimeNow(ctx),
}
@ -49,17 +50,6 @@ func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucke
return fmt.Errorf("put lifecycle object: %w", err)
}
hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash)
if err != nil {
return apierr.GetAPIError(apierr.ErrInvalidDigest)
}
if !bytes.Equal(hashBytes, createdObj.MD5Sum) {
n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
return apierr.GetAPIError(apierr.ErrInvalidDigest)
}
objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
objsToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objsToDeleteNotFound {
@ -129,6 +119,12 @@ func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *da
n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg)
for i := range lifecycleCfg.Rules {
if lifecycleCfg.Rules[i].Expiration != nil {
lifecycleCfg.Rules[i].Expiration.Epoch = nil
}
}
return lifecycleCfg, nil
}

8
api/layer/lifecycle_test.go
View file

@ -1,9 +1,7 @@
package layer
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/xml"
"testing"
@ -42,10 +40,8 @@ func TestBucketLifecycle(t *testing.T) {
require.NoError(t, err)
err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{
BktInfo: tc.bktInfo,
LifecycleCfg: lifecycle,
LifecycleReader: bytes.NewReader(lifecycleBytes),
MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)),
BktInfo: tc.bktInfo,
LifecycleCfg: lifecycle,
})
require.NoError(t, err)

28
internal/frostfs/frostfs.go
View file

@ -5,13 +5,13 @@ import (
"errors"
"fmt"
"io"
"math"
"strconv"
"time"
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -54,8 +54,7 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS {
// TimeToEpoch implements layer.FrostFS interface method.
func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
dur := futureTime.Sub(now)
if dur < 0 {
if futureTime.Before(now) {
return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)",
futureTime.Format(time.RFC3339), now.Format(time.RFC3339))
}
@ -65,27 +64,12 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u
return 0, 0, handleObjectError("get network info via client", err)
}
durEpoch := networkInfo.EpochDuration()
if durEpoch == 0 {
return 0, 0, errors.New("epoch duration is missing or zero")
epoch, err := util.TimeToEpoch(&networkInfo, now, futureTime)
if err != nil {
return 0, 0, err
}
curr := networkInfo.CurrentEpoch()
msPerEpoch := durEpoch * uint64(networkInfo.MsPerBlock())
epochLifetime := uint64(dur.Milliseconds()) / msPerEpoch
if uint64(dur.Milliseconds())%msPerEpoch != 0 {
epochLifetime++
}
var epoch uint64
if epochLifetime >= math.MaxUint64-curr {
epoch = math.MaxUint64
} else {
epoch = curr + epochLifetime
}
return curr, epoch, nil
return networkInfo.CurrentEpoch(), epoch, nil
}
// Container implements layer.FrostFS interface method.

39
internal/frostfs/util/util.go
View file

@ -2,9 +2,12 @@ package util
import (
"fmt"
"math"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
"github.com/nspcc-dev/neo-go/pkg/util"
)
@ -32,3 +35,39 @@ func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error)
return nns.ResolveContractHash(domain)
}
func TimeToEpoch(ni *netmap.NetworkInfo, now, t time.Time) (uint64, error) {
duration := t.Sub(now)
durationAbs := duration.Abs()
durEpoch := ni.EpochDuration()
if durEpoch == 0 {
return 0, fmt.Errorf("epoch duration is missing or zero")
}
msPerEpoch := durEpoch * uint64(ni.MsPerBlock())
epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch
if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 {
epochLifetime++
}
curr := ni.CurrentEpoch()
var epoch uint64
if duration > 0 {
if epochLifetime >= math.MaxUint64-curr {
epoch = math.MaxUint64
} else {
epoch = curr + epochLifetime
}
} else {
if epochLifetime >= curr {
epoch = 0
} else {
epoch = curr - epochLifetime
}
}
return epoch, nil
}

44
internal/frostfs/util/util_test.go Normal file
View file

@ -0,0 +1,44 @@
package util
import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/stretchr/testify/require"
)
func TestTimeToEpoch(t *testing.T) {
ni := netmap.NetworkInfo{}
ni.SetCurrentEpoch(10)
_, err := TimeToEpoch(&ni, time.Now(), time.Now())
require.Error(t, err)
ni.SetEpochDuration(60)
ni.SetMsPerBlock(1000)
epoch, err := TimeToEpoch(&ni, time.Now(), time.Now())
require.NoError(t, err)
require.Equal(t, uint64(10), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(30*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(11), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(90*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(12), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-30*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(9), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-90*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(8), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-10*time.Minute))
require.NoError(t, err)
require.Equal(t, uint64(0), epoch)
}