TrueCloudLab/frostfs-s3-gw
18
3
Fork 20
Code Issues 29 Pull requests 7 Projects Releases 23 Packages 1 Activity Actions

[#xxx] Convert expiration date to epoch

Browse source 
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
This commit is contained in:
Marina Biryukova 2024-10-14 17:03:53 +03:00
parent e35b582fe2
commit 1b171151a4
6 changed files with 127 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
api/data/lifecycle.go
View file

@ -27,9 +27,10 @@ type (
}
LifecycleExpiration struct {
Date string `xml:"Date,omitempty"`
Days *int `xml:"Days,omitempty"`
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
Date string `xml:"Date,omitempty"`
Days *int `xml:"Days,omitempty"`
Epoch *uint64 `xml:"Epoch,omitempty"`
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
}
LifecycleRuleFilter struct {

83
api/handler/lifecycle.go
View file

@ -1,9 +1,9 @@
package handler
import (
"bytes"
"encoding/base64"
"fmt"
"io"
"math"
"net/http"
"time"
@ -12,6 +12,7 @@ import (
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
)
const (
@ -43,9 +44,6 @@ func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
}
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
tee := io.TeeReader(r.Body, &buf)
ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx)
@ -56,6 +54,11 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
return
}
if _, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5)); err != nil {
h.logAndSendError(w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
@ -63,21 +66,25 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
}
cfg := new(data.LifecycleConfiguration)
if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil {
if err = h.cfg.NewXMLDecoder(r.Body).Decode(cfg); err != nil {
h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
if err = checkLifecycleConfiguration(cfg); err != nil {
networkInfo, err := h.obj.GetNetworkInfo(ctx)
if err != nil {
h.logAndSendError(w, "could not get network info", reqInfo, err)
return
}
if err = checkLifecycleConfiguration(cfg, &networkInfo); err != nil {
h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
params := &layer.PutBucketLifecycleParams{
BktInfo: bktInfo,
LifecycleCfg: cfg,
LifecycleReader: &buf,
MD5Hash: r.Header.Get(api.ContentMD5),
BktInfo: bktInfo,
LifecycleCfg: cfg,
}
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
@ -110,13 +117,13 @@ func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Re
w.WriteHeader(http.StatusNoContent)
}
func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error {
if len(cfg.Rules) > maxRules {
return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
}
ids := make(map[string]struct{}, len(cfg.Rules))
for _, rule := range cfg.Rules {
for i, rule := range cfg.Rules {
if _, ok := ids[rule.ID]; ok && rule.ID != "" {
return fmt.Errorf("duplicate 'ID': %s", rule.ID)
}
@ -160,8 +167,18 @@ func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days)
}
if _, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil {
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
if rule.Expiration.Date != "" {
parsedTime, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date)
if err != nil {
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
}
epoch, err := timeToEpoch(ni, parsedTime)
if err != nil {
return fmt.Errorf("convert time to epoch: %w", err)
}
cfg.Rules[i].Expiration.Epoch = &epoch
}
}
@ -233,3 +250,39 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
return nil
}
func timeToEpoch(ni *netmap.NetworkInfo, t time.Time) (uint64, error) {
duration := t.Sub(time.Now())
durationAbs := duration.Abs()
durEpoch := ni.EpochDuration()
if durEpoch == 0 {
return 0, fmt.Errorf("epoch duration is missing or zero")
}
msPerEpoch := durEpoch * uint64(ni.MsPerBlock())
epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch
if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 {
epochLifetime++
}
curr := ni.CurrentEpoch()
var epoch uint64
if duration > 0 {
if epochLifetime >= math.MaxUint64-curr {
epoch = math.MaxUint64
} else {
epoch = curr + epochLifetime
}
} else {
if epochLifetime >= curr {
epoch = 0
} else {
epoch = curr - epochLifetime
}
}
return epoch, nil
}

41
api/handler/lifecycle_test.go
View file

@ -15,6 +15,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/mr-tron/base58"
"github.com/stretchr/testify/require"
)
@ -376,11 +377,6 @@ func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5))
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
r.Header.Set(api.ContentMD5, "")
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
r.Header.Set(api.ContentMD5, "some-hash")
hc.Handler().PutBucketLifecycleHandler(w, r)
@ -399,6 +395,41 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) {
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
}
func TestTimeToEpoch(t *testing.T) {
ni := netmap.NetworkInfo{}
ni.SetCurrentEpoch(10)
_, err := timeToEpoch(&ni, time.Now())
require.Error(t, err)
ni.SetEpochDuration(60)
ni.SetMsPerBlock(1000)
epoch, err := timeToEpoch(&ni, time.Now())
require.NoError(t, err)
require.Equal(t, uint64(10), epoch)
epoch, err = timeToEpoch(&ni, time.Now().Add(30*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(11), epoch)
epoch, err = timeToEpoch(&ni, time.Now().Add(90*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(12), epoch)
epoch, err = timeToEpoch(&ni, time.Now().Add(-30*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(9), epoch)
epoch, err = timeToEpoch(&ni, time.Now().Add(-90*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(8), epoch)
epoch, err = timeToEpoch(&ni, time.Now().Add(-10*time.Minute))
require.NoError(t, err)
require.Equal(t, uint64(0), epoch)
}
func putBucketLifecycleConfiguration(hc *handlerContext, bktName string, cfg *data.LifecycleConfiguration) {
w := putBucketLifecycleConfigurationBase(hc, bktName, cfg)
assertStatus(hc.t, w, http.StatusOK)

2
api/layer/frostfs_mock.go
View file

@ -413,6 +413,8 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm frostfs.PrmObjectSear
func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) {
ni := netmap.NetworkInfo{}
ni.SetCurrentEpoch(t.currentEpoch)
ni.SetEpochDuration(60)
ni.SetMsPerBlock(1000)
return ni, nil
}

34
api/layer/lifecycle.go
View file

@ -3,11 +3,9 @@ package layer
import (
"bytes"
"context"
"encoding/base64"
"encoding/xml"
"errors"
"fmt"
"io"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
@ -19,16 +17,19 @@ import (
)
type PutBucketLifecycleParams struct {
BktInfo *data.BucketInfo
LifecycleCfg *data.LifecycleConfiguration
LifecycleReader io.Reader
CopiesNumbers []uint32
MD5Hash string
BktInfo *data.BucketInfo
LifecycleCfg *data.LifecycleConfiguration
CopiesNumbers []uint32
}
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
cfgBytes, err := xml.Marshal(p.LifecycleCfg)
if err != nil {
return fmt.Errorf("marshal lifecycle configuration: %w", err)
}
prm := frostfs.PrmObjectCreate{
Payload: p.LifecycleReader,
Payload: bytes.NewReader(cfgBytes),
Filepath: p.BktInfo.LifecycleConfigurationObjectName(),
CreationTime: TimeNow(ctx),
}
@ -49,17 +50,6 @@ func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucke
return fmt.Errorf("put lifecycle object: %w", err)
}
hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash)
if err != nil {
return apierr.GetAPIError(apierr.ErrInvalidDigest)
}
if !bytes.Equal(hashBytes, createdObj.MD5Sum) {
n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
return apierr.GetAPIError(apierr.ErrInvalidDigest)
}
objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
objsToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objsToDeleteNotFound {
@ -129,6 +119,12 @@ func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *da
n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg)
for i := range lifecycleCfg.Rules {
if lifecycleCfg.Rules[i].Expiration != nil {
lifecycleCfg.Rules[i].Expiration.Epoch = nil
}
}
return lifecycleCfg, nil
}

8
api/layer/lifecycle_test.go
View file

@ -1,9 +1,7 @@
package layer
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/xml"
"testing"
@ -42,10 +40,8 @@ func TestBucketLifecycle(t *testing.T) {
require.NoError(t, err)
err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{
BktInfo: tc.bktInfo,
LifecycleCfg: lifecycle,
LifecycleReader: bytes.NewReader(lifecycleBytes),
MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)),
BktInfo: tc.bktInfo,
LifecycleCfg: lifecycle,
})
require.NoError(t, err)