From 51e6fcafecfaa337e23700f07e83cbb4d16a19ca Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Mon, 14 Oct 2024 17:03:53 +0300 Subject: [PATCH 1/2] [#516] Convert expiration date to epoch Signed-off-by: Marina Biryukova --- api/data/lifecycle.go | 7 +++-- api/handler/lifecycle.go | 50 +++++++++++++++++++++--------- api/handler/lifecycle_test.go | 5 --- api/layer/frostfs_mock.go | 2 ++ api/layer/lifecycle.go | 34 +++++++++----------- api/layer/lifecycle_test.go | 8 ++--- internal/frostfs/frostfs.go | 28 ++++------------- internal/frostfs/util/util.go | 39 +++++++++++++++++++++++ internal/frostfs/util/util_test.go | 44 ++++++++++++++++++++++++++ 9 files changed, 147 insertions(+), 70 deletions(-) create mode 100644 internal/frostfs/util/util_test.go diff --git a/api/data/lifecycle.go b/api/data/lifecycle.go index cb3c7cb..4d0828f 100644 --- a/api/data/lifecycle.go +++ b/api/data/lifecycle.go @@ -27,9 +27,10 @@ type ( } LifecycleExpiration struct { - Date string `xml:"Date,omitempty"` - Days *int `xml:"Days,omitempty"` - ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"` + Date string `xml:"Date,omitempty"` + Days *int `xml:"Days,omitempty"` + Epoch *uint64 `xml:"Epoch,omitempty"` + ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"` } LifecycleRuleFilter struct { diff --git a/api/handler/lifecycle.go b/api/handler/lifecycle.go index 93fe2cc..88e96cf 100644 --- a/api/handler/lifecycle.go +++ b/api/handler/lifecycle.go @@ -1,9 +1,9 @@ package handler import ( - "bytes" + "context" + "encoding/base64" "fmt" - "io" "net/http" "time" @@ -12,6 +12,8 @@ import ( apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) const ( @@ -43,9 +45,6 @@ func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque } func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { - var buf bytes.Buffer - - tee := io.TeeReader(r.Body, &buf) ctx := r.Context() reqInfo := middleware.GetReqInfo(ctx) @@ -56,6 +55,11 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque return } + if _, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5)); err != nil { + h.logAndSendError(w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) + return + } + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) if err != nil { h.logAndSendError(w, "could not get bucket info", reqInfo, err) @@ -63,21 +67,25 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque } cfg := new(data.LifecycleConfiguration) - if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil { + if err = h.cfg.NewXMLDecoder(r.Body).Decode(cfg); err != nil { h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) return } - if err = checkLifecycleConfiguration(cfg); err != nil { + networkInfo, err := h.obj.GetNetworkInfo(ctx) + if err != nil { + h.logAndSendError(w, "could not get network info", reqInfo, err) + return + } + + if err = checkLifecycleConfiguration(ctx, cfg, &networkInfo); err != nil { h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) return } params := &layer.PutBucketLifecycleParams{ - BktInfo: bktInfo, - LifecycleCfg: cfg, - LifecycleReader: &buf, - MD5Hash: r.Header.Get(api.ContentMD5), + BktInfo: bktInfo, + LifecycleCfg: cfg, } params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint) @@ -110,13 +118,15 @@ func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Re w.WriteHeader(http.StatusNoContent) } -func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error { +func checkLifecycleConfiguration(ctx context.Context, cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error { + now := layer.TimeNow(ctx) + if len(cfg.Rules) > maxRules { return fmt.Errorf("number of rules cannot be greater than %d", maxRules) } ids := make(map[string]struct{}, len(cfg.Rules)) - for _, rule := range cfg.Rules { + for i, rule := range cfg.Rules { if _, ok := ids[rule.ID]; ok && rule.ID != "" { return fmt.Errorf("duplicate 'ID': %s", rule.ID) } @@ -160,8 +170,18 @@ func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error { return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days) } - if _, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil { - return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date) + if rule.Expiration.Date != "" { + parsedTime, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date) + if err != nil { + return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date) + } + + epoch, err := util.TimeToEpoch(ni, now, parsedTime) + if err != nil { + return fmt.Errorf("convert time to epoch: %w", err) + } + + cfg.Rules[i].Expiration.Epoch = &epoch } } diff --git a/api/handler/lifecycle_test.go b/api/handler/lifecycle_test.go index 77aef64..8cba0fc 100644 --- a/api/handler/lifecycle_test.go +++ b/api/handler/lifecycle_test.go @@ -376,11 +376,6 @@ func TestPutBucketLifecycleInvalidMD5(t *testing.T) { hc.Handler().PutBucketLifecycleHandler(w, r) assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5)) - w, r = prepareTestRequest(hc, bktName, "", lifecycle) - r.Header.Set(api.ContentMD5, "") - hc.Handler().PutBucketLifecycleHandler(w, r) - assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest)) - w, r = prepareTestRequest(hc, bktName, "", lifecycle) r.Header.Set(api.ContentMD5, "some-hash") hc.Handler().PutBucketLifecycleHandler(w, r) diff --git a/api/layer/frostfs_mock.go b/api/layer/frostfs_mock.go index 3039509..1b29fca 100644 --- a/api/layer/frostfs_mock.go +++ b/api/layer/frostfs_mock.go @@ -413,6 +413,8 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm frostfs.PrmObjectSear func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) { ni := netmap.NetworkInfo{} ni.SetCurrentEpoch(t.currentEpoch) + ni.SetEpochDuration(60) + ni.SetMsPerBlock(1000) return ni, nil } diff --git a/api/layer/lifecycle.go b/api/layer/lifecycle.go index 8901cc6..f16aded 100644 --- a/api/layer/lifecycle.go +++ b/api/layer/lifecycle.go @@ -3,11 +3,9 @@ package layer import ( "bytes" "context" - "encoding/base64" "encoding/xml" "errors" "fmt" - "io" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" @@ -19,16 +17,19 @@ import ( ) type PutBucketLifecycleParams struct { - BktInfo *data.BucketInfo - LifecycleCfg *data.LifecycleConfiguration - LifecycleReader io.Reader - CopiesNumbers []uint32 - MD5Hash string + BktInfo *data.BucketInfo + LifecycleCfg *data.LifecycleConfiguration + CopiesNumbers []uint32 } func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error { + cfgBytes, err := xml.Marshal(p.LifecycleCfg) + if err != nil { + return fmt.Errorf("marshal lifecycle configuration: %w", err) + } + prm := frostfs.PrmObjectCreate{ - Payload: p.LifecycleReader, + Payload: bytes.NewReader(cfgBytes), Filepath: p.BktInfo.LifecycleConfigurationObjectName(), CreationTime: TimeNow(ctx), } @@ -49,17 +50,6 @@ func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucke return fmt.Errorf("put lifecycle object: %w", err) } - hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash) - if err != nil { - return apierr.GetAPIError(apierr.ErrInvalidDigest) - } - - if !bytes.Equal(hashBytes, createdObj.MD5Sum) { - n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID)) - - return apierr.GetAPIError(apierr.ErrInvalidDigest) - } - objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID)) objsToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove) if err != nil && !objsToDeleteNotFound { @@ -129,6 +119,12 @@ func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *da n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg) + for i := range lifecycleCfg.Rules { + if lifecycleCfg.Rules[i].Expiration != nil { + lifecycleCfg.Rules[i].Expiration.Epoch = nil + } + } + return lifecycleCfg, nil } diff --git a/api/layer/lifecycle_test.go b/api/layer/lifecycle_test.go index 0e9be8f..b45c7fe 100644 --- a/api/layer/lifecycle_test.go +++ b/api/layer/lifecycle_test.go @@ -1,9 +1,7 @@ package layer import ( - "bytes" "crypto/md5" - "encoding/base64" "encoding/xml" "testing" @@ -42,10 +40,8 @@ func TestBucketLifecycle(t *testing.T) { require.NoError(t, err) err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{ - BktInfo: tc.bktInfo, - LifecycleCfg: lifecycle, - LifecycleReader: bytes.NewReader(lifecycleBytes), - MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)), + BktInfo: tc.bktInfo, + LifecycleCfg: lifecycle, }) require.NoError(t, err) diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go index 95ac17b..03f8a7f 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/frostfs/frostfs.go @@ -5,13 +5,13 @@ import ( "errors" "fmt" "io" - "math" "strconv" "time" objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs" frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -54,8 +54,7 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS { // TimeToEpoch implements layer.FrostFS interface method. func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) { - dur := futureTime.Sub(now) - if dur < 0 { + if futureTime.Before(now) { return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)", futureTime.Format(time.RFC3339), now.Format(time.RFC3339)) } @@ -65,27 +64,12 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u return 0, 0, handleObjectError("get network info via client", err) } - durEpoch := networkInfo.EpochDuration() - if durEpoch == 0 { - return 0, 0, errors.New("epoch duration is missing or zero") + epoch, err := util.TimeToEpoch(&networkInfo, now, futureTime) + if err != nil { + return 0, 0, err } - curr := networkInfo.CurrentEpoch() - msPerEpoch := durEpoch * uint64(networkInfo.MsPerBlock()) - - epochLifetime := uint64(dur.Milliseconds()) / msPerEpoch - if uint64(dur.Milliseconds())%msPerEpoch != 0 { - epochLifetime++ - } - - var epoch uint64 - if epochLifetime >= math.MaxUint64-curr { - epoch = math.MaxUint64 - } else { - epoch = curr + epochLifetime - } - - return curr, epoch, nil + return networkInfo.CurrentEpoch(), epoch, nil } // Container implements layer.FrostFS interface method. diff --git a/internal/frostfs/util/util.go b/internal/frostfs/util/util.go index 444504b..00a9905 100644 --- a/internal/frostfs/util/util.go +++ b/internal/frostfs/util/util.go @@ -2,9 +2,12 @@ package util import ( "fmt" + "math" "strings" + "time" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" "github.com/nspcc-dev/neo-go/pkg/util" ) @@ -32,3 +35,39 @@ func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) return nns.ResolveContractHash(domain) } + +func TimeToEpoch(ni *netmap.NetworkInfo, now, t time.Time) (uint64, error) { + duration := t.Sub(now) + durationAbs := duration.Abs() + + durEpoch := ni.EpochDuration() + if durEpoch == 0 { + return 0, fmt.Errorf("epoch duration is missing or zero") + } + + msPerEpoch := durEpoch * uint64(ni.MsPerBlock()) + epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch + + if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 { + epochLifetime++ + } + + curr := ni.CurrentEpoch() + + var epoch uint64 + if duration > 0 { + if epochLifetime >= math.MaxUint64-curr { + epoch = math.MaxUint64 + } else { + epoch = curr + epochLifetime + } + } else { + if epochLifetime >= curr { + epoch = 0 + } else { + epoch = curr - epochLifetime + } + } + + return epoch, nil +} diff --git a/internal/frostfs/util/util_test.go b/internal/frostfs/util/util_test.go new file mode 100644 index 0000000..92bcdff --- /dev/null +++ b/internal/frostfs/util/util_test.go @@ -0,0 +1,44 @@ +package util + +import ( + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/stretchr/testify/require" +) + +func TestTimeToEpoch(t *testing.T) { + ni := netmap.NetworkInfo{} + ni.SetCurrentEpoch(10) + + _, err := TimeToEpoch(&ni, time.Now(), time.Now()) + require.Error(t, err) + + ni.SetEpochDuration(60) + ni.SetMsPerBlock(1000) + + epoch, err := TimeToEpoch(&ni, time.Now(), time.Now()) + require.NoError(t, err) + require.Equal(t, uint64(10), epoch) + + epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(30*time.Second)) + require.NoError(t, err) + require.Equal(t, uint64(11), epoch) + + epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(90*time.Second)) + require.NoError(t, err) + require.Equal(t, uint64(12), epoch) + + epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-30*time.Second)) + require.NoError(t, err) + require.Equal(t, uint64(9), epoch) + + epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-90*time.Second)) + require.NoError(t, err) + require.Equal(t, uint64(8), epoch) + + epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-10*time.Minute)) + require.NoError(t, err) + require.Equal(t, uint64(0), epoch) +} -- 2.45.2 From a6372f252fb733eca7dc86b6e06979c12febd755 Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Tue, 15 Oct 2024 14:18:38 +0300 Subject: [PATCH 2/2] [#516] Check Content-Md5 of lifecycle configuration Signed-off-by: Marina Biryukova --- api/handler/lifecycle.go | 41 +++++++++++++++++++++++++++++------ api/handler/lifecycle_test.go | 16 ++++++++++++-- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/api/handler/lifecycle.go b/api/handler/lifecycle.go index 88e96cf..9bd1df8 100644 --- a/api/handler/lifecycle.go +++ b/api/handler/lifecycle.go @@ -1,9 +1,12 @@ package handler import ( + "bytes" "context" + "crypto/md5" "encoding/base64" "fmt" + "io" "net/http" "time" @@ -45,6 +48,9 @@ func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque } func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { + var buf bytes.Buffer + + tee := io.TeeReader(r.Body, &buf) ctx := r.Context() reqInfo := middleware.GetReqInfo(ctx) @@ -55,23 +61,35 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque return } - if _, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5)); err != nil { + headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5)) + if err != nil { h.logAndSendError(w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) return } + cfg := new(data.LifecycleConfiguration) + if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil { + h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) + return + } + + bodyMD5, err := getContentMD5(&buf) + if err != nil { + h.logAndSendError(w, "could not get content md5", reqInfo, err) + return + } + + if !bytes.Equal(headerMD5, bodyMD5) { + h.logAndSendError(w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) + return + } + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) if err != nil { h.logAndSendError(w, "could not get bucket info", reqInfo, err) return } - cfg := new(data.LifecycleConfiguration) - if err = h.cfg.NewXMLDecoder(r.Body).Decode(cfg); err != nil { - h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) - return - } - networkInfo, err := h.obj.GetNetworkInfo(ctx) if err != nil { h.logAndSendError(w, "could not get network info", reqInfo, err) @@ -253,3 +271,12 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error { return nil } + +func getContentMD5(reader io.Reader) ([]byte, error) { + hash := md5.New() + _, err := io.Copy(hash, reader) + if err != nil { + return nil, err + } + return hash.Sum(nil), nil +} diff --git a/api/handler/lifecycle_test.go b/api/handler/lifecycle_test.go index 8cba0fc..0794d55 100644 --- a/api/handler/lifecycle_test.go +++ b/api/handler/lifecycle_test.go @@ -1,6 +1,7 @@ package handler import ( + "bytes" "crypto/md5" "crypto/rand" "encoding/base64" @@ -376,6 +377,11 @@ func TestPutBucketLifecycleInvalidMD5(t *testing.T) { hc.Handler().PutBucketLifecycleHandler(w, r) assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5)) + w, r = prepareTestRequest(hc, bktName, "", lifecycle) + r.Header.Set(api.ContentMD5, "") + hc.Handler().PutBucketLifecycleHandler(w, r) + assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest)) + w, r = prepareTestRequest(hc, bktName, "", lifecycle) r.Header.Set(api.ContentMD5, "some-hash") hc.Handler().PutBucketLifecycleHandler(w, r) @@ -388,8 +394,14 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) { bktName := "bucket-lifecycle-invalid-xml" createBucket(hc, bktName) - w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{}) - r.Header.Set(api.ContentMD5, "") + cfg := &data.CORSConfiguration{} + body, err := xml.Marshal(cfg) + require.NoError(t, err) + contentMD5, err := getContentMD5(bytes.NewReader(body)) + require.NoError(t, err) + + w, r := prepareTestRequest(hc, bktName, "", cfg) + r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(contentMD5)) hc.Handler().PutBucketLifecycleHandler(w, r) assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML)) } -- 2.45.2