feature/lifecycle_convert_date #516

Merged
alexvanin merged 2 commits from mbiryukova/frostfs-s3-gw:feature/lifecycle_convert_date into master 2024-10-22 14:21:50 +00:00
2 changed files with 48 additions and 9 deletions
Showing only changes of commit a6372f252f - Show all commits

View file

@ -1,9 +1,12 @@
package handler package handler
import ( import (
"bytes"
"context" "context"
"crypto/md5"
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"io"
"net/http" "net/http"
"time" "time"
@ -45,6 +48,9 @@ func (h *handler) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
} }
func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
var buf bytes.Buffer
tee := io.TeeReader(r.Body, &buf)
ctx := r.Context() ctx := r.Context()
reqInfo := middleware.GetReqInfo(ctx) reqInfo := middleware.GetReqInfo(ctx)
@ -55,23 +61,35 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
return return
} }
if _, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5)); err != nil { headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
if err != nil {
h.logAndSendError(w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) h.logAndSendError(w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return return
} }
cfg := new(data.LifecycleConfiguration)
if err = h.cfg.NewXMLDecoder(tee).Decode(cfg); err != nil {
h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
bodyMD5, err := getContentMD5(&buf)
if err != nil {
h.logAndSendError(w, "could not get content md5", reqInfo, err)
return
}
if !bytes.Equal(headerMD5, bodyMD5) {
h.logAndSendError(w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil { if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err) h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return return
} }
cfg := new(data.LifecycleConfiguration)
if err = h.cfg.NewXMLDecoder(r.Body).Decode(cfg); err != nil {
h.logAndSendError(w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
networkInfo, err := h.obj.GetNetworkInfo(ctx) networkInfo, err := h.obj.GetNetworkInfo(ctx)
if err != nil { if err != nil {
h.logAndSendError(w, "could not get network info", reqInfo, err) h.logAndSendError(w, "could not get network info", reqInfo, err)
@ -253,3 +271,12 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
return nil return nil
dkirillov marked this conversation as resolved Outdated

Consider reusing this

diff --git a/api/handler/lifecycle.go b/api/handler/lifecycle.go
index bd89fe9f..720e37e2 100644
--- a/api/handler/lifecycle.go
+++ b/api/handler/lifecycle.go
@@ -2,11 +2,11 @@ package handler
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
 	"encoding/base64"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
 	"time"
 
@@ -15,6 +15,7 @@ import (
 	apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 )
 
@@ -95,7 +96,7 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	if err = checkLifecycleConfiguration(cfg, &networkInfo); err != nil {
+	if err = checkLifecycleConfiguration(ctx, cfg, &networkInfo); err != nil {
 		h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
 		return
 	}
@@ -135,7 +136,8 @@ func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Re
 	w.WriteHeader(http.StatusNoContent)
 }
 
-func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error {
+func checkLifecycleConfiguration(ctx context.Context, cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error {
+	now := layer.TimeNow(ctx)
 	if len(cfg.Rules) > maxRules {
 		return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
 	}
@@ -191,7 +193,7 @@ func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.Ne
 					return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
 				}
 
-				epoch, err := timeToEpoch(ni, parsedTime)
+				epoch, err := util.TimeToEpoch(ni, now, parsedTime)
 				if err != nil {
 					return fmt.Errorf("convert time to epoch: %w", err)
 				}
@@ -269,42 +271,6 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
 	return nil
 }
 
-func timeToEpoch(ni *netmap.NetworkInfo, t time.Time) (uint64, error) {
-	duration := t.Sub(time.Now())
-	durationAbs := duration.Abs()
-
-	durEpoch := ni.EpochDuration()
-	if durEpoch == 0 {
-		return 0, fmt.Errorf("epoch duration is missing or zero")
-	}
-
-	msPerEpoch := durEpoch * uint64(ni.MsPerBlock())
-	epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch
-
-	if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 {
-		epochLifetime++
-	}
-
-	curr := ni.CurrentEpoch()
-
-	var epoch uint64
-	if duration > 0 {
-		if epochLifetime >= math.MaxUint64-curr {
-			epoch = math.MaxUint64
-		} else {
-			epoch = curr + epochLifetime
-		}
-	} else {
-		if epochLifetime >= curr {
-			epoch = 0
-		} else {
-			epoch = curr - epochLifetime
-		}
-	}
-
-	return epoch, nil
-}
-
 func getContentMD5(reader io.Reader) ([]byte, error) {
 	hash := md5.New()
 	buf := make([]byte, 64*1024)
diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go
index 95ac17bf..03f8a7f6 100644
--- a/internal/frostfs/frostfs.go
+++ b/internal/frostfs/frostfs.go
@@ -5,13 +5,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"strconv"
 	"time"
 
 	objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
 	frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@@ -54,8 +54,7 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS {
 
 // TimeToEpoch implements layer.FrostFS interface method.
 func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
-	dur := futureTime.Sub(now)
-	if dur < 0 {
+	if futureTime.Before(now) {
 		return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)",
 			futureTime.Format(time.RFC3339), now.Format(time.RFC3339))
 	}
@@ -65,27 +64,12 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u
 		return 0, 0, handleObjectError("get network info via client", err)
 	}
 
-	durEpoch := networkInfo.EpochDuration()
-	if durEpoch == 0 {
-		return 0, 0, errors.New("epoch duration is missing or zero")
-	}
-
-	curr := networkInfo.CurrentEpoch()
-	msPerEpoch := durEpoch * uint64(networkInfo.MsPerBlock())
-
-	epochLifetime := uint64(dur.Milliseconds()) / msPerEpoch
-	if uint64(dur.Milliseconds())%msPerEpoch != 0 {
-		epochLifetime++
-	}
-
-	var epoch uint64
-	if epochLifetime >= math.MaxUint64-curr {
-		epoch = math.MaxUint64
-	} else {
-		epoch = curr + epochLifetime
+	epoch, err := util.TimeToEpoch(&networkInfo, now, futureTime)
+	if err != nil {
+		return 0, 0, err
 	}
 
-	return curr, epoch, nil
+	return networkInfo.CurrentEpoch(), epoch, nil
 }
 
 // Container implements layer.FrostFS interface method.
diff --git a/internal/frostfs/util/util.go b/internal/frostfs/util/util.go
index 444504b1..00a99059 100644
--- a/internal/frostfs/util/util.go
+++ b/internal/frostfs/util/util.go
@@ -2,9 +2,12 @@ package util
 
 import (
 	"fmt"
+	"math"
 	"strings"
+	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
 	"github.com/nspcc-dev/neo-go/pkg/util"
 )
@@ -32,3 +35,39 @@ func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error)
 
 	return nns.ResolveContractHash(domain)
 }
+
+func TimeToEpoch(ni *netmap.NetworkInfo, now, t time.Time) (uint64, error) {
+	duration := t.Sub(now)
+	durationAbs := duration.Abs()
+
+	durEpoch := ni.EpochDuration()
+	if durEpoch == 0 {
+		return 0, fmt.Errorf("epoch duration is missing or zero")
+	}
+
+	msPerEpoch := durEpoch * uint64(ni.MsPerBlock())
+	epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch
+
+	if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 {
+		epochLifetime++
+	}
+
+	curr := ni.CurrentEpoch()
+
+	var epoch uint64
+	if duration > 0 {
+		if epochLifetime >= math.MaxUint64-curr {
+			epoch = math.MaxUint64
+		} else {
+			epoch = curr + epochLifetime
+		}
+	} else {
+		if epochLifetime >= curr {
+			epoch = 0
+		} else {
+			epoch = curr - epochLifetime
+		}
+	}
+
+	return epoch, nil
+}

Consider reusing this ```diff diff --git a/api/handler/lifecycle.go b/api/handler/lifecycle.go index bd89fe9f..720e37e2 100644 --- a/api/handler/lifecycle.go +++ b/api/handler/lifecycle.go @@ -2,11 +2,11 @@ package handler import ( "bytes" + "context" "crypto/md5" "encoding/base64" "fmt" "io" - "math" "net/http" "time" @@ -15,6 +15,7 @@ import ( apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) @@ -95,7 +96,7 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque return } - if err = checkLifecycleConfiguration(cfg, &networkInfo); err != nil { + if err = checkLifecycleConfiguration(ctx, cfg, &networkInfo); err != nil { h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) return } @@ -135,7 +136,8 @@ func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Re w.WriteHeader(http.StatusNoContent) } -func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error { +func checkLifecycleConfiguration(ctx context.Context, cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error { + now := layer.TimeNow(ctx) if len(cfg.Rules) > maxRules { return fmt.Errorf("number of rules cannot be greater than %d", maxRules) } @@ -191,7 +193,7 @@ func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.Ne return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date) } - epoch, err := timeToEpoch(ni, parsedTime) + epoch, err := util.TimeToEpoch(ni, now, parsedTime) if err != nil { return fmt.Errorf("convert time to epoch: %w", err) } @@ -269,42 +271,6 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error { return nil } -func timeToEpoch(ni *netmap.NetworkInfo, t time.Time) (uint64, error) { - duration := t.Sub(time.Now()) - durationAbs := duration.Abs() - - durEpoch := ni.EpochDuration() - if durEpoch == 0 { - return 0, fmt.Errorf("epoch duration is missing or zero") - } - - msPerEpoch := durEpoch * uint64(ni.MsPerBlock()) - epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch - - if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 { - epochLifetime++ - } - - curr := ni.CurrentEpoch() - - var epoch uint64 - if duration > 0 { - if epochLifetime >= math.MaxUint64-curr { - epoch = math.MaxUint64 - } else { - epoch = curr + epochLifetime - } - } else { - if epochLifetime >= curr { - epoch = 0 - } else { - epoch = curr - epochLifetime - } - } - - return epoch, nil -} - func getContentMD5(reader io.Reader) ([]byte, error) { hash := md5.New() buf := make([]byte, 64*1024) diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go index 95ac17bf..03f8a7f6 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/frostfs/frostfs.go @@ -5,13 +5,13 @@ import ( "errors" "fmt" "io" - "math" "strconv" "time" objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs" frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -54,8 +54,7 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS { // TimeToEpoch implements layer.FrostFS interface method. func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) { - dur := futureTime.Sub(now) - if dur < 0 { + if futureTime.Before(now) { return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)", futureTime.Format(time.RFC3339), now.Format(time.RFC3339)) } @@ -65,27 +64,12 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u return 0, 0, handleObjectError("get network info via client", err) } - durEpoch := networkInfo.EpochDuration() - if durEpoch == 0 { - return 0, 0, errors.New("epoch duration is missing or zero") - } - - curr := networkInfo.CurrentEpoch() - msPerEpoch := durEpoch * uint64(networkInfo.MsPerBlock()) - - epochLifetime := uint64(dur.Milliseconds()) / msPerEpoch - if uint64(dur.Milliseconds())%msPerEpoch != 0 { - epochLifetime++ - } - - var epoch uint64 - if epochLifetime >= math.MaxUint64-curr { - epoch = math.MaxUint64 - } else { - epoch = curr + epochLifetime + epoch, err := util.TimeToEpoch(&networkInfo, now, futureTime) + if err != nil { + return 0, 0, err } - return curr, epoch, nil + return networkInfo.CurrentEpoch(), epoch, nil } // Container implements layer.FrostFS interface method. diff --git a/internal/frostfs/util/util.go b/internal/frostfs/util/util.go index 444504b1..00a99059 100644 --- a/internal/frostfs/util/util.go +++ b/internal/frostfs/util/util.go @@ -2,9 +2,12 @@ package util import ( "fmt" + "math" "strings" + "time" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" "github.com/nspcc-dev/neo-go/pkg/util" ) @@ -32,3 +35,39 @@ func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) return nns.ResolveContractHash(domain) } + +func TimeToEpoch(ni *netmap.NetworkInfo, now, t time.Time) (uint64, error) { + duration := t.Sub(now) + durationAbs := duration.Abs() + + durEpoch := ni.EpochDuration() + if durEpoch == 0 { + return 0, fmt.Errorf("epoch duration is missing or zero") + } + + msPerEpoch := durEpoch * uint64(ni.MsPerBlock()) + epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch + + if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 { + epochLifetime++ + } + + curr := ni.CurrentEpoch() + + var epoch uint64 + if duration > 0 { + if epochLifetime >= math.MaxUint64-curr { + epoch = math.MaxUint64 + } else { + epoch = curr + epochLifetime + } + } else { + if epochLifetime >= curr { + epoch = 0 + } else { + epoch = curr - epochLifetime + } + } + + return epoch, nil +} ```
} }
func getContentMD5(reader io.Reader) ([]byte, error) {
hash := md5.New()
_, err := io.Copy(hash, reader)
if err != nil {
return nil, err
}
return hash.Sum(nil), nil
}

View file

@ -1,6 +1,7 @@
package handler package handler
import ( import (
"bytes"
"crypto/md5" "crypto/md5"
"crypto/rand" "crypto/rand"
"encoding/base64" "encoding/base64"
@ -376,6 +377,11 @@ func TestPutBucketLifecycleInvalidMD5(t *testing.T) {
hc.Handler().PutBucketLifecycleHandler(w, r) hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5)) assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMissingContentMD5))
w, r = prepareTestRequest(hc, bktName, "", lifecycle)
r.Header.Set(api.ContentMD5, "")
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
w, r = prepareTestRequest(hc, bktName, "", lifecycle) w, r = prepareTestRequest(hc, bktName, "", lifecycle)
r.Header.Set(api.ContentMD5, "some-hash") r.Header.Set(api.ContentMD5, "some-hash")
hc.Handler().PutBucketLifecycleHandler(w, r) hc.Handler().PutBucketLifecycleHandler(w, r)
@ -388,8 +394,14 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) {
bktName := "bucket-lifecycle-invalid-xml" bktName := "bucket-lifecycle-invalid-xml"
createBucket(hc, bktName) createBucket(hc, bktName)
w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{}) cfg := &data.CORSConfiguration{}
r.Header.Set(api.ContentMD5, "") body, err := xml.Marshal(cfg)
require.NoError(t, err)
contentMD5, err := getContentMD5(bytes.NewReader(body))
require.NoError(t, err)
w, r := prepareTestRequest(hc, bktName, "", cfg)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(contentMD5))
hc.Handler().PutBucketLifecycleHandler(w, r) hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML)) assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
} }