feature/lifecycle_convert_date #516

Merged
alexvanin merged 2 commits from mbiryukova/frostfs-s3-gw:feature/lifecycle_convert_date into master 2024-10-22 14:21:50 +00:00
9 changed files with 179 additions and 63 deletions

View file

@ -29,6 +29,7 @@ type (
LifecycleExpiration struct {
Date string `xml:"Date,omitempty"`
Days *int `xml:"Days,omitempty"`
Epoch *uint64 `xml:"Epoch,omitempty"`
dkirillov marked this conversation as resolved Outdated

Should we add the similar field (Epochs) to AbortIncompleteMultipartUpload and NonCurrentVersionExpiration

Should we add the similar field (`Epochs`) to `AbortIncompleteMultipartUpload` and `NonCurrentVersionExpiration`

I’m not sure if this is necessary, current date is not involved in days conversion

I’m not sure if this is necessary, current date is not involved in days conversion

Probably it's better use duration not in days but in epochs (abort multipart not after 2 days but after N epochs after initiation)

Probably it's better use duration not in days but in epochs (abort multipart not after 2 days but after N epochs after initiation)

Otherwise, it seems we will have some inconsistency. For example it we tick some epochs (e.g. in tests) we will expire objects that have specific expiration date. But we won't expire object if they have Days expiration

@alexvanin

Otherwise, it seems we will have some inconsistency. For example it we tick some epochs (e.g. in tests) we will expire objects that have specific expiration date. But we won't expire object if they have `Days` expiration @alexvanin
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"`
}

View file

@ -2,6 +2,9 @@ package handler
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"fmt"
"io"
"net/http"
@ -12,6 +15,8 @@ import (
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
)
const (
@ -56,9 +61,9 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
h.logAndSendError(w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
@ -68,7 +73,30 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
return
}
if err = checkLifecycleConfiguration(cfg); err != nil {
bodyMD5, err := getContentMD5(&buf)
if err != nil {
h.logAndSendError(w, "could not get content md5", reqInfo, err)
return
}
if !bytes.Equal(headerMD5, bodyMD5) {
h.logAndSendError(w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(w, "could not get bucket info", reqInfo, err)
return
}
networkInfo, err := h.obj.GetNetworkInfo(ctx)
if err != nil {
h.logAndSendError(w, "could not get network info", reqInfo, err)
return
}
if err = checkLifecycleConfiguration(ctx, cfg, &networkInfo); err != nil {
h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
@ -76,8 +104,6 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
params := &layer.PutBucketLifecycleParams{
BktInfo: bktInfo,
LifecycleCfg: cfg,
LifecycleReader: &buf,
MD5Hash: r.Header.Get(api.ContentMD5),
}
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
@ -110,13 +136,15 @@ func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Re
w.WriteHeader(http.StatusNoContent)
}
func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
func checkLifecycleConfiguration(ctx context.Context, cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error {
now := layer.TimeNow(ctx)
if len(cfg.Rules) > maxRules {
return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
}
ids := make(map[string]struct{}, len(cfg.Rules))
for _, rule := range cfg.Rules {
for i, rule := range cfg.Rules {
if _, ok := ids[rule.ID]; ok && rule.ID != "" {
return fmt.Errorf("duplicate 'ID': %s", rule.ID)
}
@ -160,9 +188,19 @@ func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration) error {
return fmt.Errorf("expiration days must be a positive integer: %d", *rule.Expiration.Days)
}
if _, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date); rule.Expiration.Date != "" && err != nil {
if rule.Expiration.Date != "" {
parsedTime, err := time.Parse("2006-01-02T15:04:05Z", rule.Expiration.Date)
if err != nil {
return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
}
epoch, err := util.TimeToEpoch(ni, now, parsedTime)
if err != nil {
return fmt.Errorf("convert time to epoch: %w", err)
}
cfg.Rules[i].Expiration.Epoch = &epoch
}
}
if rule.NonCurrentVersionExpiration != nil {
@ -233,3 +271,12 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
return nil
dkirillov marked this conversation as resolved Outdated

Consider reusing this

diff --git a/api/handler/lifecycle.go b/api/handler/lifecycle.go
index bd89fe9f..720e37e2 100644
--- a/api/handler/lifecycle.go
+++ b/api/handler/lifecycle.go
@@ -2,11 +2,11 @@ package handler
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
 	"encoding/base64"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
 	"time"
 
@@ -15,6 +15,7 @@ import (
 	apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 )
 
@@ -95,7 +96,7 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	if err = checkLifecycleConfiguration(cfg, &networkInfo); err != nil {
+	if err = checkLifecycleConfiguration(ctx, cfg, &networkInfo); err != nil {
 		h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
 		return
 	}
@@ -135,7 +136,8 @@ func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Re
 	w.WriteHeader(http.StatusNoContent)
 }
 
-func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error {
+func checkLifecycleConfiguration(ctx context.Context, cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error {
+	now := layer.TimeNow(ctx)
 	if len(cfg.Rules) > maxRules {
 		return fmt.Errorf("number of rules cannot be greater than %d", maxRules)
 	}
@@ -191,7 +193,7 @@ func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.Ne
 					return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date)
 				}
 
-				epoch, err := timeToEpoch(ni, parsedTime)
+				epoch, err := util.TimeToEpoch(ni, now, parsedTime)
 				if err != nil {
 					return fmt.Errorf("convert time to epoch: %w", err)
 				}
@@ -269,42 +271,6 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error {
 	return nil
 }
 
-func timeToEpoch(ni *netmap.NetworkInfo, t time.Time) (uint64, error) {
-	duration := t.Sub(time.Now())
-	durationAbs := duration.Abs()
-
-	durEpoch := ni.EpochDuration()
-	if durEpoch == 0 {
-		return 0, fmt.Errorf("epoch duration is missing or zero")
-	}
-
-	msPerEpoch := durEpoch * uint64(ni.MsPerBlock())
-	epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch
-
-	if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 {
-		epochLifetime++
-	}
-
-	curr := ni.CurrentEpoch()
-
-	var epoch uint64
-	if duration > 0 {
-		if epochLifetime >= math.MaxUint64-curr {
-			epoch = math.MaxUint64
-		} else {
-			epoch = curr + epochLifetime
-		}
-	} else {
-		if epochLifetime >= curr {
-			epoch = 0
-		} else {
-			epoch = curr - epochLifetime
-		}
-	}
-
-	return epoch, nil
-}
-
 func getContentMD5(reader io.Reader) ([]byte, error) {
 	hash := md5.New()
 	buf := make([]byte, 64*1024)
diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go
index 95ac17bf..03f8a7f6 100644
--- a/internal/frostfs/frostfs.go
+++ b/internal/frostfs/frostfs.go
@@ -5,13 +5,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"strconv"
 	"time"
 
 	objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
 	frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@@ -54,8 +54,7 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS {
 
 // TimeToEpoch implements layer.FrostFS interface method.
 func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
-	dur := futureTime.Sub(now)
-	if dur < 0 {
+	if futureTime.Before(now) {
 		return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)",
 			futureTime.Format(time.RFC3339), now.Format(time.RFC3339))
 	}
@@ -65,27 +64,12 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u
 		return 0, 0, handleObjectError("get network info via client", err)
 	}
 
-	durEpoch := networkInfo.EpochDuration()
-	if durEpoch == 0 {
-		return 0, 0, errors.New("epoch duration is missing or zero")
-	}
-
-	curr := networkInfo.CurrentEpoch()
-	msPerEpoch := durEpoch * uint64(networkInfo.MsPerBlock())
-
-	epochLifetime := uint64(dur.Milliseconds()) / msPerEpoch
-	if uint64(dur.Milliseconds())%msPerEpoch != 0 {
-		epochLifetime++
-	}
-
-	var epoch uint64
-	if epochLifetime >= math.MaxUint64-curr {
-		epoch = math.MaxUint64
-	} else {
-		epoch = curr + epochLifetime
+	epoch, err := util.TimeToEpoch(&networkInfo, now, futureTime)
+	if err != nil {
+		return 0, 0, err
 	}
 
-	return curr, epoch, nil
+	return networkInfo.CurrentEpoch(), epoch, nil
 }
 
 // Container implements layer.FrostFS interface method.
diff --git a/internal/frostfs/util/util.go b/internal/frostfs/util/util.go
index 444504b1..00a99059 100644
--- a/internal/frostfs/util/util.go
+++ b/internal/frostfs/util/util.go
@@ -2,9 +2,12 @@ package util
 
 import (
 	"fmt"
+	"math"
 	"strings"
+	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
 	"github.com/nspcc-dev/neo-go/pkg/util"
 )
@@ -32,3 +35,39 @@ func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error)
 
 	return nns.ResolveContractHash(domain)
 }
+
+func TimeToEpoch(ni *netmap.NetworkInfo, now, t time.Time) (uint64, error) {
+	duration := t.Sub(now)
+	durationAbs := duration.Abs()
+
+	durEpoch := ni.EpochDuration()
+	if durEpoch == 0 {
+		return 0, fmt.Errorf("epoch duration is missing or zero")
+	}
+
+	msPerEpoch := durEpoch * uint64(ni.MsPerBlock())
+	epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch
+
+	if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 {
+		epochLifetime++
+	}
+
+	curr := ni.CurrentEpoch()
+
+	var epoch uint64
+	if duration > 0 {
+		if epochLifetime >= math.MaxUint64-curr {
+			epoch = math.MaxUint64
+		} else {
+			epoch = curr + epochLifetime
+		}
+	} else {
+		if epochLifetime >= curr {
+			epoch = 0
+		} else {
+			epoch = curr - epochLifetime
+		}
+	}
+
+	return epoch, nil
+}

Consider reusing this ```diff diff --git a/api/handler/lifecycle.go b/api/handler/lifecycle.go index bd89fe9f..720e37e2 100644 --- a/api/handler/lifecycle.go +++ b/api/handler/lifecycle.go @@ -2,11 +2,11 @@ package handler import ( "bytes" + "context" "crypto/md5" "encoding/base64" "fmt" "io" - "math" "net/http" "time" @@ -15,6 +15,7 @@ import ( apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) @@ -95,7 +96,7 @@ func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Reque return } - if err = checkLifecycleConfiguration(cfg, &networkInfo); err != nil { + if err = checkLifecycleConfiguration(ctx, cfg, &networkInfo); err != nil { h.logAndSendError(w, "invalid lifecycle configuration", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) return } @@ -135,7 +136,8 @@ func (h *handler) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Re w.WriteHeader(http.StatusNoContent) } -func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error { +func checkLifecycleConfiguration(ctx context.Context, cfg *data.LifecycleConfiguration, ni *netmap.NetworkInfo) error { + now := layer.TimeNow(ctx) if len(cfg.Rules) > maxRules { return fmt.Errorf("number of rules cannot be greater than %d", maxRules) } @@ -191,7 +193,7 @@ func checkLifecycleConfiguration(cfg *data.LifecycleConfiguration, ni *netmap.Ne return fmt.Errorf("invalid value of expiration date: %s", rule.Expiration.Date) } - epoch, err := timeToEpoch(ni, parsedTime) + epoch, err := util.TimeToEpoch(ni, now, parsedTime) if err != nil { return fmt.Errorf("convert time to epoch: %w", err) } @@ -269,42 +271,6 @@ func checkLifecycleRuleFilter(filter *data.LifecycleRuleFilter) error { return nil } -func timeToEpoch(ni *netmap.NetworkInfo, t time.Time) (uint64, error) { - duration := t.Sub(time.Now()) - durationAbs := duration.Abs() - - durEpoch := ni.EpochDuration() - if durEpoch == 0 { - return 0, fmt.Errorf("epoch duration is missing or zero") - } - - msPerEpoch := durEpoch * uint64(ni.MsPerBlock()) - epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch - - if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 { - epochLifetime++ - } - - curr := ni.CurrentEpoch() - - var epoch uint64 - if duration > 0 { - if epochLifetime >= math.MaxUint64-curr { - epoch = math.MaxUint64 - } else { - epoch = curr + epochLifetime - } - } else { - if epochLifetime >= curr { - epoch = 0 - } else { - epoch = curr - epochLifetime - } - } - - return epoch, nil -} - func getContentMD5(reader io.Reader) ([]byte, error) { hash := md5.New() buf := make([]byte, 64*1024) diff --git a/internal/frostfs/frostfs.go b/internal/frostfs/frostfs.go index 95ac17bf..03f8a7f6 100644 --- a/internal/frostfs/frostfs.go +++ b/internal/frostfs/frostfs.go @@ -5,13 +5,13 @@ import ( "errors" "fmt" "io" - "math" "strconv" "time" objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs" frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -54,8 +54,7 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS { // TimeToEpoch implements layer.FrostFS interface method. func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) { - dur := futureTime.Sub(now) - if dur < 0 { + if futureTime.Before(now) { return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)", futureTime.Format(time.RFC3339), now.Format(time.RFC3339)) } @@ -65,27 +64,12 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u return 0, 0, handleObjectError("get network info via client", err) } - durEpoch := networkInfo.EpochDuration() - if durEpoch == 0 { - return 0, 0, errors.New("epoch duration is missing or zero") - } - - curr := networkInfo.CurrentEpoch() - msPerEpoch := durEpoch * uint64(networkInfo.MsPerBlock()) - - epochLifetime := uint64(dur.Milliseconds()) / msPerEpoch - if uint64(dur.Milliseconds())%msPerEpoch != 0 { - epochLifetime++ - } - - var epoch uint64 - if epochLifetime >= math.MaxUint64-curr { - epoch = math.MaxUint64 - } else { - epoch = curr + epochLifetime + epoch, err := util.TimeToEpoch(&networkInfo, now, futureTime) + if err != nil { + return 0, 0, err } - return curr, epoch, nil + return networkInfo.CurrentEpoch(), epoch, nil } // Container implements layer.FrostFS interface method. diff --git a/internal/frostfs/util/util.go b/internal/frostfs/util/util.go index 444504b1..00a99059 100644 --- a/internal/frostfs/util/util.go +++ b/internal/frostfs/util/util.go @@ -2,9 +2,12 @@ package util import ( "fmt" + "math" "strings" + "time" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" "github.com/nspcc-dev/neo-go/pkg/util" ) @@ -32,3 +35,39 @@ func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) return nns.ResolveContractHash(domain) } + +func TimeToEpoch(ni *netmap.NetworkInfo, now, t time.Time) (uint64, error) { + duration := t.Sub(now) + durationAbs := duration.Abs() + + durEpoch := ni.EpochDuration() + if durEpoch == 0 { + return 0, fmt.Errorf("epoch duration is missing or zero") + } + + msPerEpoch := durEpoch * uint64(ni.MsPerBlock()) + epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch + + if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 { + epochLifetime++ + } + + curr := ni.CurrentEpoch() + + var epoch uint64 + if duration > 0 { + if epochLifetime >= math.MaxUint64-curr { + epoch = math.MaxUint64 + } else { + epoch = curr + epochLifetime + } + } else { + if epochLifetime >= curr { + epoch = 0 + } else { + epoch = curr - epochLifetime + } + } + + return epoch, nil +} ```
}
func getContentMD5(reader io.Reader) ([]byte, error) {
hash := md5.New()
_, err := io.Copy(hash, reader)
if err != nil {
return nil, err
}
return hash.Sum(nil), nil
}

View file

@ -1,6 +1,7 @@
package handler
import (
"bytes"
"crypto/md5"
"crypto/rand"
"encoding/base64"
@ -393,8 +394,14 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) {
bktName := "bucket-lifecycle-invalid-xml"
createBucket(hc, bktName)
w, r := prepareTestRequest(hc, bktName, "", &data.CORSConfiguration{})
r.Header.Set(api.ContentMD5, "")
cfg := &data.CORSConfiguration{}
body, err := xml.Marshal(cfg)
require.NoError(t, err)
contentMD5, err := getContentMD5(bytes.NewReader(body))
require.NoError(t, err)
w, r := prepareTestRequest(hc, bktName, "", cfg)
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString(contentMD5))
hc.Handler().PutBucketLifecycleHandler(w, r)
assertS3Error(hc.t, w, apierr.GetAPIError(apierr.ErrMalformedXML))
}

View file

@ -413,6 +413,8 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm frostfs.PrmObjectSear
func (t *TestFrostFS) NetworkInfo(context.Context) (netmap.NetworkInfo, error) {
ni := netmap.NetworkInfo{}
ni.SetCurrentEpoch(t.currentEpoch)
ni.SetEpochDuration(60)
ni.SetMsPerBlock(1000)
return ni, nil
}

View file

@ -3,11 +3,9 @@ package layer
import (
"bytes"
"context"
"encoding/base64"
"encoding/xml"
"errors"
"fmt"
"io"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
@ -21,14 +19,17 @@ import (
type PutBucketLifecycleParams struct {
BktInfo *data.BucketInfo
LifecycleCfg *data.LifecycleConfiguration
LifecycleReader io.Reader
CopiesNumbers []uint32
MD5Hash string
}
func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucketLifecycleParams) error {
cfgBytes, err := xml.Marshal(p.LifecycleCfg)
if err != nil {
return fmt.Errorf("marshal lifecycle configuration: %w", err)
}
prm := frostfs.PrmObjectCreate{
Payload: p.LifecycleReader,
Payload: bytes.NewReader(cfgBytes),
Filepath: p.BktInfo.LifecycleConfigurationObjectName(),
CreationTime: TimeNow(ctx),
}
@ -49,17 +50,6 @@ func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucke
return fmt.Errorf("put lifecycle object: %w", err)
}
hashBytes, err := base64.StdEncoding.DecodeString(p.MD5Hash)
if err != nil {
return apierr.GetAPIError(apierr.ErrInvalidDigest)
}
if !bytes.Equal(hashBytes, createdObj.MD5Sum) {
n.deleteLifecycleObject(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
return apierr.GetAPIError(apierr.ErrInvalidDigest)
}
objsToDelete, err := n.treeService.PutBucketLifecycleConfiguration(ctx, p.BktInfo, newAddress(lifecycleBkt.CID, createdObj.ID))
objsToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objsToDeleteNotFound {
@ -129,6 +119,12 @@ func (n *Layer) GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *da
n.cache.PutLifecycleConfiguration(owner, bktInfo, lifecycleCfg)
for i := range lifecycleCfg.Rules {
if lifecycleCfg.Rules[i].Expiration != nil {
lifecycleCfg.Rules[i].Expiration.Epoch = nil
}
}
return lifecycleCfg, nil
}

View file

@ -1,9 +1,7 @@
package layer
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/xml"
"testing"
@ -44,8 +42,6 @@ func TestBucketLifecycle(t *testing.T) {
err = tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{
BktInfo: tc.bktInfo,
LifecycleCfg: lifecycle,
LifecycleReader: bytes.NewReader(lifecycleBytes),
MD5Hash: base64.StdEncoding.EncodeToString(hash.Sum(nil)),
})
require.NoError(t, err)

View file

@ -5,13 +5,13 @@ import (
"errors"
"fmt"
"io"
"math"
"strconv"
"time"
objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -54,8 +54,7 @@ func NewFrostFS(p *pool.Pool, key *keys.PrivateKey) *FrostFS {
// TimeToEpoch implements layer.FrostFS interface method.
func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
dur := futureTime.Sub(now)
if dur < 0 {
if futureTime.Before(now) {
return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)",
futureTime.Format(time.RFC3339), now.Format(time.RFC3339))
}
@ -65,27 +64,12 @@ func (x *FrostFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (u
return 0, 0, handleObjectError("get network info via client", err)
}
durEpoch := networkInfo.EpochDuration()
if durEpoch == 0 {
return 0, 0, errors.New("epoch duration is missing or zero")
epoch, err := util.TimeToEpoch(&networkInfo, now, futureTime)
if err != nil {
return 0, 0, err
}
curr := networkInfo.CurrentEpoch()
msPerEpoch := durEpoch * uint64(networkInfo.MsPerBlock())
epochLifetime := uint64(dur.Milliseconds()) / msPerEpoch
if uint64(dur.Milliseconds())%msPerEpoch != 0 {
epochLifetime++
}
var epoch uint64
if epochLifetime >= math.MaxUint64-curr {
epoch = math.MaxUint64
} else {
epoch = curr + epochLifetime
}
return curr, epoch, nil
return networkInfo.CurrentEpoch(), epoch, nil
}
// Container implements layer.FrostFS interface method.

View file

@ -2,9 +2,12 @@ package util
import (
"fmt"
"math"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
"github.com/nspcc-dev/neo-go/pkg/util"
)
@ -32,3 +35,39 @@ func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error)
return nns.ResolveContractHash(domain)
}
func TimeToEpoch(ni *netmap.NetworkInfo, now, t time.Time) (uint64, error) {
duration := t.Sub(now)
durationAbs := duration.Abs()
durEpoch := ni.EpochDuration()
if durEpoch == 0 {
return 0, fmt.Errorf("epoch duration is missing or zero")
}
msPerEpoch := durEpoch * uint64(ni.MsPerBlock())
epochLifetime := uint64(durationAbs.Milliseconds()) / msPerEpoch
if uint64(durationAbs.Milliseconds())%msPerEpoch != 0 {
epochLifetime++
}
curr := ni.CurrentEpoch()
var epoch uint64
if duration > 0 {
if epochLifetime >= math.MaxUint64-curr {
epoch = math.MaxUint64
} else {
epoch = curr + epochLifetime
}
} else {
if epochLifetime >= curr {
epoch = 0
} else {
epoch = curr - epochLifetime
}
}
return epoch, nil
}

View file

@ -0,0 +1,44 @@
package util
import (
"testing"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/stretchr/testify/require"
)
func TestTimeToEpoch(t *testing.T) {
ni := netmap.NetworkInfo{}
ni.SetCurrentEpoch(10)
_, err := TimeToEpoch(&ni, time.Now(), time.Now())
require.Error(t, err)
ni.SetEpochDuration(60)
ni.SetMsPerBlock(1000)
epoch, err := TimeToEpoch(&ni, time.Now(), time.Now())
require.NoError(t, err)
require.Equal(t, uint64(10), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(30*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(11), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(90*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(12), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-30*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(9), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-90*time.Second))
require.NoError(t, err)
require.Equal(t, uint64(8), epoch)
epoch, err = TimeToEpoch(&ni, time.Now(), time.Now().Add(-10*time.Minute))
require.NoError(t, err)
require.Equal(t, uint64(0), epoch)
}