pogpp/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#726] Use client time on regular requests

Browse source 
Use `X-Amz-Date` header as `now` when
* compute expiration epoch
* set Timestamp for object and container
* forming locks
* send notifications

Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-11-08 12:12:55 +03:00 committed by Alex Vanin
parent d3702f86d1
commit 094eb12578
19 changed files with 106 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
api/auth/center.go
View file

@ -33,7 +33,13 @@ var postPolicyCredentialRegexp = regexp.MustCompile(`(?P<access_key_id>[^/]+)/(?
type (
// Center is a user authentication interface.
Center interface {
Authenticate(request *http.Request) (*accessbox.Box, error)
Authenticate(request *http.Request) (*Box, error)
}
// Box contains access box and additional info.
Box struct {
AccessBox *accessbox.Box
ClientTime time.Time
}
center struct {
@ -126,11 +132,12 @@ func (a *authHeader) getAddress() (oid.Address, error) {
return addr, nil
}
func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
func (c *center) Authenticate(r *http.Request) (*Box, error) {
var (
err error
authHdr *authHeader
signatureDateTimeStr string
needClientTime bool
)
queryValues := r.URL.Query()
@ -166,6 +173,7 @@ func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
return nil, err
}
signatureDateTimeStr = r.Header.Get(AmzDate)
needClientTime = true
}
signatureDateTime, err := time.Parse("20060102T150405Z", signatureDateTimeStr)
@ -192,7 +200,12 @@ func (c *center) Authenticate(r *http.Request) (*accessbox.Box, error) {
return nil, err
}
return box, nil
result := &Box{AccessBox: box}
if needClientTime {
result.ClientTime = signatureDateTime
}
return result, nil
}
func (c center) checkAccessKeyID(accessKeyID string) error {
@ -209,7 +222,7 @@ func (c center) checkAccessKeyID(accessKeyID string) error {
return apiErrors.GetAPIError(apiErrors.ErrAccessDenied)
}
func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
func (c *center) checkFormData(r *http.Request) (*Box, error) {
if err := r.ParseMultipartForm(maxFormSizeMemory); err != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrInvalidArgument)
}
@ -251,7 +264,7 @@ func (c *center) checkFormData(r *http.Request) (*accessbox.Box, error) {
return nil, apiErrors.GetAPIError(apiErrors.ErrSignatureDoesNotMatch)
}
return box, nil
return &Box{AccessBox: box}, nil
}
func cloneRequest(r *http.Request, authHeader *authHeader) *http.Request {

3
api/handler/api.go
View file

@ -2,6 +2,7 @@ package handler
import (
"errors"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
@ -19,7 +20,7 @@ type (
Notificator interface {
SendNotifications(topics map[string]string, p *SendNotificationParams) error
SendTestNotification(topic, bucketName, requestID, HostID string) error
SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error
}
// Config contains data which handler needs to keep.

2
api/handler/copy.go
View file

@ -184,7 +184,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
CopiesNuber: copiesNumber,
}
params.Lock, err = formObjectLock(dstBktInfo, settings.LockConfiguration, r.Header)
params.Lock, err = formObjectLock(r.Context(), dstBktInfo, settings.LockConfiguration, r.Header)
if err != nil {
h.logAndSendError(w, "could not form object lock", reqInfo, err)
return

13
api/handler/locking.go
View file

@ -1,6 +1,7 @@
package handler
import (
"context"
"encoding/xml"
"fmt"
"net/http"
@ -208,7 +209,7 @@ func (h *handler) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Reque
return
}
lock, err := formObjectLockFromRetention(retention, r.Header)
lock, err := formObjectLockFromRetention(r.Context(), retention, r.Header)
if err != nil {
h.logAndSendError(w, "invalid retention configuration", reqInfo, err)
return
@ -300,7 +301,7 @@ func checkLockConfiguration(conf *data.ObjectLockConfiguration) error {
return nil
}
func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
func formObjectLock(ctx context.Context, bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConfiguration, header http.Header) (*data.ObjectLock, error) {
if !bktInfo.ObjectLockEnabled {
if existLockHeaders(header) {
return nil, apiErrors.GetAPIError(apiErrors.ErrObjectLockConfigurationNotFound)
@ -318,7 +319,7 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
retention := &data.RetentionLock{}
defaultRetention := defaultConfig.Rule.DefaultRetention
retention.IsCompliance = defaultRetention.Mode == complianceMode
now := time.Now()
now := layer.TimeNow(ctx)
if defaultRetention.Days != 0 {
retention.Until = now.Add(time.Duration(defaultRetention.Days) * dayDuration)
} else {
@ -370,7 +371,7 @@ func formObjectLock(bktInfo *data.BucketInfo, defaultConfig *data.ObjectLockConf
objectLock.Retention.ByPassedGovernance = bypass
}
if objectLock.Retention.Until.Before(time.Now()) {
if objectLock.Retention.Until.Before(layer.TimeNow(ctx)) {
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
}
}
@ -384,7 +385,7 @@ func existLockHeaders(header http.Header) bool {
header.Get(api.AmzObjectLockRetainUntilDate) != ""
}
func formObjectLockFromRetention(retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
func formObjectLockFromRetention(ctx context.Context, retention *data.Retention, header http.Header) (*data.ObjectLock, error) {
if retention.Mode != governanceMode && retention.Mode != complianceMode {
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
}
@ -394,7 +395,7 @@ func formObjectLockFromRetention(retention *data.Retention, header http.Header)
return nil, apiErrors.GetAPIError(apiErrors.ErrMalformedXML)
}
if retentionDate.Before(time.Now()) {
if retentionDate.Before(layer.TimeNow(ctx)) {
return nil, apiErrors.GetAPIError(apiErrors.ErrPastObjectLockRetainDate)
}

8
api/handler/locking_test.go
View file

@ -19,6 +19,8 @@ import (
const defaultURL = "http://localhost/"
func TestFormObjectLock(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
bktInfo *data.BucketInfo
@ -73,7 +75,7 @@ func TestFormObjectLock(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
actualObjLock, err := formObjectLock(tc.bktInfo, tc.config, tc.header)
actualObjLock, err := formObjectLock(ctx, tc.bktInfo, tc.config, tc.header)
if tc.expectedError {
require.Error(t, err)
return
@ -86,6 +88,8 @@ func TestFormObjectLock(t *testing.T) {
}
func TestFormObjectLockFromRetention(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
retention *data.Retention
@ -132,7 +136,7 @@ func TestFormObjectLockFromRetention(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
actualObjLock, err := formObjectLockFromRetention(tc.retention, tc.header)
actualObjLock, err := formObjectLockFromRetention(ctx, tc.retention, tc.header)
if tc.expectedError {
require.Error(t, err)
return

10
api/handler/notifications.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"strings"
"time"
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-s3-gw/api"
@ -22,6 +23,7 @@ type (
BktInfo *data.BucketInfo
ReqInfo *api.ReqInfo
User string
Time time.Time
}
NotificationConfiguration struct {
@ -107,7 +109,7 @@ func (h *handler) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Re
return
}
if _, err = h.checkBucketConfiguration(conf, reqInfo); err != nil {
if _, err = h.checkBucketConfiguration(r.Context(), conf, reqInfo); err != nil {
h.logAndSendError(w, "couldn't check bucket configuration", reqInfo, err)
return
}
@ -164,13 +166,15 @@ func (h *handler) sendNotifications(ctx context.Context, p *SendNotificationPara
p.User = bearer.ResolveIssuer(*box.Gate.BearerToken).EncodeToString()
}
p.Time = layer.TimeNow(ctx)
topics := filterSubjects(conf, p.Event, p.NotificationInfo.Name)
return h.notificator.SendNotifications(topics, p)
}
// checkBucketConfiguration checks notification configuration and generates an ID for configurations with empty ids.
func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.NotificationConfiguration, r *api.ReqInfo) (completed bool, err error) {
if conf == nil {
return
}
@ -189,7 +193,7 @@ func (h *handler) checkBucketConfiguration(conf *data.NotificationConfiguration,
}
if h.cfg.NotificatorEnabled {
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host); err != nil {
if err = h.notificator.SendTestNotification(q.QueueArn, r.BucketName, r.RequestID, r.Host, layer.TimeNow(ctx)); err != nil {
return
}
} else {

2
api/handler/put.go
View file

@ -240,7 +240,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
return
}
params.Lock, err = formObjectLock(bktInfo, settings.LockConfiguration, r.Header)
params.Lock, err = formObjectLock(r.Context(), bktInfo, settings.LockConfiguration, r.Header)
if err != nil {
h.logAndSendError(w, "could not form object lock", reqInfo, err)
return

4
api/layer/container.go
View file

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"strconv"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/data"
@ -116,7 +115,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
bktInfo := &data.BucketInfo{
Name: p.Name,
Owner: ownerID,
Created: time.Now(), // this can be a little incorrect since the real time is set later
Created: TimeNow(ctx),
LocationConstraint: p.LocationConstraint,
ObjectLockEnabled: p.ObjectLockEnabled,
}
@ -138,6 +137,7 @@ func (n *layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
Policy: p.Policy,
Name: p.Name,
SessionToken: p.SessionContainerCreation,
CreationTime: bktInfo.Created,
AdditionalAttributes: attributes,
})
if err != nil {

1
api/layer/cors.go
View file

@ -41,6 +41,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
Creator: p.BktInfo.Owner,
Payload: p.Reader,
Filepath: p.BktInfo.CORSObjectName(),
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}

11
api/layer/layer.go
View file

@ -306,6 +306,15 @@ func IsAuthenticatedRequest(ctx context.Context) bool {
return ok
}
// TimeNow returns client time from request or time.Now().
func TimeNow(ctx context.Context) time.Time {
if now, ok := ctx.Value(api.ClientTime).(time.Time); ok {
return now
}
return time.Now()
}
// Owner returns owner id from BearerToken (context) or from client owner.
func (n *layer) Owner(ctx context.Context) user.ID {
if bd, ok := ctx.Value(api.BoxData).(*accessbox.Box); ok && bd != nil && bd.Gate != nil && bd.Gate.BearerToken != nil {
@ -565,7 +574,7 @@ func (n *layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
FilePath: obj.Name,
},
DeleteMarker: &data.DeleteMarkerInfo{
Created: time.Now(),
Created: TimeNow(ctx),
Owner: n.Owner(ctx),
},
IsUnversioned: settings.VersioningSuspended(),

5
api/layer/multipart_upload.go
View file

@ -143,7 +143,7 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
Key: p.Info.Key,
UploadID: p.Info.UploadID,
Owner: n.Owner(ctx),
Created: time.Now(),
Created: TimeNow(ctx),
Meta: make(map[string]string, metaSize),
CopiesNumber: p.CopiesNumber,
}
@ -205,6 +205,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
Creator: bktInfo.Owner,
Attributes: make([][2]string, 2),
Payload: p.Reader,
CreationTime: TimeNow(ctx),
CopiesNumber: multipartInfo.CopiesNumber,
}
@ -234,7 +235,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
OID: id,
Size: decSize,
ETag: hex.EncodeToString(hash),
Created: time.Now(),
Created: prm.CreationTime,
}
oldPartID, err := n.treeService.AddPart(ctx, bktInfo, multipartInfo.ID, partInfo)

14
api/layer/neofs.go
View file

@ -30,6 +30,9 @@ type PrmContainerCreate struct {
// Name for the container.
Name string
// CreationTime value for Timestamp attribute
CreationTime time.Time
// Token of the container's creation session. Nil means session absence.
SessionToken *session.Container
@ -94,6 +97,9 @@ type PrmObjectCreate struct {
// Key-value object attributes.
Attributes [][2]string
// Value for Timestamp attribute (optional).
CreationTime time.Time
// List of ids to lock (optional).
Locks []oid.ID
@ -204,11 +210,11 @@ type NeoFS interface {
// It returns any error encountered which prevented the removal request from being sent.
DeleteObject(context.Context, PrmObjectDelete) error
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided time.
// TimeToEpoch computes current epoch and the epoch that corresponds to the provided now and future time.
// Note:
// * time must be in the future
// * time will be ceil rounded to match epoch
// * future time must be after the now
// * future time will be ceil rounded to match epoch
//
// It returns any error encountered which prevented computing epochs.
TimeToEpoch(context.Context, time.Time) (uint64, uint64, error)
TimeToEpoch(ctx context.Context, now time.Time, future time.Time) (uint64, uint64, error)
}

11
api/layer/neofs_mock.go
View file

@ -75,7 +75,12 @@ func (t *TestNeoFS) CreateContainer(_ context.Context, prm PrmContainerCreate) (
cnr.SetOwner(prm.Creator)
cnr.SetPlacementPolicy(prm.Policy)
cnr.SetBasicACL(prm.BasicACL)
container.SetCreationTime(&cnr, time.Now())
creationTime := prm.CreationTime
if creationTime.IsZero() {
creationTime = time.Now()
}
container.SetCreationTime(&cnr, creationTime)
if prm.Name != "" {
var d container.Domain
@ -235,8 +240,8 @@ func (t *TestNeoFS) DeleteObject(ctx context.Context, prm PrmObjectDelete) error
return nil
}
func (t *TestNeoFS) TimeToEpoch(_ context.Context, futureTime time.Time) (uint64, uint64, error) {
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Second()), nil
func (t *TestNeoFS) TimeToEpoch(_ context.Context, now, futureTime time.Time) (uint64, uint64, error) {
return t.currentEpoch, t.currentEpoch + uint64(futureTime.Sub(now).Seconds()), nil
}
func (t *TestNeoFS) AllObjects(cnrID cid.ID) []oid.ID {

1
api/layer/notifications.go
View file

@ -30,6 +30,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
Creator: p.BktInfo.Owner,
Payload: bytes.NewReader(confXML),
Filepath: p.BktInfo.NotificationConfigurationObjectName(),
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}

4
api/layer/object.go
View file

@ -13,7 +13,6 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/minio/sio"
"github.com/nspcc-dev/neofs-s3-gw/api"
@ -234,6 +233,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
PayloadSize: uint64(p.Size),
Filepath: p.Object,
Payload: r,
CreationTime: TimeNow(ctx),
CopiesNumber: p.CopiesNumber,
}
@ -281,7 +281,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
Bucket: p.BktInfo.Name,
Name: p.Object,
Size: p.Size,
Created: time.Now(),
Created: prm.CreationTime,
Headers: p.Header,
ContentType: p.Header[api.ContentType],
HashSum: newVersion.ETag,

3
api/layer/system_object.go
View file

@ -116,6 +116,7 @@ func (n *layer) putLockObject(ctx context.Context, bktInfo *data.BucketInfo, obj
Container: bktInfo.CID,
Creator: bktInfo.Owner,
Locks: []oid.ID{objID},
CreationTime: TimeNow(ctx),
CopiesNumber: copiesNumber,
}
@ -227,7 +228,7 @@ func (n *layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
)
if lock.Retention != nil {
if _, expEpoch, err = n.neoFS.TimeToEpoch(ctx, lock.Retention.Until); err != nil {
if _, expEpoch, err = n.neoFS.TimeToEpoch(ctx, TimeNow(ctx), lock.Retention.Until); err != nil {
return nil, fmt.Errorf("fetch time to epoch: %w", err)
}

6
api/notifications/controller.go
View file

@ -197,11 +197,11 @@ func (c *Controller) SendNotifications(topics map[string]string, p *handler.Send
return nil
}
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string) error {
func (c *Controller) SendTestNotification(topic, bucketName, requestID, HostID string, now time.Time) error {
event := &TestEvent{
Service: "NeoFS S3",
Event: "s3:TestEvent",
Time: time.Now(),
Time: now,
Bucket: bucketName,
RequestID: requestID,
HostID: HostID,
@ -222,7 +222,7 @@ func prepareEvent(p *handler.SendNotificationParams) *Event {
EventVersion: EventVersion21,
EventSource: "neofs:s3",
AWSRegion: "",
EventTime: time.Now(),
EventTime: p.Time,
EventName: p.Event,
UserIdentity: UserIdentity{
PrincipalID: p.User,

8
api/user_auth.go
View file

@ -16,6 +16,9 @@ type KeyWrapper string
// BoxData is an ID used to store accessbox.Box in a context.
var BoxData = KeyWrapper("__context_box_key")
// ClientTime is an ID used to store client time.Time in a context.
var ClientTime = KeyWrapper("__context_client_time")
// AttachUserAuth adds user authentication via center to router using log for logging.
func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
router.Use(func(h http.Handler) http.Handler {
@ -35,7 +38,10 @@ func AttachUserAuth(router *mux.Router, center auth.Center, log *zap.Logger) {
return
}
} else {
ctx = context.WithValue(r.Context(), BoxData, box)
ctx = context.WithValue(r.Context(), BoxData, box.AccessBox)
if !box.ClientTime.IsZero() {
ctx = context.WithValue(ctx, ClientTime, box.ClientTime)
}
}
h.ServeHTTP(w, r.WithContext(ctx))

20
internal/neofs/neofs.go
View file

@ -52,8 +52,7 @@ func NewNeoFS(p *pool.Pool) *NeoFS {
}
// TimeToEpoch implements neofs.NeoFS interface method.
func (x *NeoFS) TimeToEpoch(ctx context.Context, futureTime time.Time) (uint64, uint64, error) {
now := time.Now()
func (x *NeoFS) TimeToEpoch(ctx context.Context, now, futureTime time.Time) (uint64, uint64, error) {
dur := futureTime.Sub(now)
if dur < 0 {
return 0, 0, fmt.Errorf("time '%s' must be in the future (after %s)",
@ -116,7 +115,12 @@ func (x *NeoFS) CreateContainer(ctx context.Context, prm layer.PrmContainerCreat
cnr.SetPlacementPolicy(prm.Policy)
cnr.SetOwner(prm.Creator)
cnr.SetBasicACL(prm.BasicACL)
container.SetCreationTime(&cnr, time.Now())
creationTime := prm.CreationTime
if creationTime.IsZero() {
creationTime = time.Now()
}
container.SetCreationTime(&cnr, creationTime)
if prm.Name != "" {
var d container.Domain
@ -227,7 +231,13 @@ func (x *NeoFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (oi
a = object.NewAttribute()
a.SetKey(object.AttributeTimestamp)
a.SetValue(strconv.FormatInt(time.Now().Unix(), 10))
creationTime := prm.CreationTime
if creationTime.IsZero() {
creationTime = time.Now()
}
a.SetValue(strconv.FormatInt(creationTime.Unix(), 10))
attrs = append(attrs, *a)
for i := range prm.Attributes {
@ -489,7 +499,7 @@ func (x *AuthmateNeoFS) ContainerExists(ctx context.Context, idCnr cid.ID) error
// TimeToEpoch implements authmate.NeoFS interface method.
func (x *AuthmateNeoFS) TimeToEpoch(ctx context.Context, futureTime time.Time) (uint64, uint64, error) {
return x.neoFS.TimeToEpoch(ctx, futureTime)
return x.neoFS.TimeToEpoch(ctx, time.Now(), futureTime)
}
// CreateContainer implements authmate.NeoFS interface method.