[#660] Add s3 static web hosting operations #660

Open
nzinkevich wants to merge 1 commit from nzinkevich/frostfs-s3-gw:feature/website_hosting into master
22 changed files with 784 additions and 18 deletions

20
api/cache/system.go vendored
View file

@ -104,6 +104,22 @@ func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfi
return result
}
func (o *SystemCache) GetWebsiteConfiguration(key string) *data.WebsiteConfiguration {
entry, err := o.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.(*data.WebsiteConfiguration)
if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil
}
return result
}
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
entry, err := o.cache.Get(key)
if err != nil {
@ -153,6 +169,10 @@ func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleC
return o.cache.Set(key, obj)
}
func (o *SystemCache) PutWebsiteConfiguration(key string, obj *data.WebsiteConfiguration) error {
return o.cache.Set(key, obj)
}
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
return o.cache.Set(key, settings)
}

View file

@ -17,12 +17,14 @@ const (
bktSettingsObject = ".s3-settings"
bktCORSConfigurationObject = ".s3-cors"
bktLifecycleConfigurationObject = ".s3-lifecycle"
bktWebsiteConfigurationObject = ".s3-website"
VersioningUnversioned = "Unversioned"
VersioningEnabled = "Enabled"
VersioningSuspended = "Suspended"
corsFilePathTemplate = "/%s.cors"
corsFilePathTemplate = "/%s.cors"
websiteFilePathTemplate = "/%s.wsh"
)
type (
@ -115,6 +117,15 @@ func (b *BucketInfo) LifecycleConfigurationObjectName() string {
return b.CID.EncodeToString() + bktLifecycleConfigurationObject
}
// WebsiteObjectFilePath returns a FilePath for a bucket CORS configuration file.
func (b *BucketInfo) WebsiteObjectFilePath() string {
return fmt.Sprintf(websiteFilePathTemplate, b.CID)
}
func (b *BucketInfo) WebsiteConfigurationObjectName() string {
return b.CID.EncodeToString() + bktWebsiteConfigurationObject
}
// VersionID returns object version from ObjectInfo.
func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() }

42
api/data/website.go Normal file
View file

@ -0,0 +1,42 @@
package data
import "encoding/xml"
type WebsiteConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ WebsiteConfiguration" json:"-"`
ErrorDocument *ErrorDocument `xml:"ErrorDocument" json:"ErrorDocument"`
IndexDocument *IndexDocument `xml:"IndexDocument" json:"IndexDocument"`
RedirectAllRequestsTo *RedirectAllRequestsTo `xml:"RedirectAllRequestsTo" json:"RedirectAllRequestsTo"`
RoutingRules []RoutingRule `xml:"RoutingRules>RoutingRule" json:"RoutingRules"`
}
type ErrorDocument struct {
Key string `xml:"Key" json:"Key"`
}
type IndexDocument struct {
Suffix string `xml:"Suffix" json:"Suffix"`
}
type RedirectAllRequestsTo struct {
HostName string `xml:"HostName" json:"HostName"`
Protocol *string `xml:"Protocol" json:"Protocol"`
}
type RoutingRule struct {
XMLName xml.Name `xml:"RoutingRule" json:"-"`
Redirect Redirect `xml:"Redirect" json:"Redirect"`
Condition *Condition `xml:"Condition" json:"Condition"`
}
type Redirect struct {
HostName string `xml:"HostName" json:"HostName"`
Protocol string `xml:"Protocol" json:"Protocol"`
ReplaceKeyPrefixWith string `xml:"ReplaceKeyPrefixWith" json:"ReplaceKeyPrefixWith"`
ReplaceKeyWith string `xml:"ReplaceKeyWith" json:"ReplaceKeyWith"`
}
type Condition struct {
HTTPErrorCodeReturnedEquals string `xml:"HttpErrorCodeReturnedEquals" json:"HttpErrorCodeReturnedEquals"`
KeyPrefixEquals string `xml:"KeyPrefixEquals" json:"KeyPrefixEquals"`
}

View file

@ -15,10 +15,6 @@ func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Requ
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
@ -35,10 +31,6 @@ func (h *handler) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Req
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}

194
api/handler/website.go Normal file
View file

@ -0,0 +1,194 @@
package handler
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"slices"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
)
func (h *handler) PutBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketWebsite")
defer span.End()
var buf bytes.Buffer
tee := io.TeeReader(r.Body, &buf)
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
reqInfo := middleware.GetReqInfo(ctx)
cfg := new(data.WebsiteConfiguration)
if err := h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil {
h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
if _, ok := r.Header[api.ContentMD5]; ok {
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
if err != nil {
h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
bodyMD5, err := getContentMD5(&buf)
if err != nil {
h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err)
return
}
if !bytes.Equal(headerMD5, bodyMD5) {
h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = checkWebsiteConfiguration(cfg); err != nil {
h.logAndSendError(ctx, w, "invalid website configuration", reqInfo, err)
return
}
params := &layer.PutWebsiteParams{
BktInfo: bktInfo,
WebsiteConfiguration: cfg,
}
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
if err = h.obj.PutBucketWebsite(ctx, params); err != nil {
h.logAndSendError(ctx, w, "could not put bucket website configuration", reqInfo, err)
return
}
}
func (h *handler) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketWebsite")
defer span.End()
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
cors, err := h.obj.GetBucketWebsite(ctx, bktInfo)
if err != nil {
h.logAndSendError(ctx, w, "could not get website configuration", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, cors); err != nil {
h.logAndSendError(ctx, w, "could not encode website configuration to response", reqInfo, err)
return
}
}
func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteBucketWebsite")
defer span.End()
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = h.obj.DeleteBucketWebsite(ctx, bktInfo); err != nil {
h.logAndSendError(ctx, w, "could not delete cors", reqInfo, err)
}
w.WriteHeader(http.StatusNoContent)
}
func checkWebsiteConfiguration(con *data.WebsiteConfiguration) error {
if con.RedirectAllRequestsTo != nil {
if con.RedirectAllRequestsTo.HostName == "" {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests hostname is empty"))
}
if con.RedirectAllRequestsTo.Protocol != nil && !slices.Contains([]string{"http", "https"}, *con.RedirectAllRequestsTo.Protocol) {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, fmt.Errorf("redirect all requests incorrect protocol: %s", *con.RedirectAllRequestsTo.Protocol))
}
} else {
if con.IndexDocument != nil && con.IndexDocument.Suffix == "" {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("index document suffix is empty"))
}
if con.ErrorDocument != nil && con.ErrorDocument.Key == "" {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("error document key is empty"))
}
if err := checkRoutingRules(con.RoutingRules); err != nil {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, err)
}
}
return nil
}
func checkRoutingRules(rules []data.RoutingRule) error {
if len(rules) == 0 {
return errors.New("no routing rules specified")
}
for _, r := range rules {
if err := checkRedirect(r); err != nil {
return err
}
if err := checkCondition(r); err != nil {
return err
}
}
return nil
}
func checkRedirect(r data.RoutingRule) error {
if checkAllEmpty(r.Redirect.ReplaceKeyPrefixWith, r.Redirect.ReplaceKeyWith, r.Redirect.HostName, r.Redirect.Protocol) {
return errors.New("empty redirect rule")
}
if r.Redirect.Protocol != "" && !slices.Contains([]string{"http", "https"}, r.Redirect.Protocol) {
return fmt.Errorf("invalid redirect protocol: %s", r.Redirect.Protocol)
}
return nil
}
func checkCondition(r data.RoutingRule) error {
if r.Condition != nil {
if checkAllEmpty(r.Condition.HTTPErrorCodeReturnedEquals, r.Condition.KeyPrefixEquals) {
return errors.New("empty condition properties")
}
}
return nil
}
func checkAllEmpty(elems ...string) bool {
for _, el := range elems {
if el != "" {
return false
}
}
return true
}

114
api/handler/website_test.go Normal file
View file

@ -0,0 +1,114 @@
package handler
import (
"errors"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)
func TestCheckWebsiteConfiguration(t *testing.T) {
tests := []struct {
name string
config data.WebsiteConfiguration
wantErr error
}{
{
name: "Valid RedirectAllRequestsTo",
config: data.WebsiteConfiguration{
RedirectAllRequestsTo: &data.RedirectAllRequestsTo{
HostName: "example.com",
Protocol: stringPtr("https"),
},
},
wantErr: nil,
},
{
name: "Invalid RedirectAllRequestsTo - wrong protocol",
config: data.WebsiteConfiguration{
RedirectAllRequestsTo: &data.RedirectAllRequestsTo{
HostName: "example.com",
Protocol: stringPtr("ftp"),
},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests incorrect protocol: ftp")),
},
{
name: "Invalid RedirectAllRequestsTo - empty hostname",
config: data.WebsiteConfiguration{
RedirectAllRequestsTo: &data.RedirectAllRequestsTo{
HostName: "",
Protocol: stringPtr("https"),
},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests hostname is empty")),
},
{
name: "Valid Index and Error Documents",
config: data.WebsiteConfiguration{
IndexDocument: &data.IndexDocument{Suffix: "index.html"},
ErrorDocument: &data.ErrorDocument{Key: "error.html"},
RoutingRules: []data.RoutingRule{{
Redirect: data.Redirect{
HostName: "example.com",
Protocol: "https",
ReplaceKeyWith: "new-key",
},
}},
},
wantErr: nil,
},
{
name: "Invalid Index Document - empty suffix",
config: data.WebsiteConfiguration{
IndexDocument: &data.IndexDocument{Suffix: ""},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("index document suffix is empty")),
},
{
name: "Invalid Routing Rules - empty rules",
config: data.WebsiteConfiguration{
RoutingRules: []data.RoutingRule{},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("no routing rules specified")),
},
{
name: "Invalid Routing Rules - invalid protocol",
config: data.WebsiteConfiguration{
RoutingRules: []data.RoutingRule{
{
Redirect: data.Redirect{
Protocol: "ftp",
},
},
},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("invalid redirect protocol: ftp")),
},
{
name: "Invalid Routing Rule - empty rule",
config: data.WebsiteConfiguration{
RoutingRules: []data.RoutingRule{{}},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("empty redirect rule")),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := checkWebsiteConfiguration(&tt.config)
if tt.wantErr == nil {
require.NoError(t, err)
} else {
require.Error(t, err)
require.Equal(t, tt.wantErr.Error(), err.Error())
}
})
}
}
func stringPtr(s string) *string {
return &s
}

View file

@ -293,6 +293,32 @@ func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) {
c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName())
}
func (c *Cache) GetWebsiteConfiguration(owner user.ID, bkt *data.BucketInfo) *data.WebsiteConfiguration {
key := bkt.WebsiteConfigurationObjectName()
if !c.accessCache.Get(owner, key) {
return nil
}
return c.systemCache.GetWebsiteConfiguration(key)
}
func (c *Cache) PutWebsiteConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.WebsiteConfiguration) {
key := bkt.WebsiteConfigurationObjectName()
if err := c.systemCache.PutWebsiteConfiguration(key, cfg); err != nil {
c.logger.Warn(logs.CouldntCacheWebsiteConfiguration, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
func (c *Cache) DeleteWebsiteConfiguration(bktInfo *data.BucketInfo) {
c.systemCache.Delete(bktInfo.WebsiteConfigurationObjectName())
}
func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo {
return c.networkCache.GetNetworkInfo()
}

View file

@ -61,6 +61,7 @@ type (
features FeatureSettings
gateKey *keys.PrivateKey
corsCnrInfo *data.BucketInfo
websiteCnrInfo *data.BucketInfo
lifecycleCnrInfo *data.BucketInfo
workerPool *ants.Pool
}
@ -75,6 +76,7 @@ type (
Features FeatureSettings
GateKey *keys.PrivateKey
CORSCnrInfo *data.BucketInfo
WebsiteCnrInfo *data.BucketInfo
LifecycleCnrInfo *data.BucketInfo
WorkerPool *ants.Pool
}
@ -153,6 +155,13 @@ type (
UserAgent string
}
// PutWebsiteParams stores PutBucketWebsite request parameters.
PutWebsiteParams struct {
BktInfo *data.BucketInfo
WebsiteConfiguration *data.WebsiteConfiguration
CopiesNumbers []uint32
}
// CopyObjectParams stores object copy request parameters.
CopyObjectParams struct {
SrcVersioned bool
@ -269,6 +278,7 @@ func NewLayer(log *zap.Logger, frostFS frostfs.FrostFS, config *Config) *Layer {
gateKey: config.GateKey,
corsCnrInfo: config.CORSCnrInfo,
lifecycleCnrInfo: config.LifecycleCnrInfo,
websiteCnrInfo: config.WebsiteCnrInfo,
workerPool: config.WorkerPool,
}
}

View file

@ -67,6 +67,9 @@ type Service interface {
GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
PutBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error)
GetBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
DeleteBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
// Compound methods for optimizations
// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.

View file

@ -38,6 +38,51 @@ type TreeServiceMock struct {
parts map[string]map[int]*data.PartInfoExtended
}
func (t *TreeServiceMock) PutBucketWebsiteConfiguration(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
systemMap = make(map[string]*data.BaseNodeVersion)
}
systemMap["website"] = &data.BaseNodeVersion{
OID: addr.Object(),
}
t.system[bktInfo.CID.EncodeToString()] = systemMap
return nil, tree.ErrNoNodeToRemove
}
func (t *TreeServiceMock) GetBucketWebsiteConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
return oid.Address{}, tree.ErrNodeNotFound
}
node, ok := systemMap["website"]
if !ok {
return oid.Address{}, tree.ErrNodeNotFound
}
return newAddress(bktInfo.CID, node.OID), nil
}
func (t *TreeServiceMock) DeleteBucketWebsiteConfiguration(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
if !ok {
return nil, tree.ErrNoNodeToRemove
}
node, ok := systemMap["website"]
if !ok {
return nil, tree.ErrNoNodeToRemove
}
delete(systemMap, "website")
return []oid.Address{newAddress(bktInfo.CID, node.OID)}, nil
}
func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
// TODO implement object tagging
lock, err := t.GetLock(ctx, bktInfo, objVersion.ID)

146
api/layer/website.go Normal file
View file

@ -0,0 +1,146 @@
package layer
import (
"bytes"
"context"
"encoding/xml"
"errors"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"go.uber.org/zap"
)
func (n *Layer) PutBucketWebsite(ctx context.Context, p *PutWebsiteParams) error {
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutBucketWebsiteConfiguration")
defer span.End()
cfgBytes, err := xml.Marshal(p.WebsiteConfiguration)
if err != nil {
return fmt.Errorf("marshal lifecycle configuration: %w", err)
}
prm := frostfs.PrmObjectCreate{
Payload: bytes.NewReader(cfgBytes),
Filepath: p.BktInfo.WebsiteConfigurationObjectName(),
CreationTime: TimeNow(ctx),
}
var websiteBkt *data.BucketInfo
if n.websiteCnrInfo == nil {
websiteBkt = p.BktInfo
prm.CopiesNumber = p.CopiesNumbers
} else {
websiteBkt = n.websiteCnrInfo
prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey
}
prm.Container = websiteBkt.CID
createdObj, err := n.objectPutAndHash(ctx, prm, websiteBkt)
if err != nil {
return fmt.Errorf("put website object: %w", err)
}
objsToDelete, err := n.treeService.PutBucketWebsiteConfiguration(ctx, p.BktInfo, newAddress(websiteBkt.CID, createdObj.ID))
objsToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objsToDeleteNotFound {
return err
}
if !objsToDeleteNotFound {
for _, addr := range objsToDelete {
n.deleteWebsiteObject(ctx, p.BktInfo, addr)
}
}
n.cache.PutWebsiteConfiguration(n.BearerOwner(ctx), p.BktInfo, p.WebsiteConfiguration)
return nil
}
func (n *Layer) GetBucketWebsite(ctx context.Context, bktInfo *data.BucketInfo) (*data.WebsiteConfiguration, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetBucketWebsite")
defer span.End()
owner := n.BearerOwner(ctx)
if cfg := n.cache.GetWebsiteConfiguration(owner, bktInfo); cfg != nil {
return cfg, nil
}
addr, err := n.treeService.GetBucketWebsiteConfiguration(ctx, bktInfo)
objNotFound := errors.Is(err, tree.ErrNodeNotFound)
if err != nil && !objNotFound {
return nil, err
}
if objNotFound {
return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration), err.Error())
}
var prmAuth frostfs.PrmAuth
lifecycleBkt := bktInfo
if !addr.Container().Equals(bktInfo.CID) {
lifecycleBkt = &data.BucketInfo{CID: addr.Container()}
prmAuth.PrivateKey = &n.gateKey.PrivateKey
}
obj, err := n.objectGetWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth)
if err != nil {
return nil, fmt.Errorf("get lifecycle object: %w", err)
}
lifecycleCfg := &data.WebsiteConfiguration{}
if err = xml.NewDecoder(obj.Payload).Decode(&lifecycleCfg); err != nil {
return nil, fmt.Errorf("unmarshal lifecycle configuration: %w", err)
}
n.cache.PutWebsiteConfiguration(owner, bktInfo, lifecycleCfg)
return lifecycleCfg, nil
}
func (n *Layer) DeleteBucketWebsite(ctx context.Context, bktInfo *data.BucketInfo) error {
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteBucketWebsite")
defer span.End()
objs, err := n.treeService.DeleteBucketWebsiteConfiguration(ctx, bktInfo)
objsNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
if err != nil && !objsNotFound {
return err
}
if !objsNotFound {
for _, addr := range objs {
n.deleteWebsiteObject(ctx, bktInfo, addr)
}
}
n.cache.DeleteWebsiteConfiguration(bktInfo)
return nil
}
// deleteLifecycleObject removes object and logs in case of error.
func (n *Layer) deleteWebsiteObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
var prmAuth frostfs.PrmAuth
websiteBkt := bktInfo
if !addr.Container().Equals(bktInfo.CID) {
websiteBkt = &data.BucketInfo{CID: addr.Container()}
prmAuth.PrivateKey = &n.gateKey.PrivateKey
}
if err := n.objectDeleteWithAuth(ctx, websiteBkt, addr.Object(), prmAuth); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteWebsiteObject, zap.Error(err),
zap.String("cid", websiteBkt.CID.EncodeToString()),
zap.String("oid", addr.Object().EncodeToString()),
logs.TagField(logs.TagExternalStorage),
)
}
}

59
api/layer/website_test.go Normal file
View file

@ -0,0 +1,59 @@
package layer
import (
"encoding/xml"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"github.com/stretchr/testify/require"
)
func TestBucketWebsite(t *testing.T) {
tc := prepareContext(t)
website := &data.WebsiteConfiguration{
XMLName: xml.Name{
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
Local: "WebsiteConfiguration",
},
ErrorDocument: &data.ErrorDocument{Key: "error.html"},
IndexDocument: &data.IndexDocument{Suffix: "index.html"},
RoutingRules: []data.RoutingRule{
{
XMLName: xml.Name{},
Condition: &data.Condition{
HTTPErrorCodeReturnedEquals: "404",
},
Redirect: data.Redirect{
HostName: "www.example.com",
Protocol: "http",
},
},
},
}
_, err := tc.layer.GetBucketWebsite(tc.ctx, tc.bktInfo)
require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration), frosterr.UnwrapErr(err))
err = tc.layer.DeleteBucketWebsite(tc.ctx, tc.bktInfo)
require.NoError(t, err)
err = tc.layer.PutBucketWebsite(tc.ctx, &PutWebsiteParams{
BktInfo: tc.bktInfo,
WebsiteConfiguration: website,
CopiesNumbers: []uint32{1},
})
require.NoError(t, err)
cfg, err := tc.layer.GetBucketWebsite(tc.ctx, tc.bktInfo)
require.NoError(t, err)
require.Equal(t, *website, *cfg)
err = tc.layer.DeleteBucketWebsite(tc.ctx, tc.bktInfo)
require.NoError(t, err)
_, err = tc.layer.GetBucketWebsite(tc.ctx, tc.bktInfo)
require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration), frosterr.UnwrapErr(err))
}

View file

@ -38,6 +38,7 @@ const (
PutBucketTaggingOperation = "PutBucketTagging"
PutBucketVersioningOperation = "PutBucketVersioning"
PutBucketNotificationOperation = "PutBucketNotification"
PutBucketWebsiteOperation = "PutBucketWebsite"
CreateBucketOperation = "CreateBucket"
DeleteMultipleObjectsOperation = "DeleteMultipleObjects"
PostObjectOperation = "PostObject"

View file

@ -53,6 +53,7 @@ type (
GetBucketReplicationHandler(http.ResponseWriter, *http.Request)
GetBucketTaggingHandler(http.ResponseWriter, *http.Request)
DeleteBucketWebsiteHandler(http.ResponseWriter, *http.Request)
PutBucketWebsiteHandler(http.ResponseWriter, *http.Request)
DeleteBucketTaggingHandler(http.ResponseWriter, *http.Request)
GetBucketObjectLockConfigHandler(http.ResponseWriter, *http.Request)
GetBucketVersioningHandler(http.ResponseWriter, *http.Request)
@ -88,7 +89,6 @@ type (
ListPartsHandler(w http.ResponseWriter, r *http.Request)
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
PatchObjectHandler(http.ResponseWriter, *http.Request)
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
ResolveCID(ctx context.Context, bucket string) (cid.ID, error)
}
@ -403,6 +403,9 @@ func bucketRouter(h Handler) chi.Router {
Add(NewFilter().
Queries(s3middleware.NotificationQuery).
Handler(named(s3middleware.PutBucketNotificationOperation, h.PutBucketNotificationHandler))).
Add(NewFilter().
Queries(s3middleware.WebsiteQuery).
Handler(named(s3middleware.PutBucketWebsiteOperation, h.PutBucketWebsiteHandler))).
Add(NewFilter().
NoQueries().
Handler(named(s3middleware.CreateBucketOperation, h.CreateBucketHandler))).

View file

@ -181,6 +181,11 @@ type handlerMock struct {
buckets map[string]*data.BucketInfo
}
func (h *handlerMock) PutBucketWebsiteHandler(http.ResponseWriter, *http.Request) {
//TODO implement me
panic("implement me")
}
type handlerResult struct {
Method string
ReqInfo *middleware.ReqInfo

View file

@ -286,6 +286,11 @@ func (a *App) initLayer(ctx context.Context) {
a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
}
wshCnrInfo, err := a.fetchContainerInfo(ctx, cfgContainersWebStaticHosting)
if err != nil {
a.log.Fatal(logs.CouldNotFetchWebStaticHostingContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
}
var lifecycleCnrInfo *data.BucketInfo
if a.config().IsSet(cfgContainersLifecycle) {
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
@ -306,6 +311,7 @@ func (a *App) initLayer(ctx context.Context) {
GateKey: a.key,
CORSCnrInfo: corsCnrInfo,
LifecycleCnrInfo: lifecycleCnrInfo,
WebsiteCnrInfo: wshCnrInfo,
WorkerPool: a.initWorkerPool(),
}

View file

@ -221,9 +221,10 @@ const (
cfgSourceIPHeader = "source_ip_header"
// Containers.
cfgContainersCORS = "containers.cors"
cfgContainersLifecycle = "containers.lifecycle"
cfgContainersAccessBox = "containers.accessbox"
cfgContainersCORS = "containers.cors"
cfgContainersLifecycle = "containers.lifecycle"
cfgContainersAccessBox = "containers.accessbox"
cfgContainersWebStaticHosting = "containers.web_static_hosting"
// Multinet.
cfgMultinetEnabled = "multinet.enabled"

View file

@ -273,6 +273,8 @@ S3_GW_RETRY_STRATEGY=exponential
# Containers properties
S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
S3_GW_CONTAINERS_LIFECYCLE=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
S3_GW_CONTAINERS_ACCESSBOX: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2
S3_GW_CONTAINERS_STATIC_WEB_HOSTING: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP
# Multinet properties
# Enable multinet support

View file

@ -322,6 +322,8 @@ retry:
containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
accessbox: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2
web_static_hosting: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP
# Multinet properties
multinet:

View file

@ -859,13 +859,15 @@ containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
accessbox: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2
web_static_hosting: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
| `accessbox` | `string` | no | | Container name to lookup accessbox if custom aws credentials is used. If not set, custom credentials are not supported. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|----------------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
| `accessbox` | `string` | no | | Container name to lookup accessbox if custom aws credentials is used. If not set, custom credentials are not supported. |
| `web_static_hosting` | `string` | no | | Container name for website configurations. |
# `vhs` section

View file

@ -48,6 +48,7 @@ const (
ServerReconnectFailed = "failed to reconnect server"
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
CouldNotFetchAccessBoxContainerInfo = "couldn't fetch AccessBox container info"
CouldNotFetchWebStaticHostingContainerInfo = "couldn't fetch web static hosting container info"
MultinetDialSuccess = "multinet dial successful"
MultinetDialFail = "multinet dial failed"
FailedToCreateWorkerPool = "failed to create worker pool"
@ -126,6 +127,7 @@ const (
UnexpectedMultiNodeIDsInSubTreeMultiParts = "unexpected multi node ids in sub tree multi parts"
FoundSeveralSystemNodes = "found several system nodes"
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
BucketWebsiteNodeHasMultipleIDs = "bucket website node has multiple ids"
UploadPart = "upload part"
FailedToSubmitTaskToPool = "failed to submit task to pool"
FailedToGetRealObjectSize = "failed to get real object size"
@ -154,6 +156,7 @@ const (
CouldntCacheCors = "couldn't cache cors"
CouldntCacheListPolicyChains = "couldn't cache list policy chains"
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration"
CouldntCacheWebsiteConfiguration = "couldn't cache website configuration"
GetBucketCors = "get bucket cors"
GetBucketInfo = "get bucket info"
ResolveBucket = "resolve bucket"
@ -180,6 +183,7 @@ const (
CouldNotFetchObjectMeta = "could not fetch object meta"
FailedToDeleteObject = "failed to delete object"
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object"
CouldntDeleteWebsiteObject = "couldn't delete website configuration object"
CouldntGetCORSObjectVersions = "couldn't get cors object versions"
)

View file

@ -123,6 +123,7 @@ const (
corsFilename = "bucket-cors"
bucketTaggingFilename = "bucket-tagging"
bucketLifecycleFilename = "bucket-lifecycle"
bucketWebsiteFilename = "bucket-website"
// versionTree -- ID of a tree with object versions.
versionTree = "version"
@ -1662,6 +1663,83 @@ func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *
return objToDelete, nil
}
func (c *Tree) PutBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
Review

We don't want to use a tree service to store the metainformation of objects of a static configuration website. Instead, we should search for the object by the FilePath attribute.

We don't want to use a tree service to store the metainformation of objects of a static configuration website. Instead, we should search for the object by the FilePath attribute.
ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutBucketWebsiteConfiguration")
defer span.End()
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketWebsiteFilename)
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
if err != nil && !isErrNotFound {
return nil, fmt.Errorf("couldn't get node: %w", err)
}
meta := make(map[string]string)
meta[FileNameKey] = bucketWebsiteFilename
meta[oidKV] = addr.Object().EncodeToString()
meta[cidKV] = addr.Container().EncodeToString()
if isErrNotFound {
if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil {
return nil, err
}
return nil, tree.ErrNoNodeToRemove
}
latest := multiNode.Latest()
ind := latest.GetLatestNodeIndex()
if latest.IsSplit() {
c.reqLogger(ctx).Error(logs.BucketWebsiteNodeHasMultipleIDs, logs.TagField(logs.TagDatapath))
}
if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
return nil, fmt.Errorf("move website node: %w", err)
}
objToDelete := make([]oid.Address, 1, len(multiNode.nodes))
objToDelete[0], err = getTreeNodeAddress(latest)
if err != nil {
return nil, fmt.Errorf("parse object addr of latest website node in tree: %w", err)
}
objToDelete = append(objToDelete, c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)...)
return objToDelete, nil
}
func (c *Tree) GetBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetBucketWebsite")
defer span.End()
node, err := c.getSystemNode(ctx, bktInfo, bucketWebsiteFilename)
if err != nil {
return oid.Address{}, fmt.Errorf("get lifecycle node: %w", err)
}
return getTreeNodeAddress(node.Latest())
}
func (c *Tree) DeleteBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteBucketWebsiteConfiguration")
defer span.End()
multiNode, err := c.getSystemNode(ctx, bktInfo, bucketWebsiteFilename)
isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
if err != nil && !isErrNotFound {
return nil, err
}
if isErrNotFound {
return nil, tree.ErrNoNodeToRemove
}
objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo)
if len(objToDelete) != len(multiNode.nodes) {
return nil, fmt.Errorf("failed to clean all old website nodes")
}
return objToDelete, nil
}
func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteMultipartUpload")
defer span.End()