[#660] Add bucket website operations

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2025-03-13 15:27:01 +03:00
parent 9edec7d573
commit 5e88016355
23 changed files with 760 additions and 30 deletions

20
api/cache/system.go vendored
View file

@ -104,6 +104,22 @@ func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfi
return result
}
func (o *SystemCache) GetWebsiteConfiguration(key string) *data.WebsiteConfiguration {
entry, err := o.cache.Get(key)
if err != nil {
return nil
}
result, ok := entry.(*data.WebsiteConfiguration)
if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil
}
return result
}
func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
entry, err := o.cache.Get(key)
if err != nil {
@ -153,6 +169,10 @@ func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleC
return o.cache.Set(key, obj)
}
func (o *SystemCache) PutWebsiteConfiguration(key string, obj *data.WebsiteConfiguration) error {
return o.cache.Set(key, obj)
}
func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error {
return o.cache.Set(key, settings)
}

View file

@ -17,12 +17,14 @@ const (
bktSettingsObject = ".s3-settings"
bktCORSConfigurationObject = ".s3-cors"
bktLifecycleConfigurationObject = ".s3-lifecycle"
bktWebsiteConfigurationObject = ".s3-website"
VersioningUnversioned = "Unversioned"
VersioningEnabled = "Enabled"
VersioningSuspended = "Suspended"
corsFilePathTemplate = "/%s.cors"
corsFilePathTemplate = "/%s.cors"
websiteFilePathTemplate = "/%s.wsh"
)
type (
@ -115,6 +117,15 @@ func (b *BucketInfo) LifecycleConfigurationObjectName() string {
return b.CID.EncodeToString() + bktLifecycleConfigurationObject
}
// WebsiteObjectFilePath returns a FilePath for a bucket CORS configuration file.
func (b *BucketInfo) WebsiteObjectFilePath() string {
return fmt.Sprintf(websiteFilePathTemplate, b.CID)
}
func (b *BucketInfo) WebsiteConfigurationObjectName() string {
return b.CID.EncodeToString() + bktWebsiteConfigurationObject
}
// VersionID returns object version from ObjectInfo.
func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() }

42
api/data/website.go Normal file
View file

@ -0,0 +1,42 @@
package data
import "encoding/xml"
type WebsiteConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ WebsiteConfiguration" json:"-"`
ErrorDocument *ErrorDocument `xml:"ErrorDocument" json:"ErrorDocument"`
IndexDocument *IndexDocument `xml:"IndexDocument" json:"IndexDocument"`
RedirectAllRequestsTo *RedirectAllRequestsTo `xml:"RedirectAllRequestsTo" json:"RedirectAllRequestsTo"`
RoutingRules []RoutingRule `xml:"RoutingRules>RoutingRule" json:"RoutingRules"`
}
type ErrorDocument struct {
Key string `xml:"Key" json:"Key"`
}
type IndexDocument struct {
Suffix string `xml:"Suffix" json:"Suffix"`
}
type RedirectAllRequestsTo struct {
HostName string `xml:"HostName" json:"HostName"`
Protocol *string `xml:"Protocol" json:"Protocol"`
}
type RoutingRule struct {
XMLName xml.Name `xml:"RoutingRule" json:"-"`
Redirect Redirect `xml:"Redirect" json:"Redirect"`
Condition *Condition `xml:"Condition" json:"Condition"`
}
type Redirect struct {
HostName string `xml:"HostName" json:"HostName"`
Protocol string `xml:"Protocol" json:"Protocol"`
ReplaceKeyPrefixWith string `xml:"ReplaceKeyPrefixWith" json:"ReplaceKeyPrefixWith"`
ReplaceKeyWith string `xml:"ReplaceKeyWith" json:"ReplaceKeyWith"`
}
type Condition struct {
HTTPErrorCodeReturnedEquals string `xml:"HttpErrorCodeReturnedEquals" json:"HttpErrorCodeReturnedEquals"`
KeyPrefixEquals string `xml:"KeyPrefixEquals" json:"KeyPrefixEquals"`
}

View file

@ -17,7 +17,7 @@ func TestHandler_ListBucketsHandler(t *testing.T) {
const defaultConstraint = "default"
region := "us-west-1"
hc := prepareWithoutCORSHandlerContext(t)
hc := prepareWithoutSysCnrsHandlerContext(t)
hc.config.putLocationConstraint(region)
props := []Bucket{

View file

@ -166,8 +166,8 @@ func (c *configMock) putLocationConstraint(constraint string) {
}
type handlerConfig struct {
cacheCfg *layer.CachesConfig
withoutCORS bool
cacheCfg *layer.CachesConfig
withoutSystemCnrs bool
}
func prepareHandlerContext(t *testing.T) *handlerContext {
@ -180,11 +180,11 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
}
}
func prepareWithoutCORSHandlerContext(t *testing.T) *handlerContext {
func prepareWithoutSysCnrsHandlerContext(t *testing.T) *handlerContext {
log := zaptest.NewLogger(t)
hc, err := prepareHandlerContextBase(&handlerConfig{
cacheCfg: layer.DefaultCachesConfigs(log),
withoutCORS: true,
cacheCfg: layer.DefaultCachesConfigs(log),
withoutSystemCnrs: true,
}, log)
require.NoError(t, err)
return &handlerContext{
@ -244,11 +244,15 @@ func prepareHandlerContextBase(config *handlerConfig, log *zap.Logger) (*handler
WorkerPool: pool,
}
if !config.withoutCORS {
if !config.withoutSystemCnrs {
layerCfg.CORSCnrInfo, err = createCORSContainer(key, tp)
if err != nil {
return nil, err
}
layerCfg.WebsiteCnrInfo, err = createWebsiteContainer(key, tp)
if err != nil {
return nil, err
}
}
var pp netmap.PlacementPolicy
@ -326,6 +330,38 @@ func createCORSContainer(key *keys.PrivateKey, tp *layer.TestFrostFS) (*data.Buc
}, nil
}
func createWebsiteContainer(key *keys.PrivateKey, tp *layer.TestFrostFS) (*data.BucketInfo, error) {
bearerToken := bearertest.Token()
err := bearerToken.Sign(key.PrivateKey)
if err != nil {
return nil, err
}
bktName := "wsh"
res, err := tp.CreateContainer(middleware.SetBox(context.Background(), &middleware.Box{AccessBox: &accessbox.Box{
Gate: &accessbox.GateData{
BearerToken: &bearerToken,
GateKey: key.PublicKey(),
},
}}), frostfs.PrmContainerCreate{
Name: bktName,
Policy: getPlacementPolicy(),
})
if err != nil {
return nil, err
}
var owner user.ID
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
return &data.BucketInfo{
Name: bktName,
Owner: owner,
CID: res.ContainerID,
HomomorphicHashDisabled: res.HomomorphicHashDisabled,
}, nil
}
func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {
minCacheCfg := &cache.Config{
Size: 1,

View file

@ -15,10 +15,6 @@ func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Requ
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
@ -35,10 +31,6 @@ func (h *handler) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Req
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}
func (h *handler) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented))
}

194
api/handler/website.go Normal file
View file

@ -0,0 +1,194 @@
package handler
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"slices"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
)
func (h *handler) PutBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketWebsite")
defer span.End()
var buf bytes.Buffer
tee := io.TeeReader(r.Body, &buf)
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
reqInfo := middleware.GetReqInfo(ctx)
cfg := new(data.WebsiteConfiguration)
if err := h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil {
h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error()))
return
}
if _, ok := r.Header[api.ContentMD5]; ok {
headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5))
if err != nil {
h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
bodyMD5, err := getContentMD5(&buf)
if err != nil {
h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err)
return
}
if !bytes.Equal(headerMD5, bodyMD5) {
h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest))
return
}
}
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = checkWebsiteConfiguration(cfg); err != nil {
h.logAndSendError(ctx, w, "invalid website configuration", reqInfo, err)
return
}
params := &layer.PutWebsiteParams{
BktInfo: bktInfo,
WebsiteConfiguration: cfg,
}
params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint)
if err != nil {
h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err)
return
}
if err = h.obj.PutBucketWebsite(ctx, params); err != nil {
h.logAndSendError(ctx, w, "could not put bucket website configuration", reqInfo, err)
return
}
}
func (h *handler) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketWebsite")
defer span.End()
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
cors, err := h.obj.GetBucketWebsiteConfig(ctx, bktInfo)
if err != nil {
h.logAndSendError(ctx, w, "could not get website configuration", reqInfo, err)
return
}
if err = middleware.EncodeToResponse(w, cors); err != nil {
h.logAndSendError(ctx, w, "could not encode website configuration to response", reqInfo, err)
return
}
}
func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteBucketWebsite")
defer span.End()
ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag)
reqInfo := middleware.GetReqInfo(ctx)
bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
if err != nil {
h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err)
return
}
if err = h.obj.DeleteBucketWebsite(ctx, bktInfo); err != nil {
h.logAndSendError(ctx, w, "could not delete cors", reqInfo, err)
}
w.WriteHeader(http.StatusNoContent)
}
func checkWebsiteConfiguration(con *data.WebsiteConfiguration) error {
if con.RedirectAllRequestsTo != nil {
if con.RedirectAllRequestsTo.HostName == "" {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests hostname is empty"))
}
if con.RedirectAllRequestsTo.Protocol != nil && !slices.Contains([]string{"http", "https"}, *con.RedirectAllRequestsTo.Protocol) {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, fmt.Errorf("redirect all requests incorrect protocol: %s", *con.RedirectAllRequestsTo.Protocol))
}
} else {
if con.IndexDocument != nil && con.IndexDocument.Suffix == "" {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("index document suffix is empty"))
}
if con.ErrorDocument != nil && con.ErrorDocument.Key == "" {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("error document key is empty"))
}
if err := checkRoutingRules(con.RoutingRules); err != nil {
return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, err)
}
}
return nil
}
func checkRoutingRules(rules []data.RoutingRule) error {
if len(rules) == 0 {
return errors.New("no routing rules specified")
}
for _, r := range rules {
if err := checkRedirect(r); err != nil {
return err
}
if err := checkCondition(r); err != nil {
return err
}
}
return nil
}
func checkRedirect(r data.RoutingRule) error {
if checkAllEmpty(r.Redirect.ReplaceKeyPrefixWith, r.Redirect.ReplaceKeyWith, r.Redirect.HostName, r.Redirect.Protocol) {
return errors.New("empty redirect rule")
}
if r.Redirect.Protocol != "" && !slices.Contains([]string{"http", "https"}, r.Redirect.Protocol) {
return fmt.Errorf("invalid redirect protocol: %s", r.Redirect.Protocol)
}
return nil
}
func checkCondition(r data.RoutingRule) error {
if r.Condition != nil {
if checkAllEmpty(r.Condition.HTTPErrorCodeReturnedEquals, r.Condition.KeyPrefixEquals) {
return errors.New("empty condition properties")
}
}
return nil
}
func checkAllEmpty(elems ...string) bool {
for _, el := range elems {
if el != "" {
return false
}
}
return true
}

114
api/handler/website_test.go Normal file
View file

@ -0,0 +1,114 @@
package handler
import (
"errors"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"github.com/stretchr/testify/require"
)
func TestCheckWebsiteConfiguration(t *testing.T) {
tests := []struct {
name string
config data.WebsiteConfiguration
wantErr error
}{
{
name: "Valid RedirectAllRequestsTo",
config: data.WebsiteConfiguration{
RedirectAllRequestsTo: &data.RedirectAllRequestsTo{
HostName: "example.com",
Protocol: stringPtr("https"),
},
},
wantErr: nil,
},
{
name: "Invalid RedirectAllRequestsTo - wrong protocol",
config: data.WebsiteConfiguration{
RedirectAllRequestsTo: &data.RedirectAllRequestsTo{
HostName: "example.com",
Protocol: stringPtr("ftp"),
},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests incorrect protocol: ftp")),
},
{
name: "Invalid RedirectAllRequestsTo - empty hostname",
config: data.WebsiteConfiguration{
RedirectAllRequestsTo: &data.RedirectAllRequestsTo{
HostName: "",
Protocol: stringPtr("https"),
},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests hostname is empty")),
},
{
name: "Valid Index and Error Documents",
config: data.WebsiteConfiguration{
IndexDocument: &data.IndexDocument{Suffix: "index.html"},
ErrorDocument: &data.ErrorDocument{Key: "error.html"},
RoutingRules: []data.RoutingRule{{
Redirect: data.Redirect{
HostName: "example.com",
Protocol: "https",
ReplaceKeyWith: "new-key",
},
}},
},
wantErr: nil,
},
{
name: "Invalid Index Document - empty suffix",
config: data.WebsiteConfiguration{
IndexDocument: &data.IndexDocument{Suffix: ""},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("index document suffix is empty")),
},
{
name: "Invalid Routing Rules - empty rules",
config: data.WebsiteConfiguration{
RoutingRules: []data.RoutingRule{},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("no routing rules specified")),
},
{
name: "Invalid Routing Rules - invalid protocol",
config: data.WebsiteConfiguration{
RoutingRules: []data.RoutingRule{
{
Redirect: data.Redirect{
Protocol: "ftp",
},
},
},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("invalid redirect protocol: ftp")),
},
{
name: "Invalid Routing Rule - empty rule",
config: data.WebsiteConfiguration{
RoutingRules: []data.RoutingRule{{}},
},
wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("empty redirect rule")),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := checkWebsiteConfiguration(&tt.config)
if tt.wantErr == nil {
require.NoError(t, err)
} else {
require.Error(t, err)
require.Equal(t, tt.wantErr.Error(), err.Error())
}
})
}
}
func stringPtr(s string) *string {
return &s
}

View file

@ -293,6 +293,32 @@ func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) {
c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName())
}
func (c *Cache) GetWebsiteConfiguration(owner user.ID, bkt *data.BucketInfo) *data.WebsiteConfiguration {
key := bkt.WebsiteConfigurationObjectName()
if !c.accessCache.Get(owner, key) {
return nil
}
return c.systemCache.GetWebsiteConfiguration(key)
}
func (c *Cache) PutWebsiteConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.WebsiteConfiguration) {
key := bkt.WebsiteConfigurationObjectName()
if err := c.systemCache.PutWebsiteConfiguration(key, cfg); err != nil {
c.logger.Warn(logs.CouldntCacheWebsiteConfiguration, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
func (c *Cache) DeleteWebsiteConfiguration(bktInfo *data.BucketInfo) {
c.systemCache.Delete(bktInfo.WebsiteConfigurationObjectName())
}
func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo {
return c.networkCache.GetNetworkInfo()
}

View file

@ -61,6 +61,7 @@ type (
features FeatureSettings
gateKey *keys.PrivateKey
corsCnrInfo *data.BucketInfo
websiteCnrInfo *data.BucketInfo
lifecycleCnrInfo *data.BucketInfo
workerPool *ants.Pool
}
@ -75,6 +76,7 @@ type (
Features FeatureSettings
GateKey *keys.PrivateKey
CORSCnrInfo *data.BucketInfo
WebsiteCnrInfo *data.BucketInfo
LifecycleCnrInfo *data.BucketInfo
WorkerPool *ants.Pool
}
@ -153,6 +155,13 @@ type (
UserAgent string
}
// PutWebsiteParams stores PutBucketWebsite request parameters.
PutWebsiteParams struct {
BktInfo *data.BucketInfo
WebsiteConfiguration *data.WebsiteConfiguration
CopiesNumbers []uint32
}
// CopyObjectParams stores object copy request parameters.
CopyObjectParams struct {
SrcVersioned bool
@ -269,6 +278,7 @@ func NewLayer(log *zap.Logger, frostFS frostfs.FrostFS, config *Config) *Layer {
gateKey: config.GateKey,
corsCnrInfo: config.CORSCnrInfo,
lifecycleCnrInfo: config.LifecycleCnrInfo,
websiteCnrInfo: config.WebsiteCnrInfo,
workerPool: config.WorkerPool,
}
}
@ -929,6 +939,9 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
}
n.deleteCORSVersions(ctx, p.BktInfo)
if err = n.DeleteBucketWebsite(ctx, p.BktInfo); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteWebsiteObject, zap.Error(err))
}
if treeErr == nil && !lifecycleObj.Container().Equals(p.BktInfo.CID) {
n.deleteLifecycleObject(ctx, p.BktInfo, lifecycleObj)

View file

@ -227,6 +227,46 @@ func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo, decoder func(
return cors, nil
}
func (n *Layer) getWebsiteConfigs(ctx context.Context, bkt *data.BucketInfo) (*data.WebsiteConfiguration, error) {
owner := n.BearerOwner(ctx)
if cors := n.cache.GetWebsiteConfiguration(owner, bkt); cors != nil {
return cors, nil
}
websiteVersions, err := n.getSystemObjectVersions(ctx, n.websiteCnrInfo.CID, bkt.WebsiteObjectFilePath())
if err != nil {
return nil, err
}
var (
prmAuth frostfs.PrmAuth
objID oid.ID
lastWebsite = websiteVersions.GetLast()
)
if lastWebsite == nil {
return nil, fmt.Errorf("get website versions: %w", apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration))
}
prmAuth.PrivateKey = &n.gateKey.PrivateKey
websiteBkt := n.websiteCnrInfo
objID = lastWebsite.ObjID
obj, err := n.objectGetWithAuth(ctx, websiteBkt, objID, prmAuth)
if err != nil {
return nil, fmt.Errorf("get website configuration object: %w", err)
}
website := &data.WebsiteConfiguration{}
if err = xml.NewDecoder(obj.Payload).Decode(&website); err != nil {
return nil, fmt.Errorf("unmarshal website configuration: %w", err)
}
n.cache.PutWebsiteConfiguration(owner, bkt, website)
return website, nil
}
func (n *Layer) getCORSVersions(ctx context.Context, bkt *data.BucketInfo) (*crdt.ObjectVersions, error) {
corsVersions, err := n.frostFS.SearchObjects(ctx, frostfs.PrmObjectSearch{
Container: n.corsCnrInfo.CID,
@ -277,6 +317,56 @@ func (n *Layer) getCORSVersions(ctx context.Context, bkt *data.BucketInfo) (*crd
return versions, nil
}
func (n *Layer) getSystemObjectVersions(ctx context.Context, systemCID cid.ID, filePath string) (*crdt.ObjectVersions, error) {
systemVersions, err := n.frostFS.SearchObjects(ctx, frostfs.PrmObjectSearch{
Container: systemCID,
ExactAttribute: [2]string{object.AttributeFilePath, filePath},
})
if err != nil {
return nil, fmt.Errorf("search system objects: %w", err)
}
versions := crdt.NewObjectVersions(filePath)
versions.SetLessFunc(func(ov1, ov2 *crdt.ObjectVersion) bool {
versionID1, versionID2 := ov1.VersionID(), ov2.VersionID()
timestamp1, timestamp2 := ov1.Headers[object.AttributeTimestamp], ov2.Headers[object.AttributeTimestamp]
if ov1.CreationEpoch != ov2.CreationEpoch {
return ov1.CreationEpoch < ov2.CreationEpoch
}
if len(timestamp1) > 0 && len(timestamp2) > 0 && timestamp1 != timestamp2 {
unixTime1, err := strconv.ParseInt(timestamp1, 10, 64)
if err != nil {
return versionID1 < versionID2
}
unixTime2, err := strconv.ParseInt(timestamp2, 10, 64)
if err != nil {
return versionID1 < versionID2
}
return unixTime1 < unixTime2
}
return versionID1 < versionID2
})
for _, id := range systemVersions {
objVersion, err := n.frostFS.HeadObject(ctx, frostfs.PrmObjectHead{
Container: systemCID,
Object: id,
})
if err != nil {
return nil, fmt.Errorf("head system object '%s': %w", id.EncodeToString(), err)
}
versions.AppendVersion(crdt.NewObjectVersion(objVersion))
}
return versions, nil
}
func lockObjectKey(objVersion *data.ObjectVersion) string {
// todo reconsider forming name since versionID can be "null" or ""
return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID

View file

@ -162,6 +162,13 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
})
require.NoError(t, err)
websiteName := "wsh"
wsh, err := tp.CreateContainer(ctx, frostfs.PrmContainerCreate{
Name: websiteName,
Policy: getPlacementPolicy(),
})
require.NoError(t, err)
config := DefaultCachesConfigs(logger)
if len(cachesConfig) != 0 {
config = cachesConfig[0]
@ -171,11 +178,13 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
user.IDFromKey(&owner, key.PrivateKey.PublicKey)
layerCfg := &Config{
Cache: NewCache(config),
AnonKey: AnonymousKey{Key: key},
TreeService: NewTreeService(),
Features: &FeatureSettingsMock{},
GateOwner: owner,
Cache: NewCache(config),
AnonKey: AnonymousKey{Key: key},
TreeService: NewTreeService(),
Features: &FeatureSettingsMock{},
WebsiteCnrInfo: &data.BucketInfo{CID: wsh.ContainerID},
GateOwner: owner,
GateKey: key,
}
return &testContext{

98
api/layer/website.go Normal file
View file

@ -0,0 +1,98 @@
package layer
import (
"bytes"
"context"
"encoding/xml"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"go.uber.org/zap"
)
func (n *Layer) PutBucketWebsite(ctx context.Context, p *PutWebsiteParams) error {
ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutBucketWebsiteConfig")
defer span.End()
// search for existing configurations
versions, err := n.getSystemObjectVersions(ctx, n.websiteCnrInfo.CID, p.BktInfo.WebsiteObjectFilePath())
if err != nil {
return err
}
cfgBytes, err := xml.Marshal(p.WebsiteConfiguration)
if err != nil {
return fmt.Errorf("marshal website configuration: %w", err)
}
prm := frostfs.PrmObjectCreate{
Payload: bytes.NewReader(cfgBytes),
Filepath: p.BktInfo.WebsiteObjectFilePath(),
CreationTime: TimeNow(ctx),
}
prm.Container = n.websiteCnrInfo.CID
_, err = n.objectPutAndHash(ctx, prm, n.websiteCnrInfo)
if err != nil {
return fmt.Errorf("put website object: %w", err)
}
n.cache.PutWebsiteConfiguration(n.BearerOwner(ctx), p.BktInfo, p.WebsiteConfiguration)
// delete old configurations
if objsToDelete := versions.GetSorted(); len(objsToDelete) > 0 {
for _, version := range objsToDelete {
addr := newAddress(n.websiteCnrInfo.CID, version.ObjID)
n.deleteWebsiteObject(ctx, n.websiteCnrInfo, addr)
}
}
return nil
}
func (n *Layer) GetBucketWebsiteConfig(ctx context.Context, bktInfo *data.BucketInfo) (*data.WebsiteConfiguration, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetBucketWebsiteConfig")
defer span.End()
return n.getWebsiteConfigs(ctx, bktInfo)
}
func (n *Layer) DeleteBucketWebsite(ctx context.Context, bktInfo *data.BucketInfo) error {
ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteBucketWebsiteConfig")
defer span.End()
versions, err := n.getSystemObjectVersions(ctx, n.websiteCnrInfo.CID, bktInfo.WebsiteObjectFilePath())
if err != nil {
return err
}
for _, version := range versions.GetSorted() {
addr := newAddress(n.websiteCnrInfo.CID, version.ObjID)
n.deleteWebsiteObject(ctx, bktInfo, addr)
}
n.cache.DeleteWebsiteConfiguration(bktInfo)
return nil
}
// deleteLifecycleObject removes object and logs in case of error.
func (n *Layer) deleteWebsiteObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) {
var prmAuth frostfs.PrmAuth
websiteBkt := bktInfo
if !addr.Container().Equals(bktInfo.CID) {
websiteBkt = &data.BucketInfo{CID: addr.Container()}
prmAuth.PrivateKey = &n.gateKey.PrivateKey
}
if err := n.objectDeleteWithAuth(ctx, websiteBkt, addr.Object(), prmAuth); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteWebsiteObject, zap.Error(err),
zap.String("cid", websiteBkt.CID.EncodeToString()),
zap.String("oid", addr.Object().EncodeToString()),
logs.TagField(logs.TagExternalStorage),
)
}
}

59
api/layer/website_test.go Normal file
View file

@ -0,0 +1,59 @@
package layer
import (
"encoding/xml"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
"github.com/stretchr/testify/require"
)
func TestBucketWebsite(t *testing.T) {
tc := prepareContext(t)
website := &data.WebsiteConfiguration{
XMLName: xml.Name{
Space: `http://s3.amazonaws.com/doc/2006-03-01/`,
Local: "WebsiteConfiguration",
},
ErrorDocument: &data.ErrorDocument{Key: "error.html"},
IndexDocument: &data.IndexDocument{Suffix: "index.html"},
RoutingRules: []data.RoutingRule{
{
XMLName: xml.Name{},
Condition: &data.Condition{
HTTPErrorCodeReturnedEquals: "404",
},
Redirect: data.Redirect{
HostName: "www.example.com",
Protocol: "http",
},
},
},
}
_, err := tc.layer.GetBucketWebsiteConfig(tc.ctx, tc.bktInfo)
require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration), frosterr.UnwrapErr(err))
err = tc.layer.DeleteBucketWebsite(tc.ctx, tc.bktInfo)
require.NoError(t, err)
err = tc.layer.PutBucketWebsite(tc.ctx, &PutWebsiteParams{
BktInfo: tc.bktInfo,
WebsiteConfiguration: website,
CopiesNumbers: []uint32{1},
})
require.NoError(t, err)
cfg, err := tc.layer.GetBucketWebsiteConfig(tc.ctx, tc.bktInfo)
require.NoError(t, err)
require.Equal(t, *website, *cfg)
err = tc.layer.DeleteBucketWebsite(tc.ctx, tc.bktInfo)
require.NoError(t, err)
_, err = tc.layer.GetBucketWebsiteConfig(tc.ctx, tc.bktInfo)
require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration), frosterr.UnwrapErr(err))
}

View file

@ -38,6 +38,7 @@ const (
PutBucketTaggingOperation = "PutBucketTagging"
PutBucketVersioningOperation = "PutBucketVersioning"
PutBucketNotificationOperation = "PutBucketNotification"
PutBucketWebsiteOperation = "PutBucketWebsite"
CreateBucketOperation = "CreateBucket"
DeleteMultipleObjectsOperation = "DeleteMultipleObjects"
PostObjectOperation = "PostObject"

View file

@ -53,6 +53,7 @@ type (
GetBucketReplicationHandler(http.ResponseWriter, *http.Request)
GetBucketTaggingHandler(http.ResponseWriter, *http.Request)
DeleteBucketWebsiteHandler(http.ResponseWriter, *http.Request)
PutBucketWebsiteHandler(http.ResponseWriter, *http.Request)
DeleteBucketTaggingHandler(http.ResponseWriter, *http.Request)
GetBucketObjectLockConfigHandler(http.ResponseWriter, *http.Request)
GetBucketVersioningHandler(http.ResponseWriter, *http.Request)
@ -88,7 +89,6 @@ type (
ListPartsHandler(w http.ResponseWriter, r *http.Request)
ListMultipartUploadsHandler(http.ResponseWriter, *http.Request)
PatchObjectHandler(http.ResponseWriter, *http.Request)
ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error)
ResolveCID(ctx context.Context, bucket string) (cid.ID, error)
}
@ -403,6 +403,9 @@ func bucketRouter(h Handler) chi.Router {
Add(NewFilter().
Queries(s3middleware.NotificationQuery).
Handler(named(s3middleware.PutBucketNotificationOperation, h.PutBucketNotificationHandler))).
Add(NewFilter().
Queries(s3middleware.WebsiteQuery).
Handler(named(s3middleware.PutBucketWebsiteOperation, h.PutBucketWebsiteHandler))).
Add(NewFilter().
NoQueries().
Handler(named(s3middleware.CreateBucketOperation, h.CreateBucketHandler))).

View file

@ -181,6 +181,11 @@ type handlerMock struct {
buckets map[string]*data.BucketInfo
}
func (h *handlerMock) PutBucketWebsiteHandler(http.ResponseWriter, *http.Request) {
//TODO implement me
panic("implement me")
}
type handlerResult struct {
Method string
ReqInfo *middleware.ReqInfo

View file

@ -286,6 +286,11 @@ func (a *App) initLayer(ctx context.Context) {
a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
}
wshCnrInfo, err := a.fetchContainerInfo(ctx, cfgContainersWebStaticHosting)
if err != nil {
a.log.Fatal(logs.CouldNotFetchWebStaticHostingContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
}
var lifecycleCnrInfo *data.BucketInfo
if a.config().IsSet(cfgContainersLifecycle) {
lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle)
@ -306,6 +311,7 @@ func (a *App) initLayer(ctx context.Context) {
GateKey: a.key,
CORSCnrInfo: corsCnrInfo,
LifecycleCnrInfo: lifecycleCnrInfo,
WebsiteCnrInfo: wshCnrInfo,
WorkerPool: a.initWorkerPool(),
}

View file

@ -221,9 +221,10 @@ const (
cfgSourceIPHeader = "source_ip_header"
// Containers.
cfgContainersCORS = "containers.cors"
cfgContainersLifecycle = "containers.lifecycle"
cfgContainersAccessBox = "containers.accessbox"
cfgContainersCORS = "containers.cors"
cfgContainersLifecycle = "containers.lifecycle"
cfgContainersAccessBox = "containers.accessbox"
cfgContainersWebStaticHosting = "containers.web_static_hosting"
// Multinet.
cfgMultinetEnabled = "multinet.enabled"

View file

@ -273,6 +273,8 @@ S3_GW_RETRY_STRATEGY=exponential
# Containers properties
S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
S3_GW_CONTAINERS_LIFECYCLE=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
S3_GW_CONTAINERS_ACCESSBOX: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2
S3_GW_CONTAINERS_STATIC_WEB_HOSTING: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP
# Multinet properties
# Enable multinet support

View file

@ -322,6 +322,8 @@ retry:
containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
accessbox: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2
web_static_hosting: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP
# Multinet properties
multinet:

View file

@ -859,13 +859,15 @@ containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
accessbox: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2
web_static_hosting: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
| `accessbox` | `string` | no | | Container name to lookup accessbox if custom aws credentials is used. If not set, custom credentials are not supported. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|----------------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
| `accessbox` | `string` | no | | Container name to lookup accessbox if custom aws credentials is used. If not set, custom credentials are not supported. |
| `web_static_hosting` | `string` | no | | Container name for website configurations. |
# `vhs` section

View file

@ -48,6 +48,7 @@ const (
ServerReconnectFailed = "failed to reconnect server"
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
CouldNotFetchAccessBoxContainerInfo = "couldn't fetch AccessBox container info"
CouldNotFetchWebStaticHostingContainerInfo = "couldn't fetch web static hosting container info"
MultinetDialSuccess = "multinet dial successful"
MultinetDialFail = "multinet dial failed"
FailedToCreateWorkerPool = "failed to create worker pool"
@ -126,6 +127,7 @@ const (
UnexpectedMultiNodeIDsInSubTreeMultiParts = "unexpected multi node ids in sub tree multi parts"
FoundSeveralSystemNodes = "found several system nodes"
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
BucketWebsiteNodeHasMultipleIDs = "bucket website node has multiple ids"
UploadPart = "upload part"
FailedToSubmitTaskToPool = "failed to submit task to pool"
FailedToGetRealObjectSize = "failed to get real object size"
@ -154,6 +156,7 @@ const (
CouldntCacheCors = "couldn't cache cors"
CouldntCacheListPolicyChains = "couldn't cache list policy chains"
CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration"
CouldntCacheWebsiteConfiguration = "couldn't cache website configuration"
GetBucketCors = "get bucket cors"
GetBucketInfo = "get bucket info"
ResolveBucket = "resolve bucket"
@ -180,6 +183,7 @@ const (
CouldNotFetchObjectMeta = "could not fetch object meta"
FailedToDeleteObject = "failed to delete object"
CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object"
CouldntDeleteWebsiteObject = "couldn't delete website configuration object"
CouldntGetCORSObjectVersions = "couldn't get cors object versions"
)