From 0fe63f810f9a65c87109bdefcb142a748069b714 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Thu, 13 Mar 2025 15:27:01 +0300 Subject: [PATCH] [#660] Add bucket website operations Signed-off-by: Nikita Zinkevich --- api/cache/system.go | 20 ++++ api/data/info.go | 13 ++- api/data/website.go | 42 +++++++ api/handler/unimplemented.go | 8 -- api/handler/website.go | 194 +++++++++++++++++++++++++++++++++ api/handler/website_test.go | 114 +++++++++++++++++++ api/layer/cache.go | 26 +++++ api/layer/layer.go | 10 ++ api/layer/tree/tree_service.go | 3 + api/layer/tree_mock.go | 45 ++++++++ api/layer/website.go | 146 +++++++++++++++++++++++++ api/layer/website_test.go | 59 ++++++++++ api/middleware/constants.go | 1 + api/router.go | 5 +- api/router_mock_test.go | 5 + cmd/s3-gw/app.go | 6 + cmd/s3-gw/app_settings.go | 7 +- config/config.env | 2 + config/config.yaml | 2 + docs/configuration.md | 12 +- internal/logs/logs.go | 4 + pkg/service/tree/tree.go | 78 +++++++++++++ 22 files changed, 784 insertions(+), 18 deletions(-) create mode 100644 api/data/website.go create mode 100644 api/handler/website.go create mode 100644 api/handler/website_test.go create mode 100644 api/layer/website.go create mode 100644 api/layer/website_test.go diff --git a/api/cache/system.go b/api/cache/system.go index 4e0ad20c..412f5375 100644 --- a/api/cache/system.go +++ b/api/cache/system.go @@ -104,6 +104,22 @@ func (o *SystemCache) GetLifecycleConfiguration(key string) *data.LifecycleConfi return result } +func (o *SystemCache) GetWebsiteConfiguration(key string) *data.WebsiteConfiguration { + entry, err := o.cache.Get(key) + if err != nil { + return nil + } + + result, ok := entry.(*data.WebsiteConfiguration) + if !ok { + o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)), + zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath)) + return nil + } + + return result +} + func (o *SystemCache) GetSettings(key string) *data.BucketSettings { entry, err := o.cache.Get(key) if err != nil { @@ -153,6 +169,10 @@ func (o *SystemCache) PutLifecycleConfiguration(key string, obj *data.LifecycleC return o.cache.Set(key, obj) } +func (o *SystemCache) PutWebsiteConfiguration(key string, obj *data.WebsiteConfiguration) error { + return o.cache.Set(key, obj) +} + func (o *SystemCache) PutSettings(key string, settings *data.BucketSettings) error { return o.cache.Set(key, settings) } diff --git a/api/data/info.go b/api/data/info.go index c9f9df69..5d4ef47a 100644 --- a/api/data/info.go +++ b/api/data/info.go @@ -17,12 +17,14 @@ const ( bktSettingsObject = ".s3-settings" bktCORSConfigurationObject = ".s3-cors" bktLifecycleConfigurationObject = ".s3-lifecycle" + bktWebsiteConfigurationObject = ".s3-website" VersioningUnversioned = "Unversioned" VersioningEnabled = "Enabled" VersioningSuspended = "Suspended" - corsFilePathTemplate = "/%s.cors" + corsFilePathTemplate = "/%s.cors" + websiteFilePathTemplate = "/%s.wsh" ) type ( @@ -115,6 +117,15 @@ func (b *BucketInfo) LifecycleConfigurationObjectName() string { return b.CID.EncodeToString() + bktLifecycleConfigurationObject } +// WebsiteObjectFilePath returns a FilePath for a bucket CORS configuration file. +func (b *BucketInfo) WebsiteObjectFilePath() string { + return fmt.Sprintf(websiteFilePathTemplate, b.CID) +} + +func (b *BucketInfo) WebsiteConfigurationObjectName() string { + return b.CID.EncodeToString() + bktWebsiteConfigurationObject +} + // VersionID returns object version from ObjectInfo. func (o *ObjectInfo) VersionID() string { return o.ID.EncodeToString() } diff --git a/api/data/website.go b/api/data/website.go new file mode 100644 index 00000000..98f0984b --- /dev/null +++ b/api/data/website.go @@ -0,0 +1,42 @@ +package data + +import "encoding/xml" + +type WebsiteConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ WebsiteConfiguration" json:"-"` + ErrorDocument *ErrorDocument `xml:"ErrorDocument" json:"ErrorDocument"` + IndexDocument *IndexDocument `xml:"IndexDocument" json:"IndexDocument"` + RedirectAllRequestsTo *RedirectAllRequestsTo `xml:"RedirectAllRequestsTo" json:"RedirectAllRequestsTo"` + RoutingRules []RoutingRule `xml:"RoutingRules>RoutingRule" json:"RoutingRules"` +} + +type ErrorDocument struct { + Key string `xml:"Key" json:"Key"` +} + +type IndexDocument struct { + Suffix string `xml:"Suffix" json:"Suffix"` +} + +type RedirectAllRequestsTo struct { + HostName string `xml:"HostName" json:"HostName"` + Protocol *string `xml:"Protocol" json:"Protocol"` +} + +type RoutingRule struct { + XMLName xml.Name `xml:"RoutingRule" json:"-"` + Redirect Redirect `xml:"Redirect" json:"Redirect"` + Condition *Condition `xml:"Condition" json:"Condition"` +} + +type Redirect struct { + HostName string `xml:"HostName" json:"HostName"` + Protocol string `xml:"Protocol" json:"Protocol"` + ReplaceKeyPrefixWith string `xml:"ReplaceKeyPrefixWith" json:"ReplaceKeyPrefixWith"` + ReplaceKeyWith string `xml:"ReplaceKeyWith" json:"ReplaceKeyWith"` +} + +type Condition struct { + HTTPErrorCodeReturnedEquals string `xml:"HttpErrorCodeReturnedEquals" json:"HttpErrorCodeReturnedEquals"` + KeyPrefixEquals string `xml:"KeyPrefixEquals" json:"KeyPrefixEquals"` +} diff --git a/api/handler/unimplemented.go b/api/handler/unimplemented.go index b4d8c404..322fe429 100644 --- a/api/handler/unimplemented.go +++ b/api/handler/unimplemented.go @@ -15,10 +15,6 @@ func (h *handler) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Requ h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } -func (h *handler) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - func (h *handler) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } @@ -35,10 +31,6 @@ func (h *handler) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Req h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } -func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { - h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) -} - func (h *handler) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { h.logAndSendError(r.Context(), w, "not implemented", middleware.GetReqInfo(r.Context()), errors.GetAPIError(errors.ErrNotImplemented)) } diff --git a/api/handler/website.go b/api/handler/website.go new file mode 100644 index 00000000..e4cb8d76 --- /dev/null +++ b/api/handler/website.go @@ -0,0 +1,194 @@ +package handler + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "slices" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" +) + +func (h *handler) PutBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.PutBucketWebsite") + defer span.End() + + var buf bytes.Buffer + + tee := io.TeeReader(r.Body, &buf) + ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag) + reqInfo := middleware.GetReqInfo(ctx) + + cfg := new(data.WebsiteConfiguration) + if err := h.cfg.NewXMLDecoder(tee, r.UserAgent()).Decode(cfg); err != nil { + h.logAndSendError(ctx, w, "could not decode body", reqInfo, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrMalformedXML), err.Error())) + return + } + + if _, ok := r.Header[api.ContentMD5]; ok { + headerMD5, err := base64.StdEncoding.DecodeString(r.Header.Get(api.ContentMD5)) + if err != nil { + h.logAndSendError(ctx, w, "invalid Content-MD5", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) + return + } + + bodyMD5, err := getContentMD5(&buf) + if err != nil { + h.logAndSendError(ctx, w, "could not get content md5", reqInfo, err) + return + } + + if !bytes.Equal(headerMD5, bodyMD5) { + h.logAndSendError(ctx, w, "Content-MD5 does not match", reqInfo, apierr.GetAPIError(apierr.ErrInvalidDigest)) + return + } + } + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err) + return + } + + if err = checkWebsiteConfiguration(cfg); err != nil { + h.logAndSendError(ctx, w, "invalid website configuration", reqInfo, err) + return + } + + params := &layer.PutWebsiteParams{ + BktInfo: bktInfo, + WebsiteConfiguration: cfg, + } + + params.CopiesNumbers, err = h.pickCopiesNumbers(parseMetadata(r), reqInfo.Namespace, bktInfo.LocationConstraint) + if err != nil { + h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err) + return + } + + if err = h.obj.PutBucketWebsite(ctx, params); err != nil { + h.logAndSendError(ctx, w, "could not put bucket website configuration", reqInfo, err) + return + } +} + +func (h *handler) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.GetBucketWebsite") + defer span.End() + + ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag) + reqInfo := middleware.GetReqInfo(ctx) + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err) + return + } + + cors, err := h.obj.GetBucketWebsite(ctx, bktInfo) + if err != nil { + h.logAndSendError(ctx, w, "could not get website configuration", reqInfo, err) + return + } + + if err = middleware.EncodeToResponse(w, cors); err != nil { + h.logAndSendError(ctx, w, "could not encode website configuration to response", reqInfo, err) + return + } +} + +func (h *handler) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.StartSpanFromContext(r.Context(), "handler.DeleteBucketWebsite") + defer span.End() + + ctx = qostagging.ContextWithIOTag(ctx, util.InternalIOTag) + reqInfo := middleware.GetReqInfo(ctx) + + bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName) + if err != nil { + h.logAndSendError(ctx, w, "could not get bucket info", reqInfo, err) + return + } + + if err = h.obj.DeleteBucketWebsite(ctx, bktInfo); err != nil { + h.logAndSendError(ctx, w, "could not delete cors", reqInfo, err) + } + + w.WriteHeader(http.StatusNoContent) +} + +func checkWebsiteConfiguration(con *data.WebsiteConfiguration) error { + if con.RedirectAllRequestsTo != nil { + if con.RedirectAllRequestsTo.HostName == "" { + return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests hostname is empty")) + } + if con.RedirectAllRequestsTo.Protocol != nil && !slices.Contains([]string{"http", "https"}, *con.RedirectAllRequestsTo.Protocol) { + return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, fmt.Errorf("redirect all requests incorrect protocol: %s", *con.RedirectAllRequestsTo.Protocol)) + } + } else { + if con.IndexDocument != nil && con.IndexDocument.Suffix == "" { + return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("index document suffix is empty")) + } + if con.ErrorDocument != nil && con.ErrorDocument.Key == "" { + return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("error document key is empty")) + } + if err := checkRoutingRules(con.RoutingRules); err != nil { + return apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, err) + } + } + return nil +} + +func checkRoutingRules(rules []data.RoutingRule) error { + if len(rules) == 0 { + return errors.New("no routing rules specified") + } + for _, r := range rules { + if err := checkRedirect(r); err != nil { + return err + } + if err := checkCondition(r); err != nil { + return err + } + } + + return nil +} + +func checkRedirect(r data.RoutingRule) error { + if checkAllEmpty(r.Redirect.ReplaceKeyPrefixWith, r.Redirect.ReplaceKeyWith, r.Redirect.HostName, r.Redirect.Protocol) { + return errors.New("empty redirect rule") + } + if r.Redirect.Protocol != "" && !slices.Contains([]string{"http", "https"}, r.Redirect.Protocol) { + return fmt.Errorf("invalid redirect protocol: %s", r.Redirect.Protocol) + } + return nil +} + +func checkCondition(r data.RoutingRule) error { + if r.Condition != nil { + if checkAllEmpty(r.Condition.HTTPErrorCodeReturnedEquals, r.Condition.KeyPrefixEquals) { + return errors.New("empty condition properties") + } + } + return nil +} + +func checkAllEmpty(elems ...string) bool { + for _, el := range elems { + if el != "" { + return false + } + } + return true +} diff --git a/api/handler/website_test.go b/api/handler/website_test.go new file mode 100644 index 00000000..11b5b649 --- /dev/null +++ b/api/handler/website_test.go @@ -0,0 +1,114 @@ +package handler + +import ( + "errors" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "github.com/stretchr/testify/require" +) + +func TestCheckWebsiteConfiguration(t *testing.T) { + tests := []struct { + name string + config data.WebsiteConfiguration + wantErr error + }{ + { + name: "Valid RedirectAllRequestsTo", + config: data.WebsiteConfiguration{ + RedirectAllRequestsTo: &data.RedirectAllRequestsTo{ + HostName: "example.com", + Protocol: stringPtr("https"), + }, + }, + wantErr: nil, + }, + { + name: "Invalid RedirectAllRequestsTo - wrong protocol", + config: data.WebsiteConfiguration{ + RedirectAllRequestsTo: &data.RedirectAllRequestsTo{ + HostName: "example.com", + Protocol: stringPtr("ftp"), + }, + }, + wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests incorrect protocol: ftp")), + }, + { + name: "Invalid RedirectAllRequestsTo - empty hostname", + config: data.WebsiteConfiguration{ + RedirectAllRequestsTo: &data.RedirectAllRequestsTo{ + HostName: "", + Protocol: stringPtr("https"), + }, + }, + wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("redirect all requests hostname is empty")), + }, + { + name: "Valid Index and Error Documents", + config: data.WebsiteConfiguration{ + IndexDocument: &data.IndexDocument{Suffix: "index.html"}, + ErrorDocument: &data.ErrorDocument{Key: "error.html"}, + RoutingRules: []data.RoutingRule{{ + Redirect: data.Redirect{ + HostName: "example.com", + Protocol: "https", + ReplaceKeyWith: "new-key", + }, + }}, + }, + wantErr: nil, + }, + { + name: "Invalid Index Document - empty suffix", + config: data.WebsiteConfiguration{ + IndexDocument: &data.IndexDocument{Suffix: ""}, + }, + wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("index document suffix is empty")), + }, + { + name: "Invalid Routing Rules - empty rules", + config: data.WebsiteConfiguration{ + RoutingRules: []data.RoutingRule{}, + }, + wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("no routing rules specified")), + }, + { + name: "Invalid Routing Rules - invalid protocol", + config: data.WebsiteConfiguration{ + RoutingRules: []data.RoutingRule{ + { + Redirect: data.Redirect{ + Protocol: "ftp", + }, + }, + }, + }, + wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("invalid redirect protocol: ftp")), + }, + { + name: "Invalid Routing Rule - empty rule", + config: data.WebsiteConfiguration{ + RoutingRules: []data.RoutingRule{{}}, + }, + wantErr: apierr.GetAPIErrorWithError(apierr.ErrInvalidArgument, errors.New("empty redirect rule")), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkWebsiteConfiguration(&tt.config) + if tt.wantErr == nil { + require.NoError(t, err) + } else { + require.Error(t, err) + require.Equal(t, tt.wantErr.Error(), err.Error()) + } + }) + } +} + +func stringPtr(s string) *string { + return &s +} diff --git a/api/layer/cache.go b/api/layer/cache.go index 1995da9c..4418b3dd 100644 --- a/api/layer/cache.go +++ b/api/layer/cache.go @@ -293,6 +293,32 @@ func (c *Cache) DeleteLifecycleConfiguration(bktInfo *data.BucketInfo) { c.systemCache.Delete(bktInfo.LifecycleConfigurationObjectName()) } +func (c *Cache) GetWebsiteConfiguration(owner user.ID, bkt *data.BucketInfo) *data.WebsiteConfiguration { + key := bkt.WebsiteConfigurationObjectName() + + if !c.accessCache.Get(owner, key) { + return nil + } + + return c.systemCache.GetWebsiteConfiguration(key) +} + +func (c *Cache) PutWebsiteConfiguration(owner user.ID, bkt *data.BucketInfo, cfg *data.WebsiteConfiguration) { + key := bkt.WebsiteConfigurationObjectName() + + if err := c.systemCache.PutWebsiteConfiguration(key, cfg); err != nil { + c.logger.Warn(logs.CouldntCacheWebsiteConfiguration, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath)) + } + + if err := c.accessCache.Put(owner, key); err != nil { + c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath)) + } +} + +func (c *Cache) DeleteWebsiteConfiguration(bktInfo *data.BucketInfo) { + c.systemCache.Delete(bktInfo.WebsiteConfigurationObjectName()) +} + func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo { return c.networkCache.GetNetworkInfo() } diff --git a/api/layer/layer.go b/api/layer/layer.go index b658d51e..8217433a 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -61,6 +61,7 @@ type ( features FeatureSettings gateKey *keys.PrivateKey corsCnrInfo *data.BucketInfo + websiteCnrInfo *data.BucketInfo lifecycleCnrInfo *data.BucketInfo workerPool *ants.Pool } @@ -75,6 +76,7 @@ type ( Features FeatureSettings GateKey *keys.PrivateKey CORSCnrInfo *data.BucketInfo + WebsiteCnrInfo *data.BucketInfo LifecycleCnrInfo *data.BucketInfo WorkerPool *ants.Pool } @@ -153,6 +155,13 @@ type ( UserAgent string } + // PutWebsiteParams stores PutBucketWebsite request parameters. + PutWebsiteParams struct { + BktInfo *data.BucketInfo + WebsiteConfiguration *data.WebsiteConfiguration + CopiesNumbers []uint32 + } + // CopyObjectParams stores object copy request parameters. CopyObjectParams struct { SrcVersioned bool @@ -269,6 +278,7 @@ func NewLayer(log *zap.Logger, frostFS frostfs.FrostFS, config *Config) *Layer { gateKey: config.GateKey, corsCnrInfo: config.CORSCnrInfo, lifecycleCnrInfo: config.LifecycleCnrInfo, + websiteCnrInfo: config.WebsiteCnrInfo, workerPool: config.WorkerPool, } } diff --git a/api/layer/tree/tree_service.go b/api/layer/tree/tree_service.go index f3b6f6e3..7cea37ca 100644 --- a/api/layer/tree/tree_service.go +++ b/api/layer/tree/tree_service.go @@ -67,6 +67,9 @@ type Service interface { GetBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) + PutBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) + GetBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) + DeleteBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) // Compound methods for optimizations // GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation. diff --git a/api/layer/tree_mock.go b/api/layer/tree_mock.go index 4d27da44..ce83ddeb 100644 --- a/api/layer/tree_mock.go +++ b/api/layer/tree_mock.go @@ -38,6 +38,51 @@ type TreeServiceMock struct { parts map[string]map[int]*data.PartInfoExtended } +func (t *TreeServiceMock) PutBucketWebsiteConfiguration(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) { + systemMap, ok := t.system[bktInfo.CID.EncodeToString()] + if !ok { + systemMap = make(map[string]*data.BaseNodeVersion) + } + + systemMap["website"] = &data.BaseNodeVersion{ + OID: addr.Object(), + } + + t.system[bktInfo.CID.EncodeToString()] = systemMap + + return nil, tree.ErrNoNodeToRemove +} + +func (t *TreeServiceMock) GetBucketWebsiteConfiguration(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) { + systemMap, ok := t.system[bktInfo.CID.EncodeToString()] + if !ok { + return oid.Address{}, tree.ErrNodeNotFound + } + + node, ok := systemMap["website"] + if !ok { + return oid.Address{}, tree.ErrNodeNotFound + } + + return newAddress(bktInfo.CID, node.OID), nil +} + +func (t *TreeServiceMock) DeleteBucketWebsiteConfiguration(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) { + systemMap, ok := t.system[bktInfo.CID.EncodeToString()] + if !ok { + return nil, tree.ErrNoNodeToRemove + } + + node, ok := systemMap["website"] + if !ok { + return nil, tree.ErrNoNodeToRemove + } + + delete(systemMap, "website") + + return []oid.Address{newAddress(bktInfo.CID, node.OID)}, nil +} + func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) { // TODO implement object tagging lock, err := t.GetLock(ctx, bktInfo, objVersion.ID) diff --git a/api/layer/website.go b/api/layer/website.go new file mode 100644 index 00000000..049d414b --- /dev/null +++ b/api/layer/website.go @@ -0,0 +1,146 @@ +package layer + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" +) + +func (n *Layer) PutBucketWebsite(ctx context.Context, p *PutWebsiteParams) error { + ctx, span := tracing.StartSpanFromContext(ctx, "layer.PutBucketWebsiteConfiguration") + defer span.End() + + cfgBytes, err := xml.Marshal(p.WebsiteConfiguration) + if err != nil { + return fmt.Errorf("marshal lifecycle configuration: %w", err) + } + + prm := frostfs.PrmObjectCreate{ + Payload: bytes.NewReader(cfgBytes), + Filepath: p.BktInfo.WebsiteConfigurationObjectName(), + CreationTime: TimeNow(ctx), + } + + var websiteBkt *data.BucketInfo + if n.websiteCnrInfo == nil { + websiteBkt = p.BktInfo + prm.CopiesNumber = p.CopiesNumbers + } else { + websiteBkt = n.websiteCnrInfo + prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey + } + + prm.Container = websiteBkt.CID + + createdObj, err := n.objectPutAndHash(ctx, prm, websiteBkt) + if err != nil { + return fmt.Errorf("put website object: %w", err) + } + + objsToDelete, err := n.treeService.PutBucketWebsiteConfiguration(ctx, p.BktInfo, newAddress(websiteBkt.CID, createdObj.ID)) + objsToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove) + if err != nil && !objsToDeleteNotFound { + return err + } + + if !objsToDeleteNotFound { + for _, addr := range objsToDelete { + n.deleteWebsiteObject(ctx, p.BktInfo, addr) + } + } + + n.cache.PutWebsiteConfiguration(n.BearerOwner(ctx), p.BktInfo, p.WebsiteConfiguration) + + return nil +} + +func (n *Layer) GetBucketWebsite(ctx context.Context, bktInfo *data.BucketInfo) (*data.WebsiteConfiguration, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "layer.GetBucketWebsite") + defer span.End() + + owner := n.BearerOwner(ctx) + if cfg := n.cache.GetWebsiteConfiguration(owner, bktInfo); cfg != nil { + return cfg, nil + } + + addr, err := n.treeService.GetBucketWebsiteConfiguration(ctx, bktInfo) + objNotFound := errors.Is(err, tree.ErrNodeNotFound) + if err != nil && !objNotFound { + return nil, err + } + + if objNotFound { + return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration), err.Error()) + } + + var prmAuth frostfs.PrmAuth + lifecycleBkt := bktInfo + if !addr.Container().Equals(bktInfo.CID) { + lifecycleBkt = &data.BucketInfo{CID: addr.Container()} + prmAuth.PrivateKey = &n.gateKey.PrivateKey + } + + obj, err := n.objectGetWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth) + if err != nil { + return nil, fmt.Errorf("get lifecycle object: %w", err) + } + + lifecycleCfg := &data.WebsiteConfiguration{} + + if err = xml.NewDecoder(obj.Payload).Decode(&lifecycleCfg); err != nil { + return nil, fmt.Errorf("unmarshal lifecycle configuration: %w", err) + } + + n.cache.PutWebsiteConfiguration(owner, bktInfo, lifecycleCfg) + + return lifecycleCfg, nil +} + +func (n *Layer) DeleteBucketWebsite(ctx context.Context, bktInfo *data.BucketInfo) error { + ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteBucketWebsite") + defer span.End() + + objs, err := n.treeService.DeleteBucketWebsiteConfiguration(ctx, bktInfo) + objsNotFound := errors.Is(err, tree.ErrNoNodeToRemove) + if err != nil && !objsNotFound { + return err + } + if !objsNotFound { + for _, addr := range objs { + n.deleteWebsiteObject(ctx, bktInfo, addr) + } + } + + n.cache.DeleteWebsiteConfiguration(bktInfo) + + return nil +} + +// deleteLifecycleObject removes object and logs in case of error. +func (n *Layer) deleteWebsiteObject(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) { + var prmAuth frostfs.PrmAuth + websiteBkt := bktInfo + if !addr.Container().Equals(bktInfo.CID) { + websiteBkt = &data.BucketInfo{CID: addr.Container()} + prmAuth.PrivateKey = &n.gateKey.PrivateKey + } + + if err := n.objectDeleteWithAuth(ctx, websiteBkt, addr.Object(), prmAuth); err != nil { + n.reqLogger(ctx).Error(logs.CouldntDeleteWebsiteObject, zap.Error(err), + zap.String("cid", websiteBkt.CID.EncodeToString()), + zap.String("oid", addr.Object().EncodeToString()), + logs.TagField(logs.TagExternalStorage), + ) + } +} diff --git a/api/layer/website_test.go b/api/layer/website_test.go new file mode 100644 index 00000000..4a64be72 --- /dev/null +++ b/api/layer/website_test.go @@ -0,0 +1,59 @@ +package layer + +import ( + "encoding/xml" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" + apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" + frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors" + "github.com/stretchr/testify/require" +) + +func TestBucketWebsite(t *testing.T) { + tc := prepareContext(t) + + website := &data.WebsiteConfiguration{ + XMLName: xml.Name{ + Space: `http://s3.amazonaws.com/doc/2006-03-01/`, + Local: "WebsiteConfiguration", + }, + ErrorDocument: &data.ErrorDocument{Key: "error.html"}, + IndexDocument: &data.IndexDocument{Suffix: "index.html"}, + RoutingRules: []data.RoutingRule{ + { + XMLName: xml.Name{}, + Condition: &data.Condition{ + HTTPErrorCodeReturnedEquals: "404", + }, + Redirect: data.Redirect{ + HostName: "www.example.com", + Protocol: "http", + }, + }, + }, + } + + _, err := tc.layer.GetBucketWebsite(tc.ctx, tc.bktInfo) + require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration), frosterr.UnwrapErr(err)) + + err = tc.layer.DeleteBucketWebsite(tc.ctx, tc.bktInfo) + require.NoError(t, err) + + err = tc.layer.PutBucketWebsite(tc.ctx, &PutWebsiteParams{ + BktInfo: tc.bktInfo, + WebsiteConfiguration: website, + CopiesNumbers: []uint32{1}, + }) + require.NoError(t, err) + + cfg, err := tc.layer.GetBucketWebsite(tc.ctx, tc.bktInfo) + require.NoError(t, err) + require.Equal(t, *website, *cfg) + + err = tc.layer.DeleteBucketWebsite(tc.ctx, tc.bktInfo) + require.NoError(t, err) + + _, err = tc.layer.GetBucketWebsite(tc.ctx, tc.bktInfo) + require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchWebsiteConfiguration), frosterr.UnwrapErr(err)) +} diff --git a/api/middleware/constants.go b/api/middleware/constants.go index 2c8cdfc6..bf260613 100644 --- a/api/middleware/constants.go +++ b/api/middleware/constants.go @@ -38,6 +38,7 @@ const ( PutBucketTaggingOperation = "PutBucketTagging" PutBucketVersioningOperation = "PutBucketVersioning" PutBucketNotificationOperation = "PutBucketNotification" + PutBucketWebsiteOperation = "PutBucketWebsite" CreateBucketOperation = "CreateBucket" DeleteMultipleObjectsOperation = "DeleteMultipleObjects" PostObjectOperation = "PostObject" diff --git a/api/router.go b/api/router.go index 832f9405..722d8e04 100644 --- a/api/router.go +++ b/api/router.go @@ -53,6 +53,7 @@ type ( GetBucketReplicationHandler(http.ResponseWriter, *http.Request) GetBucketTaggingHandler(http.ResponseWriter, *http.Request) DeleteBucketWebsiteHandler(http.ResponseWriter, *http.Request) + PutBucketWebsiteHandler(http.ResponseWriter, *http.Request) DeleteBucketTaggingHandler(http.ResponseWriter, *http.Request) GetBucketObjectLockConfigHandler(http.ResponseWriter, *http.Request) GetBucketVersioningHandler(http.ResponseWriter, *http.Request) @@ -88,7 +89,6 @@ type ( ListPartsHandler(w http.ResponseWriter, r *http.Request) ListMultipartUploadsHandler(http.ResponseWriter, *http.Request) PatchObjectHandler(http.ResponseWriter, *http.Request) - ResolveBucket(ctx context.Context, bucket string) (*data.BucketInfo, error) ResolveCID(ctx context.Context, bucket string) (cid.ID, error) } @@ -403,6 +403,9 @@ func bucketRouter(h Handler) chi.Router { Add(NewFilter(). Queries(s3middleware.NotificationQuery). Handler(named(s3middleware.PutBucketNotificationOperation, h.PutBucketNotificationHandler))). + Add(NewFilter(). + Queries(s3middleware.WebsiteQuery). + Handler(named(s3middleware.PutBucketWebsiteOperation, h.PutBucketWebsiteHandler))). Add(NewFilter(). NoQueries(). Handler(named(s3middleware.CreateBucketOperation, h.CreateBucketHandler))). diff --git a/api/router_mock_test.go b/api/router_mock_test.go index 5d1320ec..86f84fe0 100644 --- a/api/router_mock_test.go +++ b/api/router_mock_test.go @@ -181,6 +181,11 @@ type handlerMock struct { buckets map[string]*data.BucketInfo } +func (h *handlerMock) PutBucketWebsiteHandler(http.ResponseWriter, *http.Request) { + //TODO implement me + panic("implement me") +} + type handlerResult struct { Method string ReqInfo *middleware.ReqInfo diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index d81babf2..3c62abac 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -286,6 +286,11 @@ func (a *App) initLayer(ctx context.Context) { a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp)) } + wshCnrInfo, err := a.fetchContainerInfo(ctx, cfgContainersWebStaticHosting) + if err != nil { + a.log.Fatal(logs.CouldNotFetchWebStaticHostingContainerInfo, zap.Error(err), logs.TagField(logs.TagApp)) + } + var lifecycleCnrInfo *data.BucketInfo if a.config().IsSet(cfgContainersLifecycle) { lifecycleCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersLifecycle) @@ -306,6 +311,7 @@ func (a *App) initLayer(ctx context.Context) { GateKey: a.key, CORSCnrInfo: corsCnrInfo, LifecycleCnrInfo: lifecycleCnrInfo, + WebsiteCnrInfo: wshCnrInfo, WorkerPool: a.initWorkerPool(), } diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index 3c02c554..946e0c47 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -221,9 +221,10 @@ const ( cfgSourceIPHeader = "source_ip_header" // Containers. - cfgContainersCORS = "containers.cors" - cfgContainersLifecycle = "containers.lifecycle" - cfgContainersAccessBox = "containers.accessbox" + cfgContainersCORS = "containers.cors" + cfgContainersLifecycle = "containers.lifecycle" + cfgContainersAccessBox = "containers.accessbox" + cfgContainersWebStaticHosting = "containers.web_static_hosting" // Multinet. cfgMultinetEnabled = "multinet.enabled" diff --git a/config/config.env b/config/config.env index 53171673..cacd4039 100644 --- a/config/config.env +++ b/config/config.env @@ -273,6 +273,8 @@ S3_GW_RETRY_STRATEGY=exponential # Containers properties S3_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj S3_GW_CONTAINERS_LIFECYCLE=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj +S3_GW_CONTAINERS_ACCESSBOX: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2 +S3_GW_CONTAINERS_STATIC_WEB_HOSTING: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP # Multinet properties # Enable multinet support diff --git a/config/config.yaml b/config/config.yaml index 28e83654..41b4e681 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -322,6 +322,8 @@ retry: containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + accessbox: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2 + web_static_hosting: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP # Multinet properties multinet: diff --git a/docs/configuration.md b/docs/configuration.md index d3662fba..e8e0fe83 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -859,13 +859,15 @@ containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj accessbox: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2 + web_static_hosting: 9kjQRA9VcGc3EJqwyismcyNSnpG47i9nnMGtyDcNVgmP ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------| -| `cors` | `string` | no | | Container name for CORS configurations. | -| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. | -| `accessbox` | `string` | no | | Container name to lookup accessbox if custom aws credentials is used. If not set, custom credentials are not supported. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|----------------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------| +| `cors` | `string` | no | | Container name for CORS configurations. | +| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. | +| `accessbox` | `string` | no | | Container name to lookup accessbox if custom aws credentials is used. If not set, custom credentials are not supported. | +| `web_static_hosting` | `string` | no | | Container name for website configurations. | # `vhs` section diff --git a/internal/logs/logs.go b/internal/logs/logs.go index c0db6fe3..a2c0e32e 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -48,6 +48,7 @@ const ( ServerReconnectFailed = "failed to reconnect server" CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" CouldNotFetchAccessBoxContainerInfo = "couldn't fetch AccessBox container info" + CouldNotFetchWebStaticHostingContainerInfo = "couldn't fetch web static hosting container info" MultinetDialSuccess = "multinet dial successful" MultinetDialFail = "multinet dial failed" FailedToCreateWorkerPool = "failed to create worker pool" @@ -126,6 +127,7 @@ const ( UnexpectedMultiNodeIDsInSubTreeMultiParts = "unexpected multi node ids in sub tree multi parts" FoundSeveralSystemNodes = "found several system nodes" BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids" + BucketWebsiteNodeHasMultipleIDs = "bucket website node has multiple ids" UploadPart = "upload part" FailedToSubmitTaskToPool = "failed to submit task to pool" FailedToGetRealObjectSize = "failed to get real object size" @@ -154,6 +156,7 @@ const ( CouldntCacheCors = "couldn't cache cors" CouldntCacheListPolicyChains = "couldn't cache list policy chains" CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration" + CouldntCacheWebsiteConfiguration = "couldn't cache website configuration" GetBucketCors = "get bucket cors" GetBucketInfo = "get bucket info" ResolveBucket = "resolve bucket" @@ -180,6 +183,7 @@ const ( CouldNotFetchObjectMeta = "could not fetch object meta" FailedToDeleteObject = "failed to delete object" CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object" + CouldntDeleteWebsiteObject = "couldn't delete website configuration object" CouldntGetCORSObjectVersions = "couldn't get cors object versions" ) diff --git a/pkg/service/tree/tree.go b/pkg/service/tree/tree.go index 2b09d9e1..e6563119 100644 --- a/pkg/service/tree/tree.go +++ b/pkg/service/tree/tree.go @@ -123,6 +123,7 @@ const ( corsFilename = "bucket-cors" bucketTaggingFilename = "bucket-tagging" bucketLifecycleFilename = "bucket-lifecycle" + bucketWebsiteFilename = "bucket-website" // versionTree -- ID of a tree with object versions. versionTree = "version" @@ -1662,6 +1663,83 @@ func (c *Tree) DeleteBucketLifecycleConfiguration(ctx context.Context, bktInfo * return objToDelete, nil } +func (c *Tree) PutBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutBucketWebsiteConfiguration") + defer span.End() + + multiNode, err := c.getSystemNode(ctx, bktInfo, bucketWebsiteFilename) + isErrNotFound := errors.Is(err, tree.ErrNodeNotFound) + if err != nil && !isErrNotFound { + return nil, fmt.Errorf("couldn't get node: %w", err) + } + + meta := make(map[string]string) + meta[FileNameKey] = bucketWebsiteFilename + meta[oidKV] = addr.Object().EncodeToString() + meta[cidKV] = addr.Container().EncodeToString() + + if isErrNotFound { + if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil { + return nil, err + } + return nil, tree.ErrNoNodeToRemove + } + + latest := multiNode.Latest() + ind := latest.GetLatestNodeIndex() + if latest.IsSplit() { + c.reqLogger(ctx).Error(logs.BucketWebsiteNodeHasMultipleIDs, logs.TagField(logs.TagDatapath)) + } + + if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil { + return nil, fmt.Errorf("move website node: %w", err) + } + + objToDelete := make([]oid.Address, 1, len(multiNode.nodes)) + objToDelete[0], err = getTreeNodeAddress(latest) + if err != nil { + return nil, fmt.Errorf("parse object addr of latest website node in tree: %w", err) + } + + objToDelete = append(objToDelete, c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)...) + + return objToDelete, nil +} + +func (c *Tree) GetBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetBucketWebsite") + defer span.End() + + node, err := c.getSystemNode(ctx, bktInfo, bucketWebsiteFilename) + if err != nil { + return oid.Address{}, fmt.Errorf("get lifecycle node: %w", err) + } + + return getTreeNodeAddress(node.Latest()) +} + +func (c *Tree) DeleteBucketWebsiteConfiguration(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteBucketWebsiteConfiguration") + defer span.End() + + multiNode, err := c.getSystemNode(ctx, bktInfo, bucketWebsiteFilename) + isErrNotFound := errors.Is(err, tree.ErrNodeNotFound) + if err != nil && !isErrNotFound { + return nil, err + } + + if isErrNotFound { + return nil, tree.ErrNoNodeToRemove + } + + objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo) + if len(objToDelete) != len(multiNode.nodes) { + return nil, fmt.Errorf("failed to clean all old website nodes") + } + + return objToDelete, nil +} + func (c *Tree) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartInfo *data.MultipartInfo) error { ctx, span := tracing.StartSpanFromContext(ctx, "tree.DeleteMultipartUpload") defer span.End() -- 2.45.3