diff --git a/CHANGELOG.md b/CHANGELOG.md index f7cdd20..9774c52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ This document outlines major changes between releases. ## [Unreleased] +### Added +- Add support for virtual hosted style addressing (#446) + ## [0.30.0] - Kangshung -2024-07-19 ### Fixed diff --git a/api/middleware/address_style.go b/api/middleware/address_style.go new file mode 100644 index 0000000..274e66e --- /dev/null +++ b/api/middleware/address_style.go @@ -0,0 +1,135 @@ +package middleware + +import ( + "net/http" + "net/url" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" + "go.uber.org/zap" +) + +const wildcardPlaceholder = "" + +type VHSSettings interface { + Domains() []string + GlobalVHS() bool + VHSNamespacesEnabled() map[string]bool +} + +func PrepareAddressStyle(settings VHSSettings, log *zap.Logger) Func { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + reqInfo := GetReqInfo(ctx) + reqLogger := reqLogOrDefault(ctx, log) + + if isVHSAddress(settings.GlobalVHS(), settings.VHSNamespacesEnabled(), reqInfo.Namespace) { + prepareVHSAddress(reqInfo, r, settings) + } else { + preparePathStyleAddress(reqInfo, r, reqLogger) + } + + h.ServeHTTP(w, r) + }) + } +} + +func isVHSAddress(enabledFlag bool, vhsNamespaces map[string]bool, namespace string) bool { + result := enabledFlag + + if v, ok := vhsNamespaces[namespace]; ok { + result = v + } + + return result +} + +func prepareVHSAddress(reqInfo *ReqInfo, r *http.Request, settings VHSSettings) { + reqInfo.RequestVHSEnabled = true + bktName, match := checkDomain(r.Host, settings.Domains()) + if match { + if bktName == "" { + reqInfo.RequestType = noneType + } else { + if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" { + reqInfo.RequestType = objectType + reqInfo.ObjectName = objName + reqInfo.BucketName = bktName + } else { + reqInfo.RequestType = bucketType + reqInfo.BucketName = bktName + } + } + } else { + parts := strings.Split(r.Host, ".") + reqInfo.BucketName = parts[0] + + if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" { + reqInfo.RequestType = objectType + reqInfo.ObjectName = objName + } else { + reqInfo.RequestType = bucketType + } + } +} + +func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.Logger) { + bktObj := strings.TrimPrefix(r.URL.Path, "/") + if bktObj == "" { + reqInfo.RequestType = noneType + } else if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" { + reqInfo.RequestType = objectType + reqInfo.BucketName = bktObj[:ind] + reqInfo.ObjectName = bktObj[ind+1:] + + if r.URL.RawPath != "" { + // we have to do this because of + // https://github.com/go-chi/chi/issues/641 + // https://github.com/go-chi/chi/issues/642 + if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil { + reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err)) + } else { + reqInfo.ObjectName = obj + } + } + } else { + reqInfo.RequestType = bucketType + reqInfo.BucketName = strings.TrimSuffix(bktObj, "/") + } +} + +func checkDomain(host string, domains []string) (bktName string, match bool) { + partsHost := strings.Split(host, ".") + for _, pattern := range domains { + partsPattern := strings.Split(pattern, ".") + bktName, match = compareMatch(partsHost, partsPattern) + if match { + break + } + } + return +} + +func compareMatch(host, pattern []string) (bktName string, match bool) { + if len(host) < len(pattern) { + return "", false + } + + i, j := len(host)-1, len(pattern)-1 + for j >= 0 && (pattern[j] == wildcardPlaceholder || host[i] == pattern[j]) { + i-- + j-- + } + + switch { + case i == -1: + return "", true + case i == 0 && j == 0 && host[i] != pattern[j]: + return "", false + case i == 0: + return host[0], true + default: + return "", false + } +} diff --git a/api/middleware/policy.go b/api/middleware/policy.go index eaf9ffe..df9c7e5 100644 --- a/api/middleware/policy.go +++ b/api/middleware/policy.go @@ -73,7 +73,6 @@ type PolicyConfig struct { Storage engine.ChainRouter FrostfsID FrostFSIDInformer Settings PolicySettings - Domains []string Log *zap.Logger BucketResolver BucketResolveFunc Decoder XMLDecoder @@ -99,21 +98,21 @@ func PolicyCheck(cfg PolicyConfig) Func { } func policyCheck(r *http.Request, cfg PolicyConfig) error { - reqType, bktName, objName := getBucketObject(r, cfg.Domains) - req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqType, bktName, objName) + reqInfo := GetReqInfo(r.Context()) + + req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqInfo.RequestType, reqInfo.BucketName, reqInfo.ObjectName) if err != nil { return err } var bktInfo *data.BucketInfo - if reqType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) { - bktInfo, err = cfg.BucketResolver(r.Context(), bktName) + if reqInfo.RequestType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) { + bktInfo, err = cfg.BucketResolver(r.Context(), reqInfo.BucketName) if err != nil { return err } } - reqInfo := GetReqInfo(r.Context()) target := engine.NewRequestTargetWithNamespace(reqInfo.Namespace) if bktInfo != nil { cnrTarget := engine.ContainerTarget(bktInfo.CID.EncodeToString()) @@ -208,33 +207,6 @@ const ( objectType ) -func getBucketObject(r *http.Request, domains []string) (reqType ReqType, bktName string, objName string) { - for _, domain := range domains { - ind := strings.Index(r.Host, "."+domain) - if ind == -1 { - continue - } - - bkt := r.Host[:ind] - if obj := strings.TrimPrefix(r.URL.Path, "/"); obj != "" { - return objectType, bkt, obj - } - - return bucketType, bkt, "" - } - - bktObj := strings.TrimPrefix(r.URL.Path, "/") - if bktObj == "" { - return noneType, "", "" - } - - if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" { - return objectType, bktObj[:ind], bktObj[ind+1:] - } - - return bucketType, strings.TrimSuffix(bktObj, "/"), "" -} - func determineOperation(r *http.Request, reqType ReqType) (operation string) { switch reqType { case objectType: diff --git a/api/middleware/policy_test.go b/api/middleware/policy_test.go index 0c6f128..7147ae4 100644 --- a/api/middleware/policy_test.go +++ b/api/middleware/policy_test.go @@ -8,79 +8,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestReqTypeDetermination(t *testing.T) { - bkt, obj, domain := "test-bucket", "test-object", "domain" - - for _, tc := range []struct { - name string - target string - host string - domains []string - expectedType ReqType - expectedBktName string - expectedObjName string - }{ - { - name: "bucket request, path-style", - target: "/" + bkt, - expectedType: bucketType, - expectedBktName: bkt, - }, - { - name: "bucket request with slash, path-style", - target: "/" + bkt + "/", - expectedType: bucketType, - expectedBktName: bkt, - }, - { - name: "object request, path-style", - target: "/" + bkt + "/" + obj, - expectedType: objectType, - expectedBktName: bkt, - expectedObjName: obj, - }, - { - name: "object request with slash, path-style", - target: "/" + bkt + "/" + obj + "/", - expectedType: objectType, - expectedBktName: bkt, - expectedObjName: obj + "/", - }, - { - name: "none type request", - target: "/", - expectedType: noneType, - }, - { - name: "bucket request, virtual-hosted style", - target: "/", - host: bkt + "." + domain, - domains: []string{"some-domain", domain}, - expectedType: bucketType, - expectedBktName: bkt, - }, - { - name: "object request, virtual-hosted style", - target: "/" + obj, - host: bkt + "." + domain, - domains: []string{"some-domain", domain}, - expectedType: objectType, - expectedBktName: bkt, - expectedObjName: obj, - }, - } { - t.Run(tc.name, func(t *testing.T) { - r := httptest.NewRequest(http.MethodPut, tc.target, nil) - r.Host = tc.host - - reqType, bktName, objName := getBucketObject(r, tc.domains) - require.Equal(t, tc.expectedType, reqType) - require.Equal(t, tc.expectedBktName, bktName) - require.Equal(t, tc.expectedObjName, objName) - }) - } -} - func TestDetermineBucketOperation(t *testing.T) { const defaultValue = "value" diff --git a/api/middleware/reqinfo.go b/api/middleware/reqinfo.go index 3f81dc6..43a3d58 100644 --- a/api/middleware/reqinfo.go +++ b/api/middleware/reqinfo.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree" - "github.com/go-chi/chi/v5" "github.com/google/uuid" "go.uber.org/zap" "google.golang.org/grpc/metadata" @@ -28,19 +27,21 @@ type ( // ReqInfo stores the request info. ReqInfo struct { sync.RWMutex - RemoteHost string // Client Host/IP - Host string // Node Host/IP - UserAgent string // User Agent - DeploymentID string // random generated s3-deployment-id - RequestID string // x-amz-request-id - API string // API name -- GetObject PutObject NewMultipartUpload etc. - BucketName string // Bucket name - ObjectName string // Object name - TraceID string // Trace ID - URL *url.URL // Request url - Namespace string - User string // User owner id - Tagging *data.Tagging + RemoteHost string // Client Host/IP + Host string // Node Host/IP + UserAgent string // User Agent + DeploymentID string // random generated s3-deployment-id + RequestID string // x-amz-request-id + API string // API name -- GetObject PutObject NewMultipartUpload etc. + BucketName string // Bucket name + ObjectName string // Object name + TraceID string // Trace ID + URL *url.URL // Request url + Namespace string + User string // User owner id + Tagging *data.Tagging + RequestVHSEnabled bool + RequestType ReqType } // ObjectRequest represents object request data. @@ -197,57 +198,6 @@ func Request(log *zap.Logger, settings RequestSettings) Func { } } -// AddBucketName adds bucket name to ReqInfo from context. -func AddBucketName(l *zap.Logger) Func { - return func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - reqInfo := GetReqInfo(ctx) - reqInfo.BucketName = chi.URLParam(r, BucketURLPrm) - - if reqInfo.BucketName != "" { - reqLogger := reqLogOrDefault(ctx, l) - r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("bucket", reqInfo.BucketName)))) - } - - h.ServeHTTP(w, r) - }) - } -} - -// AddObjectName adds objects name to ReqInfo from context. -func AddObjectName(l *zap.Logger) Func { - return func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - reqInfo := GetReqInfo(ctx) - reqLogger := reqLogOrDefault(ctx, l) - - rctx := chi.RouteContext(ctx) - // trim leading slash (always present) - reqInfo.ObjectName = rctx.RoutePath[1:] - - if r.URL.RawPath != "" { - // we have to do this because of - // https://github.com/go-chi/chi/issues/641 - // https://github.com/go-chi/chi/issues/642 - if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil { - reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err)) - } else { - reqInfo.ObjectName = obj - } - } - - if reqInfo.ObjectName != "" { - r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName)))) - } - - h.ServeHTTP(w, r) - }) - } -} - // getSourceIP retrieves the IP from the X-Forwarded-For, X-Real-IP and RFC7239 // Forwarded headers (in that order), falls back to r.RemoteAddr when everything // else fails. diff --git a/api/router.go b/api/router.go index 0f86e2e..67b5816 100644 --- a/api/router.go +++ b/api/router.go @@ -97,6 +97,7 @@ type Settings interface { s3middleware.RequestSettings s3middleware.PolicySettings s3middleware.MetricsSettings + s3middleware.VHSSettings } type FrostFSID interface { @@ -113,9 +114,6 @@ type Config struct { MiddlewareSettings Settings - // Domains optional. If empty no virtual hosted domains will be attached. - Domains []string - FrostfsID FrostFSID FrostFSIDValidation bool @@ -142,11 +140,11 @@ func NewRouter(cfg Config) *chi.Mux { api.Use(s3middleware.FrostfsIDValidation(cfg.FrostfsID, cfg.Log)) } + api.Use(s3middleware.PrepareAddressStyle(cfg.MiddlewareSettings, cfg.Log)) api.Use(s3middleware.PolicyCheck(s3middleware.PolicyConfig{ Storage: cfg.PolicyChecker, FrostfsID: cfg.FrostfsID, Settings: cfg.MiddlewareSettings, - Domains: cfg.Domains, Log: cfg.Log, BucketResolver: cfg.Handler.ResolveBucket, Decoder: cfg.XMLDecoder, @@ -154,22 +152,41 @@ func NewRouter(cfg Config) *chi.Mux { })) defaultRouter := chi.NewRouter() - defaultRouter.Mount(fmt.Sprintf("/{%s}", s3middleware.BucketURLPrm), bucketRouter(cfg.Handler, cfg.Log)) - defaultRouter.Get("/", named("ListBuckets", cfg.Handler.ListBucketsHandler)) + defaultRouter.Mount(fmt.Sprintf("/{%s}", s3middleware.BucketURLPrm), bucketRouter(cfg.Handler)) + defaultRouter.Get("/", named(s3middleware.ListBucketsOperation, cfg.Handler.ListBucketsHandler)) attachErrorHandler(defaultRouter) - hr := NewHostBucketRouter("bucket") - hr.Default(defaultRouter) - for _, domain := range cfg.Domains { - hr.Map(domain, bucketRouter(cfg.Handler, cfg.Log)) - } - api.Mount("/", hr) + vhsRouter := bucketRouter(cfg.Handler) + router := newGlobalRouter(defaultRouter, vhsRouter) + + api.Mount("/", router) attachErrorHandler(api) return api } +type globalRouter struct { + pathStyleRouter chi.Router + vhsRouter chi.Router +} + +func newGlobalRouter(pathStyleRouter, vhsRouter chi.Router) *globalRouter { + return &globalRouter{ + pathStyleRouter: pathStyleRouter, + vhsRouter: vhsRouter, + } +} + +func (g *globalRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { + router := g.pathStyleRouter + if reqInfo := s3middleware.GetReqInfo(r.Context()); reqInfo.RequestVHSEnabled { + router = g.vhsRouter + } + + router.ServeHTTP(w, r) +} + func named(name string, handlerFunc http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { reqInfo := s3middleware.GetReqInfo(r.Context()) @@ -214,14 +231,13 @@ func attachErrorHandler(api *chi.Mux) { api.MethodNotAllowed(named("MethodNotAllowed", errorHandler)) } -func bucketRouter(h Handler, log *zap.Logger) chi.Router { +func bucketRouter(h Handler) chi.Router { bktRouter := chi.NewRouter() bktRouter.Use( - s3middleware.AddBucketName(log), s3middleware.WrapHandler(h.AppendCORSHeaders), ) - bktRouter.Mount("/", objectRouter(h, log)) + bktRouter.Mount("/", objectRouter(h)) bktRouter.Options("/", named(s3middleware.OptionsBucketOperation, h.Preflight)) @@ -293,7 +309,7 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router { Add(NewFilter(). Queries(s3middleware.VersionsQuery). Handler(named(s3middleware.ListBucketObjectVersionsOperation, h.ListBucketObjectVersionsHandler))). - DefaultHandler(named(s3middleware.ListObjectsV1Operation, h.ListObjectsV1Handler))) + DefaultHandler(listWrapper(h))) }) // PUT method handlers @@ -368,9 +384,20 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router { return bktRouter } -func objectRouter(h Handler, l *zap.Logger) chi.Router { +func listWrapper(h Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if reqInfo := s3middleware.GetReqInfo(r.Context()); reqInfo.BucketName == "" { + reqInfo.API = s3middleware.ListBucketsOperation + h.ListBucketsHandler(w, r) + } else { + reqInfo.API = s3middleware.ListObjectsV1Operation + h.ListObjectsV1Handler(w, r) + } + } +} + +func objectRouter(h Handler) chi.Router { objRouter := chi.NewRouter() - objRouter.Use(s3middleware.AddObjectName(l)) objRouter.Options("/*", named(s3middleware.OptionsObjectOperation, h.Preflight)) diff --git a/api/router_mock_test.go b/api/router_mock_test.go index c262023..4dd5d90 100644 --- a/api/router_mock_test.go +++ b/api/router_mock_test.go @@ -71,8 +71,11 @@ func (c *centerMock) Authenticate(*http.Request) (*middleware.Box, error) { } type middlewareSettingsMock struct { - denyByDefault bool - sourceIPHeader string + denyByDefault bool + sourceIPHeader string + domains []string + vhsEnabled bool + vhsNamespacesEnabled map[string]bool } func (r *middlewareSettingsMock) SourceIPHeader() string { @@ -91,6 +94,18 @@ func (r *middlewareSettingsMock) PolicyDenyByDefault() bool { return r.denyByDefault } +func (r *middlewareSettingsMock) Domains() []string { + return r.domains +} + +func (r *middlewareSettingsMock) GlobalVHS() bool { + return r.vhsEnabled +} + +func (r *middlewareSettingsMock) VHSNamespacesEnabled() map[string]bool { + return r.vhsNamespacesEnabled +} + type frostFSIDMock struct { tags map[string]string validateError bool diff --git a/api/router_test.go b/api/router_test.go index 48cf459..510db04 100644 --- a/api/router_test.go +++ b/api/router_test.go @@ -78,7 +78,6 @@ func prepareRouter(t *testing.T, opts ...option) *routerMock { Metrics: metrics.NewAppMetrics(metricsConfig), MiddlewareSettings: middlewareSettings, PolicyChecker: policyChecker, - Domains: []string{"domain1", "domain2"}, FrostfsID: &frostFSIDMock{}, XMLDecoder: &xmlMock{}, Tagging: &resourceTaggingMock{}, diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index 49adbe5..cead13b 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -105,6 +105,9 @@ type ( policyDenyByDefault bool sourceIPHeader string retryMaxAttempts int + domains []string + vhsEnabled bool + vhsNamespacesEnabled map[string]bool retryMaxBackoff time.Duration retryStrategy handler.RetryStrategy } @@ -231,6 +234,7 @@ func (s *appSettings) update(v *viper.Viper, log *zap.Logger) { s.setRetryMaxAttempts(fetchRetryMaxAttempts(v)) s.setRetryMaxBackoff(fetchRetryMaxBackoff(v)) s.setRetryStrategy(fetchRetryStrategy(v)) + s.setVHSSettings(v, log) } func (s *appSettings) updateNamespacesSettings(v *viper.Viper, log *zap.Logger) { @@ -245,6 +249,41 @@ func (s *appSettings) updateNamespacesSettings(v *viper.Viper, log *zap.Logger) s.namespaces = nsConfig.Namespaces } +func (s *appSettings) setVHSSettings(v *viper.Viper, log *zap.Logger) { + domains := fetchDomains(v, log) + vhsEnabled := v.GetBool(cfgVHSEnabled) + nsMap := fetchVHSNamespaces(v, log) + vhsNamespaces := make(map[string]bool, len(nsMap)) + for ns, flag := range nsMap { + vhsNamespaces[s.ResolveNamespaceAlias(ns)] = flag + } + + s.mu.Lock() + defer s.mu.Unlock() + + s.domains = domains + s.vhsEnabled = vhsEnabled + s.vhsNamespacesEnabled = vhsNamespaces +} + +func (s *appSettings) Domains() []string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.domains +} + +func (s *appSettings) GlobalVHS() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.vhsEnabled +} + +func (s *appSettings) VHSNamespacesEnabled() map[string]bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.vhsNamespacesEnabled +} + func (s *appSettings) BypassContentEncodingInChunks() bool { s.mu.RLock() defer s.mu.RUnlock() @@ -683,10 +722,6 @@ func (a *App) setHealthStatus() { // Serve runs HTTP server to handle S3 API requests. func (a *App) Serve(ctx context.Context) { - // Attach S3 API: - domains := a.cfg.GetStringSlice(cfgListenDomains) - a.log.Info(logs.FetchDomainsPrepareToUseAPI, zap.Strings("domains", domains)) - cfg := api.Config{ Throttle: middleware.ThrottleOpts{ Limit: a.settings.maxClient.count, @@ -696,7 +731,6 @@ func (a *App) Serve(ctx context.Context) { Center: a.ctr, Log: a.log, Metrics: a.metrics, - Domains: domains, MiddlewareSettings: a.settings, PolicyChecker: a.policyStorage, diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index 3aacea4..0b06aeb 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -30,6 +30,8 @@ import ( const ( destinationStdout = "stdout" destinationJournald = "journald" + + wildcardPlaceholder = "" ) const ( @@ -144,6 +146,9 @@ const ( // Settings. cfgListenDomains = "listen_domains" + cfgVHSEnabled = "vhs.enabled" + cfgVHSNamespaces = "vhs.namespaces" + // Peers. cfgPeers = "peers" @@ -667,6 +672,41 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo { return servers } +func fetchDomains(v *viper.Viper, log *zap.Logger) []string { + domains := validateDomains(v.GetStringSlice(cfgListenDomains), log) + + countParts := func(domain string) int { + return strings.Count(domain, ".") + } + + sort.Slice(domains, func(i, j int) bool { + return countParts(domains[i]) > countParts(domains[j]) + }) + + return domains +} + +func fetchVHSNamespaces(v *viper.Viper, log *zap.Logger) map[string]bool { + vhsNamespacesEnabled := make(map[string]bool) + nsMap := v.GetStringMap(cfgVHSNamespaces) + for ns, val := range nsMap { + if _, ok := vhsNamespacesEnabled[ns]; ok { + log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns)) + continue + } + + enabledFlag, ok := val.(bool) + if !ok { + log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns)) + continue + } + + vhsNamespacesEnabled[ns] = enabledFlag + } + + return vhsNamespacesEnabled +} + func newSettings() *viper.Viper { v := viper.New() @@ -1028,3 +1068,19 @@ func getLogLevel(v *viper.Viper) (zapcore.Level, error) { } return lvl, nil } + +func validateDomains(domains []string, log *zap.Logger) []string { + validDomains := make([]string, 0, len(domains)) +LOOP: + for _, domain := range domains { + domainParts := strings.Split(domain, ".") + for _, part := range domainParts { + if strings.ContainsAny(part, "<>") && part != wildcardPlaceholder { + log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain)) + continue LOOP + } + } + validDomains = append(validDomains, domain) + } + return validDomains +} diff --git a/cmd/s3-gw/validate_test.go b/cmd/s3-gw/validate_test.go new file mode 100644 index 0000000..fe88228 --- /dev/null +++ b/cmd/s3-gw/validate_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +func TestValidateDomains(t *testing.T) { + inputDomains := []string{ + "s3dev.frostfs.devenv", + "s3dev..frostfs.devenv", + "s3dev..frostfs.devenv", + "s3dev..frostfs.devenv", + "s3dev..frostfs.devenv", + ".frostfs.devenv", + ".frostfs.devenv>", + ".frostfs.devenv", + "s3dev.fro.dev..frostfs.devenv", + ".dev.ard>.frostfs.devenv", + } + expectedDomains := []string{ + "s3dev.frostfs.devenv", + "s3dev..frostfs.devenv", + ".frostfs.devenv", + ".dev..frostfs.devenv", + } + + actualDomains := validateDomains(inputDomains, zaptest.NewLogger(t)) + require.Equal(t, expectedDomains, actualDomains) +} diff --git a/config/config.env b/config/config.env index dd4438a..a6b8fab 100644 --- a/config/config.env +++ b/config/config.env @@ -36,8 +36,11 @@ S3_GW_SERVER_1_TLS_KEY_FILE=/path/to/tls/key # How often to reconnect to the servers S3_GW_RECONNECT_INTERVAL: 1m -# Domains to be able to use virtual-hosted-style access to bucket. -S3_GW_LISTEN_DOMAINS=s3dev.frostfs.devenv +# Domains to be able to use virtual-hosted-style access to bucket +S3_GW_LISTEN_DOMAINS="domain.com .domain.com" + +# VHS enabled flag +S3_GW_VHS_ENABLED=false # Config file S3_GW_CONFIG=/path/to/config/yaml diff --git a/config/config.yaml b/config/config.yaml index aae9e0b..957f126 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -42,6 +42,13 @@ server: # Domains to be able to use virtual-hosted-style access to bucket. listen_domains: - s3dev.frostfs.devenv + - s3dev..frostfs.devenv + +vhs: + enabled: false + namespaces: + "ns1": false + "ns2": true logger: level: debug diff --git a/docs/configuration.md b/docs/configuration.md index 23ba071..5e2f530 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -193,12 +193,14 @@ There are some custom types used for brevity: | `namespaces` | [Namespaces configuration](#namespaces-section) | | `retry` | [Retry configuration](#retry-section) | | `containers` | [Containers configuration](#containers-section) | +| `vhs` | [VHS configuration](#vhs-section) | ### General section ```yaml listen_domains: - s3dev.frostfs.devenv + - s3dev..frostfs.devenv - s3dev2.frostfs.devenv rpc_endpoint: http://morph-chain.frostfs.devenv:30333 @@ -226,7 +228,7 @@ source_ip_header: "Source-Ip" | Parameter | Type | SIGHUP reload | Default value | Description | |----------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `listen_domains` | `[]string` | no | | Domains to be able to use virtual-hosted-style access to bucket. | +| `listen_domains` | `[]string` | yes | | Domains to be able to use virtual-hosted-style access to bucket. The presence of placeholders of the type is supported. | | `rpc_endpoint` | `string` | no | | The address of the RPC host to which the gateway connects to resolve bucket names and interact with frostfs contracts (required to use the `nns` resolver and `frostfsid` contract). | | `resolve_order` | `[]string` | yes | `[dns]` | Order of bucket name resolvers to use. Available resolvers: `dns`, `nns`. | | `connect_timeout` | `duration` | no | `10s` | Timeout to connect to a node. | @@ -721,3 +723,20 @@ containers: | Parameter | Type | SIGHUP reload | Default value | Description | |-----------|----------|---------------|---------------|--------------------------------------------------------------------------------------| | `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. | + +# `vhs` section + +Configuration of virtual hosted addressing style. + +```yaml +vhs: + enabled: false + namespaces: + "ns1": false + "ns2": true +``` + +| Parameter | Type | SIGHUP reload | Default value | Description | +|--------------|-------------------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `enabled` | `bool` | yes | `false` | Enables the use of virtual host addressing for banquets at the application level. | +| `namespaces` | `map[string]bool` | yes | | A map in which the keys are the name of the namespace, and the values are the flag responsible for enabling VHS for the specified namespace. | diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d875eb7..e16ddd6 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -20,7 +20,6 @@ const ( UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go - FetchDomainsPrepareToUseAPI = "fetch domains, prepare to use API" // Info in ../../cmd/s3-gw/app.go StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go @@ -154,4 +153,7 @@ const ( FailedToParsePartInfo = "failed to parse part info" CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" CloseCredsObjectPayload = "close creds object payload" + WarnDuplicateNamespaceVHS = "duplicate namespace with enabled VHS, config value skipped" + WarnValueVHSEnabledFlagWrongType = "the value of the VHS enable flag for the namespace is of the wrong type, config value skipped" + WarnDomainContainsInvalidPlaceholder = "the domain contains an invalid placeholder, domain skipped" )