TrueCloudLab/frostfs-s3-gw
15
2
Fork 17
Code Issues 18 Pull requests 6 Projects Releases 15 Packages Activity Actions

feature/virtual-hosted-style #465

Merged
alexvanin merged 3 commits from feature/virtual-hosted-style into master 2024-08-23 08:35:06 +00:00
Conversation 0 Commits 3 Files changed 20 +929 -244
20 changed files with 929 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -4,6 +4,9 @@ This document outlines major changes between releases.
## [Unreleased]
### Added
- Add support for virtual hosted style addressing (#446, #449)
## [0.30.0] - Kangshung -2024-07-19
### Fixed

1
api/handler/api.go
View file

@ -41,7 +41,6 @@ type (
RetryMaxAttempts() int
RetryMaxBackoff() time.Duration
RetryStrategy() RetryStrategy
Domains() []string
}
FrostFSID interface {

5
api/handler/handlers_test.go
View file

@ -73,7 +73,6 @@ type configMock struct {
defaultCopiesNumbers []uint32
bypassContentEncodingInChunks bool
md5Enabled bool
domains []string
}
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
@ -137,10 +136,6 @@ func (c *configMock) RetryStrategy() RetryStrategy {
return RetryStrategyConstant
}
func (c *configMock) Domains() []string {
return c.domains
}
func prepareHandlerContext(t *testing.T) *handlerContext {
log := zaptest.NewLogger(t)
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(log), log)

16
api/handler/multipart_upload.go
View file

@ -7,7 +7,6 @@ import (
"net/url"
"path"
"strconv"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
@ -429,7 +428,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
Bucket: objInfo.Bucket,
Key: objInfo.Name,
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
Location: getObjectLocation(r, h.cfg.Domains(), reqInfo.BucketName, reqInfo.ObjectName),
Location: getObjectLocation(r, reqInfo.BucketName, reqInfo.ObjectName, reqInfo.RequestVHSEnabled),
}
if settings.VersioningEnabled() {
@ -450,7 +449,7 @@ func getURLScheme(r *http.Request) string {
}
// getObjectLocation gets the fully qualified URL of an object.
func getObjectLocation(r *http.Request, domains []string, bucket, object string) string {
func getObjectLocation(r *http.Request, bucket, object string, vhsEnabled bool) string {
proto := middleware.GetSourceScheme(r)
if proto == "" {
proto = getURLScheme(r)
@ -460,13 +459,12 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
Path: path.Join("/", bucket, object),
Scheme: proto,
}
// If domain is set then we need to use bucket DNS style.
for _, domain := range domains {
if strings.HasPrefix(r.Host, bucket+"."+domain) {
u.Path = path.Join("/", object)
break
}
// If vhs enabled then we need to use bucket DNS style.
if vhsEnabled {
u.Path = path.Join("/", object)
}
return u.String()
}

28
api/handler/multipart_upload_test.go
View file

@ -443,11 +443,11 @@ func TestUploadPartCheckContentSHA256(t *testing.T) {
func TestMultipartObjectLocation(t *testing.T) {
for _, tc := range []struct {
req *http.Request
bucket string
object string
domains []string
expected string
req *http.Request
bucket string
object string
vhsEnabled bool
expected string
}{
{
req: &http.Request{
@ -492,24 +492,24 @@ func TestMultipartObjectLocation(t *testing.T) {
req: &http.Request{
Host: "mybucket.s3dev.frostfs.devenv",
},
domains: []string{"s3dev.frostfs.devenv"},
bucket: "mybucket",
object: "test/1.txt",
expected: "http://mybucket.s3dev.frostfs.devenv/test/1.txt",
bucket: "mybucket",
object: "test/1.txt",
vhsEnabled: true,
expected: "http://mybucket.s3dev.frostfs.devenv/test/1.txt",
},
{
req: &http.Request{
Host: "mybucket.s3dev.frostfs.devenv",
Header: map[string][]string{"X-Forwarded-Scheme": {"https"}},
},
domains: []string{"s3dev.frostfs.devenv"},
bucket: "mybucket",
object: "test/1.txt",
expected: "https://mybucket.s3dev.frostfs.devenv/test/1.txt",
bucket: "mybucket",
object: "test/1.txt",
vhsEnabled: true,
expected: "https://mybucket.s3dev.frostfs.devenv/test/1.txt",
},
} {
t.Run("", func(t *testing.T) {
location := getObjectLocation(tc.req, tc.domains, tc.bucket, tc.object)
location := getObjectLocation(tc.req, tc.bucket, tc.object, tc.vhsEnabled)
require.Equal(t, tc.expected, location)
})
}

148
api/middleware/address_style.go Normal file
View file

@ -0,0 +1,148 @@
package middleware
import (
"net/http"
"net/url"
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
"go.uber.org/zap"
)
const wildcardPlaceholder = "<wildcard>"
type VHSSettings interface {
Domains() []string
GlobalVHS() bool
VHSHeader() string
ServernameHeader() string
VHSNamespacesEnabled() map[string]bool
}
func PrepareAddressStyle(settings VHSSettings, log *zap.Logger) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := GetReqInfo(ctx)
reqLogger := reqLogOrDefault(ctx, log)
headerVHSEnabled := r.Header.Get(settings.VHSHeader())
if isVHSAddress(headerVHSEnabled, settings.GlobalVHS(), settings.VHSNamespacesEnabled(), reqInfo.Namespace) {
prepareVHSAddress(reqInfo, r, settings)
} else {
preparePathStyleAddress(reqInfo, r, reqLogger)
}
h.ServeHTTP(w, r)
})
}
}
func isVHSAddress(headerVHSEnabled string, enabledFlag bool, vhsNamespaces map[string]bool, namespace string) bool {
if result, err := strconv.ParseBool(headerVHSEnabled); err == nil {
return result
}
result := enabledFlag
if v, ok := vhsNamespaces[namespace]; ok {
result = v
}
return result
}
func prepareVHSAddress(reqInfo *ReqInfo, r *http.Request, settings VHSSettings) {
reqInfo.RequestVHSEnabled = true
bktName, match := checkDomain(r.Host, getDomains(r, settings))
if match {
if bktName == "" {
reqInfo.RequestType = noneType
} else {
if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" {
reqInfo.RequestType = objectType
reqInfo.ObjectName = objName
reqInfo.BucketName = bktName
} else {
reqInfo.RequestType = bucketType
reqInfo.BucketName = bktName
}
}
} else {
parts := strings.Split(r.Host, ".")
reqInfo.BucketName = parts[0]
if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" {
reqInfo.RequestType = objectType
reqInfo.ObjectName = objName
} else {
reqInfo.RequestType = bucketType
}
}
}
func getDomains(r *http.Request, settings VHSSettings) []string {
if headerServername := r.Header.Get(settings.ServernameHeader()); headerServername != "" {
return []string{headerServername}
}
return settings.Domains()
}
func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.Logger) {
bktObj := strings.TrimPrefix(r.URL.Path, "/")
if bktObj == "" {
reqInfo.RequestType = noneType
} else if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" {
reqInfo.RequestType = objectType
reqInfo.BucketName = bktObj[:ind]
reqInfo.ObjectName = bktObj[ind+1:]
if r.URL.RawPath != "" {
// we have to do this because of
// https://github.com/go-chi/chi/issues/641
// https://github.com/go-chi/chi/issues/642
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
} else {
reqInfo.ObjectName = obj
}
}
} else {
reqInfo.RequestType = bucketType
reqInfo.BucketName = strings.TrimSuffix(bktObj, "/")
}
}
func checkDomain(host string, domains []string) (bktName string, match bool) {
partsHost := strings.Split(host, ".")
for _, pattern := range domains {
partsPattern := strings.Split(pattern, ".")
bktName, match = compareMatch(partsHost, partsPattern)
if match {
break
}
}
return
}
func compareMatch(host, pattern []string) (bktName string, match bool) {
if len(host) < len(pattern) {
return "", false
}
i, j := len(host)-1, len(pattern)-1
for j >= 0 && (pattern[j] == wildcardPlaceholder || host[i] == pattern[j]) {
i--
j--
}
switch {
case i == -1:
return "", true
case i == 0 && (j != 0 || host[i] == pattern[j]):
return host[0], true
default:
return "", false
}
}

443
api/middleware/address_style_test.go Normal file
View file

@ -0,0 +1,443 @@
package middleware
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
const (
FrostfsVHSHeader = "X-Frostfs-S3-VHS"
FrostfsServernameHeader = "X-Frostfs-Servername"
)
type VHSSettingsMock struct {
domains []string
}
func (v *VHSSettingsMock) Domains() []string {
return v.domains
}
func (v *VHSSettingsMock) GlobalVHS() bool {
return false
}
func (v *VHSSettingsMock) VHSHeader() string {
return FrostfsVHSHeader
}
func (v *VHSSettingsMock) ServernameHeader() string {
return FrostfsServernameHeader
}
func (v *VHSSettingsMock) VHSNamespacesEnabled() map[string]bool {
return make(map[string]bool)
}
func TestIsVHSAddress(t *testing.T) {
for _, tc := range []struct {
name string
headerVHSEnabled string
vhsEnabledFlag bool
vhsNamespaced map[string]bool
namespace string
expected bool
}{
{
name: "vhs disabled",
expected: false,
},
{
name: "vhs disabled for namespace",
vhsEnabledFlag: true,
vhsNamespaced: map[string]bool{
"kapusta": false,
},
namespace: "kapusta",
expected: false,
},
{
name: "vhs enabled (global vhs flag)",
vhsEnabledFlag: true,
expected: true,
},
{
name: "vhs enabled for namespace",
vhsNamespaced: map[string]bool{
"kapusta": true,
},
namespace: "kapusta",
expected: true,
},
{
name: "vhs enabled (header)",
headerVHSEnabled: "true",
vhsEnabledFlag: false,
vhsNamespaced: map[string]bool{
"kapusta": false,
},
namespace: "kapusta",
expected: true,
},
{
name: "vhs disabled (header)",
headerVHSEnabled: "false",
vhsEnabledFlag: true,
vhsNamespaced: map[string]bool{
"kapusta": true,
},
namespace: "kapusta",
expected: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
actual := isVHSAddress(tc.headerVHSEnabled, tc.vhsEnabledFlag, tc.vhsNamespaced, tc.namespace)
require.Equal(t, tc.expected, actual)
})
}
}
func TestPreparePathStyleAddress(t *testing.T) {
bkt, obj := "test-bucket", "test-object"
for _, tc := range []struct {
name string
urlParams string
expectedReqType ReqType
expectedBktName string
expectedObjName string
}{
{
name: "bucket request",
urlParams: "/" + bkt,
expectedReqType: bucketType,
expectedBktName: bkt,
},
{
name: "bucket request with slash",
urlParams: "/" + bkt + "/",
expectedReqType: bucketType,
expectedBktName: bkt,
},
{
name: "object request",
urlParams: "/" + bkt + "/" + obj,
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
{
name: "object request with slash",
urlParams: "/" + bkt + "/" + obj + "/",
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj + "/",
},
{
name: "none type request",
urlParams: "/",
expectedReqType: noneType,
},
} {
t.Run(tc.name, func(t *testing.T) {
reqInfo := &ReqInfo{}
r := httptest.NewRequest(http.MethodGet, tc.urlParams, nil)
preparePathStyleAddress(reqInfo, r, reqLogOrDefault(r.Context(), zaptest.NewLogger(t)))
require.Equal(t, tc.expectedReqType, reqInfo.RequestType)
require.Equal(t, tc.expectedBktName, reqInfo.BucketName)
require.Equal(t, tc.expectedObjName, reqInfo.ObjectName)
})
}
}
func TestPrepareVHSAddress(t *testing.T) {
bkt, obj, domain := "test-bucket", "test-object", "domain.com"
for _, tc := range []struct {
name string
domains []string
host string
urlParams string
expectedReqType ReqType
expectedBktName string
expectedObjName string
}{
{
name: "bucket request, the domain matched",
domains: []string{domain},
host: bkt + "." + domain,
urlParams: "/",
expectedReqType: bucketType,
expectedBktName: bkt,
},
{
name: "object request, the domain matched",
domains: []string{domain},
host: bkt + "." + domain,
urlParams: "/" + obj,
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
{
name: "object request with slash, the domain matched",
domains: []string{domain},
host: bkt + "." + domain,
urlParams: "/" + obj + "/",
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj + "/",
},
{
name: "list-buckets request, the domain matched",
domains: []string{domain},
host: domain,
urlParams: "/",
expectedReqType: noneType,
},
{
name: "bucket request, the domain don't match",
host: bkt + "." + domain,
urlParams: "/",
expectedReqType: bucketType,
expectedBktName: bkt,
},
{
name: "object request, the domain don't match",
host: bkt + "." + domain,
urlParams: "/" + obj,
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
{
name: "object request with slash, the domain don't match",
host: bkt + "." + domain,
urlParams: "/" + obj + "/",
expectedReqType: objectType,
expectedBktName: bkt,
expectedObjName: obj + "/",
},
{
name: "list-buckets request, the domain don't match (list-buckets isn't supported if the domains don't match)",
host: domain,
urlParams: "/",
expectedReqType: bucketType,
expectedBktName: strings.Split(domain, ".")[0],
},
} {
t.Run(tc.name, func(t *testing.T) {
reqInfo := &ReqInfo{}
vhsSettings := &VHSSettingsMock{domains: tc.domains}
r := httptest.NewRequest(http.MethodGet, tc.urlParams, nil)
r.Host = tc.host
prepareVHSAddress(reqInfo, r, vhsSettings)
require.Equal(t, tc.expectedReqType, reqInfo.RequestType)
require.Equal(t, tc.expectedBktName, reqInfo.BucketName)
require.Equal(t, tc.expectedObjName, reqInfo.ObjectName)
})
}
}
func TestCheckDomains(t *testing.T) {
for _, tc := range []struct {
name string
domains []string
requestURL string
expectedBktName string
expectedMatch bool
}{
{
name: "valid url with bktName and namespace (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.s3.kapusta.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName and namespace (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "s3.kapusta.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "invalid url with invalid bktName (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.bktB.s3.kapusta.domain.com",
expectedMatch: false,
},
{
name: "invalid url without namespace (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.s3.domain.com",
expectedMatch: false,
},
{
name: "invalid url with invalid infix (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.s4.kapusta.domain.com",
expectedMatch: false,
},
{
name: "invalid url with invalid postfix (wildcard after protocol infix)",
domains: []string{"s3.<wildcard>.domain.com"},
requestURL: "bktA.s3.kapusta.dom.su",
expectedMatch: false,
},
{
name: "valid url with bktName and namespace (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "bktA.kapusta.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName and namespace (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "kapusta.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "invalid url with invalid bktName (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "bktA.bktB.kapusta.domain.com",
expectedMatch: false,
},
{
name: "collision test - true, because we cannot clearly distinguish a namespace from a bucket (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "bktA.domain.com",
expectedMatch: true,
},
{
name: "invalid url (fewer hosts)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "domain.com",
expectedMatch: false,
},
{
name: "invalid url with invalid postfix (wildcard at the beginning of the domain)",
domains: []string{"<wildcard>.domain.com"},
requestURL: "bktA.kapusta.dom.su",
expectedMatch: false,
},
{
name: "valid url with bktName and without wildcard (root namaspace)",
domains: []string{"domain.com"},
requestURL: "bktA.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName and without wildcard (root namaspace)",
domains: []string{"domain.com"},
requestURL: "domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "invalid url with bktName without wildcard (root namaspace)",
domains: []string{"domain.com"},
requestURL: "bktA.dom.su",
expectedMatch: false,
},
{
name: "invalid url without wildcard (root namaspace)",
domains: []string{"domain.com"},
requestURL: "dom.su",
expectedMatch: false,
},
{
name: "valid url, with a sorted list of domains",
domains: []string{"s3.<wildcard>.domain.com", "<wildcard>.domain.com", "domain.com"},
requestURL: "s3.kapusta.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "valid url with bktName, multiple wildcards (wildcards at the beginning of the domain)",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "bktA.s3.kapusta.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName, multiple wildcards (wildcards at the beginning of the domain)",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "s3.kapusta.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "valid url with bktName, multiply wildcards",
domains: []string{"s3.<wildcard>.subdomain.<wildcard>.com"},
requestURL: "bktA.s3.kapusta.subdomain.domain.com",
expectedBktName: "bktA",
expectedMatch: true,
},
{
name: "valid url without bktName, multiply wildcards",
domains: []string{"s3.<wildcard>.subdomain.<wildcard>.com"},
requestURL: "s3.kapusta.subdomain.domain.com",
expectedBktName: "",
expectedMatch: true,
},
{
name: "invalid url without one wildcard",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "kapusta.domain.com",
expectedMatch: false,
},
{
name: "invalid url, multiply wildcards",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "s3.kapusta.dom.com",
expectedMatch: false,
},
{
name: "invalid url with invalid bktName, multiply wildcards",
domains: []string{"<wildcard>.<wildcard>.domain.com"},
requestURL: "bktA.bktB.s3.kapusta.domain.com",
expectedMatch: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
bktName, match := checkDomain(tc.requestURL, tc.domains)
require.Equal(t, tc.expectedBktName, bktName)
require.Equal(t, tc.expectedMatch, match)
})
}
}
func TestGetDomains(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/", nil)
settings := &VHSSettingsMock{
domains: []string{
"s3.domain.com",
"s3.<wildcard>.domain.com",
"domain.com",
},
}
t.Run("the request does not contain the X-Frostfs-Servername header", func(t *testing.T) {
actualDomains := getDomains(req, settings)
require.Equal(t, settings.domains, actualDomains)
})
serverName := "domain.com"
req.Header.Set(settings.ServernameHeader(), serverName)
t.Run("the request contains the X-Frostfs-Servername header", func(t *testing.T) {
actualDomains := getDomains(req, settings)
require.Equal(t, []string{serverName}, actualDomains)
})
}

38
api/middleware/policy.go
View file

@ -73,7 +73,6 @@ type PolicyConfig struct {
Storage engine.ChainRouter
FrostfsID FrostFSIDInformer
Settings PolicySettings
Domains []string
Log *zap.Logger
BucketResolver BucketResolveFunc
Decoder XMLDecoder
@ -99,21 +98,21 @@ func PolicyCheck(cfg PolicyConfig) Func {
}
func policyCheck(r *http.Request, cfg PolicyConfig) error {
reqType, bktName, objName := getBucketObject(r, cfg.Domains)
req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqType, bktName, objName)
reqInfo := GetReqInfo(r.Context())
req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqInfo.RequestType, reqInfo.BucketName, reqInfo.ObjectName)
if err != nil {
return err
}
var bktInfo *data.BucketInfo
if reqType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
bktInfo, err = cfg.BucketResolver(r.Context(), bktName)
if reqInfo.RequestType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
bktInfo, err = cfg.BucketResolver(r.Context(), reqInfo.BucketName)
if err != nil {
return err
}
}
reqInfo := GetReqInfo(r.Context())
target := engine.NewRequestTargetWithNamespace(reqInfo.Namespace)
if bktInfo != nil {
cnrTarget := engine.ContainerTarget(bktInfo.CID.EncodeToString())
@ -208,33 +207,6 @@ const (
objectType
)
func getBucketObject(r *http.Request, domains []string) (reqType ReqType, bktName string, objName string) {
for _, domain := range domains {
ind := strings.Index(r.Host, "."+domain)
if ind == -1 {
continue
}
bkt := r.Host[:ind]
if obj := strings.TrimPrefix(r.URL.Path, "/"); obj != "" {
return objectType, bkt, obj
}
return bucketType, bkt, ""
}
bktObj := strings.TrimPrefix(r.URL.Path, "/")
if bktObj == "" {
return noneType, "", ""
}
if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" {
return objectType, bktObj[:ind], bktObj[ind+1:]
}
return bucketType, strings.TrimSuffix(bktObj, "/"), ""
}
func determineOperation(r *http.Request, reqType ReqType) (operation string) {
switch reqType {
case objectType:

73
api/middleware/policy_test.go
View file

@ -8,79 +8,6 @@ import (
"github.com/stretchr/testify/require"
)
func TestReqTypeDetermination(t *testing.T) {
bkt, obj, domain := "test-bucket", "test-object", "domain"
for _, tc := range []struct {
name string
target string
host string
domains []string
expectedType ReqType
expectedBktName string
expectedObjName string
}{
{
name: "bucket request, path-style",
target: "/" + bkt,
expectedType: bucketType,
expectedBktName: bkt,
},
{
name: "bucket request with slash, path-style",
target: "/" + bkt + "/",
expectedType: bucketType,
expectedBktName: bkt,
},
{
name: "object request, path-style",
target: "/" + bkt + "/" + obj,
expectedType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
{
name: "object request with slash, path-style",
target: "/" + bkt + "/" + obj + "/",
expectedType: objectType,
expectedBktName: bkt,
expectedObjName: obj + "/",
},
{
name: "none type request",
target: "/",
expectedType: noneType,
},
{
name: "bucket request, virtual-hosted style",
target: "/",
host: bkt + "." + domain,
domains: []string{"some-domain", domain},
expectedType: bucketType,
expectedBktName: bkt,
},
{
name: "object request, virtual-hosted style",
target: "/" + obj,
host: bkt + "." + domain,
domains: []string{"some-domain", domain},
expectedType: objectType,
expectedBktName: bkt,
expectedObjName: obj,
},
} {
t.Run(tc.name, func(t *testing.T) {
r := httptest.NewRequest(http.MethodPut, tc.target, nil)
r.Host = tc.host
reqType, bktName, objName := getBucketObject(r, tc.domains)
require.Equal(t, tc.expectedType, reqType)
require.Equal(t, tc.expectedBktName, bktName)
require.Equal(t, tc.expectedObjName, objName)
})
}
}
func TestDetermineBucketOperation(t *testing.T) {
const defaultValue = "value"

84
api/middleware/reqinfo.go
View file

@ -12,7 +12,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"go.uber.org/zap"
"google.golang.org/grpc/metadata"
@ -28,19 +27,21 @@ type (
// ReqInfo stores the request info.
ReqInfo struct {
sync.RWMutex
RemoteHost string // Client Host/IP
Host string // Node Host/IP
UserAgent string // User Agent
DeploymentID string // random generated s3-deployment-id
RequestID string // x-amz-request-id
API string // API name -- GetObject PutObject NewMultipartUpload etc.
BucketName string // Bucket name
ObjectName string // Object name
TraceID string // Trace ID
URL *url.URL // Request url
Namespace string
User string // User owner id
Tagging *data.Tagging
RemoteHost string // Client Host/IP
Host string // Node Host/IP
UserAgent string // User Agent
DeploymentID string // random generated s3-deployment-id
RequestID string // x-amz-request-id
API string // API name -- GetObject PutObject NewMultipartUpload etc.
BucketName string // Bucket name
ObjectName string // Object name
TraceID string // Trace ID
URL *url.URL // Request url
Namespace string
User string // User owner id
Tagging *data.Tagging
RequestVHSEnabled bool
RequestType ReqType
}
// ObjectRequest represents object request data.
@ -61,10 +62,6 @@ const (
const HdrAmzRequestID = "x-amz-request-id"
const (
BucketURLPrm = "bucket"
)
var deploymentID = uuid.Must(uuid.NewRandom())
var (
@ -202,57 +199,6 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
}
}
// AddBucketName adds bucket name to ReqInfo from context.
func AddBucketName(l *zap.Logger) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := GetReqInfo(ctx)
reqInfo.BucketName = chi.URLParam(r, BucketURLPrm)
if reqInfo.BucketName != "" {
reqLogger := reqLogOrDefault(ctx, l)
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("bucket", reqInfo.BucketName))))
}
h.ServeHTTP(w, r)
})
}
}
// AddObjectName adds objects name to ReqInfo from context.
func AddObjectName(l *zap.Logger) Func {
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqInfo := GetReqInfo(ctx)
reqLogger := reqLogOrDefault(ctx, l)
rctx := chi.RouteContext(ctx)
// trim leading slash (always present)
reqInfo.ObjectName = rctx.RoutePath[1:]
if r.URL.RawPath != "" {
// we have to do this because of
// https://github.com/go-chi/chi/issues/641
// https://github.com/go-chi/chi/issues/642
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
} else {
reqInfo.ObjectName = obj
}
}
if reqInfo.ObjectName != "" {
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName))))
}
h.ServeHTTP(w, r)
})
}
}
// getSourceIP retrieves the IP from the X-Forwarded-For, X-Real-IP and RFC7239
// Forwarded headers (in that order), falls back to r.RemoteAddr when everything
// else fails.

63
api/router.go
View file

@ -97,6 +97,7 @@ type Settings interface {
s3middleware.RequestSettings
s3middleware.PolicySettings
s3middleware.MetricsSettings
s3middleware.VHSSettings
}
type FrostFSID interface {
@ -113,9 +114,6 @@ type Config struct {
MiddlewareSettings Settings
// Domains optional. If empty no virtual hosted domains will be attached.
Domains []string
FrostfsID FrostFSID
FrostFSIDValidation bool
@ -142,11 +140,11 @@ func NewRouter(cfg Config) *chi.Mux {
api.Use(s3middleware.FrostfsIDValidation(cfg.FrostfsID, cfg.Log))
}
api.Use(s3middleware.PrepareAddressStyle(cfg.MiddlewareSettings, cfg.Log))
api.Use(s3middleware.PolicyCheck(s3middleware.PolicyConfig{
Storage: cfg.PolicyChecker,
FrostfsID: cfg.FrostfsID,
Settings: cfg.MiddlewareSettings,
Domains: cfg.Domains,
Log: cfg.Log,
BucketResolver: cfg.Handler.ResolveBucket,
Decoder: cfg.XMLDecoder,
@ -154,22 +152,41 @@ func NewRouter(cfg Config) *chi.Mux {
}))
defaultRouter := chi.NewRouter()
defaultRouter.Mount(fmt.Sprintf("/{%s}", s3middleware.BucketURLPrm), bucketRouter(cfg.Handler, cfg.Log))
defaultRouter.Get("/", named("ListBuckets", cfg.Handler.ListBucketsHandler))
defaultRouter.Mount("/{bucket}", bucketRouter(cfg.Handler))
defaultRouter.Get("/", named(s3middleware.ListBucketsOperation, cfg.Handler.ListBucketsHandler))
attachErrorHandler(defaultRouter)
hr := NewHostBucketRouter("bucket")
hr.Default(defaultRouter)
for _, domain := range cfg.Domains {
hr.Map(domain, bucketRouter(cfg.Handler, cfg.Log))
}
api.Mount("/", hr)
vhsRouter := bucketRouter(cfg.Handler)
router := newGlobalRouter(defaultRouter, vhsRouter)
api.Mount("/", router)
attachErrorHandler(api)
return api
}
type globalRouter struct {
pathStyleRouter chi.Router
vhsRouter chi.Router
}
func newGlobalRouter(pathStyleRouter, vhsRouter chi.Router) *globalRouter {
return &globalRouter{
pathStyleRouter: pathStyleRouter,
vhsRouter: vhsRouter,
}
}
func (g *globalRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
router := g.pathStyleRouter
if reqInfo := s3middleware.GetReqInfo(r.Context()); reqInfo.RequestVHSEnabled {
router = g.vhsRouter
}
router.ServeHTTP(w, r)
}
func named(name string, handlerFunc http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
reqInfo := s3middleware.GetReqInfo(r.Context())
@ -214,14 +231,13 @@ func attachErrorHandler(api *chi.Mux) {
api.MethodNotAllowed(named("MethodNotAllowed", errorHandler))
}
func bucketRouter(h Handler, log *zap.Logger) chi.Router {
func bucketRouter(h Handler) chi.Router {
bktRouter := chi.NewRouter()
bktRouter.Use(
s3middleware.AddBucketName(log),
s3middleware.WrapHandler(h.AppendCORSHeaders),
)
bktRouter.Mount("/", objectRouter(h, log))
bktRouter.Mount("/", objectRouter(h))
bktRouter.Options("/", named(s3middleware.OptionsBucketOperation, h.Preflight))
@ -293,7 +309,7 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router {
Add(NewFilter().
Queries(s3middleware.VersionsQuery).
Handler(named(s3middleware.ListBucketObjectVersionsOperation, h.ListBucketObjectVersionsHandler))).
DefaultHandler(named(s3middleware.ListObjectsV1Operation, h.ListObjectsV1Handler)))
DefaultHandler(listWrapper(h)))
})
// PUT method handlers
@ -368,9 +384,20 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router {
return bktRouter
}
func objectRouter(h Handler, l *zap.Logger) chi.Router {
func listWrapper(h Handler) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if reqInfo := s3middleware.GetReqInfo(r.Context()); reqInfo.BucketName == "" {
reqInfo.API = s3middleware.ListBucketsOperation
h.ListBucketsHandler(w, r)
} else {
reqInfo.API = s3middleware.ListObjectsV1Operation
h.ListObjectsV1Handler(w, r)
}
}
}
func objectRouter(h Handler) chi.Router {
objRouter := chi.NewRouter()
objRouter.Use(s3middleware.AddObjectName(l))
objRouter.Options("/*", named(s3middleware.OptionsObjectOperation, h.Preflight))

33
api/router_mock_test.go
View file

@ -23,7 +23,11 @@ import (
"github.com/stretchr/testify/require"
)
const FrostfsNamespaceHeader = "X-Frostfs-Namespace"
const (
FrostfsNamespaceHeader = "X-Frostfs-Namespace"
FrostfsVHSHeader = "X-Frostfs-S3-VHS"
FrostfsServernameHeader = "X-Frostfs-Servername"
)
type poolStatisticMock struct {
}
@ -71,8 +75,11 @@ func (c *centerMock) Authenticate(*http.Request) (*middleware.Box, error) {
}
type middlewareSettingsMock struct {
denyByDefault bool
sourceIPHeader string
denyByDefault bool
sourceIPHeader string
domains []string
vhsEnabled bool
vhsNamespacesEnabled map[string]bool
}
func (r *middlewareSettingsMock) SourceIPHeader() string {
@ -91,6 +98,26 @@ func (r *middlewareSettingsMock) PolicyDenyByDefault() bool {
return r.denyByDefault
}
func (r *middlewareSettingsMock) Domains() []string {
return r.domains
}
func (r *middlewareSettingsMock) GlobalVHS() bool {
return r.vhsEnabled
}
func (r *middlewareSettingsMock) VHSHeader() string {
return FrostfsVHSHeader
}
func (r *middlewareSettingsMock) ServernameHeader() string {
return FrostfsServernameHeader
}
func (r *middlewareSettingsMock) VHSNamespacesEnabled() map[string]bool {
return r.vhsNamespacesEnabled
}
type frostFSIDMock struct {
tags map[string]string
validateError bool

26
api/router_test.go
View file

@ -78,7 +78,6 @@ func prepareRouter(t *testing.T, opts ...option) *routerMock {
Metrics: metrics.NewAppMetrics(metricsConfig),
MiddlewareSettings: middlewareSettings,
PolicyChecker: policyChecker,
Domains: []string{"domain1", "domain2"},
FrostfsID: &frostFSIDMock{},
XMLDecoder: &xmlMock{},
Tagging: &resourceTaggingMock{},
@ -847,6 +846,31 @@ func TestFrostFSIDValidation(t *testing.T) {
createBucketErr(chiRouter, "", "bkt-3", nil, apiErrors.ErrInternalError)
}
func TestRouterListObjectsV2Domains(t *testing.T) {
chiRouter := prepareRouter(t, enableVHSDomains("domain.com"))
chiRouter.handler.buckets["bucket"] = &data.BucketInfo{}
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/", nil)
r.Host = "bucket.domain.com"
query := make(url.Values)
query.Set(s3middleware.ListTypeQuery, "2")
r.URL.RawQuery = query.Encode()
chiRouter.ServeHTTP(w, r)
resp := readResponse(t, w)
require.Equal(t, s3middleware.ListObjectsV2Operation, resp.Method)
}
func enableVHSDomains(domains ...string) option {
return func(cfg *Config) {
setting := cfg.MiddlewareSettings.(*middlewareSettingsMock)
setting.vhsEnabled = true
setting.domains = domains
}
}
func readResponse(t *testing.T, w *httptest.ResponseRecorder) handlerResult {
var res handlerResult

62
cmd/s3-gw/app.go
View file

@ -105,9 +105,13 @@ type (
policyDenyByDefault bool
sourceIPHeader string
retryMaxAttempts int
domains []string
vhsEnabled bool
vhsHeader string
servernameHeader string
vhsNamespacesEnabled map[string]bool
retryMaxBackoff time.Duration
retryStrategy handler.RetryStrategy
domains []string
}
maxClientsConfig struct {
@ -256,13 +260,55 @@ func (s *appSettings) updateNamespacesSettings(v *viper.Viper, log *zap.Logger)
s.namespaces = nsConfig.Namespaces
}
func (s *appSettings) setVHSSettings(v *viper.Viper, _ *zap.Logger) {
domains := v.GetStringSlice(cfgListenDomains)
func (s *appSettings) setVHSSettings(v *viper.Viper, log *zap.Logger) {
domains := fetchDomains(v, log)
vhsEnabled := v.GetBool(cfgVHSEnabled)
vhsHeader := v.GetString(cfgVHSHeader)
servernameHeader := v.GetString(cfgServernameHeader)
nsMap := fetchVHSNamespaces(v, log)
vhsNamespaces := make(map[string]bool, len(nsMap))
for ns, flag := range nsMap {
vhsNamespaces[s.ResolveNamespaceAlias(ns)] = flag
}
s.mu.Lock()
defer s.mu.Unlock()
s.domains = domains
s.vhsEnabled = vhsEnabled
s.vhsHeader = vhsHeader
s.servernameHeader = servernameHeader
s.vhsNamespacesEnabled = vhsNamespaces
}
func (s *appSettings) Domains() []string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.domains
}
func (s *appSettings) GlobalVHS() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.vhsEnabled
}
func (s *appSettings) VHSHeader() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.vhsHeader
}
func (s *appSettings) ServernameHeader() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.servernameHeader
}
func (s *appSettings) VHSNamespacesEnabled() map[string]bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.vhsNamespacesEnabled
}
func (s *appSettings) BypassContentEncodingInChunks() bool {
@ -467,12 +513,6 @@ func (s *appSettings) RetryStrategy() handler.RetryStrategy {
return s.retryStrategy
}
func (s *appSettings) Domains() []string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.domains
}
func (a *App) initAPI(ctx context.Context) {
a.initLayer(ctx)
a.initHandler()
@ -710,9 +750,6 @@ func (a *App) setHealthStatus() {
// Serve runs HTTP server to handle S3 API requests.
func (a *App) Serve(ctx context.Context) {
// Attach S3 API:
a.log.Info(logs.FetchDomainsPrepareToUseAPI, zap.Strings("domains", a.settings.Domains()))
cfg := api.Config{
Throttle: middleware.ThrottleOpts{
Limit: a.settings.maxClient.count,
@ -722,7 +759,6 @@ func (a *App) Serve(ctx context.Context) {
Center: a.ctr,
Log: a.log,
Metrics: a.metrics,
Domains: a.settings.Domains(),
MiddlewareSettings: a.settings,
PolicyChecker: a.policyStorage,

67
cmd/s3-gw/app_settings.go
View file

@ -30,6 +30,8 @@ import (
const (
destinationStdout = "stdout"
destinationJournald = "journald"
wildcardPlaceholder = "<wildcard>"
)
const (
@ -52,7 +54,9 @@ const (
defaultAccessBoxCacheRemovingCheckInterval = 5 * time.Minute
defaultNamespaceHeader = "X-Frostfs-Namespace"
defaultNamespaceHeader = "X-Frostfs-Namespace"
defaultVHSHeader = "X-Frostfs-S3-VHS"
defaultServernameHeader = "X-Frostfs-Servername"
defaultConstraintName = "default"
@ -144,6 +148,12 @@ const ( // Settings.
cfgListenDomains = "listen_domains"
// VHS.
cfgVHSEnabled = "vhs.enabled"
cfgVHSHeader = "vhs.vhs_header"
cfgServernameHeader = "vhs.servername_header"
cfgVHSNamespaces = "vhs.namespaces"
// Peers.
cfgPeers = "peers"
@ -668,6 +678,41 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo {
return servers
}
func fetchDomains(v *viper.Viper, log *zap.Logger) []string {
domains := validateDomains(v.GetStringSlice(cfgListenDomains), log)
countParts := func(domain string) int {
return strings.Count(domain, ".")
}
sort.Slice(domains, func(i, j int) bool {
return countParts(domains[i]) > countParts(domains[j])
})
return domains
}
func fetchVHSNamespaces(v *viper.Viper, log *zap.Logger) map[string]bool {
vhsNamespacesEnabled := make(map[string]bool)
nsMap := v.GetStringMap(cfgVHSNamespaces)
for ns, val := range nsMap {
if _, ok := vhsNamespacesEnabled[ns]; ok {
log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns))
continue
}
enabledFlag, ok := val.(bool)
if !ok {
log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns))
continue
}
vhsNamespacesEnabled[ns] = enabledFlag
}
return vhsNamespacesEnabled
}
func newSettings() *viper.Viper {
v := viper.New()
@ -754,6 +799,10 @@ func newSettings() *viper.Viper {
v.SetDefault(cfgRetryMaxAttempts, defaultRetryMaxAttempts)
v.SetDefault(cfgRetryMaxBackoff, defaultRetryMaxBackoff)
// vhs
v.SetDefault(cfgVHSHeader, defaultVHSHeader)
v.SetDefault(cfgServernameHeader, defaultServernameHeader)
// Bind flags
if err := bindFlags(v, flags); err != nil {
panic(fmt.Errorf("bind flags: %w", err))
@ -1029,3 +1078,19 @@ func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
}
return lvl, nil
}
func validateDomains(domains []string, log *zap.Logger) []string {
validDomains := make([]string, 0, len(domains))
LOOP:
for _, domain := range domains {
domainParts := strings.Split(domain, ".")
for _, part := range domainParts {
if strings.ContainsAny(part, "<>") && part != wildcardPlaceholder {
log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain))
continue LOOP
}
}
validDomains = append(validDomains, domain)
}
return validDomains
}

34
cmd/s3-gw/validate_test.go Normal file
View file

@ -0,0 +1,34 @@
package main
import (
"testing"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
func TestValidateDomains(t *testing.T) {
inputDomains := []string{
"s3dev.frostfs.devenv",
"s3dev.<invalid>.frostfs.devenv",
"s3dev.<wildcard>.frostfs.devenv",
"s3dev.<wildcard.frostfs.devenv",
"s3dev.wildcard>.frostfs.devenv",
"s3dev.<wild.card>.frostfs.devenv",
"<invalid>.frostfs.devenv",
"<wildcard>.frostfs.devenv>",
"<wildcard>.frostfs.devenv",
"s3dev.fro<stfs.devenv",
"<wildcard>.dev.<wildcard>.frostfs.devenv",
"<wildcard>.dev.<wildc>ard>.frostfs.devenv",
}
expectedDomains := []string{
"s3dev.frostfs.devenv",
"s3dev.<wildcard>.frostfs.devenv",
"<wildcard>.frostfs.devenv",
"<wildcard>.dev.<wildcard>.frostfs.devenv",
}
actualDomains := validateDomains(inputDomains, zaptest.NewLogger(t))
require.Equal(t, expectedDomains, actualDomains)
}

11
config/config.env
View file

@ -36,8 +36,15 @@ S3_GW_SERVER_1_TLS_KEY_FILE=/path/to/tls/key
# How often to reconnect to the servers
S3_GW_RECONNECT_INTERVAL: 1m
# Domains to be able to use virtual-hosted-style access to bucket.
S3_GW_LISTEN_DOMAINS=s3dev.frostfs.devenv
# Domains to be able to use virtual-hosted-style access to bucket
S3_GW_LISTEN_DOMAINS="domain.com <wildcard>.domain.com"
# VHS enabled flag
S3_GW_VHS_ENABLED=false
# Header for determining whether VHS is enabled for the request
S3_GW_VHS_VHS_HEADER=X-Frostfs-S3-VHS
# Header for determining servername
S3_GW_VHS_SERVERNAME_HEADER=X-Frostfs-Servername
# Config file
S3_GW_CONFIG=/path/to/config/yaml

9
config/config.yaml
View file

@ -42,6 +42,15 @@ server:
# Domains to be able to use virtual-hosted-style access to bucket.
listen_domains:
- s3dev.frostfs.devenv
- s3dev.<wildcard>.frostfs.devenv
vhs:
enabled: false
vhs_header: X-Frostfs-S3-VHS
servername_header: X-Frostfs-Servername
namespaces:
"ns1": false
"ns2": true
logger:
level: debug

25
docs/configuration.md
View file

@ -193,12 +193,14 @@ There are some custom types used for brevity:
| `namespaces` | [Namespaces configuration](#namespaces-section) |
| `retry` | [Retry configuration](#retry-section) |
| `containers` | [Containers configuration](#containers-section) |
| `vhs` | [VHS configuration](#vhs-section) |
### General section
```yaml
listen_domains:
- s3dev.frostfs.devenv
- s3dev.<wildcard>.frostfs.devenv
- s3dev2.frostfs.devenv
alexvanin commented 2024-08-21 13:52:46 +00:00
Review
Copy link

@dkirillov Have you discussed putting it into vhs section?

@dkirillov Have you discussed putting it into `vhs` section?
dkirillov commented 2024-08-23 08:16:58 +00:00
Review
Copy link

We haven't discussed. I'm ok with keeping params in listen_domains

We haven't discussed. I'm ok with keeping params in `listen_domains`
rpc_endpoint: http://morph-chain.frostfs.devenv:30333
@ -226,7 +228,7 @@ source_ip_header: "Source-Ip"
| Parameter | Type | SIGHUP reload | Default value | Description |
|----------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `listen_domains` | `[]string` | no | | Domains to be able to use virtual-hosted-style access to bucket. |
| `listen_domains` | `[]string` | yes | | Domains to be able to use virtual-hosted-style access to bucket. The presence of placeholders of the <wildcard> type is supported. |
| `rpc_endpoint` | `string` | no | | The address of the RPC host to which the gateway connects to resolve bucket names and interact with frostfs contracts (required to use the `nns` resolver and `frostfsid` contract). |
| `resolve_order` | `[]string` | yes | `[dns]` | Order of bucket name resolvers to use. Available resolvers: `dns`, `nns`. |
| `connect_timeout` | `duration` | no | `10s` | Timeout to connect to a node. |
@ -723,3 +725,24 @@ containers:
|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. |
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
# `vhs` section
Configuration of virtual hosted addressing style.
```yaml
vhs:
enabled: false
vhs_header: X-Frostfs-S3-VHS
servername_header: X-Frostfs-Servername
namespaces:
"ns1": false
"ns2": true
```
| Parameter | Type | SIGHUP reload | Default value | Description |
| ------------------- | ----------------- | ------------- | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `enabled` | `bool` | yes | `false` | Enables the use of virtual host addressing for buckets at the application level. |
| `vhs_header` | `string` | yes | `X-Frostfs-S3-VHS` | Header for determining whether VHS is enabled for the request. |
| `servername_header` | `string` | yes | `X-Frostfs-Servername` | Header for determining servername. |
| `namespaces` | `map[string]bool` | yes | | A map in which the keys are the name of the namespace, and the values are the flag responsible for enabling VHS for the specified namespace. Overrides global 'enabled' setting even when it is disabled. |

4
internal/logs/logs.go
View file

@ -20,7 +20,6 @@ const (
UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go
ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go
ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go
FetchDomainsPrepareToUseAPI = "fetch domains, prepare to use API" // Info in ../../cmd/s3-gw/app.go
StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go
StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go
@ -159,4 +158,7 @@ const (
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
GetBucketLifecycle = "get bucket lifecycle"
WarnDuplicateNamespaceVHS = "duplicate namespace with enabled VHS, config value skipped"
WarnValueVHSEnabledFlagWrongType = "the value of the VHS enable flag for the namespace is of the wrong type, config value skipped"
WarnDomainContainsInvalidPlaceholder = "the domain contains an invalid placeholder, domain skipped"
)