feature/virtual-hosted-style #465
20 changed files with 929 additions and 244 deletions
|
@ -4,6 +4,9 @@ This document outlines major changes between releases.
|
||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Add support for virtual hosted style addressing (#446, #449)
|
||||||
|
|
||||||
## [0.30.0] - Kangshung -2024-07-19
|
## [0.30.0] - Kangshung -2024-07-19
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
|
@ -41,7 +41,6 @@ type (
|
||||||
RetryMaxAttempts() int
|
RetryMaxAttempts() int
|
||||||
RetryMaxBackoff() time.Duration
|
RetryMaxBackoff() time.Duration
|
||||||
RetryStrategy() RetryStrategy
|
RetryStrategy() RetryStrategy
|
||||||
Domains() []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
FrostFSID interface {
|
FrostFSID interface {
|
||||||
|
|
|
@ -73,7 +73,6 @@ type configMock struct {
|
||||||
defaultCopiesNumbers []uint32
|
defaultCopiesNumbers []uint32
|
||||||
bypassContentEncodingInChunks bool
|
bypassContentEncodingInChunks bool
|
||||||
md5Enabled bool
|
md5Enabled bool
|
||||||
domains []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
|
func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy {
|
||||||
|
@ -137,10 +136,6 @@ func (c *configMock) RetryStrategy() RetryStrategy {
|
||||||
return RetryStrategyConstant
|
return RetryStrategyConstant
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *configMock) Domains() []string {
|
|
||||||
return c.domains
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareHandlerContext(t *testing.T) *handlerContext {
|
func prepareHandlerContext(t *testing.T) *handlerContext {
|
||||||
log := zaptest.NewLogger(t)
|
log := zaptest.NewLogger(t)
|
||||||
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(log), log)
|
return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(log), log)
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"net/url"
|
"net/url"
|
||||||
"path"
|
"path"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
|
@ -429,7 +428,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
|
||||||
Bucket: objInfo.Bucket,
|
Bucket: objInfo.Bucket,
|
||||||
Key: objInfo.Name,
|
Key: objInfo.Name,
|
||||||
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
|
ETag: data.Quote(objInfo.ETag(h.cfg.MD5Enabled())),
|
||||||
Location: getObjectLocation(r, h.cfg.Domains(), reqInfo.BucketName, reqInfo.ObjectName),
|
Location: getObjectLocation(r, reqInfo.BucketName, reqInfo.ObjectName, reqInfo.RequestVHSEnabled),
|
||||||
}
|
}
|
||||||
|
|
||||||
if settings.VersioningEnabled() {
|
if settings.VersioningEnabled() {
|
||||||
|
@ -450,7 +449,7 @@ func getURLScheme(r *http.Request) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// getObjectLocation gets the fully qualified URL of an object.
|
// getObjectLocation gets the fully qualified URL of an object.
|
||||||
func getObjectLocation(r *http.Request, domains []string, bucket, object string) string {
|
func getObjectLocation(r *http.Request, bucket, object string, vhsEnabled bool) string {
|
||||||
proto := middleware.GetSourceScheme(r)
|
proto := middleware.GetSourceScheme(r)
|
||||||
if proto == "" {
|
if proto == "" {
|
||||||
proto = getURLScheme(r)
|
proto = getURLScheme(r)
|
||||||
|
@ -460,13 +459,12 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
|
||||||
Path: path.Join("/", bucket, object),
|
Path: path.Join("/", bucket, object),
|
||||||
Scheme: proto,
|
Scheme: proto,
|
||||||
}
|
}
|
||||||
// If domain is set then we need to use bucket DNS style.
|
|
||||||
for _, domain := range domains {
|
// If vhs enabled then we need to use bucket DNS style.
|
||||||
if strings.HasPrefix(r.Host, bucket+"."+domain) {
|
if vhsEnabled {
|
||||||
u.Path = path.Join("/", object)
|
u.Path = path.Join("/", object)
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return u.String()
|
return u.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -443,11 +443,11 @@ func TestUploadPartCheckContentSHA256(t *testing.T) {
|
||||||
|
|
||||||
func TestMultipartObjectLocation(t *testing.T) {
|
func TestMultipartObjectLocation(t *testing.T) {
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
req *http.Request
|
req *http.Request
|
||||||
bucket string
|
bucket string
|
||||||
object string
|
object string
|
||||||
domains []string
|
vhsEnabled bool
|
||||||
expected string
|
expected string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
req: &http.Request{
|
req: &http.Request{
|
||||||
|
@ -492,24 +492,24 @@ func TestMultipartObjectLocation(t *testing.T) {
|
||||||
req: &http.Request{
|
req: &http.Request{
|
||||||
Host: "mybucket.s3dev.frostfs.devenv",
|
Host: "mybucket.s3dev.frostfs.devenv",
|
||||||
},
|
},
|
||||||
domains: []string{"s3dev.frostfs.devenv"},
|
bucket: "mybucket",
|
||||||
bucket: "mybucket",
|
object: "test/1.txt",
|
||||||
object: "test/1.txt",
|
vhsEnabled: true,
|
||||||
expected: "http://mybucket.s3dev.frostfs.devenv/test/1.txt",
|
expected: "http://mybucket.s3dev.frostfs.devenv/test/1.txt",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
req: &http.Request{
|
req: &http.Request{
|
||||||
Host: "mybucket.s3dev.frostfs.devenv",
|
Host: "mybucket.s3dev.frostfs.devenv",
|
||||||
Header: map[string][]string{"X-Forwarded-Scheme": {"https"}},
|
Header: map[string][]string{"X-Forwarded-Scheme": {"https"}},
|
||||||
},
|
},
|
||||||
domains: []string{"s3dev.frostfs.devenv"},
|
bucket: "mybucket",
|
||||||
bucket: "mybucket",
|
object: "test/1.txt",
|
||||||
object: "test/1.txt",
|
vhsEnabled: true,
|
||||||
expected: "https://mybucket.s3dev.frostfs.devenv/test/1.txt",
|
expected: "https://mybucket.s3dev.frostfs.devenv/test/1.txt",
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run("", func(t *testing.T) {
|
t.Run("", func(t *testing.T) {
|
||||||
location := getObjectLocation(tc.req, tc.domains, tc.bucket, tc.object)
|
location := getObjectLocation(tc.req, tc.bucket, tc.object, tc.vhsEnabled)
|
||||||
require.Equal(t, tc.expected, location)
|
require.Equal(t, tc.expected, location)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
148
api/middleware/address_style.go
Normal file
148
api/middleware/address_style.go
Normal file
|
@ -0,0 +1,148 @@
|
||||||
|
package middleware
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
const wildcardPlaceholder = "<wildcard>"
|
||||||
|
|
||||||
|
type VHSSettings interface {
|
||||||
|
Domains() []string
|
||||||
|
GlobalVHS() bool
|
||||||
|
VHSHeader() string
|
||||||
|
ServernameHeader() string
|
||||||
|
VHSNamespacesEnabled() map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func PrepareAddressStyle(settings VHSSettings, log *zap.Logger) Func {
|
||||||
|
return func(h http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
reqInfo := GetReqInfo(ctx)
|
||||||
|
reqLogger := reqLogOrDefault(ctx, log)
|
||||||
|
headerVHSEnabled := r.Header.Get(settings.VHSHeader())
|
||||||
|
|
||||||
|
if isVHSAddress(headerVHSEnabled, settings.GlobalVHS(), settings.VHSNamespacesEnabled(), reqInfo.Namespace) {
|
||||||
|
prepareVHSAddress(reqInfo, r, settings)
|
||||||
|
} else {
|
||||||
|
preparePathStyleAddress(reqInfo, r, reqLogger)
|
||||||
|
}
|
||||||
|
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isVHSAddress(headerVHSEnabled string, enabledFlag bool, vhsNamespaces map[string]bool, namespace string) bool {
|
||||||
|
if result, err := strconv.ParseBool(headerVHSEnabled); err == nil {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
result := enabledFlag
|
||||||
|
if v, ok := vhsNamespaces[namespace]; ok {
|
||||||
|
result = v
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareVHSAddress(reqInfo *ReqInfo, r *http.Request, settings VHSSettings) {
|
||||||
|
reqInfo.RequestVHSEnabled = true
|
||||||
|
bktName, match := checkDomain(r.Host, getDomains(r, settings))
|
||||||
|
if match {
|
||||||
|
if bktName == "" {
|
||||||
|
reqInfo.RequestType = noneType
|
||||||
|
} else {
|
||||||
|
if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" {
|
||||||
|
reqInfo.RequestType = objectType
|
||||||
|
reqInfo.ObjectName = objName
|
||||||
|
reqInfo.BucketName = bktName
|
||||||
|
} else {
|
||||||
|
reqInfo.RequestType = bucketType
|
||||||
|
reqInfo.BucketName = bktName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
parts := strings.Split(r.Host, ".")
|
||||||
|
reqInfo.BucketName = parts[0]
|
||||||
|
|
||||||
|
if objName := strings.TrimPrefix(r.URL.Path, "/"); objName != "" {
|
||||||
|
reqInfo.RequestType = objectType
|
||||||
|
reqInfo.ObjectName = objName
|
||||||
|
} else {
|
||||||
|
reqInfo.RequestType = bucketType
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getDomains(r *http.Request, settings VHSSettings) []string {
|
||||||
|
if headerServername := r.Header.Get(settings.ServernameHeader()); headerServername != "" {
|
||||||
|
return []string{headerServername}
|
||||||
|
}
|
||||||
|
|
||||||
|
return settings.Domains()
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.Logger) {
|
||||||
|
bktObj := strings.TrimPrefix(r.URL.Path, "/")
|
||||||
|
if bktObj == "" {
|
||||||
|
reqInfo.RequestType = noneType
|
||||||
|
} else if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" {
|
||||||
|
reqInfo.RequestType = objectType
|
||||||
|
reqInfo.BucketName = bktObj[:ind]
|
||||||
|
reqInfo.ObjectName = bktObj[ind+1:]
|
||||||
|
|
||||||
|
if r.URL.RawPath != "" {
|
||||||
|
// we have to do this because of
|
||||||
|
// https://github.com/go-chi/chi/issues/641
|
||||||
|
// https://github.com/go-chi/chi/issues/642
|
||||||
|
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
|
||||||
|
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
|
||||||
|
} else {
|
||||||
|
reqInfo.ObjectName = obj
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
reqInfo.RequestType = bucketType
|
||||||
|
reqInfo.BucketName = strings.TrimSuffix(bktObj, "/")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkDomain(host string, domains []string) (bktName string, match bool) {
|
||||||
|
partsHost := strings.Split(host, ".")
|
||||||
|
for _, pattern := range domains {
|
||||||
|
partsPattern := strings.Split(pattern, ".")
|
||||||
|
bktName, match = compareMatch(partsHost, partsPattern)
|
||||||
|
if match {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareMatch(host, pattern []string) (bktName string, match bool) {
|
||||||
|
if len(host) < len(pattern) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
i, j := len(host)-1, len(pattern)-1
|
||||||
|
for j >= 0 && (pattern[j] == wildcardPlaceholder || host[i] == pattern[j]) {
|
||||||
|
i--
|
||||||
|
j--
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case i == -1:
|
||||||
|
return "", true
|
||||||
|
case i == 0 && (j != 0 || host[i] == pattern[j]):
|
||||||
|
return host[0], true
|
||||||
|
default:
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
443
api/middleware/address_style_test.go
Normal file
443
api/middleware/address_style_test.go
Normal file
|
@ -0,0 +1,443 @@
|
||||||
|
package middleware
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.uber.org/zap/zaptest"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
FrostfsVHSHeader = "X-Frostfs-S3-VHS"
|
||||||
|
FrostfsServernameHeader = "X-Frostfs-Servername"
|
||||||
|
)
|
||||||
|
|
||||||
|
type VHSSettingsMock struct {
|
||||||
|
domains []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *VHSSettingsMock) Domains() []string {
|
||||||
|
return v.domains
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *VHSSettingsMock) GlobalVHS() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *VHSSettingsMock) VHSHeader() string {
|
||||||
|
return FrostfsVHSHeader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *VHSSettingsMock) ServernameHeader() string {
|
||||||
|
return FrostfsServernameHeader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *VHSSettingsMock) VHSNamespacesEnabled() map[string]bool {
|
||||||
|
return make(map[string]bool)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsVHSAddress(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
headerVHSEnabled string
|
||||||
|
vhsEnabledFlag bool
|
||||||
|
vhsNamespaced map[string]bool
|
||||||
|
namespace string
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "vhs disabled",
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "vhs disabled for namespace",
|
||||||
|
vhsEnabledFlag: true,
|
||||||
|
vhsNamespaced: map[string]bool{
|
||||||
|
"kapusta": false,
|
||||||
|
},
|
||||||
|
namespace: "kapusta",
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "vhs enabled (global vhs flag)",
|
||||||
|
vhsEnabledFlag: true,
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "vhs enabled for namespace",
|
||||||
|
vhsNamespaced: map[string]bool{
|
||||||
|
"kapusta": true,
|
||||||
|
},
|
||||||
|
namespace: "kapusta",
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "vhs enabled (header)",
|
||||||
|
headerVHSEnabled: "true",
|
||||||
|
vhsEnabledFlag: false,
|
||||||
|
vhsNamespaced: map[string]bool{
|
||||||
|
"kapusta": false,
|
||||||
|
},
|
||||||
|
namespace: "kapusta",
|
||||||
|
expected: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "vhs disabled (header)",
|
||||||
|
headerVHSEnabled: "false",
|
||||||
|
vhsEnabledFlag: true,
|
||||||
|
vhsNamespaced: map[string]bool{
|
||||||
|
"kapusta": true,
|
||||||
|
},
|
||||||
|
namespace: "kapusta",
|
||||||
|
expected: false,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
actual := isVHSAddress(tc.headerVHSEnabled, tc.vhsEnabledFlag, tc.vhsNamespaced, tc.namespace)
|
||||||
|
require.Equal(t, tc.expected, actual)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparePathStyleAddress(t *testing.T) {
|
||||||
|
bkt, obj := "test-bucket", "test-object"
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
urlParams string
|
||||||
|
expectedReqType ReqType
|
||||||
|
expectedBktName string
|
||||||
|
expectedObjName string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "bucket request",
|
||||||
|
urlParams: "/" + bkt,
|
||||||
|
expectedReqType: bucketType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bucket request with slash",
|
||||||
|
urlParams: "/" + bkt + "/",
|
||||||
|
expectedReqType: bucketType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "object request",
|
||||||
|
urlParams: "/" + bkt + "/" + obj,
|
||||||
|
expectedReqType: objectType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
expectedObjName: obj,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "object request with slash",
|
||||||
|
urlParams: "/" + bkt + "/" + obj + "/",
|
||||||
|
expectedReqType: objectType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
expectedObjName: obj + "/",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "none type request",
|
||||||
|
urlParams: "/",
|
||||||
|
expectedReqType: noneType,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
reqInfo := &ReqInfo{}
|
||||||
|
r := httptest.NewRequest(http.MethodGet, tc.urlParams, nil)
|
||||||
|
|
||||||
|
preparePathStyleAddress(reqInfo, r, reqLogOrDefault(r.Context(), zaptest.NewLogger(t)))
|
||||||
|
require.Equal(t, tc.expectedReqType, reqInfo.RequestType)
|
||||||
|
require.Equal(t, tc.expectedBktName, reqInfo.BucketName)
|
||||||
|
require.Equal(t, tc.expectedObjName, reqInfo.ObjectName)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareVHSAddress(t *testing.T) {
|
||||||
|
bkt, obj, domain := "test-bucket", "test-object", "domain.com"
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
domains []string
|
||||||
|
host string
|
||||||
|
urlParams string
|
||||||
|
expectedReqType ReqType
|
||||||
|
expectedBktName string
|
||||||
|
expectedObjName string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "bucket request, the domain matched",
|
||||||
|
domains: []string{domain},
|
||||||
|
host: bkt + "." + domain,
|
||||||
|
urlParams: "/",
|
||||||
|
expectedReqType: bucketType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "object request, the domain matched",
|
||||||
|
domains: []string{domain},
|
||||||
|
host: bkt + "." + domain,
|
||||||
|
urlParams: "/" + obj,
|
||||||
|
expectedReqType: objectType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
expectedObjName: obj,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "object request with slash, the domain matched",
|
||||||
|
domains: []string{domain},
|
||||||
|
host: bkt + "." + domain,
|
||||||
|
urlParams: "/" + obj + "/",
|
||||||
|
expectedReqType: objectType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
expectedObjName: obj + "/",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "list-buckets request, the domain matched",
|
||||||
|
domains: []string{domain},
|
||||||
|
host: domain,
|
||||||
|
urlParams: "/",
|
||||||
|
expectedReqType: noneType,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "bucket request, the domain don't match",
|
||||||
|
host: bkt + "." + domain,
|
||||||
|
urlParams: "/",
|
||||||
|
expectedReqType: bucketType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "object request, the domain don't match",
|
||||||
|
host: bkt + "." + domain,
|
||||||
|
urlParams: "/" + obj,
|
||||||
|
expectedReqType: objectType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
expectedObjName: obj,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "object request with slash, the domain don't match",
|
||||||
|
host: bkt + "." + domain,
|
||||||
|
urlParams: "/" + obj + "/",
|
||||||
|
expectedReqType: objectType,
|
||||||
|
expectedBktName: bkt,
|
||||||
|
expectedObjName: obj + "/",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "list-buckets request, the domain don't match (list-buckets isn't supported if the domains don't match)",
|
||||||
|
host: domain,
|
||||||
|
urlParams: "/",
|
||||||
|
expectedReqType: bucketType,
|
||||||
|
expectedBktName: strings.Split(domain, ".")[0],
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
reqInfo := &ReqInfo{}
|
||||||
|
vhsSettings := &VHSSettingsMock{domains: tc.domains}
|
||||||
|
r := httptest.NewRequest(http.MethodGet, tc.urlParams, nil)
|
||||||
|
r.Host = tc.host
|
||||||
|
|
||||||
|
prepareVHSAddress(reqInfo, r, vhsSettings)
|
||||||
|
require.Equal(t, tc.expectedReqType, reqInfo.RequestType)
|
||||||
|
require.Equal(t, tc.expectedBktName, reqInfo.BucketName)
|
||||||
|
require.Equal(t, tc.expectedObjName, reqInfo.ObjectName)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckDomains(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
domains []string
|
||||||
|
requestURL string
|
||||||
|
expectedBktName string
|
||||||
|
expectedMatch bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid url with bktName and namespace (wildcard after protocol infix)",
|
||||||
|
domains: []string{"s3.<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.s3.kapusta.domain.com",
|
||||||
|
expectedBktName: "bktA",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url without bktName and namespace (wildcard after protocol infix)",
|
||||||
|
domains: []string{"s3.<wildcard>.domain.com"},
|
||||||
|
requestURL: "s3.kapusta.domain.com",
|
||||||
|
expectedBktName: "",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url with invalid bktName (wildcard after protocol infix)",
|
||||||
|
domains: []string{"s3.<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.bktB.s3.kapusta.domain.com",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url without namespace (wildcard after protocol infix)",
|
||||||
|
domains: []string{"s3.<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.s3.domain.com",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url with invalid infix (wildcard after protocol infix)",
|
||||||
|
domains: []string{"s3.<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.s4.kapusta.domain.com",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url with invalid postfix (wildcard after protocol infix)",
|
||||||
|
domains: []string{"s3.<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.s3.kapusta.dom.su",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url with bktName and namespace (wildcard at the beginning of the domain)",
|
||||||
|
domains: []string{"<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.kapusta.domain.com",
|
||||||
|
expectedBktName: "bktA",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url without bktName and namespace (wildcard at the beginning of the domain)",
|
||||||
|
domains: []string{"<wildcard>.domain.com"},
|
||||||
|
requestURL: "kapusta.domain.com",
|
||||||
|
expectedBktName: "",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url with invalid bktName (wildcard at the beginning of the domain)",
|
||||||
|
domains: []string{"<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.bktB.kapusta.domain.com",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "collision test - true, because we cannot clearly distinguish a namespace from a bucket (wildcard at the beginning of the domain)",
|
||||||
|
domains: []string{"<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.domain.com",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url (fewer hosts)",
|
||||||
|
domains: []string{"<wildcard>.domain.com"},
|
||||||
|
requestURL: "domain.com",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url with invalid postfix (wildcard at the beginning of the domain)",
|
||||||
|
domains: []string{"<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.kapusta.dom.su",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url with bktName and without wildcard (root namaspace)",
|
||||||
|
domains: []string{"domain.com"},
|
||||||
|
requestURL: "bktA.domain.com",
|
||||||
|
expectedBktName: "bktA",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url without bktName and without wildcard (root namaspace)",
|
||||||
|
domains: []string{"domain.com"},
|
||||||
|
requestURL: "domain.com",
|
||||||
|
expectedBktName: "",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url with bktName without wildcard (root namaspace)",
|
||||||
|
domains: []string{"domain.com"},
|
||||||
|
requestURL: "bktA.dom.su",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url without wildcard (root namaspace)",
|
||||||
|
domains: []string{"domain.com"},
|
||||||
|
requestURL: "dom.su",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url, with a sorted list of domains",
|
||||||
|
domains: []string{"s3.<wildcard>.domain.com", "<wildcard>.domain.com", "domain.com"},
|
||||||
|
requestURL: "s3.kapusta.domain.com",
|
||||||
|
expectedBktName: "",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url with bktName, multiple wildcards (wildcards at the beginning of the domain)",
|
||||||
|
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.s3.kapusta.domain.com",
|
||||||
|
expectedBktName: "bktA",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url without bktName, multiple wildcards (wildcards at the beginning of the domain)",
|
||||||
|
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
||||||
|
requestURL: "s3.kapusta.domain.com",
|
||||||
|
expectedBktName: "",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url with bktName, multiply wildcards",
|
||||||
|
domains: []string{"s3.<wildcard>.subdomain.<wildcard>.com"},
|
||||||
|
requestURL: "bktA.s3.kapusta.subdomain.domain.com",
|
||||||
|
expectedBktName: "bktA",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid url without bktName, multiply wildcards",
|
||||||
|
domains: []string{"s3.<wildcard>.subdomain.<wildcard>.com"},
|
||||||
|
requestURL: "s3.kapusta.subdomain.domain.com",
|
||||||
|
expectedBktName: "",
|
||||||
|
expectedMatch: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url without one wildcard",
|
||||||
|
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
||||||
|
requestURL: "kapusta.domain.com",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url, multiply wildcards",
|
||||||
|
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
||||||
|
requestURL: "s3.kapusta.dom.com",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid url with invalid bktName, multiply wildcards",
|
||||||
|
domains: []string{"<wildcard>.<wildcard>.domain.com"},
|
||||||
|
requestURL: "bktA.bktB.s3.kapusta.domain.com",
|
||||||
|
expectedMatch: false,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
bktName, match := checkDomain(tc.requestURL, tc.domains)
|
||||||
|
require.Equal(t, tc.expectedBktName, bktName)
|
||||||
|
require.Equal(t, tc.expectedMatch, match)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDomains(t *testing.T) {
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||||
|
settings := &VHSSettingsMock{
|
||||||
|
domains: []string{
|
||||||
|
"s3.domain.com",
|
||||||
|
"s3.<wildcard>.domain.com",
|
||||||
|
"domain.com",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("the request does not contain the X-Frostfs-Servername header", func(t *testing.T) {
|
||||||
|
actualDomains := getDomains(req, settings)
|
||||||
|
require.Equal(t, settings.domains, actualDomains)
|
||||||
|
})
|
||||||
|
|
||||||
|
serverName := "domain.com"
|
||||||
|
req.Header.Set(settings.ServernameHeader(), serverName)
|
||||||
|
|
||||||
|
t.Run("the request contains the X-Frostfs-Servername header", func(t *testing.T) {
|
||||||
|
actualDomains := getDomains(req, settings)
|
||||||
|
require.Equal(t, []string{serverName}, actualDomains)
|
||||||
|
})
|
||||||
|
}
|
|
@ -73,7 +73,6 @@ type PolicyConfig struct {
|
||||||
Storage engine.ChainRouter
|
Storage engine.ChainRouter
|
||||||
FrostfsID FrostFSIDInformer
|
FrostfsID FrostFSIDInformer
|
||||||
Settings PolicySettings
|
Settings PolicySettings
|
||||||
Domains []string
|
|
||||||
Log *zap.Logger
|
Log *zap.Logger
|
||||||
BucketResolver BucketResolveFunc
|
BucketResolver BucketResolveFunc
|
||||||
Decoder XMLDecoder
|
Decoder XMLDecoder
|
||||||
|
@ -99,21 +98,21 @@ func PolicyCheck(cfg PolicyConfig) Func {
|
||||||
}
|
}
|
||||||
|
|
||||||
func policyCheck(r *http.Request, cfg PolicyConfig) error {
|
func policyCheck(r *http.Request, cfg PolicyConfig) error {
|
||||||
reqType, bktName, objName := getBucketObject(r, cfg.Domains)
|
reqInfo := GetReqInfo(r.Context())
|
||||||
req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqType, bktName, objName)
|
|
||||||
|
req, userKey, userGroups, err := getPolicyRequest(r, cfg, reqInfo.RequestType, reqInfo.BucketName, reqInfo.ObjectName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var bktInfo *data.BucketInfo
|
var bktInfo *data.BucketInfo
|
||||||
if reqType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
|
if reqInfo.RequestType != noneType && !strings.HasSuffix(req.Operation(), CreateBucketOperation) {
|
||||||
bktInfo, err = cfg.BucketResolver(r.Context(), bktName)
|
bktInfo, err = cfg.BucketResolver(r.Context(), reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
reqInfo := GetReqInfo(r.Context())
|
|
||||||
target := engine.NewRequestTargetWithNamespace(reqInfo.Namespace)
|
target := engine.NewRequestTargetWithNamespace(reqInfo.Namespace)
|
||||||
if bktInfo != nil {
|
if bktInfo != nil {
|
||||||
cnrTarget := engine.ContainerTarget(bktInfo.CID.EncodeToString())
|
cnrTarget := engine.ContainerTarget(bktInfo.CID.EncodeToString())
|
||||||
|
@ -208,33 +207,6 @@ const (
|
||||||
objectType
|
objectType
|
||||||
)
|
)
|
||||||
|
|
||||||
func getBucketObject(r *http.Request, domains []string) (reqType ReqType, bktName string, objName string) {
|
|
||||||
for _, domain := range domains {
|
|
||||||
ind := strings.Index(r.Host, "."+domain)
|
|
||||||
if ind == -1 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
bkt := r.Host[:ind]
|
|
||||||
if obj := strings.TrimPrefix(r.URL.Path, "/"); obj != "" {
|
|
||||||
return objectType, bkt, obj
|
|
||||||
}
|
|
||||||
|
|
||||||
return bucketType, bkt, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
bktObj := strings.TrimPrefix(r.URL.Path, "/")
|
|
||||||
if bktObj == "" {
|
|
||||||
return noneType, "", ""
|
|
||||||
}
|
|
||||||
|
|
||||||
if ind := strings.IndexByte(bktObj, '/'); ind != -1 && bktObj[ind+1:] != "" {
|
|
||||||
return objectType, bktObj[:ind], bktObj[ind+1:]
|
|
||||||
}
|
|
||||||
|
|
||||||
return bucketType, strings.TrimSuffix(bktObj, "/"), ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func determineOperation(r *http.Request, reqType ReqType) (operation string) {
|
func determineOperation(r *http.Request, reqType ReqType) (operation string) {
|
||||||
switch reqType {
|
switch reqType {
|
||||||
case objectType:
|
case objectType:
|
||||||
|
|
|
@ -8,79 +8,6 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestReqTypeDetermination(t *testing.T) {
|
|
||||||
bkt, obj, domain := "test-bucket", "test-object", "domain"
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
target string
|
|
||||||
host string
|
|
||||||
domains []string
|
|
||||||
expectedType ReqType
|
|
||||||
expectedBktName string
|
|
||||||
expectedObjName string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "bucket request, path-style",
|
|
||||||
target: "/" + bkt,
|
|
||||||
expectedType: bucketType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "bucket request with slash, path-style",
|
|
||||||
target: "/" + bkt + "/",
|
|
||||||
expectedType: bucketType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request, path-style",
|
|
||||||
target: "/" + bkt + "/" + obj,
|
|
||||||
expectedType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request with slash, path-style",
|
|
||||||
target: "/" + bkt + "/" + obj + "/",
|
|
||||||
expectedType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj + "/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "none type request",
|
|
||||||
target: "/",
|
|
||||||
expectedType: noneType,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "bucket request, virtual-hosted style",
|
|
||||||
target: "/",
|
|
||||||
host: bkt + "." + domain,
|
|
||||||
domains: []string{"some-domain", domain},
|
|
||||||
expectedType: bucketType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "object request, virtual-hosted style",
|
|
||||||
target: "/" + obj,
|
|
||||||
host: bkt + "." + domain,
|
|
||||||
domains: []string{"some-domain", domain},
|
|
||||||
expectedType: objectType,
|
|
||||||
expectedBktName: bkt,
|
|
||||||
expectedObjName: obj,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
r := httptest.NewRequest(http.MethodPut, tc.target, nil)
|
|
||||||
r.Host = tc.host
|
|
||||||
|
|
||||||
reqType, bktName, objName := getBucketObject(r, tc.domains)
|
|
||||||
require.Equal(t, tc.expectedType, reqType)
|
|
||||||
require.Equal(t, tc.expectedBktName, bktName)
|
|
||||||
require.Equal(t, tc.expectedObjName, objName)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDetermineBucketOperation(t *testing.T) {
|
func TestDetermineBucketOperation(t *testing.T) {
|
||||||
const defaultValue = "value"
|
const defaultValue = "value"
|
||||||
|
|
||||||
|
|
|
@ -12,7 +12,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
"google.golang.org/grpc/metadata"
|
"google.golang.org/grpc/metadata"
|
||||||
|
@ -28,19 +27,21 @@ type (
|
||||||
// ReqInfo stores the request info.
|
// ReqInfo stores the request info.
|
||||||
ReqInfo struct {
|
ReqInfo struct {
|
||||||
sync.RWMutex
|
sync.RWMutex
|
||||||
RemoteHost string // Client Host/IP
|
RemoteHost string // Client Host/IP
|
||||||
Host string // Node Host/IP
|
Host string // Node Host/IP
|
||||||
UserAgent string // User Agent
|
UserAgent string // User Agent
|
||||||
DeploymentID string // random generated s3-deployment-id
|
DeploymentID string // random generated s3-deployment-id
|
||||||
RequestID string // x-amz-request-id
|
RequestID string // x-amz-request-id
|
||||||
API string // API name -- GetObject PutObject NewMultipartUpload etc.
|
API string // API name -- GetObject PutObject NewMultipartUpload etc.
|
||||||
BucketName string // Bucket name
|
BucketName string // Bucket name
|
||||||
ObjectName string // Object name
|
ObjectName string // Object name
|
||||||
TraceID string // Trace ID
|
TraceID string // Trace ID
|
||||||
URL *url.URL // Request url
|
URL *url.URL // Request url
|
||||||
Namespace string
|
Namespace string
|
||||||
User string // User owner id
|
User string // User owner id
|
||||||
Tagging *data.Tagging
|
Tagging *data.Tagging
|
||||||
|
RequestVHSEnabled bool
|
||||||
|
RequestType ReqType
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObjectRequest represents object request data.
|
// ObjectRequest represents object request data.
|
||||||
|
@ -61,10 +62,6 @@ const (
|
||||||
|
|
||||||
const HdrAmzRequestID = "x-amz-request-id"
|
const HdrAmzRequestID = "x-amz-request-id"
|
||||||
|
|
||||||
const (
|
|
||||||
BucketURLPrm = "bucket"
|
|
||||||
)
|
|
||||||
|
|
||||||
var deploymentID = uuid.Must(uuid.NewRandom())
|
var deploymentID = uuid.Must(uuid.NewRandom())
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -202,57 +199,6 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddBucketName adds bucket name to ReqInfo from context.
|
|
||||||
func AddBucketName(l *zap.Logger) Func {
|
|
||||||
return func(h http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
ctx := r.Context()
|
|
||||||
|
|
||||||
reqInfo := GetReqInfo(ctx)
|
|
||||||
reqInfo.BucketName = chi.URLParam(r, BucketURLPrm)
|
|
||||||
|
|
||||||
if reqInfo.BucketName != "" {
|
|
||||||
reqLogger := reqLogOrDefault(ctx, l)
|
|
||||||
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("bucket", reqInfo.BucketName))))
|
|
||||||
}
|
|
||||||
|
|
||||||
h.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddObjectName adds objects name to ReqInfo from context.
|
|
||||||
func AddObjectName(l *zap.Logger) Func {
|
|
||||||
return func(h http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
ctx := r.Context()
|
|
||||||
reqInfo := GetReqInfo(ctx)
|
|
||||||
reqLogger := reqLogOrDefault(ctx, l)
|
|
||||||
|
|
||||||
rctx := chi.RouteContext(ctx)
|
|
||||||
// trim leading slash (always present)
|
|
||||||
reqInfo.ObjectName = rctx.RoutePath[1:]
|
|
||||||
|
|
||||||
if r.URL.RawPath != "" {
|
|
||||||
// we have to do this because of
|
|
||||||
// https://github.com/go-chi/chi/issues/641
|
|
||||||
// https://github.com/go-chi/chi/issues/642
|
|
||||||
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
|
|
||||||
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
|
|
||||||
} else {
|
|
||||||
reqInfo.ObjectName = obj
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if reqInfo.ObjectName != "" {
|
|
||||||
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName))))
|
|
||||||
}
|
|
||||||
|
|
||||||
h.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// getSourceIP retrieves the IP from the X-Forwarded-For, X-Real-IP and RFC7239
|
// getSourceIP retrieves the IP from the X-Forwarded-For, X-Real-IP and RFC7239
|
||||||
// Forwarded headers (in that order), falls back to r.RemoteAddr when everything
|
// Forwarded headers (in that order), falls back to r.RemoteAddr when everything
|
||||||
// else fails.
|
// else fails.
|
||||||
|
|
|
@ -97,6 +97,7 @@ type Settings interface {
|
||||||
s3middleware.RequestSettings
|
s3middleware.RequestSettings
|
||||||
s3middleware.PolicySettings
|
s3middleware.PolicySettings
|
||||||
s3middleware.MetricsSettings
|
s3middleware.MetricsSettings
|
||||||
|
s3middleware.VHSSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
type FrostFSID interface {
|
type FrostFSID interface {
|
||||||
|
@ -113,9 +114,6 @@ type Config struct {
|
||||||
|
|
||||||
MiddlewareSettings Settings
|
MiddlewareSettings Settings
|
||||||
|
|
||||||
// Domains optional. If empty no virtual hosted domains will be attached.
|
|
||||||
Domains []string
|
|
||||||
|
|
||||||
FrostfsID FrostFSID
|
FrostfsID FrostFSID
|
||||||
|
|
||||||
FrostFSIDValidation bool
|
FrostFSIDValidation bool
|
||||||
|
@ -142,11 +140,11 @@ func NewRouter(cfg Config) *chi.Mux {
|
||||||
api.Use(s3middleware.FrostfsIDValidation(cfg.FrostfsID, cfg.Log))
|
api.Use(s3middleware.FrostfsIDValidation(cfg.FrostfsID, cfg.Log))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
api.Use(s3middleware.PrepareAddressStyle(cfg.MiddlewareSettings, cfg.Log))
|
||||||
api.Use(s3middleware.PolicyCheck(s3middleware.PolicyConfig{
|
api.Use(s3middleware.PolicyCheck(s3middleware.PolicyConfig{
|
||||||
Storage: cfg.PolicyChecker,
|
Storage: cfg.PolicyChecker,
|
||||||
FrostfsID: cfg.FrostfsID,
|
FrostfsID: cfg.FrostfsID,
|
||||||
Settings: cfg.MiddlewareSettings,
|
Settings: cfg.MiddlewareSettings,
|
||||||
Domains: cfg.Domains,
|
|
||||||
Log: cfg.Log,
|
Log: cfg.Log,
|
||||||
BucketResolver: cfg.Handler.ResolveBucket,
|
BucketResolver: cfg.Handler.ResolveBucket,
|
||||||
Decoder: cfg.XMLDecoder,
|
Decoder: cfg.XMLDecoder,
|
||||||
|
@ -154,22 +152,41 @@ func NewRouter(cfg Config) *chi.Mux {
|
||||||
}))
|
}))
|
||||||
|
|
||||||
defaultRouter := chi.NewRouter()
|
defaultRouter := chi.NewRouter()
|
||||||
defaultRouter.Mount(fmt.Sprintf("/{%s}", s3middleware.BucketURLPrm), bucketRouter(cfg.Handler, cfg.Log))
|
defaultRouter.Mount("/{bucket}", bucketRouter(cfg.Handler))
|
||||||
defaultRouter.Get("/", named("ListBuckets", cfg.Handler.ListBucketsHandler))
|
defaultRouter.Get("/", named(s3middleware.ListBucketsOperation, cfg.Handler.ListBucketsHandler))
|
||||||
attachErrorHandler(defaultRouter)
|
attachErrorHandler(defaultRouter)
|
||||||
|
|
||||||
hr := NewHostBucketRouter("bucket")
|
vhsRouter := bucketRouter(cfg.Handler)
|
||||||
hr.Default(defaultRouter)
|
router := newGlobalRouter(defaultRouter, vhsRouter)
|
||||||
for _, domain := range cfg.Domains {
|
|
||||||
hr.Map(domain, bucketRouter(cfg.Handler, cfg.Log))
|
api.Mount("/", router)
|
||||||
}
|
|
||||||
api.Mount("/", hr)
|
|
||||||
|
|
||||||
attachErrorHandler(api)
|
attachErrorHandler(api)
|
||||||
|
|
||||||
return api
|
return api
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type globalRouter struct {
|
||||||
|
pathStyleRouter chi.Router
|
||||||
|
vhsRouter chi.Router
|
||||||
|
}
|
||||||
|
|
||||||
|
func newGlobalRouter(pathStyleRouter, vhsRouter chi.Router) *globalRouter {
|
||||||
|
return &globalRouter{
|
||||||
|
pathStyleRouter: pathStyleRouter,
|
||||||
|
vhsRouter: vhsRouter,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *globalRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
router := g.pathStyleRouter
|
||||||
|
if reqInfo := s3middleware.GetReqInfo(r.Context()); reqInfo.RequestVHSEnabled {
|
||||||
|
router = g.vhsRouter
|
||||||
|
}
|
||||||
|
|
||||||
|
router.ServeHTTP(w, r)
|
||||||
|
}
|
||||||
|
|
||||||
func named(name string, handlerFunc http.HandlerFunc) http.HandlerFunc {
|
func named(name string, handlerFunc http.HandlerFunc) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
reqInfo := s3middleware.GetReqInfo(r.Context())
|
reqInfo := s3middleware.GetReqInfo(r.Context())
|
||||||
|
@ -214,14 +231,13 @@ func attachErrorHandler(api *chi.Mux) {
|
||||||
api.MethodNotAllowed(named("MethodNotAllowed", errorHandler))
|
api.MethodNotAllowed(named("MethodNotAllowed", errorHandler))
|
||||||
}
|
}
|
||||||
|
|
||||||
func bucketRouter(h Handler, log *zap.Logger) chi.Router {
|
func bucketRouter(h Handler) chi.Router {
|
||||||
bktRouter := chi.NewRouter()
|
bktRouter := chi.NewRouter()
|
||||||
bktRouter.Use(
|
bktRouter.Use(
|
||||||
s3middleware.AddBucketName(log),
|
|
||||||
s3middleware.WrapHandler(h.AppendCORSHeaders),
|
s3middleware.WrapHandler(h.AppendCORSHeaders),
|
||||||
)
|
)
|
||||||
|
|
||||||
bktRouter.Mount("/", objectRouter(h, log))
|
bktRouter.Mount("/", objectRouter(h))
|
||||||
|
|
||||||
bktRouter.Options("/", named(s3middleware.OptionsBucketOperation, h.Preflight))
|
bktRouter.Options("/", named(s3middleware.OptionsBucketOperation, h.Preflight))
|
||||||
|
|
||||||
|
@ -293,7 +309,7 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router {
|
||||||
Add(NewFilter().
|
Add(NewFilter().
|
||||||
Queries(s3middleware.VersionsQuery).
|
Queries(s3middleware.VersionsQuery).
|
||||||
Handler(named(s3middleware.ListBucketObjectVersionsOperation, h.ListBucketObjectVersionsHandler))).
|
Handler(named(s3middleware.ListBucketObjectVersionsOperation, h.ListBucketObjectVersionsHandler))).
|
||||||
DefaultHandler(named(s3middleware.ListObjectsV1Operation, h.ListObjectsV1Handler)))
|
DefaultHandler(listWrapper(h)))
|
||||||
})
|
})
|
||||||
|
|
||||||
// PUT method handlers
|
// PUT method handlers
|
||||||
|
@ -368,9 +384,20 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router {
|
||||||
return bktRouter
|
return bktRouter
|
||||||
}
|
}
|
||||||
|
|
||||||
func objectRouter(h Handler, l *zap.Logger) chi.Router {
|
func listWrapper(h Handler) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if reqInfo := s3middleware.GetReqInfo(r.Context()); reqInfo.BucketName == "" {
|
||||||
|
reqInfo.API = s3middleware.ListBucketsOperation
|
||||||
|
h.ListBucketsHandler(w, r)
|
||||||
|
} else {
|
||||||
|
reqInfo.API = s3middleware.ListObjectsV1Operation
|
||||||
|
h.ListObjectsV1Handler(w, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectRouter(h Handler) chi.Router {
|
||||||
objRouter := chi.NewRouter()
|
objRouter := chi.NewRouter()
|
||||||
objRouter.Use(s3middleware.AddObjectName(l))
|
|
||||||
|
|
||||||
objRouter.Options("/*", named(s3middleware.OptionsObjectOperation, h.Preflight))
|
objRouter.Options("/*", named(s3middleware.OptionsObjectOperation, h.Preflight))
|
||||||
|
|
||||||
|
|
|
@ -23,7 +23,11 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
const FrostfsNamespaceHeader = "X-Frostfs-Namespace"
|
const (
|
||||||
|
FrostfsNamespaceHeader = "X-Frostfs-Namespace"
|
||||||
|
FrostfsVHSHeader = "X-Frostfs-S3-VHS"
|
||||||
|
FrostfsServernameHeader = "X-Frostfs-Servername"
|
||||||
|
)
|
||||||
|
|
||||||
type poolStatisticMock struct {
|
type poolStatisticMock struct {
|
||||||
}
|
}
|
||||||
|
@ -71,8 +75,11 @@ func (c *centerMock) Authenticate(*http.Request) (*middleware.Box, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
type middlewareSettingsMock struct {
|
type middlewareSettingsMock struct {
|
||||||
denyByDefault bool
|
denyByDefault bool
|
||||||
sourceIPHeader string
|
sourceIPHeader string
|
||||||
|
domains []string
|
||||||
|
vhsEnabled bool
|
||||||
|
vhsNamespacesEnabled map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *middlewareSettingsMock) SourceIPHeader() string {
|
func (r *middlewareSettingsMock) SourceIPHeader() string {
|
||||||
|
@ -91,6 +98,26 @@ func (r *middlewareSettingsMock) PolicyDenyByDefault() bool {
|
||||||
return r.denyByDefault
|
return r.denyByDefault
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *middlewareSettingsMock) Domains() []string {
|
||||||
|
return r.domains
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *middlewareSettingsMock) GlobalVHS() bool {
|
||||||
|
return r.vhsEnabled
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *middlewareSettingsMock) VHSHeader() string {
|
||||||
|
return FrostfsVHSHeader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *middlewareSettingsMock) ServernameHeader() string {
|
||||||
|
return FrostfsServernameHeader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *middlewareSettingsMock) VHSNamespacesEnabled() map[string]bool {
|
||||||
|
return r.vhsNamespacesEnabled
|
||||||
|
}
|
||||||
|
|
||||||
type frostFSIDMock struct {
|
type frostFSIDMock struct {
|
||||||
tags map[string]string
|
tags map[string]string
|
||||||
validateError bool
|
validateError bool
|
||||||
|
|
|
@ -78,7 +78,6 @@ func prepareRouter(t *testing.T, opts ...option) *routerMock {
|
||||||
Metrics: metrics.NewAppMetrics(metricsConfig),
|
Metrics: metrics.NewAppMetrics(metricsConfig),
|
||||||
MiddlewareSettings: middlewareSettings,
|
MiddlewareSettings: middlewareSettings,
|
||||||
PolicyChecker: policyChecker,
|
PolicyChecker: policyChecker,
|
||||||
Domains: []string{"domain1", "domain2"},
|
|
||||||
FrostfsID: &frostFSIDMock{},
|
FrostfsID: &frostFSIDMock{},
|
||||||
XMLDecoder: &xmlMock{},
|
XMLDecoder: &xmlMock{},
|
||||||
Tagging: &resourceTaggingMock{},
|
Tagging: &resourceTaggingMock{},
|
||||||
|
@ -847,6 +846,31 @@ func TestFrostFSIDValidation(t *testing.T) {
|
||||||
createBucketErr(chiRouter, "", "bkt-3", nil, apiErrors.ErrInternalError)
|
createBucketErr(chiRouter, "", "bkt-3", nil, apiErrors.ErrInternalError)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRouterListObjectsV2Domains(t *testing.T) {
|
||||||
|
chiRouter := prepareRouter(t, enableVHSDomains("domain.com"))
|
||||||
|
|
||||||
|
chiRouter.handler.buckets["bucket"] = &data.BucketInfo{}
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||||
|
r.Host = "bucket.domain.com"
|
||||||
|
query := make(url.Values)
|
||||||
|
query.Set(s3middleware.ListTypeQuery, "2")
|
||||||
|
r.URL.RawQuery = query.Encode()
|
||||||
|
|
||||||
|
chiRouter.ServeHTTP(w, r)
|
||||||
|
resp := readResponse(t, w)
|
||||||
|
require.Equal(t, s3middleware.ListObjectsV2Operation, resp.Method)
|
||||||
|
}
|
||||||
|
|
||||||
|
func enableVHSDomains(domains ...string) option {
|
||||||
|
return func(cfg *Config) {
|
||||||
|
setting := cfg.MiddlewareSettings.(*middlewareSettingsMock)
|
||||||
|
setting.vhsEnabled = true
|
||||||
|
setting.domains = domains
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func readResponse(t *testing.T, w *httptest.ResponseRecorder) handlerResult {
|
func readResponse(t *testing.T, w *httptest.ResponseRecorder) handlerResult {
|
||||||
var res handlerResult
|
var res handlerResult
|
||||||
|
|
||||||
|
|
|
@ -105,9 +105,13 @@ type (
|
||||||
policyDenyByDefault bool
|
policyDenyByDefault bool
|
||||||
sourceIPHeader string
|
sourceIPHeader string
|
||||||
retryMaxAttempts int
|
retryMaxAttempts int
|
||||||
|
domains []string
|
||||||
|
vhsEnabled bool
|
||||||
|
vhsHeader string
|
||||||
|
servernameHeader string
|
||||||
|
vhsNamespacesEnabled map[string]bool
|
||||||
retryMaxBackoff time.Duration
|
retryMaxBackoff time.Duration
|
||||||
retryStrategy handler.RetryStrategy
|
retryStrategy handler.RetryStrategy
|
||||||
domains []string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
maxClientsConfig struct {
|
maxClientsConfig struct {
|
||||||
|
@ -256,13 +260,55 @@ func (s *appSettings) updateNamespacesSettings(v *viper.Viper, log *zap.Logger)
|
||||||
s.namespaces = nsConfig.Namespaces
|
s.namespaces = nsConfig.Namespaces
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *appSettings) setVHSSettings(v *viper.Viper, _ *zap.Logger) {
|
func (s *appSettings) setVHSSettings(v *viper.Viper, log *zap.Logger) {
|
||||||
domains := v.GetStringSlice(cfgListenDomains)
|
domains := fetchDomains(v, log)
|
||||||
|
vhsEnabled := v.GetBool(cfgVHSEnabled)
|
||||||
|
vhsHeader := v.GetString(cfgVHSHeader)
|
||||||
|
servernameHeader := v.GetString(cfgServernameHeader)
|
||||||
|
nsMap := fetchVHSNamespaces(v, log)
|
||||||
|
vhsNamespaces := make(map[string]bool, len(nsMap))
|
||||||
|
for ns, flag := range nsMap {
|
||||||
|
vhsNamespaces[s.ResolveNamespaceAlias(ns)] = flag
|
||||||
|
}
|
||||||
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
s.domains = domains
|
s.domains = domains
|
||||||
|
s.vhsEnabled = vhsEnabled
|
||||||
|
s.vhsHeader = vhsHeader
|
||||||
|
s.servernameHeader = servernameHeader
|
||||||
|
s.vhsNamespacesEnabled = vhsNamespaces
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) Domains() []string {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
return s.domains
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) GlobalVHS() bool {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
return s.vhsEnabled
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) VHSHeader() string {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
return s.vhsHeader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) ServernameHeader() string {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
return s.servernameHeader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) VHSNamespacesEnabled() map[string]bool {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
return s.vhsNamespacesEnabled
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *appSettings) BypassContentEncodingInChunks() bool {
|
func (s *appSettings) BypassContentEncodingInChunks() bool {
|
||||||
|
@ -467,12 +513,6 @@ func (s *appSettings) RetryStrategy() handler.RetryStrategy {
|
||||||
return s.retryStrategy
|
return s.retryStrategy
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *appSettings) Domains() []string {
|
|
||||||
s.mu.RLock()
|
|
||||||
defer s.mu.RUnlock()
|
|
||||||
return s.domains
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *App) initAPI(ctx context.Context) {
|
func (a *App) initAPI(ctx context.Context) {
|
||||||
a.initLayer(ctx)
|
a.initLayer(ctx)
|
||||||
a.initHandler()
|
a.initHandler()
|
||||||
|
@ -710,9 +750,6 @@ func (a *App) setHealthStatus() {
|
||||||
|
|
||||||
// Serve runs HTTP server to handle S3 API requests.
|
// Serve runs HTTP server to handle S3 API requests.
|
||||||
func (a *App) Serve(ctx context.Context) {
|
func (a *App) Serve(ctx context.Context) {
|
||||||
// Attach S3 API:
|
|
||||||
a.log.Info(logs.FetchDomainsPrepareToUseAPI, zap.Strings("domains", a.settings.Domains()))
|
|
||||||
|
|
||||||
cfg := api.Config{
|
cfg := api.Config{
|
||||||
Throttle: middleware.ThrottleOpts{
|
Throttle: middleware.ThrottleOpts{
|
||||||
Limit: a.settings.maxClient.count,
|
Limit: a.settings.maxClient.count,
|
||||||
|
@ -722,7 +759,6 @@ func (a *App) Serve(ctx context.Context) {
|
||||||
Center: a.ctr,
|
Center: a.ctr,
|
||||||
Log: a.log,
|
Log: a.log,
|
||||||
Metrics: a.metrics,
|
Metrics: a.metrics,
|
||||||
Domains: a.settings.Domains(),
|
|
||||||
|
|
||||||
MiddlewareSettings: a.settings,
|
MiddlewareSettings: a.settings,
|
||||||
PolicyChecker: a.policyStorage,
|
PolicyChecker: a.policyStorage,
|
||||||
|
|
|
@ -30,6 +30,8 @@ import (
|
||||||
const (
|
const (
|
||||||
destinationStdout = "stdout"
|
destinationStdout = "stdout"
|
||||||
destinationJournald = "journald"
|
destinationJournald = "journald"
|
||||||
|
|
||||||
|
wildcardPlaceholder = "<wildcard>"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -52,7 +54,9 @@ const (
|
||||||
|
|
||||||
defaultAccessBoxCacheRemovingCheckInterval = 5 * time.Minute
|
defaultAccessBoxCacheRemovingCheckInterval = 5 * time.Minute
|
||||||
|
|
||||||
defaultNamespaceHeader = "X-Frostfs-Namespace"
|
defaultNamespaceHeader = "X-Frostfs-Namespace"
|
||||||
|
defaultVHSHeader = "X-Frostfs-S3-VHS"
|
||||||
|
defaultServernameHeader = "X-Frostfs-Servername"
|
||||||
|
|
||||||
defaultConstraintName = "default"
|
defaultConstraintName = "default"
|
||||||
|
|
||||||
|
@ -144,6 +148,12 @@ const ( // Settings.
|
||||||
|
|
||||||
cfgListenDomains = "listen_domains"
|
cfgListenDomains = "listen_domains"
|
||||||
|
|
||||||
|
// VHS.
|
||||||
|
cfgVHSEnabled = "vhs.enabled"
|
||||||
|
cfgVHSHeader = "vhs.vhs_header"
|
||||||
|
cfgServernameHeader = "vhs.servername_header"
|
||||||
|
cfgVHSNamespaces = "vhs.namespaces"
|
||||||
|
|
||||||
// Peers.
|
// Peers.
|
||||||
cfgPeers = "peers"
|
cfgPeers = "peers"
|
||||||
|
|
||||||
|
@ -668,6 +678,41 @@ func fetchServers(v *viper.Viper, log *zap.Logger) []ServerInfo {
|
||||||
return servers
|
return servers
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func fetchDomains(v *viper.Viper, log *zap.Logger) []string {
|
||||||
|
domains := validateDomains(v.GetStringSlice(cfgListenDomains), log)
|
||||||
|
|
||||||
|
countParts := func(domain string) int {
|
||||||
|
return strings.Count(domain, ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(domains, func(i, j int) bool {
|
||||||
|
return countParts(domains[i]) > countParts(domains[j])
|
||||||
|
})
|
||||||
|
|
||||||
|
return domains
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchVHSNamespaces(v *viper.Viper, log *zap.Logger) map[string]bool {
|
||||||
|
vhsNamespacesEnabled := make(map[string]bool)
|
||||||
|
nsMap := v.GetStringMap(cfgVHSNamespaces)
|
||||||
|
for ns, val := range nsMap {
|
||||||
|
if _, ok := vhsNamespacesEnabled[ns]; ok {
|
||||||
|
log.Warn(logs.WarnDuplicateNamespaceVHS, zap.String("namespace", ns))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
enabledFlag, ok := val.(bool)
|
||||||
|
if !ok {
|
||||||
|
log.Warn(logs.WarnValueVHSEnabledFlagWrongType, zap.String("namespace", ns))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
vhsNamespacesEnabled[ns] = enabledFlag
|
||||||
|
}
|
||||||
|
|
||||||
|
return vhsNamespacesEnabled
|
||||||
|
}
|
||||||
|
|
||||||
func newSettings() *viper.Viper {
|
func newSettings() *viper.Viper {
|
||||||
v := viper.New()
|
v := viper.New()
|
||||||
|
|
||||||
|
@ -754,6 +799,10 @@ func newSettings() *viper.Viper {
|
||||||
v.SetDefault(cfgRetryMaxAttempts, defaultRetryMaxAttempts)
|
v.SetDefault(cfgRetryMaxAttempts, defaultRetryMaxAttempts)
|
||||||
v.SetDefault(cfgRetryMaxBackoff, defaultRetryMaxBackoff)
|
v.SetDefault(cfgRetryMaxBackoff, defaultRetryMaxBackoff)
|
||||||
|
|
||||||
|
// vhs
|
||||||
|
v.SetDefault(cfgVHSHeader, defaultVHSHeader)
|
||||||
|
v.SetDefault(cfgServernameHeader, defaultServernameHeader)
|
||||||
|
|
||||||
// Bind flags
|
// Bind flags
|
||||||
if err := bindFlags(v, flags); err != nil {
|
if err := bindFlags(v, flags); err != nil {
|
||||||
panic(fmt.Errorf("bind flags: %w", err))
|
panic(fmt.Errorf("bind flags: %w", err))
|
||||||
|
@ -1029,3 +1078,19 @@ func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
|
||||||
}
|
}
|
||||||
return lvl, nil
|
return lvl, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateDomains(domains []string, log *zap.Logger) []string {
|
||||||
|
validDomains := make([]string, 0, len(domains))
|
||||||
|
LOOP:
|
||||||
|
for _, domain := range domains {
|
||||||
|
domainParts := strings.Split(domain, ".")
|
||||||
|
for _, part := range domainParts {
|
||||||
|
if strings.ContainsAny(part, "<>") && part != wildcardPlaceholder {
|
||||||
|
log.Warn(logs.WarnDomainContainsInvalidPlaceholder, zap.String("domain", domain))
|
||||||
|
continue LOOP
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validDomains = append(validDomains, domain)
|
||||||
|
}
|
||||||
|
return validDomains
|
||||||
|
}
|
||||||
|
|
34
cmd/s3-gw/validate_test.go
Normal file
34
cmd/s3-gw/validate_test.go
Normal file
|
@ -0,0 +1,34 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.uber.org/zap/zaptest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateDomains(t *testing.T) {
|
||||||
|
inputDomains := []string{
|
||||||
|
"s3dev.frostfs.devenv",
|
||||||
|
"s3dev.<invalid>.frostfs.devenv",
|
||||||
|
"s3dev.<wildcard>.frostfs.devenv",
|
||||||
|
"s3dev.<wildcard.frostfs.devenv",
|
||||||
|
"s3dev.wildcard>.frostfs.devenv",
|
||||||
|
"s3dev.<wild.card>.frostfs.devenv",
|
||||||
|
"<invalid>.frostfs.devenv",
|
||||||
|
"<wildcard>.frostfs.devenv>",
|
||||||
|
"<wildcard>.frostfs.devenv",
|
||||||
|
"s3dev.fro<stfs.devenv",
|
||||||
|
"<wildcard>.dev.<wildcard>.frostfs.devenv",
|
||||||
|
"<wildcard>.dev.<wildc>ard>.frostfs.devenv",
|
||||||
|
}
|
||||||
|
expectedDomains := []string{
|
||||||
|
"s3dev.frostfs.devenv",
|
||||||
|
"s3dev.<wildcard>.frostfs.devenv",
|
||||||
|
"<wildcard>.frostfs.devenv",
|
||||||
|
"<wildcard>.dev.<wildcard>.frostfs.devenv",
|
||||||
|
}
|
||||||
|
|
||||||
|
actualDomains := validateDomains(inputDomains, zaptest.NewLogger(t))
|
||||||
|
require.Equal(t, expectedDomains, actualDomains)
|
||||||
|
}
|
|
@ -36,8 +36,15 @@ S3_GW_SERVER_1_TLS_KEY_FILE=/path/to/tls/key
|
||||||
# How often to reconnect to the servers
|
# How often to reconnect to the servers
|
||||||
S3_GW_RECONNECT_INTERVAL: 1m
|
S3_GW_RECONNECT_INTERVAL: 1m
|
||||||
|
|
||||||
# Domains to be able to use virtual-hosted-style access to bucket.
|
# Domains to be able to use virtual-hosted-style access to bucket
|
||||||
S3_GW_LISTEN_DOMAINS=s3dev.frostfs.devenv
|
S3_GW_LISTEN_DOMAINS="domain.com <wildcard>.domain.com"
|
||||||
|
|
||||||
|
# VHS enabled flag
|
||||||
|
S3_GW_VHS_ENABLED=false
|
||||||
|
# Header for determining whether VHS is enabled for the request
|
||||||
|
S3_GW_VHS_VHS_HEADER=X-Frostfs-S3-VHS
|
||||||
|
# Header for determining servername
|
||||||
|
S3_GW_VHS_SERVERNAME_HEADER=X-Frostfs-Servername
|
||||||
|
|
||||||
# Config file
|
# Config file
|
||||||
S3_GW_CONFIG=/path/to/config/yaml
|
S3_GW_CONFIG=/path/to/config/yaml
|
||||||
|
|
|
@ -42,6 +42,15 @@ server:
|
||||||
# Domains to be able to use virtual-hosted-style access to bucket.
|
# Domains to be able to use virtual-hosted-style access to bucket.
|
||||||
listen_domains:
|
listen_domains:
|
||||||
- s3dev.frostfs.devenv
|
- s3dev.frostfs.devenv
|
||||||
|
- s3dev.<wildcard>.frostfs.devenv
|
||||||
|
|
||||||
|
vhs:
|
||||||
|
enabled: false
|
||||||
|
vhs_header: X-Frostfs-S3-VHS
|
||||||
|
servername_header: X-Frostfs-Servername
|
||||||
|
namespaces:
|
||||||
|
"ns1": false
|
||||||
|
"ns2": true
|
||||||
|
|
||||||
logger:
|
logger:
|
||||||
level: debug
|
level: debug
|
||||||
|
|
|
@ -193,12 +193,14 @@ There are some custom types used for brevity:
|
||||||
| `namespaces` | [Namespaces configuration](#namespaces-section) |
|
| `namespaces` | [Namespaces configuration](#namespaces-section) |
|
||||||
| `retry` | [Retry configuration](#retry-section) |
|
| `retry` | [Retry configuration](#retry-section) |
|
||||||
| `containers` | [Containers configuration](#containers-section) |
|
| `containers` | [Containers configuration](#containers-section) |
|
||||||
|
| `vhs` | [VHS configuration](#vhs-section) |
|
||||||
|
|
||||||
### General section
|
### General section
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
listen_domains:
|
listen_domains:
|
||||||
- s3dev.frostfs.devenv
|
- s3dev.frostfs.devenv
|
||||||
|
- s3dev.<wildcard>.frostfs.devenv
|
||||||
- s3dev2.frostfs.devenv
|
- s3dev2.frostfs.devenv
|
||||||
|
|||||||
|
|
||||||
rpc_endpoint: http://morph-chain.frostfs.devenv:30333
|
rpc_endpoint: http://morph-chain.frostfs.devenv:30333
|
||||||
|
@ -226,7 +228,7 @@ source_ip_header: "Source-Ip"
|
||||||
|
|
||||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|----------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|----------------------------------|------------|---------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `listen_domains` | `[]string` | no | | Domains to be able to use virtual-hosted-style access to bucket. |
|
| `listen_domains` | `[]string` | yes | | Domains to be able to use virtual-hosted-style access to bucket. The presence of placeholders of the <wildcard> type is supported. |
|
||||||
| `rpc_endpoint` | `string` | no | | The address of the RPC host to which the gateway connects to resolve bucket names and interact with frostfs contracts (required to use the `nns` resolver and `frostfsid` contract). |
|
| `rpc_endpoint` | `string` | no | | The address of the RPC host to which the gateway connects to resolve bucket names and interact with frostfs contracts (required to use the `nns` resolver and `frostfsid` contract). |
|
||||||
| `resolve_order` | `[]string` | yes | `[dns]` | Order of bucket name resolvers to use. Available resolvers: `dns`, `nns`. |
|
| `resolve_order` | `[]string` | yes | `[dns]` | Order of bucket name resolvers to use. Available resolvers: `dns`, `nns`. |
|
||||||
| `connect_timeout` | `duration` | no | `10s` | Timeout to connect to a node. |
|
| `connect_timeout` | `duration` | no | `10s` | Timeout to connect to a node. |
|
||||||
|
@ -723,3 +725,24 @@ containers:
|
||||||
|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------|
|
|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------|
|
||||||
| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. |
|
| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. |
|
||||||
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
|
| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. |
|
||||||
|
|
||||||
|
# `vhs` section
|
||||||
|
|
||||||
|
Configuration of virtual hosted addressing style.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
vhs:
|
||||||
|
enabled: false
|
||||||
|
vhs_header: X-Frostfs-S3-VHS
|
||||||
|
servername_header: X-Frostfs-Servername
|
||||||
|
namespaces:
|
||||||
|
"ns1": false
|
||||||
|
"ns2": true
|
||||||
|
```
|
||||||
|
|
||||||
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|
| ------------------- | ----------------- | ------------- | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
|
| `enabled` | `bool` | yes | `false` | Enables the use of virtual host addressing for buckets at the application level. |
|
||||||
|
| `vhs_header` | `string` | yes | `X-Frostfs-S3-VHS` | Header for determining whether VHS is enabled for the request. |
|
||||||
|
| `servername_header` | `string` | yes | `X-Frostfs-Servername` | Header for determining servername. |
|
||||||
|
| `namespaces` | `map[string]bool` | yes | | A map in which the keys are the name of the namespace, and the values are the flag responsible for enabling VHS for the specified namespace. Overrides global 'enabled' setting even when it is disabled. |
|
||||||
|
|
|
@ -20,7 +20,6 @@ const (
|
||||||
UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go
|
UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go
|
||||||
ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go
|
ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go
|
||||||
ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go
|
ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go
|
||||||
FetchDomainsPrepareToUseAPI = "fetch domains, prepare to use API" // Info in ../../cmd/s3-gw/app.go
|
|
||||||
StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go
|
StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go
|
||||||
StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go
|
StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go
|
||||||
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go
|
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
@ -159,4 +158,7 @@ const (
|
||||||
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
|
CouldNotFetchLifecycleContainerInfo = "couldn't fetch lifecycle container info"
|
||||||
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
|
BucketLifecycleNodeHasMultipleIDs = "bucket lifecycle node has multiple ids"
|
||||||
GetBucketLifecycle = "get bucket lifecycle"
|
GetBucketLifecycle = "get bucket lifecycle"
|
||||||
|
WarnDuplicateNamespaceVHS = "duplicate namespace with enabled VHS, config value skipped"
|
||||||
|
WarnValueVHSEnabledFlagWrongType = "the value of the VHS enable flag for the namespace is of the wrong type, config value skipped"
|
||||||
|
WarnDomainContainsInvalidPlaceholder = "the domain contains an invalid placeholder, domain skipped"
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in a new issue
@dkirillov Have you discussed putting it into
vhs
section?We haven't discussed. I'm ok with keeping params in
listen_domains