TrueCloudLab/frostfs-s3-gw
15
3
Fork 18
Code Issues 28 Pull requests 4 Projects Releases 17 Packages Activity Actions

[#449] Add support headers for vhs and servername
All checks were successful
/ DCO (pull_request) Successful in 52s
Details
/ Vulncheck (pull_request) Successful in 1m9s
Details
/ Builds (1.21) (pull_request) Successful in 1m29s
Details
/ Builds (1.22) (pull_request) Successful in 1m28s
Details
/ Lint (pull_request) Successful in 2m18s
Details
/ Tests (1.21) (pull_request) Successful in 1m28s
Details
/ Tests (1.22) (pull_request) Successful in 1m29s
Details

Browse source 
Signed-off-by: Roman Loginov <r.loginov@yadro.com>
This commit is contained in:
Roman Loginov 2024-08-01 16:24:47 +03:00
parent c0d624df68
commit fea911c417
9 changed files with 141 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -5,7 +5,7 @@ This document outlines major changes between releases.
## [Unreleased]
### Added
- Add support for virtual hosted style addressing (#446)
- Add support for virtual hosted style addressing (#446, #449)
## [0.30.0] - Kangshung -2024-07-19

23
api/middleware/address_style.go
View file

@ -3,6 +3,7 @@ package middleware
import (
"net/http"
"net/url"
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
@ -14,6 +15,8 @@ const wildcardPlaceholder = "<wildcard>"
type VHSSettings interface {
Domains() []string
GlobalVHS() bool
VHSHeader() string
ServernameHeader() string
VHSNamespacesEnabled() map[string]bool
}
@ -23,8 +26,9 @@ func PrepareAddressStyle(settings VHSSettings, log *zap.Logger) Func {
ctx := r.Context()
reqInfo := GetReqInfo(ctx)
reqLogger := reqLogOrDefault(ctx, log)
headerVHSEnabled := r.Header.Get(settings.VHSHeader())
if isVHSAddress(settings.GlobalVHS(), settings.VHSNamespacesEnabled(), reqInfo.Namespace) {
if isVHSAddress(headerVHSEnabled, settings.GlobalVHS(), settings.VHSNamespacesEnabled(), reqInfo.Namespace) {
prepareVHSAddress(reqInfo, r, settings)
} else {
preparePathStyleAddress(reqInfo, r, reqLogger)
@ -35,9 +39,12 @@ func PrepareAddressStyle(settings VHSSettings, log *zap.Logger) Func {
}
}
func isVHSAddress(enabledFlag bool, vhsNamespaces map[string]bool, namespace string) bool {
result := enabledFlag
func isVHSAddress(headerVHSEnabled string, enabledFlag bool, vhsNamespaces map[string]bool, namespace string) bool {
if result, err := strconv.ParseBool(headerVHSEnabled); err == nil {
return result
}
result := enabledFlag
if v, ok := vhsNamespaces[namespace]; ok {
result = v
}
@ -47,7 +54,7 @@ func isVHSAddress(enabledFlag bool, vhsNamespaces map[string]bool, namespace str
func prepareVHSAddress(reqInfo *ReqInfo, r *http.Request, settings VHSSettings) {
reqInfo.RequestVHSEnabled = true
bktName, match := checkDomain(r.Host, settings.Domains())
bktName, match := checkDomain(r.Host, getDomains(r, settings))
if match {
if bktName == "" {
reqInfo.RequestType = noneType
@ -74,6 +81,14 @@ func prepareVHSAddress(reqInfo *ReqInfo, r *http.Request, settings VHSSettings)
}
}
func getDomains(r *http.Request, settings VHSSettings) []string {
if headerServername := r.Header.Get(settings.ServernameHeader()); headerServername != "" {
return []string{headerServername}
}
return settings.Domains()
}
func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.Logger) {
bktObj := strings.TrimPrefix(r.URL.Path, "/")
if bktObj == "" {

70
api/middleware/address_style_test.go
View file

@ -10,6 +10,11 @@ import (
"go.uber.org/zap/zaptest"
)
const (
FrostfsVHSHeader = "X-Frostfs-S3-VHS"
FrostfsServernameHeader = "X-Frostfs-Servername"
)
type VHSSettingsMock struct {
domains []string
}
@ -22,17 +27,26 @@ func (v *VHSSettingsMock) GlobalVHS() bool {
return false
}
func (v *VHSSettingsMock) VHSHeader() string {
return FrostfsVHSHeader
}
func (v *VHSSettingsMock) ServernameHeader() string {
return FrostfsServernameHeader
}
func (v *VHSSettingsMock) VHSNamespacesEnabled() map[string]bool {
return make(map[string]bool)
}
func TestIsVHSAddress(t *testing.T) {
for _, tc := range []struct {
name string
vhsEnabledFlag bool
vhsNamespaced map[string]bool
namespace string
expected bool
name string
headerVHSEnabled string
vhsEnabledFlag bool
vhsNamespaced map[string]bool
namespace string
expected bool
}{
{
name: "vhs disabled",
@ -60,9 +74,29 @@ func TestIsVHSAddress(t *testing.T) {
namespace: "kapusta",
expected: true,
},
{
name: "vhs enabled (header)",
headerVHSEnabled: "true",
vhsEnabledFlag: false,
vhsNamespaced: map[string]bool{
"kapusta": false,
},
namespace: "kapusta",
expected: true,
},
{
name: "vhs disabled (header)",
headerVHSEnabled: "false",
vhsEnabledFlag: true,
vhsNamespaced: map[string]bool{
"kapusta": true,
},
namespace: "kapusta",
expected: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
actual := isVHSAddress(tc.vhsEnabledFlag, tc.vhsNamespaced, tc.namespace)
actual := isVHSAddress(tc.headerVHSEnabled, tc.vhsEnabledFlag, tc.vhsNamespaced, tc.namespace)
require.Equal(t, tc.expected, actual)
})
}
@ -383,3 +417,27 @@ func TestCheckDomains(t *testing.T) {
})
}
}
func TestGetDomains(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/", nil)
settings := &VHSSettingsMock{
domains: []string{
"s3.domain.com",
"s3.<wildcard>.domain.com",
"domain.com",
},
}
t.Run("the request does not contain the X-Frostfs-Servername header", func(t *testing.T) {
actualDomains := getDomains(req, settings)
require.Equal(t, settings.domains, actualDomains)
})
serverName := "domain.com"
req.Header.Set(settings.ServernameHeader(), serverName)
t.Run("the request contains the X-Frostfs-Servername header", func(t *testing.T) {
actualDomains := getDomains(req, settings)
require.Equal(t, []string{serverName}, actualDomains)
})
}

14
api/router_mock_test.go
View file

@ -23,7 +23,11 @@ import (
"github.com/stretchr/testify/require"
)
const FrostfsNamespaceHeader = "X-Frostfs-Namespace"
const (
FrostfsNamespaceHeader = "X-Frostfs-Namespace"
FrostfsVHSHeader = "X-Frostfs-S3-VHS"
FrostfsServernameHeader = "X-Frostfs-Servername"
)
type poolStatisticMock struct {
}
@ -102,6 +106,14 @@ func (r *middlewareSettingsMock) GlobalVHS() bool {
return r.vhsEnabled
}
func (r *middlewareSettingsMock) VHSHeader() string {
return FrostfsVHSHeader
}
func (r *middlewareSettingsMock) ServernameHeader() string {
return FrostfsServernameHeader
}
func (r *middlewareSettingsMock) VHSNamespacesEnabled() map[string]bool {
return r.vhsNamespacesEnabled
}

18
cmd/s3-gw/app.go
View file

@ -107,6 +107,8 @@ type (
retryMaxAttempts int
domains []string
vhsEnabled bool
vhsHeader string
servernameHeader string
vhsNamespacesEnabled map[string]bool
retryMaxBackoff time.Duration
retryStrategy handler.RetryStrategy
@ -252,6 +254,8 @@ func (s *appSettings) updateNamespacesSettings(v *viper.Viper, log *zap.Logger)
func (s *appSettings) setVHSSettings(v *viper.Viper, log *zap.Logger) {
domains := fetchDomains(v, log)
vhsEnabled := v.GetBool(cfgVHSEnabled)
vhsHeader := v.GetString(cfgVHSHeader)
servernameHeader := v.GetString(cfgServernameHeader)
nsMap := fetchVHSNamespaces(v, log)
vhsNamespaces := make(map[string]bool, len(nsMap))
for ns, flag := range nsMap {
@ -263,6 +267,8 @@ func (s *appSettings) setVHSSettings(v *viper.Viper, log *zap.Logger) {
s.domains = domains
s.vhsEnabled = vhsEnabled
s.vhsHeader = vhsHeader
s.servernameHeader = servernameHeader
s.vhsNamespacesEnabled = vhsNamespaces
}
@ -278,6 +284,18 @@ func (s *appSettings) GlobalVHS() bool {
return s.vhsEnabled
}
func (s *appSettings) VHSHeader() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.vhsHeader
}
func (s *appSettings) ServernameHeader() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.servernameHeader
}
func (s *appSettings) VHSNamespacesEnabled() map[string]bool {
s.mu.RLock()
defer s.mu.RUnlock()

15
cmd/s3-gw/app_settings.go
View file

@ -54,7 +54,9 @@ const (
defaultAccessBoxCacheRemovingCheckInterval = 5 * time.Minute
defaultNamespaceHeader = "X-Frostfs-Namespace"
defaultNamespaceHeader = "X-Frostfs-Namespace"
defaultVHSHeader = "X-Frostfs-S3-VHS"
defaultServernameHeader = "X-Frostfs-Servername"
defaultConstraintName = "default"
@ -146,8 +148,11 @@ const ( // Settings.
cfgListenDomains = "listen_domains"
cfgVHSEnabled = "vhs.enabled"
cfgVHSNamespaces = "vhs.namespaces"
// VHS.
cfgVHSEnabled = "vhs.enabled"
cfgVHSHeader = "vhs.vhs_header"
cfgServernameHeader = "vhs.servername_header"
cfgVHSNamespaces = "vhs.namespaces"
// Peers.
cfgPeers = "peers"
@ -793,6 +798,10 @@ func newSettings() *viper.Viper {
v.SetDefault(cfgRetryMaxAttempts, defaultRetryMaxAttempts)
v.SetDefault(cfgRetryMaxBackoff, defaultRetryMaxBackoff)
// vhs
v.SetDefault(cfgVHSHeader, defaultVHSHeader)
v.SetDefault(cfgServernameHeader, defaultServernameHeader)
// Bind flags
if err := bindFlags(v, flags); err != nil {
panic(fmt.Errorf("bind flags: %w", err))

4
config/config.env
View file

@ -41,6 +41,10 @@ S3_GW_LISTEN_DOMAINS="domain.com <wildcard>.domain.com"
# VHS enabled flag
S3_GW_VHS_ENABLED=false
# Header for determining whether VHS is enabled for the request
S3_GW_VHS_VHS_HEADER=X-Frostfs-S3-VHS
# Header for determining servername
S3_GW_VHS_SERVERNAME_HEADER=X-Frostfs-Servername
# Config file
S3_GW_CONFIG=/path/to/config/yaml

2
config/config.yaml
View file

@ -46,6 +46,8 @@ listen_domains:
vhs:
enabled: false
vhs_header: X-Frostfs-S3-VHS
servername_header: X-Frostfs-Servername
namespaces:
"ns1": false
"ns2": true

12
docs/configuration.md
View file

@ -731,12 +731,16 @@ Configuration of virtual hosted addressing style.
```yaml
vhs:
enabled: false
vhs_header: X-Frostfs-S3-VHS
servername_header: X-Frostfs-Servername
namespaces:
"ns1": false
"ns2": true
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|--------------|-------------------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------|
| `enabled` | `bool` | yes | `false` | Enables the use of virtual host addressing for banquets at the application level. |
| `namespaces` | `map[string]bool` | yes | | A map in which the keys are the name of the namespace, and the values are the flag responsible for enabling VHS for the specified namespace. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|---------------------|-------------------|---------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
| `enabled` | `bool` | yes | `false` | Enables the use of virtual host addressing for buckets at the application level. |
| `vhs_header` | `string` | yes | `X-Frostfs-S3-VHS` | Header for determining whether VHS is enabled for the request. |
| `servername_header` | `string` | yes | `X-Frostfs-Servername` | Header for determining servername. |
| `namespaces` | `map[string]bool` | yes | | A map in which the keys are the name of the namespace, and the values are the flag responsible for enabling VHS for the specified namespace. |