TrueCloudLab/frostfs-http-gw
17
1
Fork 18
Code Issues 9 Pull requests 3 Projects Releases 16 Activity Actions

[#212] Support CORS container for CORS settings
All checks were successful
/ DCO (pull_request) Successful in 31s
Details
/ Vulncheck (pull_request) Successful in 45s
Details
/ Builds (pull_request) Successful in 1m2s
Details
/ OCI image (pull_request) Successful in 1m25s
Details
/ Lint (pull_request) Successful in 2m23s
Details
/ Tests (pull_request) Successful in 53s
Details
/ Integration tests (pull_request) Successful in 5m24s
Details
/ Vulncheck (push) Successful in 47s
Details
/ Builds (push) Successful in 1m2s
Details
/ OCI image (push) Successful in 1m21s
Details
/ Lint (push) Successful in 1m56s
Details
/ Tests (push) Successful in 59s
Details
/ Integration tests (push) Successful in 5m32s
Details

Browse source 
Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
This commit is contained in:
Marina Biryukova 2025-02-27 12:13:12 +03:00
parent 9cf2a4f0e0
commit 9ef6b06e91
18 changed files with 1204 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

206
cmd/http-gw/app.go
View file

@ -17,6 +17,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
@ -30,6 +31,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
@ -65,6 +67,8 @@ type (
settings *appSettings
loggerSettings *loggerSettings
bucketCache *cache.BucketCache
handle *handler.Handler
corsCnrID cid.ID
servers []Server
unbindServers []ServerInfo
@ -105,12 +109,7 @@ type (
bufferMaxSizeForPut uint64
namespaceHeader string
defaultNamespaces []string
corsAllowOrigin string
corsAllowMethods []string
corsAllowHeaders []string
corsExposeHeaders []string
corsAllowCredentials bool
corsMaxAge int
cors *data.CORSRule
enableFilepathFallback bool
}
@ -122,15 +121,6 @@ type (
logLevel zap.AtomicLevel
tagsConfig *tagsConfig
}
CORS struct {
AllowOrigin string
AllowMethods []string
AllowHeaders []string
ExposeHeaders []string
AllowCredentials bool
MaxAge int
}
)
func newLogLevel(v *viper.Viper) zap.AtomicLevel {
@ -251,6 +241,7 @@ func newApp(ctx context.Context, cfg *appCfg) App {
a.initResolver()
a.initMetrics()
a.initTracing(ctx)
a.initContainers(ctx)
return a
}
@ -259,6 +250,14 @@ func (a *app) config() *viper.Viper {
return a.cfg.config()
}
func (a *app) initContainers(ctx context.Context) {
corsCnrID, err := a.fetchContainerID(ctx, cfgContainersCORS)
if err != nil {
a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
}
a.corsCnrID = *corsCnrID
}
func (a *app) initAppSettings(lc *logLevelConfig) {
a.settings = &appSettings{
reconnectInterval: fetchReconnectInterval(a.config()),
@ -278,12 +277,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
namespaceHeader := v.GetString(cfgResolveNamespaceHeader)
defaultNamespaces := fetchDefaultNamespaces(v)
indexPage, indexEnabled := fetchIndexPageTemplate(v, l)
corsAllowOrigin := v.GetString(cfgCORSAllowOrigin)
corsAllowMethods := v.GetStringSlice(cfgCORSAllowMethods)
corsAllowHeaders := v.GetStringSlice(cfgCORSAllowHeaders)
corsExposeHeaders := v.GetStringSlice(cfgCORSExposeHeaders)
corsAllowCredentials := v.GetBool(cfgCORSAllowCredentials)
corsMaxAge := fetchCORSMaxAge(v)
cors := fetchCORSConfig(v)
enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback)
s.mu.Lock()
@ -298,12 +292,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
s.defaultNamespaces = defaultNamespaces
s.returnIndexPage = indexEnabled
s.indexPageTemplate = indexPage
s.corsAllowOrigin = corsAllowOrigin
s.corsAllowMethods = corsAllowMethods
s.corsAllowHeaders = corsAllowHeaders
s.corsExposeHeaders = corsExposeHeaders
s.corsAllowCredentials = corsAllowCredentials
s.corsMaxAge = corsMaxAge
s.cors = cors
s.enableFilepathFallback = enableFilepathFallback
}
@ -350,26 +339,33 @@ func (s *appSettings) IndexPageTemplate() string {
return s.indexPageTemplate
}
func (s *appSettings) CORS() CORS {
func (s *appSettings) CORS() *data.CORSRule {
s.mu.RLock()
defer s.mu.RUnlock()
allowMethods := make([]string, len(s.corsAllowMethods))
copy(allowMethods, s.corsAllowMethods)
if s.cors == nil {
return nil
}
allowHeaders := make([]string, len(s.corsAllowHeaders))
copy(allowHeaders, s.corsAllowHeaders)
allowMethods := make([]string, len(s.cors.AllowedMethods))
copy(allowMethods, s.cors.AllowedMethods)
exposeHeaders := make([]string, len(s.corsExposeHeaders))
copy(exposeHeaders, s.corsExposeHeaders)
allowHeaders := make([]string, len(s.cors.AllowedHeaders))
copy(allowHeaders, s.cors.AllowedHeaders)
return CORS{
AllowOrigin: s.corsAllowOrigin,
AllowMethods: allowMethods,
AllowHeaders: allowHeaders,
ExposeHeaders: exposeHeaders,
AllowCredentials: s.corsAllowCredentials,
MaxAge: s.corsMaxAge,
exposeHeaders := make([]string, len(s.cors.ExposeHeaders))
copy(exposeHeaders, s.cors.ExposeHeaders)
allowOrigins := make([]string, len(s.cors.AllowedOrigins))
copy(allowOrigins, s.cors.AllowedOrigins)
return &data.CORSRule{
AllowedOrigins: allowOrigins,
AllowedMethods: allowMethods,
AllowedHeaders: allowHeaders,
ExposeHeaders: exposeHeaders,
AllowedCredentials: s.cors.AllowedCredentials,
MaxAgeSeconds: s.cors.MaxAgeSeconds,
}
}
@ -391,15 +387,15 @@ func (s *appSettings) NamespaceHeader() string {
return s.namespaceHeader
}
func (s *appSettings) FormContainerZone(ns string) (zone string, isDefault bool) {
func (s *appSettings) FormContainerZone(ns string) string {
s.mu.RLock()
namespaces := s.defaultNamespaces
s.mu.RUnlock()
if slices.Contains(namespaces, ns) {
return v2container.SysAttributeZoneDefault, true
return v2container.SysAttributeZoneDefault
}
return ns + ".ns", false
return ns + ".ns"
}
func (s *appSettings) EnableFilepathFallback() bool {
@ -420,7 +416,6 @@ func (a *app) getResolverConfig() ([]string, *resolver.Config) {
resolveCfg := &resolver.Config{
FrostFS: frostfs.NewResolverFrostFS(a.pool),
RPCAddress: a.config().GetString(cfgRPCEndpoint),
Settings: a.settings,
}
order := a.config().GetStringSlice(cfgResolveOrder)
@ -606,10 +601,8 @@ func (a *app) Serve() {
close(a.webDone)
}()
handle := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool)
// Configure router.
a.configureRouter(handle)
a.configureRouter(workerPool)
a.startServices()
a.initServers(a.ctx)
@ -730,7 +723,9 @@ func (a *app) stopServices() {
}
}
func (a *app) configureRouter(h *handler.Handler) {
func (a *app) configureRouter(workerPool *ants.Pool) {
a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool)
r := router.New()
r.RedirectTrailingSlash = true
r.NotFound = func(r *fasthttp.RequestCtx) {
@ -740,21 +735,21 @@ func (a *app) configureRouter(h *handler.Handler) {
handler.ResponseError(r, "Method Not Allowed", fasthttp.StatusMethodNotAllowed)
}
r.POST("/upload/{cid}", a.addMiddlewares(h.Upload))
r.OPTIONS("/upload/{cid}", a.addPreflight())
r.POST("/upload/{cid}", a.addMiddlewares(a.handle.Upload))
r.OPTIONS("/upload/{cid}", a.addPreflight(a.handle.Preflight))
a.log.Info(logs.AddedPathUploadCid, logs.TagField(logs.TagApp))
r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(h.DownloadByAddressOrBucketName))
r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(h.HeadByAddressOrBucketName))
r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight())
r.GET("/get/{cid}/{oid:*}", a.addMiddlewares(a.handle.DownloadByAddressOrBucketName))
r.HEAD("/get/{cid}/{oid:*}", a.addMiddlewares(a.handle.HeadByAddressOrBucketName))
r.OPTIONS("/get/{cid}/{oid:*}", a.addPreflight(a.handle.Preflight))
a.log.Info(logs.AddedPathGetCidOid, logs.TagField(logs.TagApp))
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.DownloadByAttribute))
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(h.HeadByAttribute))
r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight())
r.GET("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(a.handle.DownloadByAttribute))
r.HEAD("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addMiddlewares(a.handle.HeadByAttribute))
r.OPTIONS("/get_by_attribute/{cid}/{attr_key}/{attr_val:*}", a.addPreflight(a.handle.Preflight))
a.log.Info(logs.AddedPathGetByAttributeCidAttrKeyAttrVal, logs.TagField(logs.TagApp))
r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadZip))
r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight())
r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(h.DownloadTar))
r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight())
r.GET("/zip/{cid}/{prefix:*}", a.addMiddlewares(a.handle.DownloadZip))
r.OPTIONS("/zip/{cid}/{prefix:*}", a.addPreflight(a.handle.Preflight))
r.GET("/tar/{cid}/{prefix:*}", a.addMiddlewares(a.handle.DownloadTar))
r.OPTIONS("/tar/{cid}/{prefix:*}", a.addPreflight(a.handle.Preflight))
a.log.Info(logs.AddedPathZipCidPrefix, logs.TagField(logs.TagApp))
a.webServer.Handler = r.Handler
@ -777,14 +772,14 @@ func (a *app) addMiddlewares(h fasthttp.RequestHandler) fasthttp.RequestHandler
return h
}
func (a *app) addPreflight() fasthttp.RequestHandler {
func (a *app) addPreflight(h fasthttp.RequestHandler) fasthttp.RequestHandler {
list := []func(fasthttp.RequestHandler) fasthttp.RequestHandler{
a.tracer,
a.logger,
a.canonicalizer,
a.reqNamespace,
}
h := a.preflightHandler
for i := len(list) - 1; i >= 0; i-- {
h = list[i](h)
}
@ -792,46 +787,16 @@ func (a *app) addPreflight() fasthttp.RequestHandler {
return h
}
func (a *app) preflightHandler(c *fasthttp.RequestCtx) {
cors := a.settings.CORS()
setCORSHeaders(c, cors)
}
func (a *app) cors(h fasthttp.RequestHandler) fasthttp.RequestHandler {
return func(c *fasthttp.RequestCtx) {
h(c)
code := c.Response.StatusCode()
if code >= fasthttp.StatusOK && code < fasthttp.StatusMultipleChoices {
cors := a.settings.CORS()
setCORSHeaders(c, cors)
a.handle.SetCORSHeaders(c)
}
}
}
func setCORSHeaders(c *fasthttp.RequestCtx, cors CORS) {
c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(cors.MaxAge))
if len(cors.AllowOrigin) != 0 {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, cors.AllowOrigin)
}
if len(cors.AllowMethods) != 0 {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(cors.AllowMethods, ","))
}
if len(cors.AllowHeaders) != 0 {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, strings.Join(cors.AllowHeaders, ","))
}
if len(cors.ExposeHeaders) != 0 {
c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(cors.ExposeHeaders, ","))
}
if cors.AllowCredentials {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true")
}
}
func (a *app) logger(h fasthttp.RequestHandler) fasthttp.RequestHandler {
return func(req *fasthttp.RequestCtx) {
requiredFields := []zap.Field{zap.Uint64("id", req.ID())}
@ -930,11 +895,13 @@ func (a *app) reqNamespace(h fasthttp.RequestHandler) fasthttp.RequestHandler {
func (a *app) AppParams() *handler.AppParams {
return &handler.AppParams{
Logger: a.log,
FrostFS: frostfs.NewFrostFS(a.pool),
Owner: a.owner,
Resolver: a.resolver,
Cache: a.bucketCache,
Logger: a.log,
FrostFS: frostfs.NewFrostFS(a.pool),
Owner: a.owner,
Resolver: a.resolver,
Cache: a.bucketCache,
CORSCnrID: a.corsCnrID,
CORSCache: cache.NewCORSCache(getCORSCacheOptions(a.config(), a.log)),
}
}
@ -1135,3 +1102,44 @@ func (a *app) tryReconnect(ctx context.Context, sr *fasthttp.Server) bool {
return len(a.unbindServers) == 0
}
func (a *app) fetchContainerID(ctx context.Context, cfgKey string) (id *cid.ID, err error) {
cnrID, err := a.resolveContainerID(ctx, cfgKey)
if err != nil {
return nil, err
}
err = checkContainerExists(ctx, *cnrID, a.pool)
if err != nil {
return nil, err
}
return cnrID, nil
}
func (a *app) resolveContainerID(ctx context.Context, cfgKey string) (*cid.ID, error) {
containerString := a.config().GetString(cfgKey)
id := new(cid.ID)
if err := id.DecodeString(containerString); err != nil {
i := strings.Index(containerString, ".")
if i < 0 {
return nil, fmt.Errorf("invalid container address: %s", containerString)
}
if id, err = a.resolver.Resolve(ctx, containerString[i+1:], containerString[:i]); err != nil {
return nil, fmt.Errorf("resolve container address %s: %w", containerString, err)
}
}
return id, nil
}
func checkContainerExists(ctx context.Context, id cid.ID, frostFSPool *pool.Pool) error {
prm := pool.PrmContainerGet{
ContainerID: id,
}
_, err := frostFSPool.GetContainer(ctx, prm)
return err
}

21
cmd/http-gw/integration_test.go
View file

@ -43,9 +43,10 @@ type putResponse struct {
}
const (
testContainerName = "friendly"
testListenAddress = "localhost:8082"
testHost = "http://" + testListenAddress
testContainerName = "friendly"
testListenAddress = "localhost:8082"
testHost = "http://" + testListenAddress
testCORSContainerName = "cors"
)
func TestIntegration(t *testing.T) {
@ -76,10 +77,14 @@ func TestIntegration(t *testing.T) {
registerUser(t, ctx, aioContainer, file.Name())
}
// Creating CORS container
clientPool := getPool(ctx, t, key)
_, err = createContainer(ctx, t, clientPool, ownerID, testCORSContainerName)
require.NoError(t, err, version)
// See the logs from the command execution.
server, cancel := runServer(file.Name())
clientPool := getPool(ctx, t, key)
CID, err := createContainer(ctx, t, clientPool, ownerID)
CID, err := createContainer(ctx, t, clientPool, ownerID, testContainerName)
require.NoError(t, err, version)
jsonToken, binaryToken := makeBearerTokens(t, key, ownerID, version)
@ -110,6 +115,8 @@ func runServer(pathToWallet string) (App, context.CancelFunc) {
v.config().Set(cfgWalletPath, pathToWallet)
v.config().Set(cfgWalletPassphrase, "")
v.config().Set(cfgContainersCORS, testCORSContainerName+"."+containerv2.SysAttributeZoneDefault)
application := newApp(cancelCtx, v)
go application.Serve()
@ -477,7 +484,7 @@ func getPool(ctx context.Context, t *testing.T, key *keys.PrivateKey) *pool.Pool
return clientPool
}
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID) (cid.ID, error) {
func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, ownerID user.ID, name string) (cid.ID, error) {
var policy netmap.PlacementPolicy
err := policy.DecodeString("REP 1")
require.NoError(t, err)
@ -491,7 +498,7 @@ func createContainer(ctx context.Context, t *testing.T, clientPool *pool.Pool, o
container.SetCreationTime(&cnr, time.Now())
var domain container.Domain
domain.SetName(testContainerName)
domain.SetName(name)
cnr.SetAttribute(containerv2.SysAttributeName, domain.Name())
cnr.SetAttribute(containerv2.SysAttributeZone, domain.Zone())

43
cmd/http-gw/settings.go
View file

@ -16,6 +16,7 @@ import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
@ -155,18 +156,21 @@ const (
cfgBucketsCacheLifetime = "cache.buckets.lifetime"
cfgBucketsCacheSize = "cache.buckets.size"
cfgNetmapCacheLifetime = "cache.netmap.lifetime"
cfgCORSCacheLifetime = "cache.cors.lifetime"
cfgCORSCacheSize = "cache.cors.size"
// Bucket resolving options.
cfgResolveNamespaceHeader = "resolve_bucket.namespace_header"
cfgResolveDefaultNamespaces = "resolve_bucket.default_namespaces"
// CORS.
cfgCORSAllowOrigin = "cors.allow_origin"
cfgCORSAllowMethods = "cors.allow_methods"
cfgCORSAllowHeaders = "cors.allow_headers"
cfgCORSExposeHeaders = "cors.expose_headers"
cfgCORSAllowCredentials = "cors.allow_credentials"
cfgCORSMaxAge = "cors.max_age"
cfgCORS = "cors"
cfgCORSAllowOrigin = cfgCORS + ".allow_origin"
cfgCORSAllowMethods = cfgCORS + ".allow_methods"
cfgCORSAllowHeaders = cfgCORS + ".allow_headers"
cfgCORSExposeHeaders = cfgCORS + ".expose_headers"
cfgCORSAllowCredentials = cfgCORS + ".allow_credentials"
cfgCORSMaxAge = cfgCORS + ".max_age"
// Multinet.
cfgMultinetEnabled = "multinet.enabled"
@ -179,6 +183,9 @@ const (
cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback"
cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support"
// Containers.
cfgContainersCORS = "containers.cors"
// Command line args.
cmdHelp = "help"
cmdVersion = "version"
@ -759,6 +766,15 @@ func getNetmapCacheOptions(v *viper.Viper, l *zap.Logger) *cache.NetmapCacheConf
return cacheCfg
}
func getCORSCacheOptions(v *viper.Viper, l *zap.Logger) *cache.Config {
cacheCfg := cache.DefaultCORSConfig(l)
cacheCfg.Lifetime = fetchCacheLifetime(v, l, cfgCORSCacheLifetime, cacheCfg.Lifetime)
cacheCfg.Size = fetchCacheSize(v, l, cfgCORSCacheSize, cacheCfg.Size)
return cacheCfg
}
func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue time.Duration) time.Duration {
if v.IsSet(cfgEntry) {
lifetime := v.GetDuration(cfgEntry)
@ -854,3 +870,18 @@ func fetchArchiveCompression(v *viper.Viper) bool {
}
return v.GetBool(cfgArchiveCompression)
}
func fetchCORSConfig(v *viper.Viper) *data.CORSRule {
if !v.IsSet(cfgCORS) {
return nil
}
return &data.CORSRule{
AllowedOrigins: []string{v.GetString(cfgCORSAllowOrigin)},
AllowedMethods: v.GetStringSlice(cfgCORSAllowMethods),
AllowedHeaders: v.GetStringSlice(cfgCORSAllowHeaders),
ExposeHeaders: v.GetStringSlice(cfgCORSExposeHeaders),
AllowedCredentials: v.GetBool(cfgCORSAllowCredentials),
MaxAgeSeconds: fetchCORSMaxAge(v),
}
}

6
config/config.env
View file

@ -129,6 +129,9 @@ HTTP_GW_CACHE_BUCKETS_LIFETIME=1m
HTTP_GW_CACHE_BUCKETS_SIZE=1000
# Cache which stores netmap
HTTP_GW_CACHE_NETMAP_LIFETIME=1m
# Cache which stores container CORS configurations
HTTP_GW_CACHE_CORS_LIFETIME=5m
HTTP_GW_CACHE_CORS_SIZE=1000
# Header to determine zone to resolve bucket name
HTTP_GW_RESOLVE_BUCKET_NAMESPACE_HEADER=X-Frostfs-Namespace
@ -172,3 +175,6 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl
HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true
# Containers properties
HTTP_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj

7
config/config.yaml
View file

@ -156,6 +156,10 @@ cache:
# Cache which stores netmap
netmap:
lifetime: 1m
# Cache which stores container CORS configurations
cors:
lifetime: 5m
size: 1000
resolve_bucket:
namespace_header: X-Frostfs-Namespace
@ -191,3 +195,6 @@ features:
enable_filepath_fallback: false
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
tree_pool_netmap_support: true
containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj

20
docs/gate-configuration.md
View file

@ -60,6 +60,7 @@ $ cat http.log
| `index_page` | [Index page configuration](#index_page-section) |
| `multinet` | [Multinet configuration](#multinet-section) |
| `features` | [Features configuration](#features-section) |
| `containers` | [Containers configuration](#containers-section) |
# General section
@ -382,12 +383,16 @@ cache:
size: 1000
netmap:
lifetime: 1m
cors:
lifetime: 5m
size: 1000
```
| Parameter | Type | Default value | Description |
|-----------|-----------------------------------|---------------------------------|---------------------------------------------------------------------------|
| `buckets` | [Cache config](#cache-subsection) | `lifetime: 60s`<br>`size: 1000` | Cache which contains mapping of bucket name to bucket info. |
| `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. |
| `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`<br>`size: 1000` | Cache which stores container CORS configurations. |
#### `cache` subsection
@ -441,7 +446,7 @@ index_page:
# `cors` section
Parameters for CORS (used in OPTIONS requests and responses in all handlers).
If values are not set, headers will not be included to response.
If values are not set, settings from CORS container will be used.
```yaml
cors:
@ -515,3 +520,16 @@ features:
|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. |
| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. |
# `containers` section
Section for well-known containers to store data and settings.
```yaml
containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------|----------|---------------|---------------|-----------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |

62
internal/cache/cors.go vendored Normal file
View file

@ -0,0 +1,62 @@
package cache
import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/bluele/gcache"
"go.uber.org/zap"
)
// CORSCache contains cache with CORS objects.
type CORSCache struct {
cache gcache.Cache
logger *zap.Logger
}
const (
// DefaultCORSCacheSize is a default maximum number of entries in cache.
DefaultCORSCacheSize = 1e3
// DefaultCORSCacheLifetime is a default lifetime of entries in cache.
DefaultCORSCacheLifetime = 5 * time.Minute
)
// DefaultCORSConfig returns new default cache expiration values.
func DefaultCORSConfig(logger *zap.Logger) *Config {
return &Config{
Size: DefaultCORSCacheSize,
Lifetime: DefaultCORSCacheLifetime,
Logger: logger,
}
}
// NewCORSCache creates an object of CORSCache.
func NewCORSCache(config *Config) *CORSCache {
gc := gcache.New(config.Size).LRU().Expiration(config.Lifetime).Build()
return &CORSCache{cache: gc, logger: config.Logger}
}
// Get returns a cached object.
func (o *CORSCache) Get(cnrID cid.ID) *data.CORSConfiguration {
entry, err := o.cache.Get(cnrID)
if err != nil {
return nil
}
result, ok := entry.(*data.CORSConfiguration)
if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil
}
return result
}
// Put puts an object to cache.
func (o *CORSCache) Put(cnrID cid.ID, cors *data.CORSConfiguration) error {
return o.cache.Set(cnrID, cors)
}

18
internal/data/cors.go Normal file
View file

@ -0,0 +1,18 @@
package data
type (
// CORSConfiguration stores CORS configuration of a request.
CORSConfiguration struct {
CORSRules []CORSRule `xml:"CORSRule" json:"CORSRules"`
}
// CORSRule stores rules for CORS configuration.
CORSRule struct {
AllowedHeaders []string `xml:"AllowedHeader" json:"AllowedHeaders"`
AllowedMethods []string `xml:"AllowedMethod" json:"AllowedMethods"`
AllowedOrigins []string `xml:"AllowedOrigin" json:"AllowedOrigins"`
ExposeHeaders []string `xml:"ExposeHeader" json:"ExposeHeaders"`
MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty" json:"MaxAgeSeconds,omitempty"`
AllowedCredentials bool `xml:"AllowedCredentials,omitempty" json:"AllowedCredentials,omitempty"`
}
)

342
internal/handler/cors.go Normal file
View file

@ -0,0 +1,342 @@
package handler
import (
"context"
"encoding/xml"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
const (
internalIOTag = "internal"
corsFilePathTemplate = "/%s.cors"
wildcard = "*"
)
var errNoCORS = errors.New("no CORS objects found")
func (h *Handler) Preflight(c *fasthttp.RequestCtx) {
ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.Preflight")
defer span.End()
ctx = qostagging.ContextWithIOTag(ctx, internalIOTag)
cidParam, _ := c.UserValue("cid").(string)
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
log := reqLog.With(zap.String("cid", cidParam))
origin := c.Request.Header.Peek(fasthttp.HeaderOrigin)
if len(origin) == 0 {
log.Error(logs.EmptyOriginRequestHeader, logs.TagField(logs.TagDatapath))
ResponseError(c, "Origin request header needed", fasthttp.StatusBadRequest)
return
}
method := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod)
if len(method) == 0 {
log.Error(logs.EmptyAccessControlRequestMethodHeader, logs.TagField(logs.TagDatapath))
ResponseError(c, "Access-Control-Request-Method request header needed", fasthttp.StatusBadRequest)
return
}
corsRule := h.config.CORS()
if corsRule != nil {
setCORSHeadersFromRule(c, corsRule)
return
}
corsConfig, err := h.getCORSConfig(ctx, log, cidParam)
if err != nil {
log.Error(logs.CouldNotGetCORSConfiguration, zap.Error(err), logs.TagField(logs.TagDatapath))
status := fasthttp.StatusInternalServerError
if errors.Is(err, errNoCORS) {
status = fasthttp.StatusNotFound
}
ResponseError(c, "could not get CORS configuration: "+err.Error(), status)
return
}
var headers []string
requestHeaders := c.Request.Header.Peek(fasthttp.HeaderAccessControlRequestHeaders)
if len(requestHeaders) > 0 {
headers = strings.Split(string(requestHeaders), ", ")
}
for _, rule := range corsConfig.CORSRules {
for _, o := range rule.AllowedOrigins {
if o == string(origin) || o == wildcard {
for _, m := range rule.AllowedMethods {
if m == string(method) {
if !checkSubslice(rule.AllowedHeaders, headers) {
continue
}
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin))
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", "))
if headers != nil {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, string(requestHeaders))
}
if rule.ExposeHeaders != nil {
c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(rule.ExposeHeaders, ", "))
}
if rule.MaxAgeSeconds > 0 || rule.MaxAgeSeconds == -1 {
c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(rule.MaxAgeSeconds))
}
if o != wildcard {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true")
}
return
}
}
}
}
}
log.Error(logs.CORSRuleWasNotMatched, logs.TagField(logs.TagDatapath))
ResponseError(c, "Forbidden", fasthttp.StatusForbidden)
}
func (h *Handler) SetCORSHeaders(c *fasthttp.RequestCtx) {
ctx, span := tracing.StartSpanFromContext(utils.GetContextFromRequest(c), "handler.SetCORSHeaders")
defer span.End()
origin := c.Request.Header.Peek(fasthttp.HeaderOrigin)
if len(origin) == 0 {
return
}
ctx = qostagging.ContextWithIOTag(ctx, internalIOTag)
cidParam, _ := c.UserValue("cid").(string)
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
log := reqLog.With(zap.String("cid", cidParam))
corsRule := h.config.CORS()
if corsRule != nil {
setCORSHeadersFromRule(c, corsRule)
return
}
corsConfig, err := h.getCORSConfig(ctx, log, cidParam)
if err != nil {
log.Error(logs.CouldNotGetCORSConfiguration, zap.Error(err), logs.TagField(logs.TagDatapath))
return
}
var withCredentials bool
if tkn, err := tokens.LoadBearerToken(ctx); err == nil && tkn != nil {
withCredentials = true
}
for _, rule := range corsConfig.CORSRules {
for _, o := range rule.AllowedOrigins {
if o == string(origin) {
for _, m := range rule.AllowedMethods {
if m == string(c.Method()) {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin))
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", "))
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true")
c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin)
return
}
}
}
if o == wildcard {
for _, m := range rule.AllowedMethods {
if m == string(c.Method()) {
if withCredentials {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin))
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true")
c.Response.Header.Set(fasthttp.HeaderVary, fasthttp.HeaderOrigin)
} else {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, o)
}
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", "))
return
}
}
}
}
}
}
func (h *Handler) getCORSConfig(ctx context.Context, log *zap.Logger, cidStr string) (*data.CORSConfiguration, error) {
cnrID, err := h.resolveContainer(ctx, cidStr)
if err != nil {
return nil, fmt.Errorf("resolve container '%s': %w", cidStr, err)
}
if cors := h.corsCache.Get(*cnrID); cors != nil {
return cors, nil
}
objID, err := h.getLastCORSObject(ctx, *cnrID)
if err != nil {
return nil, fmt.Errorf("get last cors object: %w", err)
}
var addr oid.Address
addr.SetContainer(h.corsCnrID)
addr.SetObject(objID)
corsObj, err := h.frostfs.GetObject(ctx, PrmObjectGet{
PrmAuth: PrmAuth{
BearerToken: bearerToken(ctx),
},
Address: addr,
})
if err != nil {
return nil, fmt.Errorf("get cors object '%s': %w", addr.EncodeToString(), err)
}
corsConfig := &data.CORSConfiguration{}
if err = xml.NewDecoder(corsObj.Payload).Decode(corsConfig); err != nil {
return nil, fmt.Errorf("decode cors object: %w", err)
}
if err = h.corsCache.Put(*cnrID, corsConfig); err != nil {
log.Warn(logs.CouldntCacheCors, zap.Error(err), logs.TagField(logs.TagDatapath))
}
return corsConfig, nil
}
func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID, error) {
filters := object.NewSearchFilters()
filters.AddRootFilter()
filters.AddFilter(object.AttributeFilePath, fmt.Sprintf(corsFilePathTemplate, cnrID), object.MatchStringEqual)
prmAuth := PrmAuth{
BearerToken: bearerToken(ctx),
}
res, err := h.frostfs.SearchObjects(ctx, PrmObjectSearch{
PrmAuth: prmAuth,
Container: h.corsCnrID,
Filters: filters,
})
if err != nil {
return oid.ID{}, fmt.Errorf("search cors versions: %w", err)
}
defer res.Close()
var (
addr oid.Address
obj *object.Object
headErr error
objs = make([]*object.Object, 0)
)
addr.SetContainer(h.corsCnrID)
err = res.Iterate(func(id oid.ID) bool {
addr.SetObject(id)
obj, headErr = h.frostfs.HeadObject(ctx, PrmObjectHead{
PrmAuth: prmAuth,
Address: addr,
})
if headErr != nil {
headErr = fmt.Errorf("head cors object '%s': %w", addr.EncodeToString(), headErr)
return true
}
objs = append(objs, obj)
return false
})
if err != nil {
return oid.ID{}, fmt.Errorf("iterate cors objects: %w", err)
}
if headErr != nil {
return oid.ID{}, headErr
}
if len(objs) == 0 {
return oid.ID{}, errNoCORS
}
sort.Slice(objs, func(i, j int) bool {
versionID1, _ := objs[i].ID()
versionID2, _ := objs[j].ID()
timestamp1 := utils.GetAttributeValue(objs[i].Attributes(), object.AttributeTimestamp)
timestamp2 := utils.GetAttributeValue(objs[j].Attributes(), object.AttributeTimestamp)
if objs[i].CreationEpoch() != objs[j].CreationEpoch() {
return objs[i].CreationEpoch() < objs[j].CreationEpoch()
}
if len(timestamp1) > 0 && len(timestamp2) > 0 && timestamp1 != timestamp2 {
unixTime1, err := strconv.ParseInt(timestamp1, 10, 64)
if err != nil {
return versionID1.EncodeToString() < versionID2.EncodeToString()
}
unixTime2, err := strconv.ParseInt(timestamp2, 10, 64)
if err != nil {
return versionID1.EncodeToString() < versionID2.EncodeToString()
}
return unixTime1 < unixTime2
}
return versionID1.EncodeToString() < versionID2.EncodeToString()
})
objID, _ := objs[len(objs)-1].ID()
return objID, nil
}
func setCORSHeadersFromRule(c *fasthttp.RequestCtx, cors *data.CORSRule) {
c.Response.Header.Set(fasthttp.HeaderAccessControlMaxAge, strconv.Itoa(cors.MaxAgeSeconds))
if len(cors.AllowedOrigins) != 0 {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, cors.AllowedOrigins[0])
}
if len(cors.AllowedMethods) != 0 {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(cors.AllowedMethods, ", "))
}
if len(cors.AllowedHeaders) != 0 {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowHeaders, strings.Join(cors.AllowedHeaders, ", "))
}
if len(cors.ExposeHeaders) != 0 {
c.Response.Header.Set(fasthttp.HeaderAccessControlExposeHeaders, strings.Join(cors.ExposeHeaders, ", "))
}
if cors.AllowedCredentials {
c.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true")
}
}
func checkSubslice(slice []string, subSlice []string) bool {
if sliceContains(slice, wildcard) {
return true
}
if len(subSlice) > len(slice) {
return false
}
for _, r := range subSlice {
if !sliceContains(slice, r) {
return false
}
}
return true
}
func sliceContains(slice []string, str string) bool {
for _, s := range slice {
if s == str {
return true
}
}
return false
}

440
internal/handler/cors_test.go Normal file
View file

@ -0,0 +1,440 @@
package handler
import (
"encoding/base64"
"encoding/xml"
"fmt"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
"github.com/valyala/fasthttp"
)
func TestPreflight(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-preflight"
cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private)
require.NoError(t, err)
hc.frostfs.SetContainer(cnrID, cnr)
var epoch uint64
t.Run("CORS object", func(t *testing.T) {
for _, tc := range []struct {
name string
corsConfig *data.CORSConfiguration
requestHeaders map[string]string
expectedHeaders map[string]string
status int
}{
{
name: "no CORS configuration",
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
fasthttp.HeaderAccessControlExposeHeaders: "",
fasthttp.HeaderAccessControlMaxAge: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
},
status: fasthttp.StatusNotFound,
},
{
name: "specific allowed origin",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"http://example.com"},
AllowedMethods: []string{"GET", "HEAD"},
AllowedHeaders: []string{"Content-Type"},
ExposeHeaders: []string{"x-amz-*", "X-Amz-*"},
MaxAgeSeconds: 900,
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
fasthttp.HeaderAccessControlRequestHeaders: "Content-Type",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "http://example.com",
fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD",
fasthttp.HeaderAccessControlAllowHeaders: "Content-Type",
fasthttp.HeaderAccessControlExposeHeaders: "x-amz-*, X-Amz-*",
fasthttp.HeaderAccessControlMaxAge: "900",
fasthttp.HeaderAccessControlAllowCredentials: "true",
},
status: fasthttp.StatusOK,
},
{
name: "wildcard allowed origin",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "HEAD"},
AllowedHeaders: []string{"Content-Type"},
ExposeHeaders: []string{"x-amz-*", "X-Amz-*"},
MaxAgeSeconds: 900,
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "http://example.com",
fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD",
fasthttp.HeaderAccessControlAllowHeaders: "",
fasthttp.HeaderAccessControlExposeHeaders: "x-amz-*, X-Amz-*",
fasthttp.HeaderAccessControlMaxAge: "900",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
status: fasthttp.StatusOK,
},
{
name: "not allowed header",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "HEAD"},
AllowedHeaders: []string{"Content-Type"},
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
fasthttp.HeaderAccessControlRequestMethod: "GET",
fasthttp.HeaderAccessControlRequestHeaders: "Authorization",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
fasthttp.HeaderAccessControlExposeHeaders: "",
fasthttp.HeaderAccessControlMaxAge: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
status: fasthttp.StatusForbidden,
},
{
name: "empty Origin header",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "HEAD"},
},
},
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
fasthttp.HeaderAccessControlExposeHeaders: "",
fasthttp.HeaderAccessControlMaxAge: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
status: fasthttp.StatusBadRequest,
},
{
name: "empty Access-Control-Request-Method header",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "HEAD"},
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
fasthttp.HeaderAccessControlExposeHeaders: "",
fasthttp.HeaderAccessControlMaxAge: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
status: fasthttp.StatusBadRequest,
},
} {
t.Run(tc.name, func(t *testing.T) {
if tc.corsConfig != nil {
epoch++
setCORSObject(t, hc, cnrID, tc.corsConfig, epoch)
}
r := prepareCORSRequest(t, bktName, tc.requestHeaders)
hc.Handler().Preflight(r)
require.Equal(t, tc.status, r.Response.StatusCode())
for k, v := range tc.expectedHeaders {
require.Equal(t, v, string(r.Response.Header.Peek(k)))
}
})
}
})
t.Run("CORS config", func(t *testing.T) {
hc.cfg.cors = &data.CORSRule{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "HEAD"},
AllowedHeaders: []string{"Content-Type", "Content-Encoding"},
ExposeHeaders: []string{"x-amz-*", "X-Amz-*"},
MaxAgeSeconds: 900,
AllowedCredentials: true,
}
r := prepareCORSRequest(t, bktName, map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
fasthttp.HeaderAccessControlRequestMethod: "GET",
})
hc.Handler().Preflight(r)
require.Equal(t, fasthttp.StatusOK, r.Response.StatusCode())
require.Equal(t, "900", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlMaxAge)))
require.Equal(t, "*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowOrigin)))
require.Equal(t, "GET, HEAD", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowMethods)))
require.Equal(t, "Content-Type, Content-Encoding", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowHeaders)))
require.Equal(t, "x-amz-*, X-Amz-*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlExposeHeaders)))
require.Equal(t, "true", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowCredentials)))
})
}
func TestSetCORSHeaders(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-set-cors-headers"
cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private)
require.NoError(t, err)
hc.frostfs.SetContainer(cnrID, cnr)
var epoch uint64
t.Run("CORS object", func(t *testing.T) {
for _, tc := range []struct {
name string
corsConfig *data.CORSConfiguration
requestHeaders map[string]string
expectedHeaders map[string]string
}{
{
name: "empty Origin header",
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderVary: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
},
{
name: "no CORS configuration",
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderVary: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
},
},
{
name: "specific allowed origin",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"http://example.com"},
AllowedMethods: []string{"GET", "HEAD"},
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "http://example.com",
fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD",
fasthttp.HeaderVary: fasthttp.HeaderOrigin,
fasthttp.HeaderAccessControlAllowCredentials: "true",
},
},
{
name: "wildcard allowed origin, with credentials",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "HEAD"},
},
},
},
requestHeaders: func() map[string]string {
tkn := new(bearer.Token)
err = tkn.Sign(hc.key.PrivateKey)
require.NoError(t, err)
t64 := base64.StdEncoding.EncodeToString(tkn.Marshal())
require.NotEmpty(t, t64)
return map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
fasthttp.HeaderAuthorization: "Bearer " + t64,
}
}(),
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "http://example.com",
fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD",
fasthttp.HeaderVary: fasthttp.HeaderOrigin,
fasthttp.HeaderAccessControlAllowCredentials: "true",
},
},
{
name: "wildcard allowed origin, without credentials",
corsConfig: &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "HEAD"},
},
},
},
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://example.com",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "*",
fasthttp.HeaderAccessControlAllowMethods: "GET, HEAD",
fasthttp.HeaderVary: "",
fasthttp.HeaderAccessControlAllowCredentials: "",
},
},
} {
t.Run(tc.name, func(t *testing.T) {
epoch++
setCORSObject(t, hc, cnrID, tc.corsConfig, epoch)
r := prepareCORSRequest(t, bktName, tc.requestHeaders)
hc.Handler().SetCORSHeaders(r)
require.Equal(t, fasthttp.StatusOK, r.Response.StatusCode())
for k, v := range tc.expectedHeaders {
require.Equal(t, v, string(r.Response.Header.Peek(k)))
}
})
}
})
t.Run("CORS config", func(t *testing.T) {
hc.cfg.cors = &data.CORSRule{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "HEAD"},
AllowedHeaders: []string{"Content-Type", "Content-Encoding"},
ExposeHeaders: []string{"x-amz-*", "X-Amz-*"},
MaxAgeSeconds: 900,
AllowedCredentials: true,
}
r := prepareCORSRequest(t, bktName, map[string]string{fasthttp.HeaderOrigin: "http://example.com"})
hc.Handler().SetCORSHeaders(r)
require.Equal(t, "900", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlMaxAge)))
require.Equal(t, "*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowOrigin)))
require.Equal(t, "GET, HEAD", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowMethods)))
require.Equal(t, "Content-Type, Content-Encoding", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowHeaders)))
require.Equal(t, "x-amz-*, X-Amz-*", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlExposeHeaders)))
require.Equal(t, "true", string(r.Response.Header.Peek(fasthttp.HeaderAccessControlAllowCredentials)))
})
}
func TestCheckSubslice(t *testing.T) {
for _, tc := range []struct {
name string
allowed []string
actual []string
expected bool
}{
{
name: "empty allowed slice",
allowed: []string{},
actual: []string{"str1", "str2", "str3"},
expected: false,
},
{
name: "empty actual slice",
allowed: []string{"str1", "str2", "str3"},
actual: []string{},
expected: true,
},
{
name: "allowed wildcard",
allowed: []string{"str", "*"},
actual: []string{"str1", "str2", "str3"},
expected: true,
},
{
name: "similar allowed and actual",
allowed: []string{"str1", "str2", "str3"},
actual: []string{"str1", "str2", "str3"},
expected: true,
},
{
name: "allowed actual",
allowed: []string{"str", "str1", "str2", "str4"},
actual: []string{"str1", "str2"},
expected: true,
},
{
name: "not allowed actual",
allowed: []string{"str", "str1", "str2", "str4"},
actual: []string{"str1", "str5"},
expected: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.expected, checkSubslice(tc.allowed, tc.actual))
})
}
}
func setCORSObject(t *testing.T, hc *handlerContext, cnrID cid.ID, corsConfig *data.CORSConfiguration, epoch uint64) {
payload, err := xml.Marshal(corsConfig)
require.NoError(t, err)
a := object.NewAttribute()
a.SetKey(object.AttributeFilePath)
a.SetValue(fmt.Sprintf(corsFilePathTemplate, cnrID))
objID := oidtest.ID()
obj := object.New()
obj.SetAttributes(*a)
obj.SetOwnerID(hc.owner)
obj.SetPayload(payload)
obj.SetPayloadSize(uint64(len(payload)))
obj.SetContainerID(hc.corsCnr)
obj.SetID(objID)
obj.SetCreationEpoch(epoch)
var addr oid.Address
addr.SetObject(objID)
addr.SetContainer(hc.corsCnr)
hc.frostfs.SetObject(addr, obj)
}

4
internal/handler/frostfs_mock.go
View file

@ -52,6 +52,10 @@ func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) {
t.containers[cnrID.EncodeToString()] = cnr
}
func (t *TestFrostFS) SetObject(addr oid.Address, obj *object.Object) {
t.objects[addr.EncodeToString()] = obj
}
// AllowUserOperation grants access to object operations.
// Empty userID and objID means any user and object respectively.
func (t *TestFrostFS) AllowUserOperation(cnrID cid.ID, userID user.ID, op acl.Op, objID oid.ID) {

29
internal/handler/handler.go
View file

@ -36,6 +36,8 @@ type Config interface {
BufferMaxSizeForPut() uint64
NamespaceHeader() string
EnableFilepathFallback() bool
FormContainerZone(string) string
CORS() *data.CORSRule
}
// PrmContainer groups parameters of FrostFS.Container operation.
@ -158,7 +160,7 @@ type FrostFS interface {
}
type ContainerResolver interface {
Resolve(ctx context.Context, name string) (*cid.ID, error)
Resolve(ctx context.Context, zone, name string) (*cid.ID, error)
}
type Handler struct {
@ -170,14 +172,18 @@ type Handler struct {
tree layer.TreeService
cache *cache.BucketCache
workerPool *ants.Pool
corsCnrID cid.ID
corsCache *cache.CORSCache
}
type AppParams struct {
Logger *zap.Logger
FrostFS FrostFS
Owner *user.ID
Resolver ContainerResolver
Cache *cache.BucketCache
Logger *zap.Logger
FrostFS FrostFS
Owner *user.ID
Resolver ContainerResolver
Cache *cache.BucketCache
CORSCnrID cid.ID
CORSCache *cache.CORSCache
}
func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
@ -190,6 +196,8 @@ func New(params *AppParams, config Config, tree layer.TreeService, workerPool *a
tree: tree,
cache: params.Cache,
workerPool: workerPool,
corsCnrID: params.CORSCnrID,
corsCache: params.CORSCache,
}
}
@ -352,7 +360,14 @@ func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*ci
cnrID := new(cid.ID)
err := cnrID.DecodeString(containerID)
if err != nil {
cnrID, err = h.containerResolver.Resolve(ctx, containerID)
var namespace string
namespace, err = middleware.GetNamespace(ctx)
if err != nil {
return nil, err
}
zone := h.config.FormContainerZone(namespace)
cnrID, err = h.containerResolver.Resolve(ctx, zone, containerID)
if err != nil && strings.Contains(err.Error(), "not found") {
err = fmt.Errorf("%w: %s", new(apistatus.ContainerNotFound), err.Error())
}

60
internal/handler/handler_test.go
View file

@ -16,7 +16,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -61,6 +63,7 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d
type configMock struct {
additionalSearch bool
cors *data.CORSRule
}
func (c *configMock) DefaultTimestamp() bool {
@ -99,9 +102,18 @@ func (c *configMock) EnableFilepathFallback() bool {
return c.additionalSearch
}
func (c *configMock) FormContainerZone(string) string {
return v2container.SysAttributeZoneDefault
}
func (c *configMock) CORS() *data.CORSRule {
return c.cors
}
type handlerContext struct {
key *keys.PrivateKey
owner user.ID
key *keys.PrivateKey
owner user.ID
corsCnr cid.ID
h *Handler
frostfs *TestFrostFS
@ -131,10 +143,12 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) {
testFrostFS := NewTestFrostFS(key)
testResolver := &resolver.Resolver{Name: "test_resolver"}
testResolver.SetResolveFunc(func(_ context.Context, name string) (*cid.ID, error) {
testResolver.SetResolveFunc(func(_ context.Context, _, name string) (*cid.ID, error) {
return testFrostFS.ContainerID(name)
})
cnrID := createCORSContainer(owner, testFrostFS)
params := &AppParams{
Logger: logger,
FrostFS: testFrostFS,
@ -145,6 +159,12 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) {
Lifetime: 1,
Logger: logger,
}, false),
CORSCnrID: cnrID,
CORSCache: cache.NewCORSCache(&cache.Config{
Size: 1,
Lifetime: 1,
Logger: logger,
}),
}
treeMock := newTreeService()
@ -159,6 +179,7 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) {
return &handlerContext{
key: key,
owner: owner,
corsCnr: cnrID,
h: handler,
frostfs: testFrostFS,
tree: treeMock,
@ -166,6 +187,20 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) {
}, nil
}
func createCORSContainer(owner user.ID, frostfs *TestFrostFS) cid.ID {
var cnr container.Container
cnr.Init()
cnr.SetOwner(owner)
cnrID := cidtest.ID()
frostfs.SetContainer(cnrID, &cnr)
frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectSearch, oid.ID{})
frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectHead, oid.ID{})
frostfs.AllowUserOperation(cnrID, owner, acl.OpObjectGet, oid.ID{})
return cnrID
}
func (hc *handlerContext) prepareContainer(name string, basicACL acl.Basic) (cid.ID, *container.Container, error) {
var pp netmap.PlacementPolicy
err := pp.DecodeString("REP 1")
@ -486,6 +521,25 @@ func prepareGetRequest(ctx context.Context, bucket, objID string) *fasthttp.Requ
return r
}
func prepareCORSRequest(t *testing.T, bucket string, headers map[string]string) *fasthttp.RequestCtx {
ctx := context.Background()
ctx = middleware.SetNamespace(ctx, "")
r := new(fasthttp.RequestCtx)
r.SetUserValue("cid", bucket)
for k, v := range headers {
r.Request.Header.Set(k, v)
}
ctx, err := tokens.StoreBearerTokenAppCtx(ctx, r)
require.NoError(t, err)
utils.SetContextToRequest(ctx, r)
return r
}
func prepareGetByAttributeRequest(ctx context.Context, bucket, attrKey, attrVal string) *fasthttp.RequestCtx {
r := new(fasthttp.RequestCtx)
utils.SetContextToRequest(ctx, r)

68
internal/logs/logs.go
View file

@ -72,41 +72,47 @@ const (
TagsLogConfigWontBeUpdated = "tags log config won't be updated"
FailedToReadIndexPageTemplate = "failed to read index page template"
SetCustomIndexPageTemplate = "set custom index page template"
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
)
// Log messages with the "datapath" tag.
const (
CouldntParseCreationDate = "couldn't parse creation date"
CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload"
FailedToAddObjectToArchive = "failed to add object to archive"
CloseZipWriter = "close zip writer"
IgnorePartEmptyFormName = "ignore part, empty form name"
IgnorePartEmptyFilename = "ignore part, empty filename"
CouldNotParseClientTime = "could not parse client time"
CouldNotPrepareExpirationHeader = "could not prepare expiration header"
CouldNotEncodeResponse = "could not encode response"
AddAttributeToResultObject = "add attribute to result object"
Request = "request"
CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token"
CouldntPutBucketIntoCache = "couldn't put bucket info into cache"
FailedToIterateOverResponse = "failed to iterate over search response"
InvalidCacheEntryType = "invalid cache entry type"
FailedToUnescapeQuery = "failed to unescape query"
CouldntCacheNetmap = "couldn't cache netmap"
FailedToCloseReader = "failed to close reader"
FailedToFilterHeaders = "failed to filter headers"
FailedToReadFileFromTar = "failed to read file from tar"
FailedToGetAttributes = "failed to get attributes"
CloseGzipWriter = "close gzip writer"
CloseTarWriter = "close tar writer"
FailedToCreateGzipReader = "failed to create gzip reader"
GzipReaderSelected = "gzip reader selected"
CouldNotReceiveMultipartForm = "could not receive multipart/form"
ObjectsNotFound = "objects not found"
IteratingOverSelectedObjectsFailed = "iterating over selected objects failed"
CouldNotGetBucket = "could not get bucket"
CouldNotResolveContainerID = "could not resolve container id"
FailedToSumbitTaskToPool = "failed to submit task to pool"
CouldntParseCreationDate = "couldn't parse creation date"
CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload"
FailedToAddObjectToArchive = "failed to add object to archive"
CloseZipWriter = "close zip writer"
IgnorePartEmptyFormName = "ignore part, empty form name"
IgnorePartEmptyFilename = "ignore part, empty filename"
CouldNotParseClientTime = "could not parse client time"
CouldNotPrepareExpirationHeader = "could not prepare expiration header"
CouldNotEncodeResponse = "could not encode response"
AddAttributeToResultObject = "add attribute to result object"
Request = "request"
CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token"
CouldntPutBucketIntoCache = "couldn't put bucket info into cache"
FailedToIterateOverResponse = "failed to iterate over search response"
InvalidCacheEntryType = "invalid cache entry type"
FailedToUnescapeQuery = "failed to unescape query"
CouldntCacheNetmap = "couldn't cache netmap"
FailedToCloseReader = "failed to close reader"
FailedToFilterHeaders = "failed to filter headers"
FailedToReadFileFromTar = "failed to read file from tar"
FailedToGetAttributes = "failed to get attributes"
CloseGzipWriter = "close gzip writer"
CloseTarWriter = "close tar writer"
FailedToCreateGzipReader = "failed to create gzip reader"
GzipReaderSelected = "gzip reader selected"
CouldNotReceiveMultipartForm = "could not receive multipart/form"
ObjectsNotFound = "objects not found"
IteratingOverSelectedObjectsFailed = "iterating over selected objects failed"
CouldNotGetBucket = "could not get bucket"
CouldNotResolveContainerID = "could not resolve container id"
FailedToSumbitTaskToPool = "failed to submit task to pool"
CouldNotGetCORSConfiguration = "could not get cors configuration"
EmptyOriginRequestHeader = "empty Origin request header"
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
CORSRuleWasNotMatched = "cors rule was not matched"
CouldntCacheCors = "couldn't cache cors"
)
// Log messages with the "external_storage" tag.

13
internal/service/frostfs/frostfs.go
View file

@ -10,7 +10,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
@ -21,8 +20,6 @@ import (
"google.golang.org/grpc/status"
)
const clientIOTag = "client"
// FrostFS represents virtual connection to the FrostFS network.
// It is used to provide an interface to dependent packages
// which work with FrostFS.
@ -70,7 +67,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm handler.PrmObjectCreate)
prmPut.UseBearer(*prm.BearerToken)
}
idObj, err := x.pool.PutObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmPut)
idObj, err := x.pool.PutObject(ctx, prmPut)
if err != nil {
return oid.ID{}, handleObjectError("save object via connection pool", err)
}
@ -103,7 +100,7 @@ func (x *FrostFS) HeadObject(ctx context.Context, prm handler.PrmObjectHead) (*o
prmHead.UseBearer(*prm.BearerToken)
}
res, err := x.pool.HeadObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmHead)
res, err := x.pool.HeadObject(ctx, prmHead)
if err != nil {
return nil, handleObjectError("read object header via connection pool", err)
}
@ -123,7 +120,7 @@ func (x *FrostFS) GetObject(ctx context.Context, prm handler.PrmObjectGet) (*han
prmGet.UseBearer(*prm.BearerToken)
}
res, err := x.pool.GetObject(qostagging.ContextWithIOTag(ctx, clientIOTag), prmGet)
res, err := x.pool.GetObject(ctx, prmGet)
if err != nil {
return nil, handleObjectError("init full object reading via connection pool", err)
}
@ -148,7 +145,7 @@ func (x *FrostFS) RangeObject(ctx context.Context, prm handler.PrmObjectRange) (
prmRange.UseBearer(*prm.BearerToken)
}
res, err := x.pool.ObjectRange(qostagging.ContextWithIOTag(ctx, clientIOTag), prmRange)
res, err := x.pool.ObjectRange(ctx, prmRange)
if err != nil {
return nil, handleObjectError("init payload range reading via connection pool", err)
}
@ -169,7 +166,7 @@ func (x *FrostFS) SearchObjects(ctx context.Context, prm handler.PrmObjectSearch
prmSearch.UseBearer(*prm.BearerToken)
}
res, err := x.pool.SearchObjects(qostagging.ContextWithIOTag(ctx, clientIOTag), prmSearch)
res, err := x.pool.SearchObjects(ctx, prmSearch)
if err != nil {
return nil, handleObjectError("init object search via connection pool", err)
}

5
internal/service/frostfs/tree_pool_wrapper.go
View file

@ -10,7 +10,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
qostagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
apitree "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/tree"
treepool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool/tree"
)
@ -62,7 +61,7 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([
BearerToken: getBearer(ctx),
}
nodes, err := w.p.GetNodes(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm)
nodes, err := w.p.GetNodes(ctx, poolPrm)
if err != nil {
return nil, handleError(err)
}
@ -121,7 +120,7 @@ func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo,
poolPrm.RootID = nil
}
subTreeReader, err := w.p.GetSubTree(qostagging.ContextWithIOTag(ctx, clientIOTag), poolPrm)
subTreeReader, err := w.p.GetSubTree(ctx, poolPrm)
if err != nil {
return nil, handleError(err)
}

52
resolver/resolver.go
View file

@ -6,7 +6,7 @@ import (
"fmt"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
@ -29,14 +29,9 @@ type FrostFS interface {
SystemDNS(context.Context) (string, error)
}
type Settings interface {
FormContainerZone(ns string) (zone string, isDefault bool)
}
type Config struct {
FrostFS FrostFS
RPCAddress string
Settings Settings
}
type ContainerResolver struct {
@ -46,15 +41,15 @@ type ContainerResolver struct {
type Resolver struct {
Name string
resolve func(context.Context, string) (*cid.ID, error)
resolve func(context.Context, string, string) (*cid.ID, error)
}
func (r *Resolver) SetResolveFunc(fn func(context.Context, string) (*cid.ID, error)) {
func (r *Resolver) SetResolveFunc(fn func(context.Context, string, string) (*cid.ID, error)) {
r.resolve = fn
}
func (r *Resolver) Resolve(ctx context.Context, name string) (*cid.ID, error) {
return r.resolve(ctx, name)
func (r *Resolver) Resolve(ctx context.Context, zone, name string) (*cid.ID, error) {
return r.resolve(ctx, zone, name)
}
func NewContainerResolver(resolverNames []string, cfg *Config) (*ContainerResolver, error) {
@ -81,13 +76,13 @@ func createResolvers(resolverNames []string, cfg *Config) ([]*Resolver, error) {
return resolvers, nil
}
func (r *ContainerResolver) Resolve(ctx context.Context, cnrName string) (*cid.ID, error) {
func (r *ContainerResolver) Resolve(ctx context.Context, cnrZone, cnrName string) (*cid.ID, error) {
r.mu.RLock()
defer r.mu.RUnlock()
var err error
for _, resolver := range r.resolvers {
cnrID, resolverErr := resolver.Resolve(ctx, cnrName)
cnrID, resolverErr := resolver.Resolve(ctx, cnrZone, cnrName)
if resolverErr != nil {
resolverErr = fmt.Errorf("%s: %w", resolver.Name, resolverErr)
if err == nil {
@ -141,34 +136,25 @@ func (r *ContainerResolver) equals(resolverNames []string) bool {
func newResolver(name string, cfg *Config) (*Resolver, error) {
switch name {
case DNSResolver:
return NewDNSResolver(cfg.FrostFS, cfg.Settings)
return NewDNSResolver(cfg.FrostFS)
case NNSResolver:
return NewNNSResolver(cfg.RPCAddress, cfg.Settings)
return NewNNSResolver(cfg.RPCAddress)
default:
return nil, fmt.Errorf("unknown resolver: %s", name)
}
}
func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) {
func NewDNSResolver(frostFS FrostFS) (*Resolver, error) {
if frostFS == nil {
return nil, fmt.Errorf("pool must not be nil for DNS resolver")
}
if settings == nil {
return nil, fmt.Errorf("resolver settings must not be nil for DNS resolver")
}
var dns ns.DNS
resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) {
resolveFunc := func(ctx context.Context, zone, name string) (*cid.ID, error) {
var err error
namespace, err := middleware.GetNamespace(ctx)
if err != nil {
return nil, err
}
zone, isDefault := settings.FormContainerZone(namespace)
if isDefault {
if zone == v2container.SysAttributeZoneDefault {
zone, err = frostFS.SystemDNS(ctx)
if err != nil {
return nil, fmt.Errorf("read system DNS parameter of the FrostFS: %w", err)
@ -190,13 +176,10 @@ func NewDNSResolver(frostFS FrostFS, settings Settings) (*Resolver, error) {
}, nil
}
func NewNNSResolver(rpcAddress string, settings Settings) (*Resolver, error) {
func NewNNSResolver(rpcAddress string) (*Resolver, error) {
if rpcAddress == "" {
return nil, fmt.Errorf("rpc address must not be empty for NNS resolver")
}
if settings == nil {
return nil, fmt.Errorf("resolver settings must not be nil for NNS resolver")
}
var nns ns.NNS
@ -204,16 +187,9 @@ func NewNNSResolver(rpcAddress string, settings Settings) (*Resolver, error) {
return nil, fmt.Errorf("could not dial nns: %w", err)
}
resolveFunc := func(ctx context.Context, name string) (*cid.ID, error) {
resolveFunc := func(_ context.Context, zone, name string) (*cid.ID, error) {
var d container.Domain
d.SetName(name)
namespace, err := middleware.GetNamespace(ctx)
if err != nil {
return nil, err
}
zone, _ := settings.FormContainerZone(namespace)
d.SetZone(zone)
cnrID, err := nns.ResolveContainerDomain(d)

11
utils/attributes.go
View file

@ -11,6 +11,8 @@ import (
"time"
"unicode"
"unicode/utf8"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
)
type EpochDurations struct {
@ -256,3 +258,12 @@ func (t systemTransformer) updateExpirationHeader(headers map[string]string, dur
headers[t.expirationEpochAttr()] = strconv.FormatUint(expirationEpoch, 10)
}
func GetAttributeValue(attrs []object.Attribute, key string) string {
for _, attr := range attrs {
if attr.Key() == key {
return attr.Value()
}
}
return ""
}