TrueCloudLab/frostfs-http-gw
17
1
Fork 18
Code Issues 11 Pull requests 1 Projects Releases 16 Packages 1 Activity Actions

[#XX] get/head: Middleware refactor

Browse source 
Add:
 * search index.html
 * fallback by leading slash
Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2025-04-22 18:16:23 +03:00
parent ee628617a3
commit 5fd351d559
11 changed files with 311 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
cmd/http-gw/app.go
View file

@ -100,17 +100,18 @@ type (
workerPoolSize int
logLevelConfig *logLevelConfig
mu sync.RWMutex
defaultTimestamp bool
archiveCompression bool
clientCut bool
returnIndexPage bool
indexPageTemplate string
bufferMaxSizeForPut uint64
namespaceHeader string
defaultNamespaces []string
cors *data.CORSRule
enableFilepathFallback bool
mu sync.RWMutex
defaultTimestamp bool
archiveCompression bool
clientCut bool
returnIndexPage bool
indexPageTemplate string
bufferMaxSizeForPut uint64
namespaceHeader string
defaultNamespaces []string
cors *data.CORSRule
enableFilepathFallback bool
enableFilepathSlashFallback bool
}
tagsConfig struct {
@ -296,6 +297,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
indexPage, indexEnabled := fetchIndexPageTemplate(v, l)
cors := fetchCORSConfig(v)
enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback)
enableFilepathSlashFallback := v.GetBool(cfgFeaturesEnableFilepathSlashFallback)
s.mu.Lock()
defer s.mu.Unlock()
@ -311,6 +313,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
s.indexPageTemplate = indexPage
s.cors = cors
s.enableFilepathFallback = enableFilepathFallback
s.enableFilepathSlashFallback = enableFilepathSlashFallback
}
func (s *loggerSettings) DroppedLogsInc() {
@ -421,6 +424,12 @@ func (s *appSettings) EnableFilepathFallback() bool {
return s.enableFilepathFallback
}
func (s *appSettings) EnableFilepathSlashFallback() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.enableFilepathSlashFallback
}
func (a *app) initResolver() {
var err error
a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig())

5
cmd/http-gw/settings.go
View file

@ -180,8 +180,9 @@ const (
cfgMultinetSubnets = "multinet.subnets"
// Feature.
cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback"
cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support"
cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback"
cfgFeaturesEnableFilepathSlashFallback = "features.enable_filepath_slash_fallback"
cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support"
// Containers.
cfgContainersCORS = "containers.cors"

2
config/config.env
View file

@ -174,6 +174,8 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl
# Enable using fallback path to search for a object by attribute
HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false
# See description in docs/gate-configuration.md
HTTP_GW_FEATURES_ENABLE_FILEPATH_SLASH_FALLBACK=false
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true

2
config/config.yaml
View file

@ -192,6 +192,8 @@ multinet:
features:
# Enable using fallback path to search for a object by attribute
enable_filepath_fallback: false
# See description in docs/gate-configuration.md
enable_filepath_slash_fallback: false
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
tree_pool_netmap_support: true

69
docs/gate-configuration.md
View file

@ -8,7 +8,6 @@ There are some custom types used for brevity:
* `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` (
milliseconds).
# Reload on SIGHUP
Some config values can be reloaded on SIGHUP signal.
@ -163,7 +162,6 @@ server:
| `tls.cert_file` | `string` | yes | | Path to the TLS certificate. |
| `tls.key_file` | `string` | yes | | Path to the key. |
# `logger` section
```yaml
@ -177,7 +175,7 @@ logger:
interval: 1s
tags:
- names: "app,datapath"
level: info
level: info
- names: "external_storage_tree"
```
@ -235,7 +233,6 @@ web:
| `stream_request_body` | `bool` | `true` | Enables request body streaming, and calls the handler sooner when given body is larger than the current limit. |
| `max_request_body_size` | `int` | `4194304` | Maximum request body size. The server rejects requests with bodies exceeding this limit. |
# `upload-header` section
```yaml
@ -271,7 +268,6 @@ archive:
|---------------|--------|---------------|---------------|------------------------------------------------------------------|
| `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. |
# `pprof` section
Contains configuration for the `pprof` profiler.
@ -320,14 +316,13 @@ tracing:
```
| Parameter | Type | SIGHUP reload | Default value | Description |
| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|--------------|----------------------------------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------|
| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. |
| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). |
| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. |
| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. |
| `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. |
#### `attributes` subsection
```yaml
@ -338,12 +333,13 @@ tracing:
value: value
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-----------------------|----------|---------------|---------------|----------------------------------------------------------|
| `key` | `string` | yes | | Attribute key. |
| `value` | `string` | yes | | Attribute value. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|----------|---------------|---------------|------------------|
| `key` | `string` | yes | | Attribute key. |
| `value` | `string` | yes | | Attribute value. |
# `runtime` section
Contains runtime parameters.
```yaml
@ -372,7 +368,6 @@ frostfs:
| `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. |
| `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. |
### `cache` section
```yaml
@ -393,7 +388,6 @@ cache:
| `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. |
| `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`<br>`size: 1000` | Cache which stores container CORS configurations. |
#### `cache` subsection
```yaml
@ -406,7 +400,6 @@ size: 1000
| `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. |
| `size` | `int` | depends on cache | LRU cache size. |
# `resolve_bucket` section
Bucket name resolving parameters from and to container ID.
@ -417,10 +410,10 @@ resolve_bucket:
default_namespaces: [ "", "root" ]
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. |
| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|----------------------|------------|---------------|-----------------------|--------------------------------------------------|
| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. |
| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. |
# `index_page` section
@ -450,9 +443,9 @@ If values are not set, settings from CORS container will be used.
```yaml
cors:
allow_origin: "*"
allow_methods: ["GET", "HEAD"]
allow_headers: ["Authorization"]
expose_headers: ["*"]
allow_methods: [ "GET", "HEAD" ]
allow_headers: [ "Authorization" ]
expose_headers: [ "*" ]
allow_credentials: false
max_age: 600
```
@ -472,15 +465,15 @@ Configuration of multinet support.
```yaml
multinet:
enabled: false
balancer: roundrobin
restrict: false
fallback_delay: 300ms
subnets:
- mask: 1.2.3.4/24
source_ips:
- 1.2.3.4
- 1.2.3.5
enabled: false
balancer: roundrobin
restrict: false
fallback_delay: 300ms
subnets:
- mask: 1.2.3.4/24
source_ips:
- 1.2.3.4
- 1.2.3.5
```
| Parameter | Type | SIGHUP reload | Default value | Description |
@ -512,13 +505,15 @@ Contains parameters for enabling features.
```yaml
features:
enable_filepath_fallback: true
enable_filepath_slash_fallback: false
tree_pool_netmap_support: true
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. |
| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------------|--------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. |
| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) if object with provided `FilePath`/`FileName` wasn't found. This fallback goes `before enable_filepath_fallback`. |
| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. |
# `containers` section
@ -529,6 +524,6 @@ containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------|----------|---------------|---------------|-----------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|----------|---------------|---------------|-----------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |

29
internal/handler/browse.go
View file

@ -12,7 +12,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -161,6 +160,7 @@ func urlencode(path string) string {
type GetObjectsResponse struct {
objects []ResponseObject
hasErrors bool
isNative bool
}
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
@ -226,7 +226,8 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
log := h.reqLogger(ctx)
dirs := make(map[string]struct{})
result := &GetObjectsResponse{
objects: make([]ResponseObject, 0, 100),
objects: make([]ResponseObject, 0, 100),
isNative: true,
}
for objExt := range resp {
if objExt.Error != nil {
@ -322,28 +323,16 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID,
}
type browseParams struct {
bucketInfo *data.BucketInfo
prefix string
isNative bool
listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error)
bucketInfo *data.BucketInfo
prefix string
objects *GetObjectsResponse
}
func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) {
const S3Protocol = "s3"
const FrostfsProtocol = "frostfs"
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("bucket", p.bucketInfo.Name),
zap.String("container", p.bucketInfo.CID.EncodeToString()),
zap.String("prefix", p.prefix),
))
resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToListObjects, err)
return
}
objects := resp.objects
objects := p.objects.objects
sort.Slice(objects, func(i, j int) bool {
if objects[i].IsDir == objects[j].IsDir {
return objects[i].FileName < objects[j].FileName
@ -363,7 +352,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p
}
bucketName := p.bucketInfo.Name
protocol := S3Protocol
if p.isNative {
if p.objects.isNative {
bucketName = p.bucketInfo.CID.EncodeToString()
protocol = FrostfsProtocol
}
@ -372,7 +361,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p
Prefix: p.prefix,
Objects: objects,
Protocol: protocol,
HasErrors: resp.hasErrors,
HasErrors: p.objects.hasErrors,
}); err != nil {
h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err)
return

165
internal/handler/download.go
View file

@ -10,6 +10,7 @@ import (
"fmt"
"io"
"net/url"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
@ -31,13 +32,18 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) {
cidParam := req.UserValue("cid").(string)
oidParam := req.UserValue("oid").(string)
downloadParam := req.QueryArgs().GetBool("download")
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("cid", cidParam),
zap.String("oid", oidParam),
))
path, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
@ -50,18 +56,159 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) {
return
}
var objID oid.ID
if checkS3Err == nil && shouldDownload(oidParam, downloadParam) {
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile)
} else if err = objID.DecodeString(oidParam); err == nil {
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile)
prm := MiddlewareParam{
Context: ctx,
Request: req,
BktInfo: bktInfo,
Path: path,
}
indexPageEnabled := h.config.IndexPageEnabled()
if checkS3Err == nil {
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byS3PathMiddleware(h.receiveFile, noopFormer), Enabled: true},
Middleware{Func: h.byS3PathMiddleware(h.receiveFile, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsS3), Enabled: indexPageEnabled},
)
} else {
h.browseIndex(ctx, req, cidParam, oidParam, checkS3Err != nil)
slashFallbackEnabled := h.config.EnableFilepathSlashFallback()
fileNameFallbackEnabled := h.config.EnableFilepathFallback()
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byAddressMiddleware(h.receiveFile), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, noopFormer), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled},
Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsNative), Enabled: indexPageEnabled},
)
}
}
func shouldDownload(oidParam string, downloadParam bool) bool {
return !isDir(oidParam) || downloadParam
type MiddlewareFunc func(param MiddlewareParam) bool
type MiddlewareParam struct {
Context context.Context
Request *fasthttp.RequestCtx
BktInfo *data.BucketInfo
Path string
}
type Middleware struct {
Func MiddlewareFunc
Enabled bool
}
func run(prm MiddlewareParam, defaultMiddleware MiddlewareFunc, middlewares ...Middleware) {
for _, m := range middlewares {
if m.Enabled && !m.Func(prm) {
return
}
}
defaultMiddleware(prm)
}
func indexFormer(path string) string {
indexPath := path
if indexPath != "" && !strings.HasSuffix(indexPath, "/") {
indexPath += "/"
}
return indexPath + "index.html"
}
func reverseLeadingSlash(path string) string {
if path == "" || path == "/" {
return path
}
if path[0] == '/' {
return path[1:]
}
return "/" + path
}
func noopFormer(path string) string {
return path
}
func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), pathFormer func(string) string) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byS3Path")
defer span.End()
path := pathFormer(prm.Path)
foundOID, err := h.tree.GetLatestVersion(ctx, &prm.BktInfo.CID, path)
if err == nil {
if foundOID.IsDeleteMarker {
h.logAndSendError(ctx, prm.Request, logs.IndexWasDeleted, ErrObjectNotFound)
return false
}
addr := newAddress(prm.BktInfo.CID, foundOID.OID)
handler(ctx, prm.Request, addr)
return false
}
if !errors.Is(err, layer.ErrNodeNotFound) {
h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path))
return false
}
return true
}
}
func (h *Handler) byAttributeSearchMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), attr string, pathFormer func(string) string) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAttributeSearch")
defer span.End()
path := pathFormer(prm.Path)
res, err := h.search(ctx, prm.BktInfo.CID, attr, path, object.MatchStringEqual)
if err != nil {
h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err)
return false
}
defer res.Close()
buf := make([]oid.ID, 1)
n, err := res.Read(buf)
if err == nil && n > 0 {
addr := newAddress(prm.BktInfo.CID, buf[0])
handler(ctx, prm.Request, addr)
return false
}
if !errors.Is(err, io.EOF) {
h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err)
return false
}
return true
}
}
func (h *Handler) byAddressMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAddress")
defer span.End()
var objID oid.ID
if objID.DecodeString(prm.Path) == nil {
handler(ctx, prm.Request, newAddress(prm.BktInfo.CID, objID))
return false
}
return true
}
}
// DownloadByAttribute handles attribute-based download requests.

63
internal/handler/handler.go
View file

@ -35,6 +35,7 @@ type Config interface {
BufferMaxSizeForPut() uint64
NamespaceHeader() string
EnableFilepathFallback() bool
EnableFilepathSlashFallback() bool
FormContainerZone(string) string
CORS() *data.CORSRule
}
@ -216,11 +217,11 @@ func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx,
// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that
// resolves object address from S3-like path <bucket name>/<object key>.
func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) {
func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, bktInfo *data.BucketInfo, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) {
ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path")
defer span.End()
foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path)
foundOID, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, path)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path))
return
@ -230,7 +231,7 @@ func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID
return
}
addr := newAddress(cnrID, foundOID.OID)
addr := newAddress(bktInfo.CID, foundOID.OID)
handler(ctx, req, addr)
}
@ -418,37 +419,31 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
return bktInfo, err
}
func (h *Handler) browseIndex(ctx context.Context, req *fasthttp.RequestCtx, cidParam, oidParam string, isNativeList bool) {
ctx, span := tracing.StartSpanFromContext(ctx, "handler.browseIndex")
defer span.End()
type ListFunc func(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error)
if !h.config.IndexPageEnabled() {
req.SetStatusCode(fasthttp.StatusNotFound)
return
func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
defer span.End()
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("bucket", prm.BktInfo.Name),
zap.String("container", prm.BktInfo.CID.EncodeToString()),
zap.String("prefix", prm.Path),
))
objects, err := fn(ctx, prm.BktInfo, prm.Path)
if err != nil {
h.logAndSendError(ctx, prm.Request, logs.FailedToListObjects, err)
return false
}
h.browseObjects(ctx, prm.Request, browseParams{
bucketInfo: prm.BktInfo,
prefix: prm.Path,
objects: objects,
})
return false
}
unescapedKey, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapeOIDParam, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
return
}
listFunc := h.getDirObjectsS3
if isNativeList {
// tree probe failed, trying to use native
listFunc = h.getDirObjectsNative
}
h.browseObjects(ctx, req, browseParams{
bucketInfo: bktInfo,
prefix: unescapedKey,
listObjects: listFunc,
isNative: isNativeList,
})
}

17
internal/handler/handler_test.go
View file

@ -62,8 +62,9 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d
}
type configMock struct {
additionalSearch bool
cors *data.CORSRule
additionalFilenameSearch bool
additionalSlashSearch bool
cors *data.CORSRule
}
func (c *configMock) DefaultTimestamp() bool {
@ -99,7 +100,11 @@ func (c *configMock) NamespaceHeader() string {
}
func (c *configMock) EnableFilepathFallback() bool {
return c.additionalSearch
return c.additionalFilenameSearch
}
func (c *configMock) EnableFilepathSlashFallback() bool {
return c.additionalSlashSearch
}
func (c *configMock) FormContainerZone(string) string {
@ -327,7 +332,7 @@ func TestBasic(t *testing.T) {
func TestFindObjectByAttribute(t *testing.T) {
hc := prepareHandlerContext(t)
hc.cfg.additionalSearch = true
hc.cfg.additionalFilenameSearch = true
bktName := "bucket"
cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended)
@ -407,7 +412,7 @@ func TestFindObjectByAttribute(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
obj.SetAttributes(tc.firstAttr, tc.secondAttr)
hc.cfg.additionalSearch = tc.additionalSearch
hc.cfg.additionalFilenameSearch = tc.additionalSearch
objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue)
if tc.err != "" {
@ -476,7 +481,7 @@ func TestNeedSearchByFileName(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
hc.cfg.additionalSearch = tc.additionalSearch
hc.cfg.additionalFilenameSearch = tc.additionalSearch
res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal)
require.Equal(t, tc.expected, res)

45
internal/handler/head.go
View file

@ -5,6 +5,7 @@ import (
"errors"
"io"
"net/http"
"net/url"
"strconv"
"time"
@ -128,6 +129,12 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) {
zap.String("oid", oidParam),
))
path, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
@ -140,9 +147,38 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) {
return
}
prm := MiddlewareParam{
Context: ctx,
Request: req,
BktInfo: bktInfo,
Path: path,
}
indexPageEnabled := h.config.IndexPageEnabled()
if checkS3Err == nil {
run(prm, h.errorMiddleware(logs.ObjectNotFound, layer.ErrNodeNotFound),
Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true},
Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled},
)
} else {
slashFallbackEnabled := h.config.EnableFilepathSlashFallback()
fileNameFallbackEnabled := h.config.EnableFilepathFallback()
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byAddressMiddleware(h.headObject), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, noopFormer), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled},
)
}
var objID oid.ID
if checkS3Err == nil {
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject)
h.byS3Path(ctx, req, bktInfo, oidParam, h.headObject)
} else if err = objID.DecodeString(oidParam); err == nil {
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject)
} else {
@ -157,3 +193,10 @@ func (h *Handler) HeadByAttribute(req *fasthttp.RequestCtx) {
h.byAttribute(ctx, req, h.headObject)
}
func (h *Handler) errorMiddleware(msg string, err error) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
h.logAndSendError(prm.Context, prm.Request, msg, err)
return false
}
}

4
internal/logs/logs.go
View file

@ -108,7 +108,9 @@ const (
FailedToGetBucketInfo = "could not get bucket info"
FailedToSubmitTaskToPool = "failed to submit task to pool"
ObjectWasDeleted = "object was deleted"
IndexWasDeleted = "index was deleted"
FailedToGetLatestVersionOfObject = "failed to get latest version of object"
FailedToGetLatestVersionOfIndexObject = "failed to get latest version of index object"
FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists"
FailedToListObjects = "failed to list objects"
FailedToParseTemplate = "failed to parse template"
@ -118,7 +120,7 @@ const (
FailedToGetObject = "failed to get object"
FailedToGetObjectPayload = "failed to get object payload"
FailedToFindObjectByAttribute = "failed to get find object by attribute"
FailedToUnescapeOIDParam = "failed to unescape oid param"
FailedToUnescapePath = "failed to unescape path"
InvalidOIDParam = "invalid oid param"
CouldNotGetCORSConfiguration = "could not get cors configuration"
EmptyOriginRequestHeader = "empty Origin request header"