diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index ed16234..f603d3b 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -100,17 +100,18 @@ type ( workerPoolSize int logLevelConfig *logLevelConfig - mu sync.RWMutex - defaultTimestamp bool - archiveCompression bool - clientCut bool - returnIndexPage bool - indexPageTemplate string - bufferMaxSizeForPut uint64 - namespaceHeader string - defaultNamespaces []string - cors *data.CORSRule - enableFilepathFallback bool + mu sync.RWMutex + defaultTimestamp bool + archiveCompression bool + clientCut bool + returnIndexPage bool + indexPageTemplate string + bufferMaxSizeForPut uint64 + namespaceHeader string + defaultNamespaces []string + cors *data.CORSRule + enableFilepathFallback bool + enableFilepathSlashFallback bool } tagsConfig struct { @@ -296,6 +297,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { indexPage, indexEnabled := fetchIndexPageTemplate(v, l) cors := fetchCORSConfig(v) enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback) + enableFilepathSlashFallback := v.GetBool(cfgFeaturesEnableFilepathSlashFallback) s.mu.Lock() defer s.mu.Unlock() @@ -311,6 +313,7 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) { s.indexPageTemplate = indexPage s.cors = cors s.enableFilepathFallback = enableFilepathFallback + s.enableFilepathSlashFallback = enableFilepathSlashFallback } func (s *loggerSettings) DroppedLogsInc() { @@ -421,6 +424,12 @@ func (s *appSettings) EnableFilepathFallback() bool { return s.enableFilepathFallback } +func (s *appSettings) EnableFilepathSlashFallback() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.enableFilepathSlashFallback +} + func (a *app) initResolver() { var err error a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 814a14e..07722de 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -180,8 +180,9 @@ const ( cfgMultinetSubnets = "multinet.subnets" // Feature. - cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" - cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" + cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" + cfgFeaturesEnableFilepathSlashFallback = "features.enable_filepath_slash_fallback" + cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support" // Containers. cfgContainersCORS = "containers.cors" diff --git a/config/config.env b/config/config.env index 72492d8..a86f3e8 100644 --- a/config/config.env +++ b/config/config.env @@ -174,6 +174,8 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl # Enable using fallback path to search for a object by attribute HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false +# See description in docs/gate-configuration.md +HTTP_GW_FEATURES_ENABLE_FILEPATH_SLASH_FALLBACK=false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true diff --git a/config/config.yaml b/config/config.yaml index ccd025e..bb01d47 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -192,6 +192,8 @@ multinet: features: # Enable using fallback path to search for a object by attribute enable_filepath_fallback: false + # See description in docs/gate-configuration.md + enable_filepath_slash_fallback: false # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service tree_pool_netmap_support: true diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md index 1dec574..3a058ae 100644 --- a/docs/gate-configuration.md +++ b/docs/gate-configuration.md @@ -8,7 +8,6 @@ There are some custom types used for brevity: * `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` ( milliseconds). - # Reload on SIGHUP Some config values can be reloaded on SIGHUP signal. @@ -163,7 +162,6 @@ server: | `tls.cert_file` | `string` | yes | | Path to the TLS certificate. | | `tls.key_file` | `string` | yes | | Path to the key. | - # `logger` section ```yaml @@ -177,7 +175,7 @@ logger: interval: 1s tags: - names: "app,datapath" - level: info + level: info - names: "external_storage_tree" ``` @@ -235,7 +233,6 @@ web: | `stream_request_body` | `bool` | `true` | Enables request body streaming, and calls the handler sooner when given body is larger than the current limit. | | `max_request_body_size` | `int` | `4194304` | Maximum request body size. The server rejects requests with bodies exceeding this limit. | - # `upload-header` section ```yaml @@ -271,7 +268,6 @@ archive: |---------------|--------|---------------|---------------|------------------------------------------------------------------| | `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. | - # `pprof` section Contains configuration for the `pprof` profiler. @@ -320,14 +316,13 @@ tracing: ``` | Parameter | Type | SIGHUP reload | Default value | Description | -| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +|--------------|----------------------------------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| | `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | | `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | | `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. | - #### `attributes` subsection ```yaml @@ -338,12 +333,13 @@ tracing: value: value ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-----------------------|----------|---------------|---------------|----------------------------------------------------------| -| `key` | `string` | yes | | Attribute key. | -| `value` | `string` | yes | | Attribute value. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|------------------| +| `key` | `string` | yes | | Attribute key. | +| `value` | `string` | yes | | Attribute value. | # `runtime` section + Contains runtime parameters. ```yaml @@ -372,7 +368,6 @@ frostfs: | `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | | `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. | - ### `cache` section ```yaml @@ -393,7 +388,6 @@ cache: | `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. | | `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`
`size: 1000` | Cache which stores container CORS configurations. | - #### `cache` subsection ```yaml @@ -406,7 +400,6 @@ size: 1000 | `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. | | `size` | `int` | depends on cache | LRU cache size. | - # `resolve_bucket` section Bucket name resolving parameters from and to container ID. @@ -417,10 +410,10 @@ resolve_bucket: default_namespaces: [ "", "root" ] ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------| -| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | -| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|----------------------|------------|---------------|-----------------------|--------------------------------------------------| +| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | +| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | # `index_page` section @@ -450,9 +443,9 @@ If values are not set, settings from CORS container will be used. ```yaml cors: allow_origin: "*" - allow_methods: ["GET", "HEAD"] - allow_headers: ["Authorization"] - expose_headers: ["*"] + allow_methods: [ "GET", "HEAD" ] + allow_headers: [ "Authorization" ] + expose_headers: [ "*" ] allow_credentials: false max_age: 600 ``` @@ -472,15 +465,15 @@ Configuration of multinet support. ```yaml multinet: - enabled: false - balancer: roundrobin - restrict: false - fallback_delay: 300ms - subnets: - - mask: 1.2.3.4/24 - source_ips: - - 1.2.3.4 - - 1.2.3.5 + enabled: false + balancer: roundrobin + restrict: false + fallback_delay: 300ms + subnets: + - mask: 1.2.3.4/24 + source_ips: + - 1.2.3.4 + - 1.2.3.5 ``` | Parameter | Type | SIGHUP reload | Default value | Description | @@ -512,13 +505,15 @@ Contains parameters for enabling features. ```yaml features: enable_filepath_fallback: true + enable_filepath_slash_fallback: false tree_pool_netmap_support: true ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. | -| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------------------------------------|--------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. | +| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) if object with provided `FilePath`/`FileName` wasn't found. This fallback goes `before enable_filepath_fallback`. | +| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. | # `containers` section @@ -529,6 +524,6 @@ containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------|----------|---------------|---------------|-----------------------------------------| -| `cors` | `string` | no | | Container name for CORS configurations. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------|----------|---------------|---------------|-----------------------------------------| +| `cors` | `string` | no | | Container name for CORS configurations. | diff --git a/internal/handler/browse.go b/internal/handler/browse.go index ebe9004..e1fc59d 100644 --- a/internal/handler/browse.go +++ b/internal/handler/browse.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -161,6 +160,7 @@ func urlencode(path string) string { type GetObjectsResponse struct { objects []ResponseObject hasErrors bool + isNative bool } func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { @@ -226,7 +226,8 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck log := h.reqLogger(ctx) dirs := make(map[string]struct{}) result := &GetObjectsResponse{ - objects: make([]ResponseObject, 0, 100), + objects: make([]ResponseObject, 0, 100), + isNative: true, } for objExt := range resp { if objExt.Error != nil { @@ -322,28 +323,16 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID, } type browseParams struct { - bucketInfo *data.BucketInfo - prefix string - isNative bool - listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error) + bucketInfo *data.BucketInfo + prefix string + objects *GetObjectsResponse } func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) { const S3Protocol = "s3" const FrostfsProtocol = "frostfs" - ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( - zap.String("bucket", p.bucketInfo.Name), - zap.String("container", p.bucketInfo.CID.EncodeToString()), - zap.String("prefix", p.prefix), - )) - resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToListObjects, err) - return - } - - objects := resp.objects + objects := p.objects.objects sort.Slice(objects, func(i, j int) bool { if objects[i].IsDir == objects[j].IsDir { return objects[i].FileName < objects[j].FileName @@ -363,7 +352,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p } bucketName := p.bucketInfo.Name protocol := S3Protocol - if p.isNative { + if p.objects.isNative { bucketName = p.bucketInfo.CID.EncodeToString() protocol = FrostfsProtocol } @@ -372,7 +361,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p Prefix: p.prefix, Objects: objects, Protocol: protocol, - HasErrors: resp.hasErrors, + HasErrors: p.objects.hasErrors, }); err != nil { h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err) return diff --git a/internal/handler/download.go b/internal/handler/download.go index 114bf34..301d10f 100644 --- a/internal/handler/download.go +++ b/internal/handler/download.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net/url" + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" @@ -31,13 +32,18 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { cidParam := req.UserValue("cid").(string) oidParam := req.UserValue("oid").(string) - downloadParam := req.QueryArgs().GetBool("download") ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( zap.String("cid", cidParam), zap.String("oid", oidParam), )) + path, err := url.QueryUnescape(oidParam) + if err != nil { + h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err) + return + } + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) @@ -50,18 +56,159 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) { return } - var objID oid.ID - if checkS3Err == nil && shouldDownload(oidParam, downloadParam) { - h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile) - } else if err = objID.DecodeString(oidParam); err == nil { - h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile) + prm := MiddlewareParam{ + Context: ctx, + Request: req, + BktInfo: bktInfo, + Path: path, + } + + indexPageEnabled := h.config.IndexPageEnabled() + + if checkS3Err == nil { + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byS3PathMiddleware(h.receiveFile, noopFormer), Enabled: true}, + Middleware{Func: h.byS3PathMiddleware(h.receiveFile, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsS3), Enabled: indexPageEnabled}, + ) } else { - h.browseIndex(ctx, req, cidParam, oidParam, checkS3Err != nil) + slashFallbackEnabled := h.config.EnableFilepathSlashFallback() + fileNameFallbackEnabled := h.config.EnableFilepathFallback() + + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byAddressMiddleware(h.receiveFile), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, noopFormer), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled}, + Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsNative), Enabled: indexPageEnabled}, + ) } } -func shouldDownload(oidParam string, downloadParam bool) bool { - return !isDir(oidParam) || downloadParam +type MiddlewareFunc func(param MiddlewareParam) bool + +type MiddlewareParam struct { + Context context.Context + Request *fasthttp.RequestCtx + BktInfo *data.BucketInfo + Path string +} + +type Middleware struct { + Func MiddlewareFunc + Enabled bool +} + +func run(prm MiddlewareParam, defaultMiddleware MiddlewareFunc, middlewares ...Middleware) { + for _, m := range middlewares { + if m.Enabled && !m.Func(prm) { + return + } + } + + defaultMiddleware(prm) +} + +func indexFormer(path string) string { + indexPath := path + if indexPath != "" && !strings.HasSuffix(indexPath, "/") { + indexPath += "/" + } + + return indexPath + "index.html" +} + +func reverseLeadingSlash(path string) string { + if path == "" || path == "/" { + return path + } + + if path[0] == '/' { + return path[1:] + } + + return "/" + path +} + +func noopFormer(path string) string { + return path +} + +func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), pathFormer func(string) string) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byS3Path") + defer span.End() + + path := pathFormer(prm.Path) + + foundOID, err := h.tree.GetLatestVersion(ctx, &prm.BktInfo.CID, path) + if err == nil { + if foundOID.IsDeleteMarker { + h.logAndSendError(ctx, prm.Request, logs.IndexWasDeleted, ErrObjectNotFound) + return false + } + + addr := newAddress(prm.BktInfo.CID, foundOID.OID) + handler(ctx, prm.Request, addr) + return false + } + + if !errors.Is(err, layer.ErrNodeNotFound) { + h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path)) + return false + } + + return true + } +} + +func (h *Handler) byAttributeSearchMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), attr string, pathFormer func(string) string) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAttributeSearch") + defer span.End() + + path := pathFormer(prm.Path) + + res, err := h.search(ctx, prm.BktInfo.CID, attr, path, object.MatchStringEqual) + if err != nil { + h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err) + return false + } + defer res.Close() + + buf := make([]oid.ID, 1) + n, err := res.Read(buf) + if err == nil && n > 0 { + addr := newAddress(prm.BktInfo.CID, buf[0]) + handler(ctx, prm.Request, addr) + return false + } + + if !errors.Is(err, io.EOF) { + h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err) + return false + } + + return true + } +} + +func (h *Handler) byAddressMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAddress") + defer span.End() + + var objID oid.ID + if objID.DecodeString(prm.Path) == nil { + handler(ctx, prm.Request, newAddress(prm.BktInfo.CID, objID)) + return false + } + + return true + } } // DownloadByAttribute handles attribute-based download requests. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index a982bc2..b0daf44 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -35,6 +35,7 @@ type Config interface { BufferMaxSizeForPut() uint64 NamespaceHeader() string EnableFilepathFallback() bool + EnableFilepathSlashFallback() bool FormContainerZone(string) string CORS() *data.CORSRule } @@ -216,11 +217,11 @@ func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, // byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that // resolves object address from S3-like path /. -func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { +func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, bktInfo *data.BucketInfo, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path") defer span.End() - foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path) + foundOID, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, path) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path)) return @@ -230,7 +231,7 @@ func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID return } - addr := newAddress(cnrID, foundOID.OID) + addr := newAddress(bktInfo.CID, foundOID.OID) handler(ctx, req, addr) } @@ -418,37 +419,31 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket return bktInfo, err } -func (h *Handler) browseIndex(ctx context.Context, req *fasthttp.RequestCtx, cidParam, oidParam string, isNativeList bool) { - ctx, span := tracing.StartSpanFromContext(ctx, "handler.browseIndex") - defer span.End() +type ListFunc func(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) - if !h.config.IndexPageEnabled() { - req.SetStatusCode(fasthttp.StatusNotFound) - return +func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex") + defer span.End() + + ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( + zap.String("bucket", prm.BktInfo.Name), + zap.String("container", prm.BktInfo.CID.EncodeToString()), + zap.String("prefix", prm.Path), + )) + + objects, err := fn(ctx, prm.BktInfo, prm.Path) + if err != nil { + h.logAndSendError(ctx, prm.Request, logs.FailedToListObjects, err) + return false + } + + h.browseObjects(ctx, prm.Request, browseParams{ + bucketInfo: prm.BktInfo, + prefix: prm.Path, + objects: objects, + }) + + return false } - - unescapedKey, err := url.QueryUnescape(oidParam) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToUnescapeOIDParam, err) - return - } - - bktInfo, err := h.getBucketInfo(ctx, cidParam) - if err != nil { - h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) - return - } - - listFunc := h.getDirObjectsS3 - if isNativeList { - // tree probe failed, trying to use native - listFunc = h.getDirObjectsNative - } - - h.browseObjects(ctx, req, browseParams{ - bucketInfo: bktInfo, - prefix: unescapedKey, - listObjects: listFunc, - isNative: isNativeList, - }) } diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 93cb1d9..fc75d69 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -62,8 +62,9 @@ func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*d } type configMock struct { - additionalSearch bool - cors *data.CORSRule + additionalFilenameSearch bool + additionalSlashSearch bool + cors *data.CORSRule } func (c *configMock) DefaultTimestamp() bool { @@ -99,7 +100,11 @@ func (c *configMock) NamespaceHeader() string { } func (c *configMock) EnableFilepathFallback() bool { - return c.additionalSearch + return c.additionalFilenameSearch +} + +func (c *configMock) EnableFilepathSlashFallback() bool { + return c.additionalSlashSearch } func (c *configMock) FormContainerZone(string) string { @@ -327,7 +332,7 @@ func TestBasic(t *testing.T) { func TestFindObjectByAttribute(t *testing.T) { hc := prepareHandlerContext(t) - hc.cfg.additionalSearch = true + hc.cfg.additionalFilenameSearch = true bktName := "bucket" cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) @@ -407,7 +412,7 @@ func TestFindObjectByAttribute(t *testing.T) { t.Run(tc.name, func(t *testing.T) { obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] obj.SetAttributes(tc.firstAttr, tc.secondAttr) - hc.cfg.additionalSearch = tc.additionalSearch + hc.cfg.additionalFilenameSearch = tc.additionalSearch objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue) if tc.err != "" { @@ -476,7 +481,7 @@ func TestNeedSearchByFileName(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - hc.cfg.additionalSearch = tc.additionalSearch + hc.cfg.additionalFilenameSearch = tc.additionalSearch res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal) require.Equal(t, tc.expected, res) diff --git a/internal/handler/head.go b/internal/handler/head.go index 11d45fc..e130124 100644 --- a/internal/handler/head.go +++ b/internal/handler/head.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/http" + "net/url" "strconv" "time" @@ -128,6 +129,12 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { zap.String("oid", oidParam), )) + path, err := url.QueryUnescape(oidParam) + if err != nil { + h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err) + return + } + bktInfo, err := h.getBucketInfo(ctx, cidParam) if err != nil { h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) @@ -140,9 +147,38 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) { return } + prm := MiddlewareParam{ + Context: ctx, + Request: req, + BktInfo: bktInfo, + Path: path, + } + + indexPageEnabled := h.config.IndexPageEnabled() + + if checkS3Err == nil { + run(prm, h.errorMiddleware(logs.ObjectNotFound, layer.ErrNodeNotFound), + Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true}, + Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled}, + ) + } else { + slashFallbackEnabled := h.config.EnableFilepathSlashFallback() + fileNameFallbackEnabled := h.config.EnableFilepathFallback() + + run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound), + Middleware{Func: h.byAddressMiddleware(h.headObject), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, noopFormer), Enabled: true}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled}, + Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled}, + ) + } + var objID oid.ID if checkS3Err == nil { - h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject) + h.byS3Path(ctx, req, bktInfo, oidParam, h.headObject) } else if err = objID.DecodeString(oidParam); err == nil { h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject) } else { @@ -157,3 +193,10 @@ func (h *Handler) HeadByAttribute(req *fasthttp.RequestCtx) { h.byAttribute(ctx, req, h.headObject) } + +func (h *Handler) errorMiddleware(msg string, err error) MiddlewareFunc { + return func(prm MiddlewareParam) bool { + h.logAndSendError(prm.Context, prm.Request, msg, err) + return false + } +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3e9b931..213e7c7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -108,7 +108,9 @@ const ( FailedToGetBucketInfo = "could not get bucket info" FailedToSubmitTaskToPool = "failed to submit task to pool" ObjectWasDeleted = "object was deleted" + IndexWasDeleted = "index was deleted" FailedToGetLatestVersionOfObject = "failed to get latest version of object" + FailedToGetLatestVersionOfIndexObject = "failed to get latest version of index object" FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists" FailedToListObjects = "failed to list objects" FailedToParseTemplate = "failed to parse template" @@ -118,7 +120,7 @@ const ( FailedToGetObject = "failed to get object" FailedToGetObjectPayload = "failed to get object payload" FailedToFindObjectByAttribute = "failed to get find object by attribute" - FailedToUnescapeOIDParam = "failed to unescape oid param" + FailedToUnescapePath = "failed to unescape path" InvalidOIDParam = "invalid oid param" CouldNotGetCORSConfiguration = "could not get cors configuration" EmptyOriginRequestHeader = "empty Origin request header"