Compare commits

..

No commits in common. "master" and "feature/cors_container" have entirely different histories.

21 changed files with 522 additions and 1017 deletions

View file

@ -22,8 +22,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net"
containerClient "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/contracts/container"
contractsUtil "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/contracts/util"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
@ -41,7 +39,6 @@ import (
"github.com/nspcc-dev/neo-go/cli/flags"
"github.com/nspcc-dev/neo-go/cli/input"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/rpcclient"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/wallet"
"github.com/panjf2000/ants/v2"
@ -103,18 +100,17 @@ type (
workerPoolSize int
logLevelConfig *logLevelConfig
mu sync.RWMutex
defaultTimestamp bool
archiveCompression bool
clientCut bool
returnIndexPage bool
indexPageTemplate string
bufferMaxSizeForPut uint64
namespaceHeader string
defaultNamespaces []string
cors *data.CORSRule
enableFilepathFallback bool
enableFilepathSlashFallback bool
mu sync.RWMutex
defaultTimestamp bool
archiveCompression bool
clientCut bool
returnIndexPage bool
indexPageTemplate string
bufferMaxSizeForPut uint64
namespaceHeader string
defaultNamespaces []string
cors *data.CORSRule
enableFilepathFallback bool
}
tagsConfig struct {
@ -279,14 +275,6 @@ func (a *app) initContainers(ctx context.Context) {
a.corsCnrID = *corsCnrID
}
func (a *app) initRPCClient(ctx context.Context) *rpcclient.Client {
rpcCli, err := rpcclient.New(ctx, a.config().GetString(cfgRPCEndpoint), rpcclient.Options{})
if err != nil {
a.log.Fatal(logs.InitRPCClientFailed, zap.Error(err), logs.TagField(logs.TagApp))
}
return rpcCli
}
func (a *app) initAppSettings(lc *logLevelConfig) {
a.settings = &appSettings{
reconnectInterval: fetchReconnectInterval(a.config()),
@ -308,7 +296,6 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
indexPage, indexEnabled := fetchIndexPageTemplate(v, l)
cors := fetchCORSConfig(v)
enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback)
enableFilepathSlashFallback := v.GetBool(cfgFeaturesEnableFilepathSlashFallback)
s.mu.Lock()
defer s.mu.Unlock()
@ -324,7 +311,6 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
s.indexPageTemplate = indexPage
s.cors = cors
s.enableFilepathFallback = enableFilepathFallback
s.enableFilepathSlashFallback = enableFilepathSlashFallback
}
func (s *loggerSettings) DroppedLogsInc() {
@ -435,12 +421,6 @@ func (s *appSettings) EnableFilepathFallback() bool {
return s.enableFilepathFallback
}
func (s *appSettings) EnableFilepathSlashFallback() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.enableFilepathSlashFallback
}
func (a *app) initResolver() {
var err error
a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig())
@ -761,22 +741,7 @@ func (a *app) stopServices() {
}
func (a *app) configureRouter(workerPool *ants.Pool) {
rpcCli := a.initRPCClient(a.ctx)
cnrContractName := a.config().GetString(cfgContractsContainerName)
rpcEndpoint := a.config().GetString(cfgRPCEndpoint)
cnrAddr, err := contractsUtil.ResolveContractHash(cnrContractName, rpcEndpoint)
if err != nil {
a.log.Fatal(logs.FailedToResolveContractHash, zap.Error(err), logs.TagField(logs.TagApp))
}
cnrClient, err := containerClient.New(containerClient.Config{
ContractHash: cnrAddr,
Key: a.key,
RPCClient: rpcCli,
})
if err != nil {
a.log.Fatal(logs.InitContainerContractFailed, zap.Error(err), logs.TagField(logs.TagApp))
}
a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), cnrClient, workerPool)
a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool)
r := router.New()
r.RedirectTrailingSlash = true

View file

@ -62,8 +62,6 @@ const (
defaultMultinetFallbackDelay = 300 * time.Millisecond
defaultContainerContractName = "container.frostfs"
cfgServer = "server"
cfgTLSEnabled = "tls.enabled"
cfgTLSCertFile = "tls.cert_file"
@ -182,9 +180,8 @@ const (
cfgMultinetSubnets = "multinet.subnets"
// Feature.
cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback"
cfgFeaturesEnableFilepathSlashFallback = "features.enable_filepath_slash_fallback"
cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support"
cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback"
cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support"
// Containers.
cfgContainersCORS = "containers.cors"
@ -199,9 +196,6 @@ const (
cmdConfig = "config"
cmdConfigDir = "config-dir"
cmdListenAddress = "listen_address"
// Contracts.
cfgContractsContainerName = "contracts.container.name"
)
var ignore = map[string]struct{}{
@ -406,9 +400,6 @@ func setDefaults(v *viper.Viper, flags *pflag.FlagSet) {
// multinet
v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay)
// contracts
v.SetDefault(cfgContractsContainerName, defaultContainerContractName)
if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil {
v.SetDefault(cfgResolveOrder, resolveMethods)
}

View file

@ -174,13 +174,8 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl
# Enable using fallback path to search for a object by attribute
HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false
# See description in docs/gate-configuration.md
HTTP_GW_FEATURES_ENABLE_FILEPATH_SLASH_FALLBACK=false
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true
# Containers properties
HTTP_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
# Container contract hash (LE) or name in NNS.
HTTP_GW_CONTRACTS_CONTAINER_NAME=container.frostfs

View file

@ -192,15 +192,8 @@ multinet:
features:
# Enable using fallback path to search for a object by attribute
enable_filepath_fallback: false
# See description in docs/gate-configuration.md
enable_filepath_slash_fallback: false
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
tree_pool_netmap_support: true
containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
contracts:
container:
# Container contract hash (LE) or name in NNS.
name: container.frostfs

View file

@ -8,6 +8,7 @@ There are some custom types used for brevity:
* `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` (
milliseconds).
# Reload on SIGHUP
Some config values can be reloaded on SIGHUP signal.
@ -60,7 +61,6 @@ $ cat http.log
| `multinet` | [Multinet configuration](#multinet-section) |
| `features` | [Features configuration](#features-section) |
| `containers` | [Containers configuration](#containers-section) |
| `contracts` | [Contracts configuration](#contracts-section) |
# General section
@ -163,6 +163,7 @@ server:
| `tls.cert_file` | `string` | yes | | Path to the TLS certificate. |
| `tls.key_file` | `string` | yes | | Path to the key. |
# `logger` section
```yaml
@ -176,7 +177,7 @@ logger:
interval: 1s
tags:
- names: "app,datapath"
level: info
level: info
- names: "external_storage_tree"
```
@ -234,6 +235,7 @@ web:
| `stream_request_body` | `bool` | `true` | Enables request body streaming, and calls the handler sooner when given body is larger than the current limit. |
| `max_request_body_size` | `int` | `4194304` | Maximum request body size. The server rejects requests with bodies exceeding this limit. |
# `upload-header` section
```yaml
@ -269,6 +271,7 @@ archive:
|---------------|--------|---------------|---------------|------------------------------------------------------------------|
| `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. |
# `pprof` section
Contains configuration for the `pprof` profiler.
@ -317,13 +320,14 @@ tracing:
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|--------------|----------------------------------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------|
| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- |
| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. |
| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). |
| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. |
| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. |
| `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. |
#### `attributes` subsection
```yaml
@ -334,13 +338,12 @@ tracing:
value: value
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|----------|---------------|---------------|------------------|
| `key` | `string` | yes | | Attribute key. |
| `value` | `string` | yes | | Attribute value. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|-----------------------|----------|---------------|---------------|----------------------------------------------------------|
| `key` | `string` | yes | | Attribute key. |
| `value` | `string` | yes | | Attribute value. |
# `runtime` section
Contains runtime parameters.
```yaml
@ -369,6 +372,7 @@ frostfs:
| `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. |
| `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. |
### `cache` section
```yaml
@ -389,6 +393,7 @@ cache:
| `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. |
| `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`<br>`size: 1000` | Cache which stores container CORS configurations. |
#### `cache` subsection
```yaml
@ -401,6 +406,7 @@ size: 1000
| `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. |
| `size` | `int` | depends on cache | LRU cache size. |
# `resolve_bucket` section
Bucket name resolving parameters from and to container ID.
@ -411,10 +417,10 @@ resolve_bucket:
default_namespaces: [ "", "root" ]
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|----------------------|------------|---------------|-----------------------|--------------------------------------------------|
| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. |
| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. |
| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. |
# `index_page` section
@ -444,9 +450,9 @@ If values are not set, settings from CORS container will be used.
```yaml
cors:
allow_origin: "*"
allow_methods: [ "GET", "HEAD" ]
allow_headers: [ "Authorization" ]
expose_headers: [ "*" ]
allow_methods: ["GET", "HEAD"]
allow_headers: ["Authorization"]
expose_headers: ["*"]
allow_credentials: false
max_age: 600
```
@ -466,15 +472,15 @@ Configuration of multinet support.
```yaml
multinet:
enabled: false
balancer: roundrobin
restrict: false
fallback_delay: 300ms
subnets:
- mask: 1.2.3.4/24
source_ips:
- 1.2.3.4
- 1.2.3.5
enabled: false
balancer: roundrobin
restrict: false
fallback_delay: 300ms
subnets:
- mask: 1.2.3.4/24
source_ips:
- 1.2.3.4
- 1.2.3.5
```
| Parameter | Type | SIGHUP reload | Default value | Description |
@ -506,15 +512,13 @@ Contains parameters for enabling features.
```yaml
features:
enable_filepath_fallback: true
enable_filepath_slash_fallback: false
tree_pool_netmap_support: true
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. |
| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) leading slash if object with provided `FilePath`/`FileName` wasn't found. This fallback goes before `enable_filepath_fallback`. |
| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. |
| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. |
# `containers` section
@ -525,18 +529,6 @@ containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|----------|---------------|---------------|-----------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |
# `contracts` section
```yaml
contracts:
container:
name: container.frostfs
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|------------------|----------|---------------|---------------------|----------------------------------------------|
| `container.name` | `string` | no | `container.frostfs` | Container contract hash (LE) or name in NNS. |
| Parameter | Type | SIGHUP reload | Default value | Description |
|-------------|----------|---------------|---------------|-----------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. |

2
go.mod
View file

@ -3,7 +3,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw
go 1.23
require (
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121
git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc
@ -34,6 +33,7 @@ require (
require (
dario.cat/mergo v1.0.0 // indirect
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect

View file

@ -12,6 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -130,15 +131,11 @@ func parentDir(prefix string) string {
return prefix[index:]
}
func getParent(encPrefix string) string {
func trimPrefix(encPrefix string) string {
prefix, err := url.PathUnescape(encPrefix)
if err != nil {
return ""
}
if prefix != "" && prefix[len(prefix)-1] == '/' {
prefix = prefix[:len(prefix)-1]
}
slashIndex := strings.LastIndex(prefix, "/")
if slashIndex == -1 {
return ""
@ -164,15 +161,10 @@ func urlencode(path string) string {
type GetObjectsResponse struct {
objects []ResponseObject
hasErrors bool
isNative bool
}
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
if prefix != "" && prefix[len(prefix)-1] == '/' {
prefix = prefix[:len(prefix)-1]
}
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
if err != nil {
return nil, err
}
@ -193,7 +185,7 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
if obj.IsDeleteMarker {
continue
}
obj.FilePath = prefix + "/" + obj.FileName
obj.FilePath = prefix + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
result.objects = append(result.objects, obj)
}
@ -202,9 +194,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
}
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
basePath := prefix
if basePath != "" && basePath[len(basePath)-1] != '/' {
basePath += "/"
var basePath string
if ind := strings.LastIndex(prefix, "/"); ind != -1 {
basePath = prefix[:ind+1]
}
filters := object.NewSearchFilters()
@ -234,8 +226,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
log := h.reqLogger(ctx)
dirs := make(map[string]struct{})
result := &GetObjectsResponse{
objects: make([]ResponseObject, 0, 100),
isNative: true,
objects: make([]ResponseObject, 0, 100),
}
for objExt := range resp {
if objExt.Error != nil {
@ -331,16 +322,28 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID,
}
type browseParams struct {
bucketInfo *data.BucketInfo
prefix string
objects *GetObjectsResponse
bucketInfo *data.BucketInfo
prefix string
isNative bool
listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error)
}
func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) {
const S3Protocol = "s3"
const FrostfsProtocol = "frostfs"
objects := p.objects.objects
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("bucket", p.bucketInfo.Name),
zap.String("container", p.bucketInfo.CID.EncodeToString()),
zap.String("prefix", p.prefix),
))
resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToListObjects, err)
return
}
objects := resp.objects
sort.Slice(objects, func(i, j int) bool {
if objects[i].IsDir == objects[j].IsDir {
return objects[i].FileName < objects[j].FileName
@ -350,7 +353,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p
tmpl, err := template.New("index").Funcs(template.FuncMap{
"formatSize": formatSize,
"getParent": getParent,
"trimPrefix": trimPrefix,
"urlencode": urlencode,
"parentDir": parentDir,
}).Parse(h.config.IndexPageTemplate())
@ -360,21 +363,16 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p
}
bucketName := p.bucketInfo.Name
protocol := S3Protocol
if p.objects.isNative {
if p.isNative {
bucketName = p.bucketInfo.CID.EncodeToString()
protocol = FrostfsProtocol
}
prefix := p.prefix
if prefix != "" && prefix[len(prefix)-1] != '/' {
prefix += "/"
}
if err = tmpl.Execute(req, &BrowsePageData{
Container: bucketName,
Prefix: prefix,
Prefix: p.prefix,
Objects: objects,
Protocol: protocol,
HasErrors: p.objects.hasErrors,
HasErrors: resp.hasErrors,
}); err != nil {
h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err)
return

View file

@ -1,42 +0,0 @@
package handler
import (
"context"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"go.uber.org/zap"
)
func (h *Handler) containerInfo(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) {
info := &data.BucketInfo{
CID: cnrID,
Name: cnrID.EncodeToString(),
}
res, err := h.cnrContract.GetContainerByID(cnrID)
if err != nil {
return nil, fmt.Errorf("get frostfs container: %w", err)
}
cnr := *res
if domain := container.ReadDomain(cnr); domain.Name() != "" {
info.Name = domain.Name()
info.Zone = domain.Zone()
}
info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr)
info.PlacementPolicy = cnr.PlacementPolicy()
if err = h.cache.Put(info); err != nil {
h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache,
zap.String("bucket name", info.Name),
zap.Stringer("cid", info.CID),
zap.Error(err),
logs.TagField(logs.TagDatapath))
}
return info, nil
}

View file

@ -10,12 +10,11 @@ import (
"fmt"
"io"
"net/url"
"strings"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -32,18 +31,13 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) {
cidParam := req.UserValue("cid").(string)
oidParam := req.UserValue("oid").(string)
downloadParam := req.QueryArgs().GetBool("download")
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("cid", cidParam),
zap.String("oid", oidParam),
))
path, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
@ -51,166 +45,23 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) {
}
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) {
if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) {
h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err)
return
}
prm := MiddlewareParam{
Context: ctx,
Request: req,
BktInfo: bktInfo,
Path: path,
}
indexPageEnabled := h.config.IndexPageEnabled()
if checkS3Err == nil {
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byS3PathMiddleware(h.receiveFile, noopFormer), Enabled: true},
Middleware{Func: h.byS3PathMiddleware(h.receiveFile, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsS3), Enabled: indexPageEnabled},
)
var objID oid.ID
if checkS3Err == nil && shouldDownload(oidParam, downloadParam) {
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile)
} else if err = objID.DecodeString(oidParam); err == nil {
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile)
} else {
slashFallbackEnabled := h.config.EnableFilepathSlashFallback()
fileNameFallbackEnabled := h.config.EnableFilepathFallback()
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byAddressMiddleware(h.receiveFile), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, noopFormer), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled},
Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsNative), Enabled: indexPageEnabled},
)
h.browseIndex(ctx, req, cidParam, oidParam, checkS3Err != nil)
}
}
type ObjectHandlerFunc func(context.Context, *fasthttp.RequestCtx, oid.Address)
type MiddlewareFunc func(param MiddlewareParam) bool
type MiddlewareParam struct {
Context context.Context
Request *fasthttp.RequestCtx
BktInfo *data.BucketInfo
Path string
}
type Middleware struct {
Func MiddlewareFunc
Enabled bool
}
func run(prm MiddlewareParam, defaultMiddleware MiddlewareFunc, middlewares ...Middleware) {
for _, m := range middlewares {
if m.Enabled && !m.Func(prm) {
return
}
}
defaultMiddleware(prm)
}
func indexFormer(path string) string {
indexPath := path
if indexPath != "" && !strings.HasSuffix(indexPath, "/") {
indexPath += "/"
}
return indexPath + "index.html"
}
func reverseLeadingSlash(path string) string {
if path == "" || path == "/" {
return path
}
if path[0] == '/' {
return path[1:]
}
return "/" + path
}
func noopFormer(path string) string {
return path
}
func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), pathFormer func(string) string) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byS3Path")
defer span.End()
path := pathFormer(prm.Path)
foundOID, err := h.tree.GetLatestVersion(ctx, &prm.BktInfo.CID, path)
if err == nil {
if foundOID.IsDeleteMarker {
h.logAndSendError(ctx, prm.Request, logs.IndexWasDeleted, ErrObjectNotFound)
return false
}
addr := newAddress(prm.BktInfo.CID, foundOID.OID)
handler(ctx, prm.Request, addr)
return false
}
if !errors.Is(err, tree.ErrNodeNotFound) {
h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path))
return false
}
return true
}
}
func (h *Handler) byAttributeSearchMiddleware(handler ObjectHandlerFunc, attr string, pathFormer func(string) string) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAttributeSearch")
defer span.End()
path := pathFormer(prm.Path)
res, err := h.search(ctx, prm.BktInfo.CID, attr, path, object.MatchStringEqual)
if err != nil {
h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err)
return false
}
defer res.Close()
buf := make([]oid.ID, 1)
n, err := res.Read(buf)
if err == nil && n > 0 {
addr := newAddress(prm.BktInfo.CID, buf[0])
handler(ctx, prm.Request, addr)
return false
}
if !errors.Is(err, io.EOF) {
h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err)
return false
}
return true
}
}
func (h *Handler) byAddressMiddleware(handler ObjectHandlerFunc) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAddress")
defer span.End()
var objID oid.ID
if objID.DecodeString(prm.Path) == nil {
handler(ctx, prm.Request, newAddress(prm.BktInfo.CID, objID))
return false
}
return true
}
func shouldDownload(oidParam string, downloadParam bool) bool {
return !isDir(oidParam) || downloadParam
}
// DownloadByAttribute handles attribute-based download requests.

View file

@ -233,16 +233,6 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (Res
return &resObjectSearchMock{res: res}, nil
}
func (t *TestFrostFS) GetContainerByID(cid cid.ID) (*container.Container, error) {
for k, v := range t.containers {
if k == cid.EncodeToString() {
return v, nil
}
}
return nil, fmt.Errorf("container does not exist %s", cid)
}
func (t *TestFrostFS) InitMultiObjectReader(context.Context, PrmInitMultiObjectReader) (io.Reader, error) {
return nil, nil
}

View file

@ -11,8 +11,8 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
@ -35,7 +35,6 @@ type Config interface {
BufferMaxSizeForPut() uint64
NamespaceHeader() string
EnableFilepathFallback() bool
EnableFilepathSlashFallback() bool
FormContainerZone(string) string
CORS() *data.CORSRule
}
@ -167,19 +166,13 @@ type ContainerResolver interface {
Resolve(ctx context.Context, zone, name string) (*cid.ID, error)
}
type ContainerContract interface {
// GetContainerByID reads a container from contract by ID.
GetContainerByID(cid.ID) (*container.Container, error)
}
type Handler struct {
log *zap.Logger
frostfs FrostFS
ownerID *user.ID
config Config
containerResolver ContainerResolver
cnrContract ContainerContract
tree *tree.Tree
tree layer.TreeService
cache *cache.BucketCache
workerPool *ants.Pool
corsCnrID cid.ID
@ -196,7 +189,7 @@ type AppParams struct {
CORSCache *cache.CORSCache
}
func New(params *AppParams, config Config, tree *tree.Tree, rpcCli ContainerContract, workerPool *ants.Pool) *Handler {
func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
return &Handler{
log: params.Logger,
frostfs: params.FrostFS,
@ -208,10 +201,39 @@ func New(params *AppParams, config Config, tree *tree.Tree, rpcCli ContainerCont
workerPool: workerPool,
corsCnrID: params.CORSCnrID,
corsCache: params.CORSCache,
cnrContract: rpcCli,
}
}
// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
// prepares request and object address to it.
func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, objID oid.ID, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) {
ctx, span := tracing.StartSpanFromContext(ctx, "handler.byNativeAddress")
defer span.End()
addr := newAddress(cnrID, objID)
handler(ctx, req, addr)
}
// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that
// resolves object address from S3-like path <bucket name>/<object key>.
func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) {
ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path")
defer span.End()
foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path))
return
}
if foundOID.IsDeleteMarker {
h.logAndSendError(ctx, req, logs.ObjectWasDeleted, ErrObjectNotFound)
return
}
addr := newAddress(cnrID, foundOID.OID)
handler(ctx, req, addr)
}
// byAttribute is a wrapper similar to byNativeAddress.
func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) {
cidParam, _ := req.UserValue("cid").(string)
@ -230,6 +252,8 @@ func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, han
return
}
val = prepareAtribute(key, val)
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", cidParam),
zap.String("attr_key", key), zap.String("attr_val", val)))
@ -267,6 +291,10 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrK
n, err := res.Read(buf)
if n == 0 {
switch {
case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal):
h.reqLogger(ctx).Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage))
attrVal = prepareAtribute(attrFileName, attrVal)
return h.findObjectByAttribute(ctx, cnrID, attrFileName, attrVal)
case errors.Is(err, io.EOF):
h.reqLogger(ctx).Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return oid.ID{}, fmt.Errorf("object not found: %w", err)
@ -279,6 +307,42 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrK
return buf[0], nil
}
func (h *Handler) needSearchByFileName(key, val string) bool {
if key != attrFilePath || !h.config.EnableFilepathFallback() {
return false
}
return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/")
}
func prepareAtribute(attrKey, attrVal string) string {
if attrKey == attrFileName {
return prepareFileName(attrVal)
}
if attrKey == attrFilePath {
return prepareFilePath(attrVal)
}
return attrVal
}
func prepareFileName(fileName string) string {
if strings.HasPrefix(fileName, "/") {
return fileName[1:]
}
return fileName
}
func prepareFilePath(filePath string) string {
if !strings.HasPrefix(filePath, "/") {
return "/" + filePath
}
return filePath
}
// resolveContainer decode container id, if it's not a valid container id
// then trey to resolve name using provided resolver.
func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) {
@ -315,34 +379,76 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string) (*dat
return nil, fmt.Errorf("resolve container: %w", err)
}
return h.containerInfo(ctx, *cnrID)
}
type ListFunc func(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error)
func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
defer span.End()
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("bucket", prm.BktInfo.Name),
zap.String("container", prm.BktInfo.CID.EncodeToString()),
zap.String("prefix", prm.Path),
))
objects, err := fn(ctx, prm.BktInfo, prm.Path)
if err != nil {
h.logAndSendError(ctx, prm.Request, logs.FailedToListObjects, err)
return false
}
h.browseObjects(ctx, prm.Request, browseParams{
bucketInfo: prm.BktInfo,
prefix: prm.Path,
objects: objects,
})
return false
bktInfo, err := h.readContainer(ctx, *cnrID)
if err != nil {
return nil, fmt.Errorf("read container: %w", err)
}
if err = h.cache.Put(bktInfo); err != nil {
h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache,
zap.String("bucket name", bktInfo.Name),
zap.Stringer("bucket cid", bktInfo.CID),
zap.Error(err),
logs.TagField(logs.TagDatapath))
}
return bktInfo, nil
}
func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) {
prm := PrmContainer{ContainerID: cnrID}
res, err := h.frostfs.Container(ctx, prm)
if err != nil {
return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err)
}
bktInfo := &data.BucketInfo{
CID: cnrID,
Name: cnrID.EncodeToString(),
}
if domain := container.ReadDomain(*res); domain.Name() != "" {
bktInfo.Name = domain.Name()
bktInfo.Zone = domain.Zone()
}
bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(*res)
bktInfo.PlacementPolicy = res.PlacementPolicy()
return bktInfo, err
}
func (h *Handler) browseIndex(ctx context.Context, req *fasthttp.RequestCtx, cidParam, oidParam string, isNativeList bool) {
ctx, span := tracing.StartSpanFromContext(ctx, "handler.browseIndex")
defer span.End()
if !h.config.IndexPageEnabled() {
req.SetStatusCode(fasthttp.StatusNotFound)
return
}
unescapedKey, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapeOIDParam, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
return
}
listFunc := h.getDirObjectsS3
if isNativeList {
// tree probe failed, trying to use native
listFunc = h.getDirObjectsNative
}
h.browseObjects(ctx, req, browseParams{
bucketInfo: bktInfo,
prefix: unescapedKey,
listObjects: listFunc,
isNative: isNativeList,
})
}

View file

@ -14,10 +14,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
@ -27,7 +26,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/panjf2000/ants/v2"
@ -37,11 +35,35 @@ import (
"go.uber.org/zap/zaptest"
)
type treeServiceMock struct {
system map[string]map[string]*data.BaseNodeVersion
}
func newTreeService() *treeServiceMock {
return &treeServiceMock{
system: make(map[string]map[string]*data.BaseNodeVersion),
}
}
func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error {
_, ok := t.system["bucket-settings"]
if !ok {
return layer.ErrNodeNotFound
}
return nil
}
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) {
return nil, "", nil
}
func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) {
return nil, nil
}
type configMock struct {
additionalFilenameSearch bool
additionalSlashSearch bool
indexEnabled bool
cors *data.CORSRule
additionalSearch bool
cors *data.CORSRule
}
func (c *configMock) DefaultTimestamp() bool {
@ -53,11 +75,11 @@ func (c *configMock) ArchiveCompression() bool {
}
func (c *configMock) IndexPageEnabled() bool {
return c.indexEnabled
return false
}
func (c *configMock) IndexPageTemplate() string {
return templates.DefaultIndexTemplate
return ""
}
func (c *configMock) IndexPageNativeTemplate() string {
@ -77,11 +99,7 @@ func (c *configMock) NamespaceHeader() string {
}
func (c *configMock) EnableFilepathFallback() bool {
return c.additionalFilenameSearch
}
func (c *configMock) EnableFilepathSlashFallback() bool {
return c.additionalSlashSearch
return c.additionalSearch
}
func (c *configMock) FormContainerZone(string) string {
@ -99,7 +117,7 @@ type handlerContext struct {
h *Handler
frostfs *TestFrostFS
tree *treeServiceClientMock
tree *treeServiceMock
cfg *configMock
}
@ -149,14 +167,14 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) {
}),
}
treeMock := newTreeServiceClientMock()
treeMock := newTreeService()
cfgMock := &configMock{}
workerPool, err := ants.NewPool(1)
if err != nil {
return nil, err
}
handler := New(params, cfgMock, tree.NewTree(treeMock, logger), testFrostFS, workerPool)
handler := New(params, cfgMock, treeMock, workerPool)
return &handlerContext{
key: key,
@ -236,7 +254,6 @@ func TestBasic(t *testing.T) {
err = json.Unmarshal(r.Response.Body(), &putRes)
require.NoError(t, err)
hc.cfg.additionalFilenameSearch = true
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
fileName := prepareObjectAttributes(object.AttributeFileName, objFileName)
filePath := prepareObjectAttributes(object.AttributeFilePath, objFilePath)
@ -247,14 +264,6 @@ func TestBasic(t *testing.T) {
r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID)
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, content, string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFilePath)
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, content, string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFileName)
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, content, string(r.Response.Body()))
})
t.Run("head", func(t *testing.T) {
@ -262,16 +271,6 @@ func TestBasic(t *testing.T) {
hc.Handler().HeadByAddressOrBucketName(r)
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFilePath)
hc.Handler().HeadByAddressOrBucketName(r)
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFileName)
hc.Handler().HeadByAddressOrBucketName(r)
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
})
t.Run("get by attribute", func(t *testing.T) {
@ -281,11 +280,11 @@ func TestBasic(t *testing.T) {
r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath)
hc.Handler().DownloadByAttribute(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
require.Equal(t, content, string(r.Response.Body()))
r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName)
hc.Handler().DownloadByAttribute(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
require.Equal(t, content, string(r.Response.Body()))
})
t.Run("head by attribute", func(t *testing.T) {
@ -296,11 +295,13 @@ func TestBasic(t *testing.T) {
r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath)
hc.Handler().HeadByAttribute(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName)
hc.Handler().HeadByAttribute(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
})
t.Run("zip", func(t *testing.T) {
@ -324,281 +325,185 @@ func TestBasic(t *testing.T) {
})
}
func prepareHandlerAndBucket(t *testing.T) (*handlerContext, cid.ID) {
func TestFindObjectByAttribute(t *testing.T) {
hc := prepareHandlerContext(t)
hc.cfg.additionalSearch = true
bktName := "bucket"
cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended)
require.NoError(t, err)
hc.frostfs.SetContainer(cnrID, cnr)
return hc, cnrID
ctx := context.Background()
ctx = middleware.SetNamespace(ctx, "")
content := "hello"
r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content)
require.NoError(t, err)
hc.Handler().Upload(r)
require.Equal(t, r.Response.StatusCode(), http.StatusOK)
var putRes putResponse
err = json.Unmarshal(r.Response.Body(), &putRes)
require.NoError(t, err)
testAttrVal1 := "/folder/cat.jpg"
testAttrVal2 := "cat.jpg"
testAttrVal3 := "test-attr-val3"
for _, tc := range []struct {
name string
firstAttr object.Attribute
secondAttr object.Attribute
reqAttrKey string
reqAttrValue string
err string
additionalSearch bool
}{
{
name: "success search by FileName",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFileName,
reqAttrValue: testAttrVal2,
additionalSearch: false,
},
{
name: "failed search by FileName",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFileName,
reqAttrValue: testAttrVal3,
err: "not found",
additionalSearch: false,
},
{
name: "success search by FilePath (with additional search)",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFilePath,
reqAttrValue: testAttrVal2,
additionalSearch: true,
},
{
name: "failed by FilePath (with additional search)",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFilePath,
reqAttrValue: testAttrVal3,
err: "not found",
additionalSearch: true,
},
{
name: "success search by FilePath with leading slash (with additional search)",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFilePath,
reqAttrValue: "/cat.jpg",
additionalSearch: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
obj.SetAttributes(tc.firstAttr, tc.secondAttr)
hc.cfg.additionalSearch = tc.additionalSearch
objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue)
if tc.err != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tc.err)
return
}
require.NoError(t, err)
require.Equal(t, putRes.ObjectID, objID.EncodeToString())
})
}
}
func TestGetObjectWithFallback(t *testing.T) {
ctx := middleware.SetNamespace(context.Background(), "")
func TestNeedSearchByFileName(t *testing.T) {
hc := prepareHandlerContext(t)
t.Run("by oid", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
for _, tc := range []struct {
name string
attrKey string
attrVal string
additionalSearch bool
expected bool
}{
{
name: "need search - not contains slash",
attrKey: attrFilePath,
attrVal: "cat.png",
additionalSearch: true,
expected: true,
},
{
name: "need search - single lead slash",
attrKey: attrFilePath,
attrVal: "/cat.png",
additionalSearch: true,
expected: true,
},
{
name: "don't need search - single slash but not lead",
attrKey: attrFilePath,
attrVal: "cats/cat.png",
additionalSearch: true,
expected: false,
},
{
name: "don't need search - more one slash",
attrKey: attrFilePath,
attrVal: "/cats/cat.png",
additionalSearch: true,
expected: false,
},
{
name: "don't need search - incorrect attribute key",
attrKey: attrFileName,
attrVal: "cat.png",
additionalSearch: true,
expected: false,
},
{
name: "don't need search - additional search disabled",
attrKey: attrFilePath,
attrVal: "cat.png",
additionalSearch: false,
expected: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
hc.cfg.additionalSearch = tc.additionalSearch
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), obj1ID.String())
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("by filepath as it is", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/filepath/obj2"))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj2")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj2.Payload()), string(r.Response.Body()))
})
t.Run("by filepath slash fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.additionalSlashSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("by filename fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.additionalFilenameSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("by filename and slash fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.additionalFilenameSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.additionalSlashSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("index fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/index.html"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.indexEnabled = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("index filename fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/index.html"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.indexEnabled = true
hc.cfg.additionalFilenameSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal)
require.Equal(t, tc.expected, res)
})
}
}
func TestIndex(t *testing.T) {
ctx := middleware.SetNamespace(context.Background(), "")
func TestPrepareFileName(t *testing.T) {
fileName := "/cat.jpg"
expected := "cat.jpg"
actual := prepareFileName(fileName)
require.Equal(t, expected, actual)
t.Run("s3", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
fileName = "cat.jpg"
actual = prepareFileName(fileName)
require.Equal(t, expected, actual)
}
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
func TestPrepareFilePath(t *testing.T) {
filePath := "cat.jpg"
expected := "/cat.jpg"
actual := prepareFilePath(filePath)
require.Equal(t, expected, actual)
hc.tree.containers[cnrID.String()] = containerInfo{
trees: map[string]map[string]nodeResponse{
"system": {"bucket-settings": nodeResponse{nodeID: 1}},
"version": {
"": nodeResponse{}, //root
"prefix": nodeResponse{
nodeID: 1,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}},
"obj1": nodeResponse{
parentID: 1,
nodeID: 2,
meta: []nodeMeta{
{key: tree.FileNameKey, value: []byte("obj1")},
{key: "OID", value: []byte(obj1ID.String())},
},
},
},
},
}
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.indexEnabled = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix")
require.Contains(t, string(r.Response.Body()), obj1ID.String())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix")
require.Contains(t, string(r.Response.Body()), obj1ID.String())
r = prepareGetRequest(ctx, "bucket", "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
})
t.Run("native", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.indexEnabled = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix")
require.Contains(t, string(r.Response.Body()), obj1ID.String())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix")
require.Contains(t, string(r.Response.Body()), obj1ID.String())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
})
filePath = "/cat.jpg"
actual = prepareFilePath(filePath)
require.Equal(t, expected, actual)
}
func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) {

View file

@ -5,12 +5,11 @@ import (
"errors"
"io"
"net/http"
"net/url"
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -129,12 +128,6 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) {
zap.String("oid", oidParam),
))
path, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
@ -142,38 +135,18 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) {
}
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) {
if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) {
h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err)
return
}
prm := MiddlewareParam{
Context: ctx,
Request: req,
BktInfo: bktInfo,
Path: path,
}
indexPageEnabled := h.config.IndexPageEnabled()
var objID oid.ID
if checkS3Err == nil {
run(prm, h.errorMiddleware(logs.ObjectNotFound, tree.ErrNodeNotFound),
Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true},
Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled},
)
h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject)
} else if err = objID.DecodeString(oidParam); err == nil {
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject)
} else {
slashFallbackEnabled := h.config.EnableFilepathSlashFallback()
fileNameFallbackEnabled := h.config.EnableFilepathFallback()
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byAddressMiddleware(h.headObject), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, noopFormer), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled},
)
h.logAndSendError(ctx, req, logs.InvalidOIDParam, err)
}
}
@ -184,10 +157,3 @@ func (h *Handler) HeadByAttribute(req *fasthttp.RequestCtx) {
h.byAttribute(ctx, req, h.headObject)
}
func (h *Handler) errorMiddleware(msg string, err error) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
h.logAndSendError(prm.Context, prm.Request, msg, err)
return false
}
}

View file

@ -1,141 +0,0 @@
package handler
import (
"context"
"errors"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
)
type nodeMeta struct {
key string
value []byte
}
func (m nodeMeta) GetKey() string {
return m.key
}
func (m nodeMeta) GetValue() []byte {
return m.value
}
type nodeResponse struct {
meta []nodeMeta
nodeID uint64
parentID uint64
timestamp uint64
}
func (n nodeResponse) GetNodeID() []uint64 {
return []uint64{n.nodeID}
}
func (n nodeResponse) GetParentID() []uint64 {
return []uint64{n.parentID}
}
func (n nodeResponse) GetTimestamp() []uint64 {
return []uint64{n.timestamp}
}
func (n nodeResponse) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.meta))
for i, value := range n.meta {
res[i] = value
}
return res
}
type containerInfo struct {
trees map[string]map[string]nodeResponse
}
type treeServiceClientMock struct {
containers map[string]containerInfo
}
func newTreeServiceClientMock() *treeServiceClientMock {
return &treeServiceClientMock{
containers: make(map[string]containerInfo),
}
}
func (t *treeServiceClientMock) GetNodes(_ context.Context, p *tree.GetNodesParams) ([]tree.NodeResponse, error) {
cnr, ok := t.containers[p.CnrID.EncodeToString()]
if !ok {
return nil, tree.ErrNodeNotFound
}
tr, ok := cnr.trees[p.TreeID]
if !ok {
return nil, tree.ErrNodeNotFound
}
node, ok := tr[strings.Join(p.Path, "/")]
if !ok {
return nil, tree.ErrNodeNotFound
}
return []tree.NodeResponse{node}, nil
}
func (t *treeServiceClientMock) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, _ bool) ([]tree.NodeResponse, error) {
cnr, ok := t.containers[bktInfo.CID.EncodeToString()]
if !ok {
return nil, tree.ErrNodeNotFound
}
tr, ok := cnr.trees[treeID]
if !ok {
return nil, tree.ErrNodeNotFound
}
if len(rootID) != 1 {
return nil, errors.New("invalid rootID")
}
var root *nodeResponse
for _, v := range tr {
if v.nodeID == rootID[0] {
root = &v
break
}
}
if root == nil {
return nil, tree.ErrNodeNotFound
}
var res []nodeResponse
if depth == 0 {
for _, v := range tr {
res = append(res, v)
}
} else {
res = append(res, *root)
depthIndex := 0
for i := uint32(0); i < depth-1; i++ {
childrenCount := 0
for _, v := range tr {
for j := range res[depthIndex:] {
if v.parentID == res[j].nodeID {
res = append(res, v)
childrenCount++
break
}
}
}
depthIndex = len(res) - childrenCount
}
}
res2 := make([]tree.NodeResponse, len(res))
for i := range res {
res2[i] = res[i]
}
return res2, nil
}

View file

@ -6,9 +6,9 @@ import (
"fmt"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -93,7 +93,7 @@ func formErrorResponse(err error) (string, int) {
switch {
case errors.Is(err, ErrAccessDenied):
return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden
case errors.Is(err, tree.ErrNodeAccessDenied):
case errors.Is(err, layer.ErrNodeAccessDenied):
return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden
case errors.Is(err, ErrQuotaLimitReached):
return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict
@ -101,7 +101,7 @@ func formErrorResponse(err error) (string, int) {
return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound
case errors.Is(err, ErrObjectNotFound):
return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound
case errors.Is(err, tree.ErrNodeNotFound):
case errors.Is(err, layer.ErrNodeNotFound):
return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound
case errors.Is(err, ErrGatewayTimeout):
return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout

View file

@ -0,0 +1,24 @@
package layer
import (
"context"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error)
CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error
}
var (
// ErrNodeNotFound is returned from Tree service in case of not found error.
ErrNodeNotFound = errors.New("not found")
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
ErrNodeAccessDenied = errors.New("access denied")
)

View file

@ -73,9 +73,6 @@ const (
FailedToReadIndexPageTemplate = "failed to read index page template"
SetCustomIndexPageTemplate = "set custom index page template"
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
InitRPCClientFailed = "init rpc client faileds"
InitContainerContractFailed = "init container contract failed"
FailedToResolveContractHash = "failed to resolve contract hash"
)
// Log messages with the "datapath" tag.
@ -110,8 +107,8 @@ const (
IteratingOverSelectedObjectsFailed = "iterating over selected objects failed"
FailedToGetBucketInfo = "could not get bucket info"
FailedToSubmitTaskToPool = "failed to submit task to pool"
IndexWasDeleted = "index was deleted"
FailedToGetLatestVersionOfIndexObject = "failed to get latest version of index object"
ObjectWasDeleted = "object was deleted"
FailedToGetLatestVersionOfObject = "failed to get latest version of object"
FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists"
FailedToListObjects = "failed to list objects"
FailedToParseTemplate = "failed to parse template"
@ -121,7 +118,8 @@ const (
FailedToGetObject = "failed to get object"
FailedToGetObjectPayload = "failed to get object payload"
FailedToFindObjectByAttribute = "failed to get find object by attribute"
FailedToUnescapePath = "failed to unescape path"
FailedToUnescapeOIDParam = "failed to unescape oid param"
InvalidOIDParam = "invalid oid param"
CouldNotGetCORSConfiguration = "could not get cors configuration"
EmptyOriginRequestHeader = "empty Origin request header"
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
@ -131,9 +129,10 @@ const (
// Log messages with the "external_storage" tag.
const (
ObjectNotFound = "object not found"
ReadObjectListFailed = "read object list failed"
ObjectUploaded = "object uploaded"
ObjectNotFound = "object not found"
ReadObjectListFailed = "read object list failed"
ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName"
ObjectUploaded = "object uploaded"
)
// Log messages with the "external_storage_tree" tag.

View file

@ -1,73 +0,0 @@
package container
import (
"fmt"
"strings"
containercontract "git.frostfs.info/TrueCloudLab/frostfs-contract/container"
containerclient "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/container"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/rpcclient"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/actor"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/wallet"
)
type Client struct {
contract *containerclient.Contract
}
type Config struct {
ContractHash util.Uint160
Key *keys.PrivateKey
RPCClient *rpcclient.Client
}
func New(cfg Config) (*Client, error) {
var err error
key := cfg.Key
if key == nil {
if key, err = keys.NewPrivateKey(); err != nil {
return nil, fmt.Errorf("generate anon private key for container contract: %w", err)
}
}
acc := wallet.NewAccountFromPrivateKey(key)
act, err := actor.NewSimple(cfg.RPCClient, acc)
if err != nil {
return nil, fmt.Errorf("create new actor: %w", err)
}
return &Client{
contract: containerclient.New(act, cfg.ContractHash),
}, nil
}
func (c *Client) GetContainerByID(cnrID cid.ID) (*container.Container, error) {
items, err := c.contract.Get(cnrID[:])
if err != nil {
if strings.Contains(err.Error(), containercontract.NotFoundError) {
return nil, fmt.Errorf("%w: %s", handler.ErrContainerNotFound, err)
}
return nil, err
}
if len(items) != 4 {
return nil, fmt.Errorf("unexpected container stack item count: %d", len(items))
}
cnrBytes, err := items[0].TryBytes()
if err != nil {
return nil, fmt.Errorf("could not get byte array of container: %w", err)
}
var cnr container.Container
if err = cnr.Unmarshal(cnrBytes); err != nil {
return nil, fmt.Errorf("can't unmarshal container: %w", err)
}
return &cnr, nil
}

View file

@ -1,34 +0,0 @@
package util
import (
"fmt"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
"github.com/nspcc-dev/neo-go/pkg/util"
)
// ResolveContractHash determine contract hash by resolving NNS name.
func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) {
if hash, err := util.Uint160DecodeStringLE(contractHash); err == nil {
return hash, nil
}
splitName := strings.Split(contractHash, ".")
if len(splitName) != 2 {
return util.Uint160{}, fmt.Errorf("invalid contract name: '%s'", contractHash)
}
var domain container.Domain
domain.SetName(splitName[0])
domain.SetZone(splitName[1])
var nns ns.NNS
if err := nns.Dial(rpcAddress); err != nil {
return util.Uint160{}, fmt.Errorf("dial nns %s: %w", rpcAddress, err)
}
defer nns.Close()
return nns.ResolveContractHash(domain)
}

View file

@ -1,9 +1,11 @@
{{$container := .Container}}
{{ $prefix := trimPrefix .Prefix }}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8"/>
<title>Index of {{.Protocol}}://{{$container}}/{{.Prefix}}</title>
<title>Index of {{.Protocol}}://{{$container}}
/{{if $prefix}}/{{$prefix}}/{{end}}</title>
<style>
.alert {
width: 80%;
@ -38,7 +40,7 @@
</style>
</head>
<body>
<h1>Index of {{.Protocol}}://{{$container}}/{{.Prefix}}</h1>
<h1>Index of {{.Protocol}}://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}</h1>
{{ if .HasErrors }}
<div class="alert">
Errors occurred while processing the request. Perhaps some objects are missing
@ -55,11 +57,11 @@
</tr>
</thead>
<tbody>
{{ $parentPrefix := getParent .Prefix }}
{{if $parentPrefix }}
{{ $trimmedPrefix := trimPrefix $prefix }}
{{if $trimmedPrefix }}
<tr>
<td>
⮐<a href="/get/{{$container}}{{ urlencode $parentPrefix }}/">..</a>
⮐<a href="/get/{{$container}}{{ urlencode $trimmedPrefix }}/">..</a>
</td>
<td></td>
<td></td>

View file

@ -7,6 +7,7 @@ import (
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
@ -51,10 +52,10 @@ type (
var (
// ErrNodeNotFound is returned from ServiceClient in case of not found error.
ErrNodeNotFound = errors.New("not found")
ErrNodeNotFound = layer.ErrNodeNotFound
// ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error.
ErrNodeAccessDenied = errors.New("access denied")
ErrNodeAccessDenied = layer.ErrNodeAccessDenied
)
const (
@ -258,7 +259,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
nodes = filterMultipartNodes(nodes)
if len(nodes) == 0 {
return nil, ErrNodeNotFound
return nil, layer.ErrNodeNotFound
}
if len(nodes) != 1 {
c.reqLogger(ctx).Warn(logs.FoundSeveralSystemTreeNodes, zap.String("name", name), logs.TagField(logs.TagExternalStorageTree))
@ -302,7 +303,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
}
if targetIndexNode == -1 {
return nil, fmt.Errorf("latest version: %w", ErrNodeNotFound)
return nil, fmt.Errorf("latest version: %w", layer.ErrNodeNotFound)
}
return nodes[targetIndexNode], nil
@ -323,23 +324,20 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator)
}
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) {
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
defer span.End()
rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator))
rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
}
return nil, err
return nil, "", err
}
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
if err != nil {
if errors.Is(err, ErrNodeNotFound) {
return nil, nil
return nil, "", nil
}
return nil, err
return nil, "", err
}
nodesMap := make(map[string][]NodeResponse, len(subTree))
@ -349,6 +347,10 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
}
fileName := GetFilename(node)
if !strings.HasPrefix(fileName, tailPrefix) {
continue
}
nodes := nodesMap[fileName]
// Add all nodes if flag latestOnly is false.
@ -372,7 +374,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
result = append(result, nodeResponseToNodeInfo(nodes)...)
}
return result, nil
return result, strings.TrimSuffix(prefix, tailPrefix), nil
}
func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo {
@ -384,6 +386,22 @@ func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo {
return nodesInfo
}
func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) {
rootID := []uint64{0}
path := strings.Split(prefix, separator)
tailPrefix := path[len(path)-1]
if len(path) > 1 {
var err error
rootID, err = c.getPrefixNodeID(ctx, bktInfo, treeID, path[:len(path)-1])
if err != nil {
return nil, "", err
}
}
return rootID, tailPrefix, nil
}
func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) ([]uint64, error) {
p := &GetNodesParams{
CnrID: bktInfo.CID,
@ -406,7 +424,7 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
}
if len(intermediateNodes) == 0 {
return nil, ErrNodeNotFound
return nil, layer.ErrNodeNotFound
}
return intermediateNodes, nil