Compare commits

..

No commits in common. "master" and "master" have entirely different histories.

31 changed files with 604 additions and 1618 deletions

View file

@ -1,4 +1,4 @@
FROM golang:1.24-alpine AS basebuilder FROM golang:1.22-alpine AS basebuilder
RUN apk add --update make bash ca-certificates RUN apk add --update make bash ca-certificates
FROM basebuilder AS builder FROM basebuilder AS builder

View file

@ -10,7 +10,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
go_versions: [ '1.23', '1.24' ] go_versions: [ '1.22', '1.23' ]
fail-fast: false fail-fast: false
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3

View file

@ -14,7 +14,7 @@ jobs:
- name: Set up Go - name: Set up Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.24' go-version: '1.23'
cache: true cache: true
- name: Install linters - name: Install linters
@ -28,7 +28,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
go_versions: [ '1.23', '1.24' ] go_versions: [ '1.22', '1.23' ]
fail-fast: false fail-fast: false
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
@ -53,7 +53,7 @@ jobs:
- name: Set up Go - name: Set up Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.24' go-version: '1.23'
- name: Run integration tests - name: Run integration tests
run: |- run: |-

View file

@ -16,7 +16,7 @@ jobs:
- name: Setup Go - name: Setup Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '1.23' go-version: '1.22'
check-latest: true check-latest: true
- name: Install govulncheck - name: Install govulncheck

View file

@ -22,6 +22,9 @@ linters-settings:
# 'default' case is present, even if all enum members aren't listed in the # 'default' case is present, even if all enum members aren't listed in the
# switch # switch
default-signifies-exhaustive: true default-signifies-exhaustive: true
govet:
# report about shadowed variables
check-shadowing: false
custom: custom:
truecloudlab-linters: truecloudlab-linters:
path: bin/external_linters.so path: bin/external_linters.so

View file

@ -4,12 +4,9 @@ This document outlines major changes between releases.
## [Unreleased] ## [Unreleased]
- Update Go to 1.23 (#228)
### Added ### Added
- Add handling quota limit reached error (#187) - Add handling quota limit reached error (#187)
- Add slash clipping for FileName attribute (#174) - Add slash clipping for FileName attribute (#174)
- Add new format of tag names config
## [0.32.3] - 2025-02-05 ## [0.32.3] - 2025-02-05

View file

@ -2,9 +2,9 @@
REPO ?= $(shell go list -m) REPO ?= $(shell go list -m)
VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") VERSION ?= $(shell git describe --tags --match "v*" --dirty --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
GO_VERSION ?= 1.23 GO_VERSION ?= 1.22
LINT_VERSION ?= 1.64.8 LINT_VERSION ?= 1.60.3
TRUECLOUDLAB_LINT_VERSION ?= 0.0.10 TRUECLOUDLAB_LINT_VERSION ?= 0.0.6
BUILD ?= $(shell date -u --iso=seconds) BUILD ?= $(shell date -u --iso=seconds)
HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs-http-gw HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs-http-gw
@ -30,10 +30,9 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \
sed "s/-/~/")-${OS_RELEASE} sed "s/-/~/")-${OS_RELEASE}
.PHONY: debpackage debclean .PHONY: debpackage debclean
FUZZING_DIR = $(shell pwd)/tests/fuzzing/files FUZZ_NGFUZZ_DIR ?= ""
NGFUZZ_REPO = https://gitflic.ru/project/yadro/ngfuzz.git
FUZZ_TIMEOUT ?= 30 FUZZ_TIMEOUT ?= 30
FUZZ_FUNCTIONS ?= "" FUZZ_FUNCTIONS ?= "all"
FUZZ_AUX ?= "" FUZZ_AUX ?= ""
# Make all binaries # Make all binaries
@ -100,22 +99,18 @@ check-ngfuzz:
exit 1; \ exit 1; \
fi fi
.PHONY: install-ngfuzz .PHONY: install-fuzzing-deps
install-ngfuzz: install-fuzzing-deps: check-clang check-ngfuzz
ifeq (,$(wildcard $(FUZZING_DIR)/ngfuzz))
@rm -rf $(FUZZING_DIR)/ngfuzz
@git clone $(NGFUZZ_REPO) $(FUZZING_DIR)/ngfuzz
@cd $(FUZZING_DIR)/ngfuzz && make
endif
.PHONY: fuzz .PHONY: fuzz
fuzz: check-clang install-ngfuzz fuzz: install-fuzzing-deps
@START_PATH=$$(pwd); \ @START_PATH=$$(pwd); \
ROOT_PATH=$$(realpath --relative-to=$(FUZZING_DIR)/ngfuzz $$START_PATH) ; \ ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \
cd $(FUZZING_DIR)/ngfuzz && \ cd $(FUZZ_NGFUZZ_DIR) && \
./bin/ngfuzz clean && \ ./ngfuzz -clean && \
env CGO_ENABLED=1 ./bin/ngfuzz fuzz --funcs $(FUZZ_FUNCTIONS) --rootdir $$ROOT_PATH --timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \ ./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \
./bin/ngfuzz coverage --rootdir $$ROOT_PATH ./ngfuzz -report
# Reformat code # Reformat code
fmt: fmt:
@ -155,7 +150,7 @@ dirty-image:
@@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR)
@rm -rf $(TMP_DIR)/linters @rm -rf $(TMP_DIR)/linters
@rmdir $(TMP_DIR) 2>/dev/null || true @rmdir $(TMP_DIR) 2>/dev/null || true
@CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION)
# Run linters # Run linters
lint: lint:

View file

@ -22,8 +22,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net"
containerClient "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/contracts/container"
contractsUtil "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/contracts/util"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/service/frostfs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
@ -41,7 +39,6 @@ import (
"github.com/nspcc-dev/neo-go/cli/flags" "github.com/nspcc-dev/neo-go/cli/flags"
"github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/cli/input"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/rpcclient"
"github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neo-go/pkg/wallet"
"github.com/panjf2000/ants/v2" "github.com/panjf2000/ants/v2"
@ -103,23 +100,21 @@ type (
workerPoolSize int workerPoolSize int
logLevelConfig *logLevelConfig logLevelConfig *logLevelConfig
mu sync.RWMutex mu sync.RWMutex
defaultTimestamp bool defaultTimestamp bool
archiveCompression bool archiveCompression bool
clientCut bool clientCut bool
returnIndexPage bool returnIndexPage bool
indexPageTemplate string indexPageTemplate string
bufferMaxSizeForPut uint64 bufferMaxSizeForPut uint64
namespaceHeader string namespaceHeader string
defaultNamespaces []string defaultNamespaces []string
cors *data.CORSRule cors *data.CORSRule
enableFilepathFallback bool enableFilepathFallback bool
enableFilepathSlashFallback bool
} }
tagsConfig struct { tagsConfig struct {
tagLogs sync.Map tagLogs sync.Map
defaultLvl zap.AtomicLevel
} }
logLevelConfig struct { logLevelConfig struct {
@ -139,34 +134,19 @@ func newLogLevel(v *viper.Viper) zap.AtomicLevel {
} }
func newTagsConfig(v *viper.Viper, ll zapcore.Level) *tagsConfig { func newTagsConfig(v *viper.Viper, ll zapcore.Level) *tagsConfig {
t := tagsConfig{defaultLvl: zap.NewAtomicLevelAt(ll)} var t tagsConfig
if err := t.update(v, ll); err != nil { if err := t.update(v, ll); err != nil {
// panic here is analogue of the similar panic during common log level initialization. // panic here is analogue of the similar panic during common log level initialization.
panic(err.Error()) panic(err.Error())
} }
return &t return &t
} }
func newLogLevelConfig(lvl zap.AtomicLevel, tagsConfig *tagsConfig) *logLevelConfig { func newLogLevelConfig(lvl zap.AtomicLevel, tagsConfig *tagsConfig) *logLevelConfig {
cfg := &logLevelConfig{ return &logLevelConfig{
logLevel: lvl, logLevel: lvl,
tagsConfig: tagsConfig, tagsConfig: tagsConfig,
} }
cfg.setMinLogLevel()
return cfg
}
func (l *logLevelConfig) setMinLogLevel() {
l.tagsConfig.tagLogs.Range(func(_, value any) bool {
v := value.(zapcore.Level)
if v < l.logLevel.Level() {
l.logLevel.SetLevel(v)
}
return true
})
} }
func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) { func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) {
@ -179,34 +159,34 @@ func (l *logLevelConfig) update(cfg *viper.Viper, log *zap.Logger) {
if err := l.tagsConfig.update(cfg, l.logLevel.Level()); err != nil { if err := l.tagsConfig.update(cfg, l.logLevel.Level()); err != nil {
log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) log.Warn(logs.TagsLogConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
} }
l.setMinLogLevel()
} }
func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool { func (t *tagsConfig) LevelEnabled(tag string, tgtLevel zapcore.Level) bool {
lvl, ok := t.tagLogs.Load(tag) lvl, ok := t.tagLogs.Load(tag)
if !ok { if !ok {
return t.defaultLvl.Enabled(tgtLevel) return false
} }
return lvl.(zapcore.Level).Enabled(tgtLevel) return lvl.(zapcore.Level).Enabled(tgtLevel)
} }
func (t *tagsConfig) DefaultEnabled(lvl zapcore.Level) bool {
return t.defaultLvl.Enabled(lvl)
}
func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error { func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error {
tags, err := fetchLogTagsConfig(cfg, ll) tags, err := fetchLogTagsConfig(cfg, ll)
if err != nil { if err != nil {
return err return err
} }
t.tagLogs.Range(func(key, _ any) bool { t.tagLogs.Range(func(key, value any) bool {
k := key.(string) k := key.(string)
v := value.(zapcore.Level)
if _, ok := tags[k]; !ok { if lvl, ok := tags[k]; ok {
if lvl != v {
t.tagLogs.Store(key, lvl)
}
} else {
t.tagLogs.Delete(key) t.tagLogs.Delete(key)
delete(tags, k)
} }
return true return true
}) })
@ -214,7 +194,6 @@ func (t *tagsConfig) update(cfg *viper.Viper, ll zapcore.Level) error {
for k, v := range tags { for k, v := range tags {
t.tagLogs.Store(k, v) t.tagLogs.Store(k, v)
} }
t.defaultLvl.SetLevel(ll)
return nil return nil
} }
@ -279,14 +258,6 @@ func (a *app) initContainers(ctx context.Context) {
a.corsCnrID = *corsCnrID a.corsCnrID = *corsCnrID
} }
func (a *app) initRPCClient(ctx context.Context) *rpcclient.Client {
rpcCli, err := rpcclient.New(ctx, a.config().GetString(cfgRPCEndpoint), rpcclient.Options{})
if err != nil {
a.log.Fatal(logs.InitRPCClientFailed, zap.Error(err), logs.TagField(logs.TagApp))
}
return rpcCli
}
func (a *app) initAppSettings(lc *logLevelConfig) { func (a *app) initAppSettings(lc *logLevelConfig) {
a.settings = &appSettings{ a.settings = &appSettings{
reconnectInterval: fetchReconnectInterval(a.config()), reconnectInterval: fetchReconnectInterval(a.config()),
@ -308,7 +279,6 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
indexPage, indexEnabled := fetchIndexPageTemplate(v, l) indexPage, indexEnabled := fetchIndexPageTemplate(v, l)
cors := fetchCORSConfig(v) cors := fetchCORSConfig(v)
enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback) enableFilepathFallback := v.GetBool(cfgFeaturesEnableFilepathFallback)
enableFilepathSlashFallback := v.GetBool(cfgFeaturesEnableFilepathSlashFallback)
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
@ -324,7 +294,6 @@ func (s *appSettings) update(v *viper.Viper, l *zap.Logger) {
s.indexPageTemplate = indexPage s.indexPageTemplate = indexPage
s.cors = cors s.cors = cors
s.enableFilepathFallback = enableFilepathFallback s.enableFilepathFallback = enableFilepathFallback
s.enableFilepathSlashFallback = enableFilepathSlashFallback
} }
func (s *loggerSettings) DroppedLogsInc() { func (s *loggerSettings) DroppedLogsInc() {
@ -435,12 +404,6 @@ func (s *appSettings) EnableFilepathFallback() bool {
return s.enableFilepathFallback return s.enableFilepathFallback
} }
func (s *appSettings) EnableFilepathSlashFallback() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.enableFilepathSlashFallback
}
func (a *app) initResolver() { func (a *app) initResolver() {
var err error var err error
a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig()) a.resolver, err = resolver.NewContainerResolver(a.getResolverConfig())
@ -709,7 +672,7 @@ func (a *app) configReload(ctx context.Context) {
return return
} }
a.settings.logLevelConfig.update(a.cfg.config(), a.log) a.settings.logLevelConfig.update(a.cfg.settings, a.log)
if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil { if err := a.settings.dialerSource.Update(fetchMultinetConfig(a.config(), a.log)); err != nil {
a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp)) a.log.Warn(logs.MultinetConfigWontBeUpdated, zap.Error(err), logs.TagField(logs.TagApp))
@ -761,22 +724,7 @@ func (a *app) stopServices() {
} }
func (a *app) configureRouter(workerPool *ants.Pool) { func (a *app) configureRouter(workerPool *ants.Pool) {
rpcCli := a.initRPCClient(a.ctx) a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), workerPool)
cnrContractName := a.config().GetString(cfgContractsContainerName)
rpcEndpoint := a.config().GetString(cfgRPCEndpoint)
cnrAddr, err := contractsUtil.ResolveContractHash(cnrContractName, rpcEndpoint)
if err != nil {
a.log.Fatal(logs.FailedToResolveContractHash, zap.Error(err), logs.TagField(logs.TagApp))
}
cnrClient, err := containerClient.New(containerClient.Config{
ContractHash: cnrAddr,
Key: a.key,
RPCClient: rpcCli,
})
if err != nil {
a.log.Fatal(logs.InitContainerContractFailed, zap.Error(err), logs.TagField(logs.TagApp))
}
a.handle = handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool), a.log), cnrClient, workerPool)
r := router.New() r := router.New()
r.RedirectTrailingSlash = true r.RedirectTrailingSlash = true

View file

@ -41,7 +41,6 @@ type zapCoreTagFilterWrapper struct {
type TagFilterSettings interface { type TagFilterSettings interface {
LevelEnabled(tag string, lvl zapcore.Level) bool LevelEnabled(tag string, lvl zapcore.Level) bool
DefaultEnabled(lvl zapcore.Level) bool
} }
func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool { func (c *zapCoreTagFilterWrapper) Enabled(level zapcore.Level) bool {
@ -64,26 +63,24 @@ func (c *zapCoreTagFilterWrapper) Check(entry zapcore.Entry, checked *zapcore.Ch
} }
func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error { func (c *zapCoreTagFilterWrapper) Write(entry zapcore.Entry, fields []zapcore.Field) error {
if c.shouldSkip(entry, fields, c.extra) { if c.shouldSkip(entry, fields) || c.shouldSkip(entry, c.extra) {
return nil return nil
} }
return c.core.Write(entry, fields) return c.core.Write(entry, fields)
} }
func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field, extra []zap.Field) bool { func (c *zapCoreTagFilterWrapper) shouldSkip(entry zapcore.Entry, fields []zap.Field) bool {
for _, field := range fields { for _, field := range fields {
if field.Key == logs.TagFieldName && field.Type == zapcore.StringType { if field.Key == logs.TagFieldName && field.Type == zapcore.StringType {
return !c.settings.LevelEnabled(field.String, entry.Level) if !c.settings.LevelEnabled(field.String, entry.Level) {
} return true
} }
for _, field := range extra { break
if field.Key == logs.TagFieldName && field.Type == zapcore.StringType {
return !c.settings.LevelEnabled(field.String, entry.Level)
} }
} }
return !c.settings.DefaultEnabled(entry.Level) return false
} }
func (c *zapCoreTagFilterWrapper) Sync() error { func (c *zapCoreTagFilterWrapper) Sync() error {
@ -130,13 +127,14 @@ func newLogEncoder() zapcore.Encoder {
// //
// See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
func newStdoutLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger { func newStdoutLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings LoggerAppSettings, tagSetting TagFilterSettings) *Logger {
stdout := zapcore.AddSync(os.Stdout) stdout := zapcore.AddSync(os.Stderr)
consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, lvl) consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, lvl)
consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, loggerSettings, tagSetting) consoleOutCore = applyZapCoreMiddlewares(consoleOutCore, v, loggerSettings, tagSetting)
return &Logger{ return &Logger{
logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), logger: zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))),
lvl: lvl,
} }
} }
@ -154,6 +152,7 @@ func newJournaldLogger(v *viper.Viper, lvl zap.AtomicLevel, loggerSettings Logge
return &Logger{ return &Logger{
logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))), logger: zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))),
lvl: lvl,
} }
} }

View file

@ -62,8 +62,6 @@ const (
defaultMultinetFallbackDelay = 300 * time.Millisecond defaultMultinetFallbackDelay = 300 * time.Millisecond
defaultContainerContractName = "container.frostfs"
cfgServer = "server" cfgServer = "server"
cfgTLSEnabled = "tls.enabled" cfgTLSEnabled = "tls.enabled"
cfgTLSCertFile = "tls.cert_file" cfgTLSCertFile = "tls.cert_file"
@ -115,7 +113,7 @@ const (
cfgLoggerTags = "logger.tags" cfgLoggerTags = "logger.tags"
cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d." cfgLoggerTagsPrefixTmpl = cfgLoggerTags + ".%d."
cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "names" cfgLoggerTagsNameTmpl = cfgLoggerTagsPrefixTmpl + "name"
cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level" cfgLoggerTagsLevelTmpl = cfgLoggerTagsPrefixTmpl + "level"
// Wallet. // Wallet.
@ -182,9 +180,8 @@ const (
cfgMultinetSubnets = "multinet.subnets" cfgMultinetSubnets = "multinet.subnets"
// Feature. // Feature.
cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback" cfgFeaturesEnableFilepathFallback = "features.enable_filepath_fallback"
cfgFeaturesEnableFilepathSlashFallback = "features.enable_filepath_slash_fallback" cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support"
cfgFeaturesTreePoolNetmapSupport = "features.tree_pool_netmap_support"
// Containers. // Containers.
cfgContainersCORS = "containers.cors" cfgContainersCORS = "containers.cors"
@ -199,9 +196,6 @@ const (
cmdConfig = "config" cmdConfig = "config"
cmdConfigDir = "config-dir" cmdConfigDir = "config-dir"
cmdListenAddress = "listen_address" cmdListenAddress = "listen_address"
// Contracts.
cfgContractsContainerName = "contracts.container.name"
) )
var ignore = map[string]struct{}{ var ignore = map[string]struct{}{
@ -214,6 +208,7 @@ var defaultTags = []string{logs.TagApp, logs.TagDatapath, logs.TagExternalStorag
type Logger struct { type Logger struct {
logger *zap.Logger logger *zap.Logger
lvl zap.AtomicLevel
} }
type appCfg struct { type appCfg struct {
@ -406,9 +401,6 @@ func setDefaults(v *viper.Viper, flags *pflag.FlagSet) {
// multinet // multinet
v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay) v.SetDefault(cfgMultinetFallbackDelay, defaultMultinetFallbackDelay)
// contracts
v.SetDefault(cfgContractsContainerName, defaultContainerContractName)
if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil { if resolveMethods, err := flags.GetStringSlice(cfgResolveOrder); err == nil {
v.SetDefault(cfgResolveOrder, resolveMethods) v.SetDefault(cfgResolveOrder, resolveMethods)
} }
@ -524,8 +516,8 @@ func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]za
res := make(map[string]zapcore.Level) res := make(map[string]zapcore.Level)
for i := 0; ; i++ { for i := 0; ; i++ {
tagNames := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i)) name := v.GetString(fmt.Sprintf(cfgLoggerTagsNameTmpl, i))
if tagNames == "" { if name == "" {
break break
} }
@ -537,12 +529,7 @@ func fetchLogTagsConfig(v *viper.Viper, defaultLvl zapcore.Level) (map[string]za
} }
} }
for _, tagName := range strings.Split(tagNames, ",") { res[name] = lvl
tagName = strings.TrimSpace(tagName)
if len(tagName) != 0 {
res[tagName] = lvl
}
}
} }
if len(res) == 0 && !v.IsSet(cfgLoggerTags) { if len(res) == 0 && !v.IsSet(cfgLoggerTags) {

View file

@ -20,9 +20,8 @@ HTTP_GW_LOGGER_SAMPLING_ENABLED=false
HTTP_GW_LOGGER_SAMPLING_INITIAL=100 HTTP_GW_LOGGER_SAMPLING_INITIAL=100
HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100 HTTP_GW_LOGGER_SAMPLING_THEREAFTER=100
HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s HTTP_GW_LOGGER_SAMPLING_INTERVAL=1s
HTTP_GW_LOGGER_TAGS_0_NAMES=app,datapath HTTP_GW_LOGGER_TAGS_0_NAME=app
HTTP_GW_LOGGER_TAGS_0_LEVEL=level HTTP_GW_LOGGER_TAGS_1_NAME=datapath
HTTP_GW_LOGGER_TAGS_1_NAME=external_storage_tree
HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443 HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:443
HTTP_GW_SERVER_0_TLS_ENABLED=false HTTP_GW_SERVER_0_TLS_ENABLED=false
@ -174,13 +173,8 @@ HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl
# Enable using fallback path to search for a object by attribute # Enable using fallback path to search for a object by attribute
HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false HTTP_GW_FEATURES_ENABLE_FILEPATH_FALLBACK=false
# See description in docs/gate-configuration.md
HTTP_GW_FEATURES_ENABLE_FILEPATH_SLASH_FALLBACK=false
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true HTTP_GW_FEATURES_TREE_POOL_NETMAP_SUPPORT=true
# Containers properties # Containers properties
HTTP_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj HTTP_GW_CONTAINERS_CORS=AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
# Container contract hash (LE) or name in NNS.
HTTP_GW_CONTRACTS_CONTAINER_NAME=container.frostfs

View file

@ -30,7 +30,8 @@ logger:
thereafter: 100 thereafter: 100
interval: 1s interval: 1s
tags: tags:
- names: app,datapath - name: app
- name: datapath
level: debug level: debug
server: server:
@ -192,15 +193,8 @@ multinet:
features: features:
# Enable using fallback path to search for a object by attribute # Enable using fallback path to search for a object by attribute
enable_filepath_fallback: false enable_filepath_fallback: false
# See description in docs/gate-configuration.md
enable_filepath_slash_fallback: false
# Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service # Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service
tree_pool_netmap_support: true tree_pool_netmap_support: true
containers: containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
contracts:
container:
# Container contract hash (LE) or name in NNS.
name: container.frostfs

View file

@ -8,6 +8,7 @@ There are some custom types used for brevity:
* `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` ( * `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` (
milliseconds). milliseconds).
# Reload on SIGHUP # Reload on SIGHUP
Some config values can be reloaded on SIGHUP signal. Some config values can be reloaded on SIGHUP signal.
@ -60,7 +61,6 @@ $ cat http.log
| `multinet` | [Multinet configuration](#multinet-section) | | `multinet` | [Multinet configuration](#multinet-section) |
| `features` | [Features configuration](#features-section) | | `features` | [Features configuration](#features-section) |
| `containers` | [Containers configuration](#containers-section) | | `containers` | [Containers configuration](#containers-section) |
| `contracts` | [Contracts configuration](#contracts-section) |
# General section # General section
@ -163,6 +163,7 @@ server:
| `tls.cert_file` | `string` | yes | | Path to the TLS certificate. | | `tls.cert_file` | `string` | yes | | Path to the TLS certificate. |
| `tls.key_file` | `string` | yes | | Path to the key. | | `tls.key_file` | `string` | yes | | Path to the key. |
# `logger` section # `logger` section
```yaml ```yaml
@ -175,9 +176,10 @@ logger:
thereafter: 100 thereafter: 100
interval: 1s interval: 1s
tags: tags:
- names: "app,datapath" - name: "app"
level: info level: info
- names: "external_storage_tree" - name: "datapath"
- name: "external_storage_tree"
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
@ -197,14 +199,14 @@ parameter. Available tags:
```yaml ```yaml
tags: tags:
- names: "app,datapath" - name: "app"
level: info level: info
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------| |-----------------------|------------|---------------|---------------------------|-------------------------------------------------------------------------------------------------------|
| `names` | `[]string` | yes | | Tag names separated by `,`. Possible values see below in `Tag values` section. | | `name` | `string` | yes | | Tag name. Possible values see below in `Tag values` section. |
| `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | | `level` | `string` | yes | Value from `logger.level` | Logging level for specific tag. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. |
### Tag values ### Tag values
@ -234,6 +236,7 @@ web:
| `stream_request_body` | `bool` | `true` | Enables request body streaming, and calls the handler sooner when given body is larger than the current limit. | | `stream_request_body` | `bool` | `true` | Enables request body streaming, and calls the handler sooner when given body is larger than the current limit. |
| `max_request_body_size` | `int` | `4194304` | Maximum request body size. The server rejects requests with bodies exceeding this limit. | | `max_request_body_size` | `int` | `4194304` | Maximum request body size. The server rejects requests with bodies exceeding this limit. |
# `upload-header` section # `upload-header` section
```yaml ```yaml
@ -269,6 +272,7 @@ archive:
|---------------|--------|---------------|---------------|------------------------------------------------------------------| |---------------|--------|---------------|---------------|------------------------------------------------------------------|
| `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. | | `compression` | `bool` | yes | `false` | Enable archive compression when download files by common prefix. |
# `pprof` section # `pprof` section
Contains configuration for the `pprof` profiler. Contains configuration for the `pprof` profiler.
@ -317,13 +321,14 @@ tracing:
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|--------------|----------------------------------------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------| | ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- |
| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. | | `enabled` | `bool` | yes | `false` | Flag to enable the tracing. |
| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). | | `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). |
| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. | | `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. |
| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. | | `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. |
| `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. | | `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. |
#### `attributes` subsection #### `attributes` subsection
```yaml ```yaml
@ -334,13 +339,12 @@ tracing:
value: value value: value
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|----------|---------------|---------------|------------------| |-----------------------|----------|---------------|---------------|----------------------------------------------------------|
| `key` | `string` | yes | | Attribute key. | | `key` | `string` | yes | | Attribute key. |
| `value` | `string` | yes | | Attribute value. | | `value` | `string` | yes | | Attribute value. |
# `runtime` section # `runtime` section
Contains runtime parameters. Contains runtime parameters.
```yaml ```yaml
@ -369,6 +373,7 @@ frostfs:
| `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. | | `buffer_max_size_for_put` | `uint64` | yes | `1048576` | Sets max buffer size for read payload in put operations. |
| `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. | | `tree_pool_max_attempts` | `uint32` | no | `0` | Sets max attempt to make successful tree request. Value 0 means the number of attempts equals to number of nodes in pool. |
### `cache` section ### `cache` section
```yaml ```yaml
@ -389,6 +394,7 @@ cache:
| `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. | | `netmap` | [Cache config](#cache-subsection) | `lifetime: 1m` | Cache which stores netmap. `netmap.size` isn't applicable for this cache. |
| `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`<br>`size: 1000` | Cache which stores container CORS configurations. | | `cors` | [Cache config](#cache-subsection) | `lifetime: 5m`<br>`size: 1000` | Cache which stores container CORS configurations. |
#### `cache` subsection #### `cache` subsection
```yaml ```yaml
@ -401,6 +407,7 @@ size: 1000
| `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. | | `lifetime` | `duration` | depends on cache | Lifetime of entries in cache. |
| `size` | `int` | depends on cache | LRU cache size. | | `size` | `int` | depends on cache | LRU cache size. |
# `resolve_bucket` section # `resolve_bucket` section
Bucket name resolving parameters from and to container ID. Bucket name resolving parameters from and to container ID.
@ -411,10 +418,10 @@ resolve_bucket:
default_namespaces: [ "", "root" ] default_namespaces: [ "", "root" ]
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|----------------------|------------|---------------|-----------------------|--------------------------------------------------| |----------------------|------------|---------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------|
| `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. | | `namespace_header` | `string` | yes | `X-Frostfs-Namespace` | Header to determine zone to resolve bucket name. |
| `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. | | `default_namespaces` | `[]string` | yes | ["","root"] | Namespaces that should be handled as default. |
# `index_page` section # `index_page` section
@ -444,9 +451,9 @@ If values are not set, settings from CORS container will be used.
```yaml ```yaml
cors: cors:
allow_origin: "*" allow_origin: "*"
allow_methods: [ "GET", "HEAD" ] allow_methods: ["GET", "HEAD"]
allow_headers: [ "Authorization" ] allow_headers: ["Authorization"]
expose_headers: [ "*" ] expose_headers: ["*"]
allow_credentials: false allow_credentials: false
max_age: 600 max_age: 600
``` ```
@ -466,15 +473,15 @@ Configuration of multinet support.
```yaml ```yaml
multinet: multinet:
enabled: false enabled: false
balancer: roundrobin balancer: roundrobin
restrict: false restrict: false
fallback_delay: 300ms fallback_delay: 300ms
subnets: subnets:
- mask: 1.2.3.4/24 - mask: 1.2.3.4/24
source_ips: source_ips:
- 1.2.3.4 - 1.2.3.4
- 1.2.3.5 - 1.2.3.5
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
@ -506,15 +513,13 @@ Contains parameters for enabling features.
```yaml ```yaml
features: features:
enable_filepath_fallback: true enable_filepath_fallback: true
enable_filepath_slash_fallback: false
tree_pool_netmap_support: true tree_pool_netmap_support: true
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|-------------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |-------------------------------------|--------|---------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FileName` attribute if object with `FilePath` attribute wasn't found. | | `features.enable_filepath_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by attribute. If the value of the `FilePath` attribute in the request contains no `/` symbols or single leading `/` symbol and the object was not found, then an attempt is made to search for the object by the attribute `FileName`. |
| `features.enable_filepath_slash_fallback` | `bool` | yes | `false` | Enable using fallback path to search for a object by `FilePath`/`FileName` with/without (depends on provided value in `FilePath`/`FileName`) leading slash if object with provided `FilePath`/`FileName` wasn't found. This fallback goes before `enable_filepath_fallback`. | | `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. |
| `features.tree_pool_netmap_support` | `bool` | no | `false` | Enable using new version of tree pool, which uses netmap to select nodes, for requests to tree service. |
# `containers` section # `containers` section
@ -525,18 +530,6 @@ containers:
cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj
``` ```
| Parameter | Type | SIGHUP reload | Default value | Description | | Parameter | Type | SIGHUP reload | Default value | Description |
|-----------|----------|---------------|---------------|-----------------------------------------| |-------------|----------|---------------|---------------|-----------------------------------------|
| `cors` | `string` | no | | Container name for CORS configurations. | | `cors` | `string` | no | | Container name for CORS configurations. |
# `contracts` section
```yaml
contracts:
container:
name: container.frostfs
```
| Parameter | Type | SIGHUP reload | Default value | Description |
|------------------|----------|---------------|---------------------|----------------------------------------------|
| `container.name` | `string` | no | `container.frostfs` | Container contract hash (LE) or name in NNS. |

4
go.mod
View file

@ -1,9 +1,8 @@
module git.frostfs.info/TrueCloudLab/frostfs-http-gw module git.frostfs.info/TrueCloudLab/frostfs-http-gw
go 1.23 go 1.22
require ( require (
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241125133852-37bd75821121
git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250317082814-87bb55f992dc
@ -34,6 +33,7 @@ require (
require ( require (
dario.cat/mergo v1.0.0 // indirect dario.cat/mergo v1.0.0 // indirect
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e // indirect
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect
git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect git.frostfs.info/TrueCloudLab/hrw v1.2.1 // indirect
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect

View file

@ -12,6 +12,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@ -130,15 +131,11 @@ func parentDir(prefix string) string {
return prefix[index:] return prefix[index:]
} }
func getParent(encPrefix string) string { func trimPrefix(encPrefix string) string {
prefix, err := url.PathUnescape(encPrefix) prefix, err := url.PathUnescape(encPrefix)
if err != nil { if err != nil {
return "" return ""
} }
if prefix != "" && prefix[len(prefix)-1] == '/' {
prefix = prefix[:len(prefix)-1]
}
slashIndex := strings.LastIndex(prefix, "/") slashIndex := strings.LastIndex(prefix, "/")
if slashIndex == -1 { if slashIndex == -1 {
return "" return ""
@ -164,15 +161,10 @@ func urlencode(path string) string {
type GetObjectsResponse struct { type GetObjectsResponse struct {
objects []ResponseObject objects []ResponseObject
hasErrors bool hasErrors bool
isNative bool
} }
func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
if prefix != "" && prefix[len(prefix)-1] == '/' { nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
prefix = prefix[:len(prefix)-1]
}
nodes, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -193,7 +185,7 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
if obj.IsDeleteMarker { if obj.IsDeleteMarker {
continue continue
} }
obj.FilePath = prefix + "/" + obj.FileName obj.FilePath = prefix + obj.FileName
obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath) obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
result.objects = append(result.objects, obj) result.objects = append(result.objects, obj)
} }
@ -202,9 +194,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
} }
func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) { func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
basePath := prefix var basePath string
if basePath != "" && basePath[len(basePath)-1] != '/' { if ind := strings.LastIndex(prefix, "/"); ind != -1 {
basePath += "/" basePath = prefix[:ind+1]
} }
filters := object.NewSearchFilters() filters := object.NewSearchFilters()
@ -234,8 +226,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
log := h.reqLogger(ctx) log := h.reqLogger(ctx)
dirs := make(map[string]struct{}) dirs := make(map[string]struct{})
result := &GetObjectsResponse{ result := &GetObjectsResponse{
objects: make([]ResponseObject, 0, 100), objects: make([]ResponseObject, 0, 100),
isNative: true,
} }
for objExt := range resp { for objExt := range resp {
if objExt.Error != nil { if objExt.Error != nil {
@ -331,16 +322,28 @@ func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID,
} }
type browseParams struct { type browseParams struct {
bucketInfo *data.BucketInfo bucketInfo *data.BucketInfo
prefix string prefix string
objects *GetObjectsResponse isNative bool
listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error)
} }
func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) { func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p browseParams) {
const S3Protocol = "s3" const S3Protocol = "s3"
const FrostfsProtocol = "frostfs" const FrostfsProtocol = "frostfs"
objects := p.objects.objects ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("bucket", p.bucketInfo.Name),
zap.String("container", p.bucketInfo.CID.EncodeToString()),
zap.String("prefix", p.prefix),
))
resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToListObjects, err)
return
}
objects := resp.objects
sort.Slice(objects, func(i, j int) bool { sort.Slice(objects, func(i, j int) bool {
if objects[i].IsDir == objects[j].IsDir { if objects[i].IsDir == objects[j].IsDir {
return objects[i].FileName < objects[j].FileName return objects[i].FileName < objects[j].FileName
@ -350,7 +353,7 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p
tmpl, err := template.New("index").Funcs(template.FuncMap{ tmpl, err := template.New("index").Funcs(template.FuncMap{
"formatSize": formatSize, "formatSize": formatSize,
"getParent": getParent, "trimPrefix": trimPrefix,
"urlencode": urlencode, "urlencode": urlencode,
"parentDir": parentDir, "parentDir": parentDir,
}).Parse(h.config.IndexPageTemplate()) }).Parse(h.config.IndexPageTemplate())
@ -360,21 +363,16 @@ func (h *Handler) browseObjects(ctx context.Context, req *fasthttp.RequestCtx, p
} }
bucketName := p.bucketInfo.Name bucketName := p.bucketInfo.Name
protocol := S3Protocol protocol := S3Protocol
if p.objects.isNative { if p.isNative {
bucketName = p.bucketInfo.CID.EncodeToString() bucketName = p.bucketInfo.CID.EncodeToString()
protocol = FrostfsProtocol protocol = FrostfsProtocol
} }
prefix := p.prefix
if prefix != "" && prefix[len(prefix)-1] != '/' {
prefix += "/"
}
if err = tmpl.Execute(req, &BrowsePageData{ if err = tmpl.Execute(req, &BrowsePageData{
Container: bucketName, Container: bucketName,
Prefix: prefix, Prefix: p.prefix,
Objects: objects, Objects: objects,
Protocol: protocol, Protocol: protocol,
HasErrors: p.objects.hasErrors, HasErrors: resp.hasErrors,
}); err != nil { }); err != nil {
h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err) h.logAndSendError(ctx, req, logs.FailedToExecuteTemplate, err)
return return

View file

@ -1,42 +0,0 @@
package handler
import (
"context"
"fmt"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"go.uber.org/zap"
)
func (h *Handler) containerInfo(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) {
info := &data.BucketInfo{
CID: cnrID,
Name: cnrID.EncodeToString(),
}
res, err := h.cnrContract.GetContainerByID(cnrID)
if err != nil {
return nil, fmt.Errorf("get frostfs container: %w", err)
}
cnr := *res
if domain := container.ReadDomain(cnr); domain.Name() != "" {
info.Name = domain.Name()
info.Zone = domain.Zone()
}
info.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(cnr)
info.PlacementPolicy = cnr.PlacementPolicy()
if err = h.cache.Put(info); err != nil {
h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache,
zap.String("bucket name", info.Name),
zap.Stringer("cid", info.CID),
zap.Error(err),
logs.TagField(logs.TagDatapath))
}
return info, nil
}

View file

@ -5,8 +5,6 @@ import (
"encoding/xml" "encoding/xml"
"errors" "errors"
"fmt" "fmt"
"regexp"
"slices"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@ -80,7 +78,7 @@ func (h *Handler) Preflight(req *fasthttp.RequestCtx) {
for _, rule := range corsConfig.CORSRules { for _, rule := range corsConfig.CORSRules {
for _, o := range rule.AllowedOrigins { for _, o := range rule.AllowedOrigins {
if o == string(origin) || o == wildcard || (strings.Contains(o, "*") && match(o, string(origin))) { if o == string(origin) || o == wildcard {
for _, m := range rule.AllowedMethods { for _, m := range rule.AllowedMethods {
if m == string(method) { if m == string(method) {
if !checkSubslice(rule.AllowedHeaders, headers) { if !checkSubslice(rule.AllowedHeaders, headers) {
@ -119,11 +117,6 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) {
return return
} }
method := req.Request.Header.Peek(fasthttp.HeaderAccessControlRequestMethod)
if len(method) == 0 {
method = req.Method()
}
ctx = qostagging.ContextWithIOTag(ctx, internalIOTag) ctx = qostagging.ContextWithIOTag(ctx, internalIOTag)
cidParam, _ := req.UserValue("cid").(string) cidParam, _ := req.UserValue("cid").(string)
reqLog := h.reqLogger(ctx) reqLog := h.reqLogger(ctx)
@ -148,9 +141,9 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) {
for _, rule := range corsConfig.CORSRules { for _, rule := range corsConfig.CORSRules {
for _, o := range rule.AllowedOrigins { for _, o := range rule.AllowedOrigins {
if o == string(origin) || (strings.Contains(o, "*") && len(o) > 1 && match(o, string(origin))) { if o == string(origin) {
for _, m := range rule.AllowedMethods { for _, m := range rule.AllowedMethods {
if m == string(method) { if m == string(req.Method()) {
req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin))
req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", ")) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", "))
req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true")
@ -161,7 +154,7 @@ func (h *Handler) SetCORSHeaders(req *fasthttp.RequestCtx) {
} }
if o == wildcard { if o == wildcard {
for _, m := range rule.AllowedMethods { for _, m := range rule.AllowedMethods {
if m == string(method) { if m == string(req.Method()) {
if withCredentials { if withCredentials {
req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin)) req.Response.Header.Set(fasthttp.HeaderAccessControlAllowOrigin, string(origin))
req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true") req.Response.Header.Set(fasthttp.HeaderAccessControlAllowCredentials, "true")
@ -197,6 +190,9 @@ func (h *Handler) getCORSConfig(ctx context.Context, log *zap.Logger, cidStr str
addr.SetContainer(h.corsCnrID) addr.SetContainer(h.corsCnrID)
addr.SetObject(objID) addr.SetObject(objID)
corsObj, err := h.frostfs.GetObject(ctx, PrmObjectGet{ corsObj, err := h.frostfs.GetObject(ctx, PrmObjectGet{
PrmAuth: PrmAuth{
BearerToken: bearerToken(ctx),
},
Address: addr, Address: addr,
}) })
if err != nil { if err != nil {
@ -220,7 +216,11 @@ func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID,
filters.AddRootFilter() filters.AddRootFilter()
filters.AddFilter(object.AttributeFilePath, fmt.Sprintf(corsFilePathTemplate, cnrID), object.MatchStringEqual) filters.AddFilter(object.AttributeFilePath, fmt.Sprintf(corsFilePathTemplate, cnrID), object.MatchStringEqual)
prmAuth := PrmAuth{
BearerToken: bearerToken(ctx),
}
res, err := h.frostfs.SearchObjects(ctx, PrmObjectSearch{ res, err := h.frostfs.SearchObjects(ctx, PrmObjectSearch{
PrmAuth: prmAuth,
Container: h.corsCnrID, Container: h.corsCnrID,
Filters: filters, Filters: filters,
}) })
@ -239,6 +239,7 @@ func (h *Handler) getLastCORSObject(ctx context.Context, cnrID cid.ID) (oid.ID,
err = res.Iterate(func(id oid.ID) bool { err = res.Iterate(func(id oid.ID) bool {
addr.SetObject(id) addr.SetObject(id)
obj, headErr = h.frostfs.HeadObject(ctx, PrmObjectHead{ obj, headErr = h.frostfs.HeadObject(ctx, PrmObjectHead{
PrmAuth: prmAuth,
Address: addr, Address: addr,
}) })
if headErr != nil { if headErr != nil {
@ -317,9 +318,12 @@ func setCORSHeadersFromRule(c *fasthttp.RequestCtx, cors *data.CORSRule) {
} }
func checkSubslice(slice []string, subSlice []string) bool { func checkSubslice(slice []string, subSlice []string) bool {
if slices.Contains(slice, wildcard) { if sliceContains(slice, wildcard) {
return true return true
} }
if len(subSlice) > len(slice) {
return false
}
for _, r := range subSlice { for _, r := range subSlice {
if !sliceContains(slice, r) { if !sliceContains(slice, r) {
return false return false
@ -330,16 +334,9 @@ func checkSubslice(slice []string, subSlice []string) bool {
func sliceContains(slice []string, str string) bool { func sliceContains(slice []string, str string) bool {
for _, s := range slice { for _, s := range slice {
if s == str || (strings.Contains(s, "*") && match(s, str)) { if s == str {
return true return true
} }
} }
return false return false
} }
func match(tmpl, str string) bool {
regexpStr := "^" + regexp.QuoteMeta(tmpl) + "$"
regexpStr = regexpStr[:strings.Index(regexpStr, "*")-1] + "." + regexpStr[strings.Index(regexpStr, "*"):]
reg := regexp.MustCompile(regexpStr)
return reg.Match([]byte(str))
}

View file

@ -4,7 +4,6 @@ import (
"encoding/base64" "encoding/base64"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"net/http"
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
@ -408,12 +407,6 @@ func TestCheckSubslice(t *testing.T) {
actual: []string{"str1", "str5"}, actual: []string{"str1", "str5"},
expected: false, expected: false,
}, },
{
name: "wildcard in allowed",
allowed: []string{"str*"},
actual: []string{"str", "str5"},
expected: true,
},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.expected, checkSubslice(tc.allowed, tc.actual)) require.Equal(t, tc.expected, checkSubslice(tc.allowed, tc.actual))
@ -421,489 +414,6 @@ func TestCheckSubslice(t *testing.T) {
} }
} }
func TestAllowedOriginWildcards(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-allowed-origin-wildcards"
cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private)
require.NoError(t, err)
hc.frostfs.SetContainer(cnrID, cnr)
cfg := &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"*suffix.example"},
AllowedMethods: []string{"GET"},
},
{
AllowedOrigins: []string{"https://*example"},
AllowedMethods: []string{"GET"},
},
{
AllowedOrigins: []string{"prefix.example*"},
AllowedMethods: []string{"GET"},
},
},
}
setCORSObject(t, hc, cnrID, cfg, 1)
for _, tc := range []struct {
name string
handler func(*fasthttp.RequestCtx)
requestHeaders map[string]string
expectedHeaders map[string]string
expectedStatus int
}{
{
name: "set cors headers, empty request cors headers",
handler: hc.Handler().SetCORSHeaders,
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
},
{
name: "set cors headers, invalid origin",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://origin.com",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
},
{
name: "set cors headers, first rule, no symbols in place of wildcard",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "suffix.example",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "suffix.example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "set cors headers, first rule, valid origin",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://suffix.example",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "http://suffix.example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "set cors headers, first rule, invalid origin",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://suffix-example",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
},
{
name: "set cors headers, second rule, no symbols in place of wildcard",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://example",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "https://example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "set cors headers, second rule, valid origin",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "https://www.example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "set cors headers, second rule, invalid origin",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
},
{
name: "set cors headers, third rule, no symbols in place of wildcard",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "prefix.example",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "prefix.example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "set cors headers, third rule, valid origin",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "prefix.example.com",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "set cors headers, third rule, invalid origin",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "www.prefix.example",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
},
{
name: "set cors headers, third rule, invalid request method in header",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "prefix.example.com",
fasthttp.HeaderAccessControlRequestMethod: "PUT",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
},
{
name: "set cors headers, third rule, valid request method in header",
handler: hc.Handler().SetCORSHeaders,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "prefix.example.com",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "preflight, empty request cors headers",
handler: hc.Handler().Preflight,
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
expectedStatus: http.StatusBadRequest,
},
{
name: "preflight, invalid origin",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://origin.com",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
expectedStatus: http.StatusForbidden,
},
{
name: "preflight, first rule, no symbols in place of wildcard",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "suffix.example",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "suffix.example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "prelight, first rule, valid origin",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://suffix.example",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "http://suffix.example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "preflight, first rule, invalid origin",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "http://suffix-example",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
expectedStatus: http.StatusForbidden,
},
{
name: "preflight, second rule, no symbols in place of wildcard",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://example",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "https://example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "preflight, second rule, valid origin",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "https://www.example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "preflight, second rule, invalid origin",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
expectedStatus: http.StatusForbidden,
},
{
name: "preflight, third rule, no symbols in place of wildcard",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "prefix.example",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "prefix.example",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "preflight, third rule, valid origin",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "prefix.example.com",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "prefix.example.com",
fasthttp.HeaderAccessControlAllowMethods: "GET",
},
},
{
name: "preflight, third rule, invalid origin",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "www.prefix.example",
fasthttp.HeaderAccessControlRequestMethod: "GET",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
expectedStatus: http.StatusForbidden,
},
{
name: "preflight, third rule, invalid request method in header",
handler: hc.Handler().Preflight,
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "prefix.example.com",
fasthttp.HeaderAccessControlRequestMethod: "PUT",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
},
expectedStatus: http.StatusForbidden,
},
} {
t.Run(tc.name, func(t *testing.T) {
r := prepareCORSRequest(t, bktName, tc.requestHeaders)
tc.handler(r)
expectedStatus := fasthttp.StatusOK
if tc.expectedStatus != 0 {
expectedStatus = tc.expectedStatus
}
require.Equal(t, expectedStatus, r.Response.StatusCode())
for k, v := range tc.expectedHeaders {
require.Equal(t, v, string(r.Response.Header.Peek(k)))
}
})
}
}
func TestAllowedHeaderWildcards(t *testing.T) {
hc := prepareHandlerContext(t)
bktName := "bucket-allowed-header-wildcards"
cnrID, cnr, err := hc.prepareContainer(bktName, acl.Private)
require.NoError(t, err)
hc.frostfs.SetContainer(cnrID, cnr)
cfg := &data.CORSConfiguration{
CORSRules: []data.CORSRule{
{
AllowedOrigins: []string{"https://www.example.com"},
AllowedMethods: []string{"HEAD"},
AllowedHeaders: []string{"*-suffix"},
},
{
AllowedOrigins: []string{"https://www.example.com"},
AllowedMethods: []string{"HEAD"},
AllowedHeaders: []string{"start-*-end"},
},
{
AllowedOrigins: []string{"https://www.example.com"},
AllowedMethods: []string{"HEAD"},
AllowedHeaders: []string{"X-Amz-*"},
},
},
}
setCORSObject(t, hc, cnrID, cfg, 1)
for _, tc := range []struct {
name string
requestHeaders map[string]string
expectedHeaders map[string]string
expectedStatus int
}{
{
name: "first rule, valid headers",
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
fasthttp.HeaderAccessControlRequestHeaders: "header-suffix, -suffix",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlAllowMethods: "HEAD",
fasthttp.HeaderAccessControlAllowHeaders: "header-suffix, -suffix",
},
},
{
name: "first rule, invalid headers",
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
fasthttp.HeaderAccessControlRequestHeaders: "header-suffix-*",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
},
expectedStatus: http.StatusForbidden,
},
{
name: "second rule, valid headers",
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
fasthttp.HeaderAccessControlRequestHeaders: "start--end, start-header-end",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlAllowMethods: "HEAD",
fasthttp.HeaderAccessControlAllowHeaders: "start--end, start-header-end",
},
},
{
name: "second rule, invalid header ending",
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
fasthttp.HeaderAccessControlRequestHeaders: "start-header-end-*",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
},
expectedStatus: http.StatusForbidden,
},
{
name: "second rule, invalid header beginning",
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
fasthttp.HeaderAccessControlRequestHeaders: "*-start-header-end",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
},
expectedStatus: http.StatusForbidden,
},
{
name: "third rule, valid headers",
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
fasthttp.HeaderAccessControlRequestHeaders: "X-Amz-Date, X-Amz-Content-Sha256",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlAllowMethods: "HEAD",
fasthttp.HeaderAccessControlAllowHeaders: "X-Amz-Date, X-Amz-Content-Sha256",
},
},
{
name: "third rule, invalid headers",
requestHeaders: map[string]string{
fasthttp.HeaderOrigin: "https://www.example.com",
fasthttp.HeaderAccessControlRequestMethod: "HEAD",
fasthttp.HeaderAccessControlRequestHeaders: "Authorization",
},
expectedHeaders: map[string]string{
fasthttp.HeaderAccessControlAllowOrigin: "",
fasthttp.HeaderAccessControlAllowMethods: "",
fasthttp.HeaderAccessControlAllowHeaders: "",
},
expectedStatus: http.StatusForbidden,
},
} {
t.Run(tc.name, func(t *testing.T) {
r := prepareCORSRequest(t, bktName, tc.requestHeaders)
hc.Handler().Preflight(r)
expectedStatus := http.StatusOK
if tc.expectedStatus != 0 {
expectedStatus = tc.expectedStatus
}
require.Equal(t, expectedStatus, r.Response.StatusCode())
for k, v := range tc.expectedHeaders {
require.Equal(t, v, string(r.Response.Header.Peek(k)))
}
})
}
}
func setCORSObject(t *testing.T, hc *handlerContext, cnrID cid.ID, corsConfig *data.CORSConfiguration, epoch uint64) { func setCORSObject(t *testing.T, hc *handlerContext, cnrID cid.ID, corsConfig *data.CORSConfiguration, epoch uint64) {
payload, err := xml.Marshal(corsConfig) payload, err := xml.Marshal(corsConfig)
require.NoError(t, err) require.NoError(t, err)

View file

@ -10,12 +10,11 @@ import (
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
"strings"
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -32,18 +31,13 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) {
cidParam := req.UserValue("cid").(string) cidParam := req.UserValue("cid").(string)
oidParam := req.UserValue("oid").(string) oidParam := req.UserValue("oid").(string)
downloadParam := req.QueryArgs().GetBool("download")
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With( ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("cid", cidParam), zap.String("cid", cidParam),
zap.String("oid", oidParam), zap.String("oid", oidParam),
)) ))
path, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam) bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil { if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
@ -51,166 +45,23 @@ func (h *Handler) DownloadByAddressOrBucketName(req *fasthttp.RequestCtx) {
} }
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) { if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) {
h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err)
return return
} }
prm := MiddlewareParam{ var objID oid.ID
Context: ctx, if checkS3Err == nil && shouldDownload(oidParam, downloadParam) {
Request: req, h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.receiveFile)
BktInfo: bktInfo, } else if err = objID.DecodeString(oidParam); err == nil {
Path: path, h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.receiveFile)
}
indexPageEnabled := h.config.IndexPageEnabled()
if checkS3Err == nil {
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byS3PathMiddleware(h.receiveFile, noopFormer), Enabled: true},
Middleware{Func: h.byS3PathMiddleware(h.receiveFile, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsS3), Enabled: indexPageEnabled},
)
} else { } else {
slashFallbackEnabled := h.config.EnableFilepathSlashFallback() h.browseIndex(ctx, req, cidParam, oidParam, checkS3Err != nil)
fileNameFallbackEnabled := h.config.EnableFilepathFallback()
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byAddressMiddleware(h.receiveFile), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, noopFormer), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.receiveFile, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled},
Middleware{Func: h.browseIndexMiddleware(h.getDirObjectsNative), Enabled: indexPageEnabled},
)
} }
} }
type ObjectHandlerFunc func(context.Context, *fasthttp.RequestCtx, oid.Address) func shouldDownload(oidParam string, downloadParam bool) bool {
return !isDir(oidParam) || downloadParam
type MiddlewareFunc func(param MiddlewareParam) bool
type MiddlewareParam struct {
Context context.Context
Request *fasthttp.RequestCtx
BktInfo *data.BucketInfo
Path string
}
type Middleware struct {
Func MiddlewareFunc
Enabled bool
}
func run(prm MiddlewareParam, defaultMiddleware MiddlewareFunc, middlewares ...Middleware) {
for _, m := range middlewares {
if m.Enabled && !m.Func(prm) {
return
}
}
defaultMiddleware(prm)
}
func indexFormer(path string) string {
indexPath := path
if indexPath != "" && !strings.HasSuffix(indexPath, "/") {
indexPath += "/"
}
return indexPath + "index.html"
}
func reverseLeadingSlash(path string) string {
if path == "" || path == "/" {
return path
}
if path[0] == '/' {
return path[1:]
}
return "/" + path
}
func noopFormer(path string) string {
return path
}
func (h *Handler) byS3PathMiddleware(handler func(context.Context, *fasthttp.RequestCtx, oid.Address), pathFormer func(string) string) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byS3Path")
defer span.End()
path := pathFormer(prm.Path)
foundOID, err := h.tree.GetLatestVersion(ctx, &prm.BktInfo.CID, path)
if err == nil {
if foundOID.IsDeleteMarker {
h.logAndSendError(ctx, prm.Request, logs.IndexWasDeleted, ErrObjectNotFound)
return false
}
addr := newAddress(prm.BktInfo.CID, foundOID.OID)
handler(ctx, prm.Request, addr)
return false
}
if !errors.Is(err, tree.ErrNodeNotFound) {
h.logAndSendError(ctx, prm.Request, logs.FailedToGetLatestVersionOfIndexObject, err, zap.String("path", path))
return false
}
return true
}
}
func (h *Handler) byAttributeSearchMiddleware(handler ObjectHandlerFunc, attr string, pathFormer func(string) string) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAttributeSearch")
defer span.End()
path := pathFormer(prm.Path)
res, err := h.search(ctx, prm.BktInfo.CID, attr, path, object.MatchStringEqual)
if err != nil {
h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err)
return false
}
defer res.Close()
buf := make([]oid.ID, 1)
n, err := res.Read(buf)
if err == nil && n > 0 {
addr := newAddress(prm.BktInfo.CID, buf[0])
handler(ctx, prm.Request, addr)
return false
}
if !errors.Is(err, io.EOF) {
h.logAndSendError(ctx, prm.Request, logs.FailedToFindObjectByAttribute, err)
return false
}
return true
}
}
func (h *Handler) byAddressMiddleware(handler ObjectHandlerFunc) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.byAddress")
defer span.End()
var objID oid.ID
if objID.DecodeString(prm.Path) == nil {
handler(ctx, prm.Request, newAddress(prm.BktInfo.CID, objID))
return false
}
return true
}
} }
// DownloadByAttribute handles attribute-based download requests. // DownloadByAttribute handles attribute-based download requests.

View file

@ -233,16 +233,6 @@ func (t *TestFrostFS) SearchObjects(_ context.Context, prm PrmObjectSearch) (Res
return &resObjectSearchMock{res: res}, nil return &resObjectSearchMock{res: res}, nil
} }
func (t *TestFrostFS) GetContainerByID(cid cid.ID) (*container.Container, error) {
for k, v := range t.containers {
if k == cid.EncodeToString() {
return v, nil
}
}
return nil, fmt.Errorf("container does not exist %s", cid)
}
func (t *TestFrostFS) InitMultiObjectReader(context.Context, PrmInitMultiObjectReader) (io.Reader, error) { func (t *TestFrostFS) InitMultiObjectReader(context.Context, PrmInitMultiObjectReader) (io.Reader, error) {
return nil, nil return nil, nil
} }

View file

@ -11,8 +11,8 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
@ -35,7 +35,6 @@ type Config interface {
BufferMaxSizeForPut() uint64 BufferMaxSizeForPut() uint64
NamespaceHeader() string NamespaceHeader() string
EnableFilepathFallback() bool EnableFilepathFallback() bool
EnableFilepathSlashFallback() bool
FormContainerZone(string) string FormContainerZone(string) string
CORS() *data.CORSRule CORS() *data.CORSRule
} }
@ -167,19 +166,13 @@ type ContainerResolver interface {
Resolve(ctx context.Context, zone, name string) (*cid.ID, error) Resolve(ctx context.Context, zone, name string) (*cid.ID, error)
} }
type ContainerContract interface {
// GetContainerByID reads a container from contract by ID.
GetContainerByID(cid.ID) (*container.Container, error)
}
type Handler struct { type Handler struct {
log *zap.Logger log *zap.Logger
frostfs FrostFS frostfs FrostFS
ownerID *user.ID ownerID *user.ID
config Config config Config
containerResolver ContainerResolver containerResolver ContainerResolver
cnrContract ContainerContract tree layer.TreeService
tree *tree.Tree
cache *cache.BucketCache cache *cache.BucketCache
workerPool *ants.Pool workerPool *ants.Pool
corsCnrID cid.ID corsCnrID cid.ID
@ -196,7 +189,7 @@ type AppParams struct {
CORSCache *cache.CORSCache CORSCache *cache.CORSCache
} }
func New(params *AppParams, config Config, tree *tree.Tree, rpcCli ContainerContract, workerPool *ants.Pool) *Handler { func New(params *AppParams, config Config, tree layer.TreeService, workerPool *ants.Pool) *Handler {
return &Handler{ return &Handler{
log: params.Logger, log: params.Logger,
frostfs: params.FrostFS, frostfs: params.FrostFS,
@ -208,10 +201,39 @@ func New(params *AppParams, config Config, tree *tree.Tree, rpcCli ContainerCont
workerPool: workerPool, workerPool: workerPool,
corsCnrID: params.CORSCnrID, corsCnrID: params.CORSCnrID,
corsCache: params.CORSCache, corsCache: params.CORSCache,
cnrContract: rpcCli,
} }
} }
// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
// prepares request and object address to it.
func (h *Handler) byNativeAddress(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, objID oid.ID, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) {
ctx, span := tracing.StartSpanFromContext(ctx, "handler.byNativeAddress")
defer span.End()
addr := newAddress(cnrID, objID)
handler(ctx, req, addr)
}
// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that
// resolves object address from S3-like path <bucket name>/<object key>.
func (h *Handler) byS3Path(ctx context.Context, req *fasthttp.RequestCtx, cnrID cid.ID, path string, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) {
ctx, span := tracing.StartSpanFromContext(ctx, "handler.byS3Path")
defer span.End()
foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetLatestVersionOfObject, err, zap.String("path", path))
return
}
if foundOID.IsDeleteMarker {
h.logAndSendError(ctx, req, logs.ObjectWasDeleted, ErrObjectNotFound)
return
}
addr := newAddress(cnrID, foundOID.OID)
handler(ctx, req, addr)
}
// byAttribute is a wrapper similar to byNativeAddress. // byAttribute is a wrapper similar to byNativeAddress.
func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) { func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, handler func(context.Context, *fasthttp.RequestCtx, oid.Address)) {
cidParam, _ := req.UserValue("cid").(string) cidParam, _ := req.UserValue("cid").(string)
@ -230,6 +252,8 @@ func (h *Handler) byAttribute(ctx context.Context, req *fasthttp.RequestCtx, han
return return
} }
val = prepareAtribute(key, val)
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", cidParam), ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(zap.String("cid", cidParam),
zap.String("attr_key", key), zap.String("attr_val", val))) zap.String("attr_key", key), zap.String("attr_val", val)))
@ -267,6 +291,10 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrK
n, err := res.Read(buf) n, err := res.Read(buf)
if n == 0 { if n == 0 {
switch { switch {
case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal):
h.reqLogger(ctx).Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage))
attrVal = prepareAtribute(attrFileName, attrVal)
return h.findObjectByAttribute(ctx, cnrID, attrFileName, attrVal)
case errors.Is(err, io.EOF): case errors.Is(err, io.EOF):
h.reqLogger(ctx).Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage)) h.reqLogger(ctx).Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return oid.ID{}, fmt.Errorf("object not found: %w", err) return oid.ID{}, fmt.Errorf("object not found: %w", err)
@ -279,6 +307,42 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, cnrID cid.ID, attrK
return buf[0], nil return buf[0], nil
} }
func (h *Handler) needSearchByFileName(key, val string) bool {
if key != attrFilePath || !h.config.EnableFilepathFallback() {
return false
}
return strings.HasPrefix(val, "/") && strings.Count(val, "/") == 1 || !strings.Contains(val, "/")
}
func prepareAtribute(attrKey, attrVal string) string {
if attrKey == attrFileName {
return prepareFileName(attrVal)
}
if attrKey == attrFilePath {
return prepareFilePath(attrVal)
}
return attrVal
}
func prepareFileName(fileName string) string {
if strings.HasPrefix(fileName, "/") {
return fileName[1:]
}
return fileName
}
func prepareFilePath(filePath string) string {
if !strings.HasPrefix(filePath, "/") {
return "/" + filePath
}
return filePath
}
// resolveContainer decode container id, if it's not a valid container id // resolveContainer decode container id, if it's not a valid container id
// then trey to resolve name using provided resolver. // then trey to resolve name using provided resolver.
func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) { func (h *Handler) resolveContainer(ctx context.Context, containerID string) (*cid.ID, error) {
@ -315,34 +379,76 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string) (*dat
return nil, fmt.Errorf("resolve container: %w", err) return nil, fmt.Errorf("resolve container: %w", err)
} }
return h.containerInfo(ctx, *cnrID) bktInfo, err := h.readContainer(ctx, *cnrID)
} if err != nil {
return nil, fmt.Errorf("read container: %w", err)
type ListFunc func(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error)
func (h *Handler) browseIndexMiddleware(fn ListFunc) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
ctx, span := tracing.StartSpanFromContext(prm.Context, "handler.browseIndex")
defer span.End()
ctx = utils.SetReqLog(ctx, h.reqLogger(ctx).With(
zap.String("bucket", prm.BktInfo.Name),
zap.String("container", prm.BktInfo.CID.EncodeToString()),
zap.String("prefix", prm.Path),
))
objects, err := fn(ctx, prm.BktInfo, prm.Path)
if err != nil {
h.logAndSendError(ctx, prm.Request, logs.FailedToListObjects, err)
return false
}
h.browseObjects(ctx, prm.Request, browseParams{
bucketInfo: prm.BktInfo,
prefix: prm.Path,
objects: objects,
})
return false
} }
if err = h.cache.Put(bktInfo); err != nil {
h.reqLogger(ctx).Warn(logs.CouldntPutBucketIntoCache,
zap.String("bucket name", bktInfo.Name),
zap.Stringer("bucket cid", bktInfo.CID),
zap.Error(err),
logs.TagField(logs.TagDatapath))
}
return bktInfo, nil
}
func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.BucketInfo, error) {
prm := PrmContainer{ContainerID: cnrID}
res, err := h.frostfs.Container(ctx, prm)
if err != nil {
return nil, fmt.Errorf("get frostfs container '%s': %w", cnrID.String(), err)
}
bktInfo := &data.BucketInfo{
CID: cnrID,
Name: cnrID.EncodeToString(),
}
if domain := container.ReadDomain(*res); domain.Name() != "" {
bktInfo.Name = domain.Name()
bktInfo.Zone = domain.Zone()
}
bktInfo.HomomorphicHashDisabled = container.IsHomomorphicHashingDisabled(*res)
bktInfo.PlacementPolicy = res.PlacementPolicy()
return bktInfo, err
}
func (h *Handler) browseIndex(ctx context.Context, req *fasthttp.RequestCtx, cidParam, oidParam string, isNativeList bool) {
ctx, span := tracing.StartSpanFromContext(ctx, "handler.browseIndex")
defer span.End()
if !h.config.IndexPageEnabled() {
req.SetStatusCode(fasthttp.StatusNotFound)
return
}
unescapedKey, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapeOIDParam, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
return
}
listFunc := h.getDirObjectsS3
if isNativeList {
// tree probe failed, trying to use native
listFunc = h.getDirObjectsNative
}
h.browseObjects(ctx, req, browseParams{
bucketInfo: bktInfo,
prefix: unescapedKey,
listObjects: listFunc,
isNative: isNativeList,
})
} }

View file

@ -14,10 +14,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler/middleware"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/templates" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
@ -27,7 +26,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/panjf2000/ants/v2" "github.com/panjf2000/ants/v2"
@ -37,11 +35,35 @@ import (
"go.uber.org/zap/zaptest" "go.uber.org/zap/zaptest"
) )
type treeServiceMock struct {
system map[string]map[string]*data.BaseNodeVersion
}
func newTreeService() *treeServiceMock {
return &treeServiceMock{
system: make(map[string]map[string]*data.BaseNodeVersion),
}
}
func (t *treeServiceMock) CheckSettingsNodeExists(context.Context, *data.BucketInfo) error {
_, ok := t.system["bucket-settings"]
if !ok {
return layer.ErrNodeNotFound
}
return nil
}
func (t *treeServiceMock) GetSubTreeByPrefix(context.Context, *data.BucketInfo, string, bool) ([]data.NodeInfo, string, error) {
return nil, "", nil
}
func (t *treeServiceMock) GetLatestVersion(context.Context, *cid.ID, string) (*data.NodeVersion, error) {
return nil, nil
}
type configMock struct { type configMock struct {
additionalFilenameSearch bool additionalSearch bool
additionalSlashSearch bool cors *data.CORSRule
indexEnabled bool
cors *data.CORSRule
} }
func (c *configMock) DefaultTimestamp() bool { func (c *configMock) DefaultTimestamp() bool {
@ -53,11 +75,11 @@ func (c *configMock) ArchiveCompression() bool {
} }
func (c *configMock) IndexPageEnabled() bool { func (c *configMock) IndexPageEnabled() bool {
return c.indexEnabled return false
} }
func (c *configMock) IndexPageTemplate() string { func (c *configMock) IndexPageTemplate() string {
return templates.DefaultIndexTemplate return ""
} }
func (c *configMock) IndexPageNativeTemplate() string { func (c *configMock) IndexPageNativeTemplate() string {
@ -77,11 +99,7 @@ func (c *configMock) NamespaceHeader() string {
} }
func (c *configMock) EnableFilepathFallback() bool { func (c *configMock) EnableFilepathFallback() bool {
return c.additionalFilenameSearch return c.additionalSearch
}
func (c *configMock) EnableFilepathSlashFallback() bool {
return c.additionalSlashSearch
} }
func (c *configMock) FormContainerZone(string) string { func (c *configMock) FormContainerZone(string) string {
@ -99,7 +117,7 @@ type handlerContext struct {
h *Handler h *Handler
frostfs *TestFrostFS frostfs *TestFrostFS
tree *treeServiceClientMock tree *treeServiceMock
cfg *configMock cfg *configMock
} }
@ -149,14 +167,14 @@ func prepareHandlerContextBase(logger *zap.Logger) (*handlerContext, error) {
}), }),
} }
treeMock := newTreeServiceClientMock() treeMock := newTreeService()
cfgMock := &configMock{} cfgMock := &configMock{}
workerPool, err := ants.NewPool(1) workerPool, err := ants.NewPool(1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
handler := New(params, cfgMock, tree.NewTree(treeMock, logger), testFrostFS, workerPool) handler := New(params, cfgMock, treeMock, workerPool)
return &handlerContext{ return &handlerContext{
key: key, key: key,
@ -236,7 +254,6 @@ func TestBasic(t *testing.T) {
err = json.Unmarshal(r.Response.Body(), &putRes) err = json.Unmarshal(r.Response.Body(), &putRes)
require.NoError(t, err) require.NoError(t, err)
hc.cfg.additionalFilenameSearch = true
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID] obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
fileName := prepareObjectAttributes(object.AttributeFileName, objFileName) fileName := prepareObjectAttributes(object.AttributeFileName, objFileName)
filePath := prepareObjectAttributes(object.AttributeFilePath, objFilePath) filePath := prepareObjectAttributes(object.AttributeFilePath, objFilePath)
@ -247,14 +264,6 @@ func TestBasic(t *testing.T) {
r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID) r = prepareGetRequest(ctx, cnrID.EncodeToString(), putRes.ObjectID)
hc.Handler().DownloadByAddressOrBucketName(r) hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, content, string(r.Response.Body())) require.Equal(t, content, string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFilePath)
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, content, string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFileName)
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, content, string(r.Response.Body()))
}) })
t.Run("head", func(t *testing.T) { t.Run("head", func(t *testing.T) {
@ -262,16 +271,6 @@ func TestBasic(t *testing.T) {
hc.Handler().HeadByAddressOrBucketName(r) hc.Handler().HeadByAddressOrBucketName(r)
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID))) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID))) require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFilePath)
hc.Handler().HeadByAddressOrBucketName(r)
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), objFileName)
hc.Handler().HeadByAddressOrBucketName(r)
require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
}) })
t.Run("get by attribute", func(t *testing.T) { t.Run("get by attribute", func(t *testing.T) {
@ -281,11 +280,11 @@ func TestBasic(t *testing.T) {
r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath)
hc.Handler().DownloadByAttribute(r) hc.Handler().DownloadByAttribute(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) require.Equal(t, content, string(r.Response.Body()))
r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName)
hc.Handler().DownloadByAttribute(r) hc.Handler().DownloadByAttribute(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) require.Equal(t, content, string(r.Response.Body()))
}) })
t.Run("head by attribute", func(t *testing.T) { t.Run("head by attribute", func(t *testing.T) {
@ -296,11 +295,13 @@ func TestBasic(t *testing.T) {
r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath) r = prepareGetByAttributeRequest(ctx, bktName, attrFileName, objFilePath)
hc.Handler().HeadByAttribute(r) hc.Handler().HeadByAttribute(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName) r = prepareGetByAttributeRequest(ctx, bktName, attrFilePath, objFileName)
hc.Handler().HeadByAttribute(r) hc.Handler().HeadByAttribute(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode()) require.Equal(t, putRes.ObjectID, string(r.Response.Header.Peek(hdrObjectID)))
require.Equal(t, putRes.ContainerID, string(r.Response.Header.Peek(hdrContainerID)))
}) })
t.Run("zip", func(t *testing.T) { t.Run("zip", func(t *testing.T) {
@ -324,281 +325,185 @@ func TestBasic(t *testing.T) {
}) })
} }
func prepareHandlerAndBucket(t *testing.T) (*handlerContext, cid.ID) { func TestFindObjectByAttribute(t *testing.T) {
hc := prepareHandlerContext(t) hc := prepareHandlerContext(t)
hc.cfg.additionalSearch = true
bktName := "bucket" bktName := "bucket"
cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended) cnrID, cnr, err := hc.prepareContainer(bktName, acl.PublicRWExtended)
require.NoError(t, err) require.NoError(t, err)
hc.frostfs.SetContainer(cnrID, cnr) hc.frostfs.SetContainer(cnrID, cnr)
return hc, cnrID ctx := context.Background()
ctx = middleware.SetNamespace(ctx, "")
content := "hello"
r, err := prepareUploadRequest(ctx, cnrID.EncodeToString(), content)
require.NoError(t, err)
hc.Handler().Upload(r)
require.Equal(t, r.Response.StatusCode(), http.StatusOK)
var putRes putResponse
err = json.Unmarshal(r.Response.Body(), &putRes)
require.NoError(t, err)
testAttrVal1 := "/folder/cat.jpg"
testAttrVal2 := "cat.jpg"
testAttrVal3 := "test-attr-val3"
for _, tc := range []struct {
name string
firstAttr object.Attribute
secondAttr object.Attribute
reqAttrKey string
reqAttrValue string
err string
additionalSearch bool
}{
{
name: "success search by FileName",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFileName,
reqAttrValue: testAttrVal2,
additionalSearch: false,
},
{
name: "failed search by FileName",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFileName,
reqAttrValue: testAttrVal3,
err: "not found",
additionalSearch: false,
},
{
name: "success search by FilePath (with additional search)",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFilePath,
reqAttrValue: testAttrVal2,
additionalSearch: true,
},
{
name: "failed by FilePath (with additional search)",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFilePath,
reqAttrValue: testAttrVal3,
err: "not found",
additionalSearch: true,
},
{
name: "success search by FilePath with leading slash (with additional search)",
firstAttr: prepareObjectAttributes(attrFilePath, testAttrVal1),
secondAttr: prepareObjectAttributes(attrFileName, testAttrVal2),
reqAttrKey: attrFilePath,
reqAttrValue: "/cat.jpg",
additionalSearch: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
obj := hc.frostfs.objects[putRes.ContainerID+"/"+putRes.ObjectID]
obj.SetAttributes(tc.firstAttr, tc.secondAttr)
hc.cfg.additionalSearch = tc.additionalSearch
objID, err := hc.Handler().findObjectByAttribute(ctx, cnrID, tc.reqAttrKey, tc.reqAttrValue)
if tc.err != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tc.err)
return
}
require.NoError(t, err)
require.Equal(t, putRes.ObjectID, objID.EncodeToString())
})
}
} }
func TestGetObjectWithFallback(t *testing.T) { func TestNeedSearchByFileName(t *testing.T) {
ctx := middleware.SetNamespace(context.Background(), "") hc := prepareHandlerContext(t)
t.Run("by oid", func(t *testing.T) { for _, tc := range []struct {
hc, cnrID := prepareHandlerAndBucket(t) name string
attrKey string
attrVal string
additionalSearch bool
expected bool
}{
{
name: "need search - not contains slash",
attrKey: attrFilePath,
attrVal: "cat.png",
additionalSearch: true,
expected: true,
},
{
name: "need search - single lead slash",
attrKey: attrFilePath,
attrVal: "/cat.png",
additionalSearch: true,
expected: true,
},
{
name: "don't need search - single slash but not lead",
attrKey: attrFilePath,
attrVal: "cats/cat.png",
additionalSearch: true,
expected: false,
},
{
name: "don't need search - more one slash",
attrKey: attrFilePath,
attrVal: "/cats/cat.png",
additionalSearch: true,
expected: false,
},
{
name: "don't need search - incorrect attribute key",
attrKey: attrFileName,
attrVal: "cat.png",
additionalSearch: true,
expected: false,
},
{
name: "don't need search - additional search disabled",
attrKey: attrFilePath,
attrVal: "cat.png",
additionalSearch: false,
expected: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
hc.cfg.additionalSearch = tc.additionalSearch
obj1ID := oidtest.ID() res := hc.h.needSearchByFileName(tc.attrKey, tc.attrVal)
obj1 := object.New() require.Equal(t, tc.expected, res)
obj1.SetID(obj1ID) })
obj1.SetPayload([]byte("obj1")) }
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), obj1ID.String())
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("by filepath as it is", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
obj2ID := oidtest.ID()
obj2 := object.New()
obj2.SetID(obj2ID)
obj2.SetPayload([]byte("obj2"))
obj2.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "/filepath/obj2"))
hc.frostfs.objects[cnrID.String()+"/"+obj2ID.String()] = obj2
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj2")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj2.Payload()), string(r.Response.Body()))
})
t.Run("by filepath slash fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.additionalSlashSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filepath/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("by filename fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.additionalFilenameSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("by filename and slash fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.additionalFilenameSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.additionalSlashSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "/filename/obj1")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("index fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "filepath/index.html"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.indexEnabled = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filepath/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
t.Run("index filename fallback", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFileName, "filename/index.html"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.indexEnabled = true
hc.cfg.additionalFilenameSearch = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "filename/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, string(obj1.Payload()), string(r.Response.Body()))
})
} }
func TestIndex(t *testing.T) { func TestPrepareFileName(t *testing.T) {
ctx := middleware.SetNamespace(context.Background(), "") fileName := "/cat.jpg"
expected := "cat.jpg"
actual := prepareFileName(fileName)
require.Equal(t, expected, actual)
t.Run("s3", func(t *testing.T) { fileName = "cat.jpg"
hc, cnrID := prepareHandlerAndBucket(t) actual = prepareFileName(fileName)
require.Equal(t, expected, actual)
}
obj1ID := oidtest.ID() func TestPrepareFilePath(t *testing.T) {
obj1 := object.New() filePath := "cat.jpg"
obj1.SetID(obj1ID) expected := "/cat.jpg"
obj1.SetPayload([]byte("obj1")) actual := prepareFilePath(filePath)
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1")) require.Equal(t, expected, actual)
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
hc.tree.containers[cnrID.String()] = containerInfo{ filePath = "/cat.jpg"
trees: map[string]map[string]nodeResponse{ actual = prepareFilePath(filePath)
"system": {"bucket-settings": nodeResponse{nodeID: 1}}, require.Equal(t, expected, actual)
"version": {
"": nodeResponse{}, //root
"prefix": nodeResponse{
nodeID: 1,
meta: []nodeMeta{{key: tree.FileNameKey, value: []byte("prefix")}}},
"obj1": nodeResponse{
parentID: 1,
nodeID: 2,
meta: []nodeMeta{
{key: tree.FileNameKey, value: []byte("obj1")},
{key: "OID", value: []byte(obj1ID.String())},
},
},
},
},
}
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.indexEnabled = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix")
require.Contains(t, string(r.Response.Body()), obj1ID.String())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/prefix")
require.Contains(t, string(r.Response.Body()), obj1ID.String())
r = prepareGetRequest(ctx, "bucket", "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of s3://bucket/dummy")
})
t.Run("native", func(t *testing.T) {
hc, cnrID := prepareHandlerAndBucket(t)
obj1ID := oidtest.ID()
obj1 := object.New()
obj1.SetID(obj1ID)
obj1.SetPayload([]byte("obj1"))
obj1.SetAttributes(prepareObjectAttributes(object.AttributeFilePath, "prefix/obj1"))
hc.frostfs.objects[cnrID.String()+"/"+obj1ID.String()] = obj1
r := prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Equal(t, fasthttp.StatusNotFound, r.Response.StatusCode())
hc.cfg.indexEnabled = true
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix")
require.Contains(t, string(r.Response.Body()), obj1ID.String())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "prefix/")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/prefix")
require.Contains(t, string(r.Response.Body()), obj1ID.String())
r = prepareGetRequest(ctx, cnrID.EncodeToString(), "dummy")
hc.Handler().DownloadByAddressOrBucketName(r)
require.Contains(t, string(r.Response.Body()), "Index of frostfs://"+cnrID.String()+"/dummy")
})
} }
func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) { func prepareUploadRequest(ctx context.Context, bucket, content string) (*fasthttp.RequestCtx, error) {

View file

@ -5,12 +5,11 @@ import (
"errors" "errors"
"io" "io"
"net/http" "net/http"
"net/url"
"strconv" "strconv"
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -129,12 +128,6 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) {
zap.String("oid", oidParam), zap.String("oid", oidParam),
)) ))
path, err := url.QueryUnescape(oidParam)
if err != nil {
h.logAndSendError(ctx, req, logs.FailedToUnescapePath, err)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidParam) bktInfo, err := h.getBucketInfo(ctx, cidParam)
if err != nil { if err != nil {
h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err) h.logAndSendError(ctx, req, logs.FailedToGetBucketInfo, err)
@ -142,38 +135,18 @@ func (h *Handler) HeadByAddressOrBucketName(req *fasthttp.RequestCtx) {
} }
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo) checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
if checkS3Err != nil && !errors.Is(checkS3Err, tree.ErrNodeNotFound) { if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) {
h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err) h.logAndSendError(ctx, req, logs.FailedToCheckIfSettingsNodeExist, checkS3Err)
return return
} }
prm := MiddlewareParam{ var objID oid.ID
Context: ctx,
Request: req,
BktInfo: bktInfo,
Path: path,
}
indexPageEnabled := h.config.IndexPageEnabled()
if checkS3Err == nil { if checkS3Err == nil {
run(prm, h.errorMiddleware(logs.ObjectNotFound, tree.ErrNodeNotFound), h.byS3Path(ctx, req, bktInfo.CID, oidParam, h.headObject)
Middleware{Func: h.byS3PathMiddleware(h.headObject, noopFormer), Enabled: true}, } else if err = objID.DecodeString(oidParam); err == nil {
Middleware{Func: h.byS3PathMiddleware(h.headObject, indexFormer), Enabled: indexPageEnabled}, h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject)
)
} else { } else {
slashFallbackEnabled := h.config.EnableFilepathSlashFallback() h.logAndSendError(ctx, req, logs.InvalidOIDParam, err)
fileNameFallbackEnabled := h.config.EnableFilepathFallback()
run(prm, h.errorMiddleware(logs.ObjectNotFound, ErrObjectNotFound),
Middleware{Func: h.byAddressMiddleware(h.headObject), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, noopFormer), Enabled: true},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, reverseLeadingSlash), Enabled: slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, noopFormer), Enabled: fileNameFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, reverseLeadingSlash), Enabled: fileNameFallbackEnabled && slashFallbackEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFilePath, indexFormer), Enabled: indexPageEnabled},
Middleware{Func: h.byAttributeSearchMiddleware(h.headObject, object.AttributeFileName, indexFormer), Enabled: fileNameFallbackEnabled && indexPageEnabled},
)
} }
} }
@ -184,10 +157,3 @@ func (h *Handler) HeadByAttribute(req *fasthttp.RequestCtx) {
h.byAttribute(ctx, req, h.headObject) h.byAttribute(ctx, req, h.headObject)
} }
func (h *Handler) errorMiddleware(msg string, err error) MiddlewareFunc {
return func(prm MiddlewareParam) bool {
h.logAndSendError(prm.Context, prm.Request, msg, err)
return false
}
}

View file

@ -1,141 +0,0 @@
package handler
import (
"context"
"errors"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
)
type nodeMeta struct {
key string
value []byte
}
func (m nodeMeta) GetKey() string {
return m.key
}
func (m nodeMeta) GetValue() []byte {
return m.value
}
type nodeResponse struct {
meta []nodeMeta
nodeID uint64
parentID uint64
timestamp uint64
}
func (n nodeResponse) GetNodeID() []uint64 {
return []uint64{n.nodeID}
}
func (n nodeResponse) GetParentID() []uint64 {
return []uint64{n.parentID}
}
func (n nodeResponse) GetTimestamp() []uint64 {
return []uint64{n.timestamp}
}
func (n nodeResponse) GetMeta() []tree.Meta {
res := make([]tree.Meta, len(n.meta))
for i, value := range n.meta {
res[i] = value
}
return res
}
type containerInfo struct {
trees map[string]map[string]nodeResponse
}
type treeServiceClientMock struct {
containers map[string]containerInfo
}
func newTreeServiceClientMock() *treeServiceClientMock {
return &treeServiceClientMock{
containers: make(map[string]containerInfo),
}
}
func (t *treeServiceClientMock) GetNodes(_ context.Context, p *tree.GetNodesParams) ([]tree.NodeResponse, error) {
cnr, ok := t.containers[p.CnrID.EncodeToString()]
if !ok {
return nil, tree.ErrNodeNotFound
}
tr, ok := cnr.trees[p.TreeID]
if !ok {
return nil, tree.ErrNodeNotFound
}
node, ok := tr[strings.Join(p.Path, "/")]
if !ok {
return nil, tree.ErrNodeNotFound
}
return []tree.NodeResponse{node}, nil
}
func (t *treeServiceClientMock) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, _ bool) ([]tree.NodeResponse, error) {
cnr, ok := t.containers[bktInfo.CID.EncodeToString()]
if !ok {
return nil, tree.ErrNodeNotFound
}
tr, ok := cnr.trees[treeID]
if !ok {
return nil, tree.ErrNodeNotFound
}
if len(rootID) != 1 {
return nil, errors.New("invalid rootID")
}
var root *nodeResponse
for _, v := range tr {
if v.nodeID == rootID[0] {
root = &v
break
}
}
if root == nil {
return nil, tree.ErrNodeNotFound
}
var res []nodeResponse
if depth == 0 {
for _, v := range tr {
res = append(res, v)
}
} else {
res = append(res, *root)
depthIndex := 0
for i := uint32(0); i < depth-1; i++ {
childrenCount := 0
for _, v := range tr {
for j := range res[depthIndex:] {
if v.parentID == res[j].nodeID {
res = append(res, v)
childrenCount++
break
}
}
}
depthIndex = len(res) - childrenCount
}
}
res2 := make([]tree.NodeResponse, len(res))
for i := range res {
res2[i] = res[i]
}
return res2, nil
}

View file

@ -6,9 +6,9 @@ import (
"fmt" "fmt"
"strings" "strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@ -93,7 +93,7 @@ func formErrorResponse(err error) (string, int) {
switch { switch {
case errors.Is(err, ErrAccessDenied): case errors.Is(err, ErrAccessDenied):
return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden return fmt.Sprintf("Storage Access Denied:\n%v", err), fasthttp.StatusForbidden
case errors.Is(err, tree.ErrNodeAccessDenied): case errors.Is(err, layer.ErrNodeAccessDenied):
return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden return fmt.Sprintf("Tree Access Denied:\n%v", err), fasthttp.StatusForbidden
case errors.Is(err, ErrQuotaLimitReached): case errors.Is(err, ErrQuotaLimitReached):
return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict return fmt.Sprintf("Quota Reached:\n%v", err), fasthttp.StatusConflict
@ -101,7 +101,7 @@ func formErrorResponse(err error) (string, int) {
return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound return fmt.Sprintf("Container Not Found:\n%v", err), fasthttp.StatusNotFound
case errors.Is(err, ErrObjectNotFound): case errors.Is(err, ErrObjectNotFound):
return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound return fmt.Sprintf("Object Not Found:\n%v", err), fasthttp.StatusNotFound
case errors.Is(err, tree.ErrNodeNotFound): case errors.Is(err, layer.ErrNodeNotFound):
return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound return fmt.Sprintf("Tree Node Not Found:\n%v", err), fasthttp.StatusNotFound
case errors.Is(err, ErrGatewayTimeout): case errors.Is(err, ErrGatewayTimeout):
return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout return fmt.Sprintf("Gateway Timeout:\n%v", err), fasthttp.StatusGatewayTimeout

View file

@ -0,0 +1,24 @@
package layer
import (
"context"
"errors"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
)
// TreeService provide interface to interact with tree service using s3 data models.
type TreeService interface {
GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*data.NodeVersion, error)
GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error)
CheckSettingsNodeExists(ctx context.Context, bktInfo *data.BucketInfo) error
}
var (
// ErrNodeNotFound is returned from Tree service in case of not found error.
ErrNodeNotFound = errors.New("not found")
// ErrNodeAccessDenied is returned from Tree service in case of access denied error.
ErrNodeAccessDenied = errors.New("access denied")
)

View file

@ -73,9 +73,6 @@ const (
FailedToReadIndexPageTemplate = "failed to read index page template" FailedToReadIndexPageTemplate = "failed to read index page template"
SetCustomIndexPageTemplate = "set custom index page template" SetCustomIndexPageTemplate = "set custom index page template"
CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info"
InitRPCClientFailed = "init rpc client faileds"
InitContainerContractFailed = "init container contract failed"
FailedToResolveContractHash = "failed to resolve contract hash"
) )
// Log messages with the "datapath" tag. // Log messages with the "datapath" tag.
@ -110,8 +107,8 @@ const (
IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" IteratingOverSelectedObjectsFailed = "iterating over selected objects failed"
FailedToGetBucketInfo = "could not get bucket info" FailedToGetBucketInfo = "could not get bucket info"
FailedToSubmitTaskToPool = "failed to submit task to pool" FailedToSubmitTaskToPool = "failed to submit task to pool"
IndexWasDeleted = "index was deleted" ObjectWasDeleted = "object was deleted"
FailedToGetLatestVersionOfIndexObject = "failed to get latest version of index object" FailedToGetLatestVersionOfObject = "failed to get latest version of object"
FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists" FailedToCheckIfSettingsNodeExist = "failed to check if settings node exists"
FailedToListObjects = "failed to list objects" FailedToListObjects = "failed to list objects"
FailedToParseTemplate = "failed to parse template" FailedToParseTemplate = "failed to parse template"
@ -121,7 +118,8 @@ const (
FailedToGetObject = "failed to get object" FailedToGetObject = "failed to get object"
FailedToGetObjectPayload = "failed to get object payload" FailedToGetObjectPayload = "failed to get object payload"
FailedToFindObjectByAttribute = "failed to get find object by attribute" FailedToFindObjectByAttribute = "failed to get find object by attribute"
FailedToUnescapePath = "failed to unescape path" FailedToUnescapeOIDParam = "failed to unescape oid param"
InvalidOIDParam = "invalid oid param"
CouldNotGetCORSConfiguration = "could not get cors configuration" CouldNotGetCORSConfiguration = "could not get cors configuration"
EmptyOriginRequestHeader = "empty Origin request header" EmptyOriginRequestHeader = "empty Origin request header"
EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header" EmptyAccessControlRequestMethodHeader = "empty Access-Control-Request-Method request header"
@ -131,9 +129,10 @@ const (
// Log messages with the "external_storage" tag. // Log messages with the "external_storage" tag.
const ( const (
ObjectNotFound = "object not found" ObjectNotFound = "object not found"
ReadObjectListFailed = "read object list failed" ReadObjectListFailed = "read object list failed"
ObjectUploaded = "object uploaded" ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName"
ObjectUploaded = "object uploaded"
) )
// Log messages with the "external_storage_tree" tag. // Log messages with the "external_storage_tree" tag.

View file

@ -1,73 +0,0 @@
package container
import (
"fmt"
"strings"
containercontract "git.frostfs.info/TrueCloudLab/frostfs-contract/container"
containerclient "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/container"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/handler"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/nspcc-dev/neo-go/pkg/rpcclient"
"github.com/nspcc-dev/neo-go/pkg/rpcclient/actor"
"github.com/nspcc-dev/neo-go/pkg/util"
"github.com/nspcc-dev/neo-go/pkg/wallet"
)
type Client struct {
contract *containerclient.Contract
}
type Config struct {
ContractHash util.Uint160
Key *keys.PrivateKey
RPCClient *rpcclient.Client
}
func New(cfg Config) (*Client, error) {
var err error
key := cfg.Key
if key == nil {
if key, err = keys.NewPrivateKey(); err != nil {
return nil, fmt.Errorf("generate anon private key for container contract: %w", err)
}
}
acc := wallet.NewAccountFromPrivateKey(key)
act, err := actor.NewSimple(cfg.RPCClient, acc)
if err != nil {
return nil, fmt.Errorf("create new actor: %w", err)
}
return &Client{
contract: containerclient.New(act, cfg.ContractHash),
}, nil
}
func (c *Client) GetContainerByID(cnrID cid.ID) (*container.Container, error) {
items, err := c.contract.Get(cnrID[:])
if err != nil {
if strings.Contains(err.Error(), containercontract.NotFoundError) {
return nil, fmt.Errorf("%w: %s", handler.ErrContainerNotFound, err)
}
return nil, err
}
if len(items) != 4 {
return nil, fmt.Errorf("unexpected container stack item count: %d", len(items))
}
cnrBytes, err := items[0].TryBytes()
if err != nil {
return nil, fmt.Errorf("could not get byte array of container: %w", err)
}
var cnr container.Container
if err = cnr.Unmarshal(cnrBytes); err != nil {
return nil, fmt.Errorf("can't unmarshal container: %w", err)
}
return &cnr, nil
}

View file

@ -1,34 +0,0 @@
package util
import (
"fmt"
"strings"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
"github.com/nspcc-dev/neo-go/pkg/util"
)
// ResolveContractHash determine contract hash by resolving NNS name.
func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) {
if hash, err := util.Uint160DecodeStringLE(contractHash); err == nil {
return hash, nil
}
splitName := strings.Split(contractHash, ".")
if len(splitName) != 2 {
return util.Uint160{}, fmt.Errorf("invalid contract name: '%s'", contractHash)
}
var domain container.Domain
domain.SetName(splitName[0])
domain.SetZone(splitName[1])
var nns ns.NNS
if err := nns.Dial(rpcAddress); err != nil {
return util.Uint160{}, fmt.Errorf("dial nns %s: %w", rpcAddress, err)
}
defer nns.Close()
return nns.ResolveContractHash(domain)
}

View file

@ -1,9 +1,11 @@
{{$container := .Container}} {{$container := .Container}}
{{ $prefix := trimPrefix .Prefix }}
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8"/> <meta charset="UTF-8"/>
<title>Index of {{.Protocol}}://{{$container}}/{{.Prefix}}</title> <title>Index of {{.Protocol}}://{{$container}}
/{{if $prefix}}/{{$prefix}}/{{end}}</title>
<style> <style>
.alert { .alert {
width: 80%; width: 80%;
@ -38,7 +40,7 @@
</style> </style>
</head> </head>
<body> <body>
<h1>Index of {{.Protocol}}://{{$container}}/{{.Prefix}}</h1> <h1>Index of {{.Protocol}}://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}</h1>
{{ if .HasErrors }} {{ if .HasErrors }}
<div class="alert"> <div class="alert">
Errors occurred while processing the request. Perhaps some objects are missing Errors occurred while processing the request. Perhaps some objects are missing
@ -55,11 +57,11 @@
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{{ $parentPrefix := getParent .Prefix }} {{ $trimmedPrefix := trimPrefix $prefix }}
{{if $parentPrefix }} {{if $trimmedPrefix }}
<tr> <tr>
<td> <td>
⮐<a href="/get/{{$container}}{{ urlencode $parentPrefix }}/">..</a> ⮐<a href="/get/{{$container}}{{ urlencode $trimmedPrefix }}/">..</a>
</td> </td>
<td></td> <td></td>
<td></td> <td></td>

View file

@ -7,6 +7,7 @@ import (
"strings" "strings"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/layer"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
@ -51,10 +52,10 @@ type (
var ( var (
// ErrNodeNotFound is returned from ServiceClient in case of not found error. // ErrNodeNotFound is returned from ServiceClient in case of not found error.
ErrNodeNotFound = errors.New("not found") ErrNodeNotFound = layer.ErrNodeNotFound
// ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error. // ErrNodeAccessDenied is returned from ServiceClient service in case of access denied error.
ErrNodeAccessDenied = errors.New("access denied") ErrNodeAccessDenied = layer.ErrNodeAccessDenied
) )
const ( const (
@ -258,7 +259,7 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name
nodes = filterMultipartNodes(nodes) nodes = filterMultipartNodes(nodes)
if len(nodes) == 0 { if len(nodes) == 0 {
return nil, ErrNodeNotFound return nil, layer.ErrNodeNotFound
} }
if len(nodes) != 1 { if len(nodes) != 1 {
c.reqLogger(ctx).Warn(logs.FoundSeveralSystemTreeNodes, zap.String("name", name), logs.TagField(logs.TagExternalStorageTree)) c.reqLogger(ctx).Warn(logs.FoundSeveralSystemTreeNodes, zap.String("name", name), logs.TagField(logs.TagExternalStorageTree))
@ -302,7 +303,7 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
} }
if targetIndexNode == -1 { if targetIndexNode == -1 {
return nil, fmt.Errorf("latest version: %w", ErrNodeNotFound) return nil, fmt.Errorf("latest version: %w", layer.ErrNodeNotFound)
} }
return nodes[targetIndexNode], nil return nodes[targetIndexNode], nil
@ -323,23 +324,20 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator) return strings.Split(objectName, separator)
} }
func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, error) { func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]data.NodeInfo, string, error) {
ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix") ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetSubTreeByPrefix")
defer span.End() defer span.End()
rootID, err := c.getPrefixNodeID(ctx, bktInfo, versionTree, strings.Split(prefix, separator)) rootID, tailPrefix, err := c.determinePrefixNode(ctx, bktInfo, versionTree, prefix)
if err != nil { if err != nil {
if errors.Is(err, ErrNodeNotFound) { return nil, "", err
return nil, nil
}
return nil, err
} }
subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false) subTree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, rootID, 2, false)
if err != nil { if err != nil {
if errors.Is(err, ErrNodeNotFound) { if errors.Is(err, ErrNodeNotFound) {
return nil, nil return nil, "", nil
} }
return nil, err return nil, "", err
} }
nodesMap := make(map[string][]NodeResponse, len(subTree)) nodesMap := make(map[string][]NodeResponse, len(subTree))
@ -349,6 +347,10 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
} }
fileName := GetFilename(node) fileName := GetFilename(node)
if !strings.HasPrefix(fileName, tailPrefix) {
continue
}
nodes := nodesMap[fileName] nodes := nodesMap[fileName]
// Add all nodes if flag latestOnly is false. // Add all nodes if flag latestOnly is false.
@ -372,7 +374,7 @@ func (c *Tree) GetSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
result = append(result, nodeResponseToNodeInfo(nodes)...) result = append(result, nodeResponseToNodeInfo(nodes)...)
} }
return result, nil return result, strings.TrimSuffix(prefix, tailPrefix), nil
} }
func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo { func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo {
@ -384,6 +386,22 @@ func nodeResponseToNodeInfo(nodes []NodeResponse) []data.NodeInfo {
return nodesInfo return nodesInfo
} }
func (c *Tree) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) ([]uint64, string, error) {
rootID := []uint64{0}
path := strings.Split(prefix, separator)
tailPrefix := path[len(path)-1]
if len(path) > 1 {
var err error
rootID, err = c.getPrefixNodeID(ctx, bktInfo, treeID, path[:len(path)-1])
if err != nil {
return nil, "", err
}
}
return rootID, tailPrefix, nil
}
func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) ([]uint64, error) { func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, treeID string, prefixPath []string) ([]uint64, error) {
p := &GetNodesParams{ p := &GetNodesParams{
CnrID: bktInfo.CID, CnrID: bktInfo.CID,
@ -406,7 +424,7 @@ func (c *Tree) getPrefixNodeID(ctx context.Context, bktInfo *data.BucketInfo, tr
} }
if len(intermediateNodes) == 0 { if len(intermediateNodes) == 0 {
return nil, ErrNodeNotFound return nil, layer.ErrNodeNotFound
} }
return intermediateNodes, nil return intermediateNodes, nil