Release v0.31.0

Signed-off-by: Alex Vanin <a.vanin@yadro.com>
[#151 ] index page: Add browse via native protocol
2024-11-20 11:09:31 +03:00 · 2024-11-19 17:33:21 +03:00 · 2024-11-18 12:48:04 +00:00 · 2024-11-15 11:41:13 +00:00 · 2024-11-06 15:19:54 +03:00 · 2024-11-05 17:51:35 +03:00
28 changed files with 691 additions and 169 deletions
--- a/.forgejo/ISSUE_TEMPLATE/bug_report.md
+++ b/.forgejo/ISSUE_TEMPLATE/bug_report.md
--- a/.forgejo/ISSUE_TEMPLATE/config.yml
+++ b/.forgejo/ISSUE_TEMPLATE/config.yml
--- a/.forgejo/ISSUE_TEMPLATE/feature_request.md
+++ b/.forgejo/ISSUE_TEMPLATE/feature_request.md
--- a/.forgejo/logo.svg
+++ b/.forgejo/logo.svg
--- a/.forgejo/workflows/builds.yml
+++ b/.forgejo/workflows/builds.yml
@ -1,4 +1,8 @@
-on: [pull_request]
+on:
+  pull_request:
+  push:
+    branches:
+      - master

 jobs:
  builds:
--- a/.forgejo/workflows/tests.yml
+++ b/.forgejo/workflows/tests.yml
@ -1,4 +1,8 @@
-on: [pull_request]
+on:
+  pull_request:
+  push:
+    branches:
+      - master

 jobs:
  lint:
--- a/.forgejo/workflows/vulncheck.yml
+++ b/.forgejo/workflows/vulncheck.yml
@ -1,4 +1,8 @@
-on: [pull_request]
+on:
+  pull_request:
+  push:
+    branches:
+      - master

 jobs:
  vulncheck:
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@ -1 +0,0 @@
-*       @alexvanin @dkirillov
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -4,13 +4,51 @@ This document outlines major changes between releases.

 ## [Unreleased]

+## [0.31.0] - Rongbuk - 2024-11-20
+
+### Fixed
+- Docker warnings during image build (#126)
+- `trace_id` parameter in logs (#148)
+- SIGHUP support for `tracing.enabled` config parameter (#157)
+
 ### Added
- Support percent-encoding for GET queries (#134)
- Add `trace_id` to logs (#148)
- Add `cors` config params (#158)
+- Vulnerability report document (#123) 
+- Root CA configuration for tracing (#139)
+- Log sampling policy configuration (#147)
+- Index page support for buckets and containers (#137, #151)
+- CORS support (#158)
+- Source IP binding configuration for FrostFS requests (#160)
+- Tracing attributes (#164)

 ### Changed
- Update go version to 1.22 (#132)
+- Updated Go version to 1.22 (#132)
+
+### Removed
+- Duplicated NNS Resolver code (#129)
+
+## [0.30.3] - 2024-10-18
+
+### Fixed
+- Get response on S3 multipart object (#142)
+
+### Added
+- Support percent-encoding for GET queries (#134)
+
+### Changed
+- Split `FrostFS` interface into separate read methods (#127)
+
+## [0.30.2] - 2024-09-03
+
+### Added
+- Fuzzing tests (#135)
+
+## [0.30.1] - 2024-08-20
+
+### Fixed
+- Error counting in pool component before connection switch (#131)
+
+### Added
+- Log of endpoint address during tree pool errors (#131)

 ## [0.30.0] - Kangshung - 2024-07-22

@ -128,4 +166,8 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
 [0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...v0.28.1
 [0.29.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.1...v0.29.0
 [0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.29.0...v0.30.0
-[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...master
+[0.30.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...v0.30.1
+[0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.1...v0.30.2
+[0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.2...v0.30.3
+[0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0
+[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...master
--- a/1
+++ b/1
@ -0,0 +1 @@
+.*       @alexvanin @dkirillov
--- a/README.md
+++ b/README.md
@ -1,5 +1,5 @@
 <p align="center">
-<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
+<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS logo">
 </p>
 <p align="center">
  <a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
--- a/2
+++ b/2
@ -1 +1 @@
-v0.30.0
+v0.31.0
--- a/cmd/http-gw/app.go
+++ b/cmd/http-gw/app.go
@ -40,6 +40,7 @@ import (
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	"github.com/nspcc-dev/neo-go/pkg/util"
 	"github.com/nspcc-dev/neo-go/pkg/wallet"
+	"github.com/panjf2000/ants/v2"
 	"github.com/spf13/viper"
 	"github.com/valyala/fasthttp"
 	"go.opentelemetry.io/otel/trace"
@ -89,6 +90,7 @@ type (
 	appSettings struct {
 		reconnectInterval time.Duration
 		dialerSource      *internalnet.DialerSource
+		workerPoolSize    int

 		mu                   sync.RWMutex
 		defaultTimestamp     bool
@ -184,6 +186,7 @@ func (a *app) initAppSettings() {
 	a.settings = &appSettings{
 		reconnectInterval: fetchReconnectInterval(a.cfg),
 		dialerSource:      getDialerSource(a.log, a.cfg),
+		workerPoolSize:    a.cfg.GetInt(cfgWorkerPoolSize),
 	}
 	a.settings.update(a.cfg, a.log)
 }
@ -490,7 +493,13 @@ func (a *app) setHealthStatus() {
 }

 func (a *app) Serve() {
-	handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)))
+	workerPool := a.initWorkerPool()
+	defer func() {
+		workerPool.Release()
+		close(a.webDone)
+	}()
+
+	handler := handler.New(a.AppParams(), a.settings, tree.NewTree(frostfs.NewPoolWrapper(a.treePool)), workerPool)

 	// Configure router.
 	a.configureRouter(handler)
@ -532,8 +541,14 @@ LOOP:
 	a.metrics.Shutdown()
 	a.stopServices()
 	a.shutdownTracing()
+}

-	close(a.webDone)
+func (a *app) initWorkerPool() *ants.Pool {
+	workerPool, err := ants.NewPool(a.settings.workerPoolSize)
+	if err != nil {
+		a.log.Fatal(logs.FailedToCreateWorkerPool, zap.Error(err))
+	}
+	return workerPool
 }

 func (a *app) shutdownTracing() {
@ -609,6 +624,7 @@ func (a *app) stopServices() {
 		svc.ShutDown(ctx)
 	}
 }
+
 func (a *app) configureRouter(handler *handler.Handler) {
 	r := router.New()
 	r.RedirectTrailingSlash = true
@ -927,6 +943,13 @@ func (a *app) initTracing(ctx context.Context) {
 		cfg.ServerCaCertPool = certPool
 	}

+	attributes, err := fetchTracingAttributes(a.cfg)
+	if err != nil {
+		a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
+		return
+	}
+	cfg.Attributes = attributes
+
 	updated, err := tracing.Setup(ctx, cfg)
 	if err != nil {
 		a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
--- a/cmd/http-gw/settings.go
+++ b/cmd/http-gw/settings.go
@ -71,6 +71,8 @@ const (
 	cfgIndexPageEnabled      = "index_page.enabled"
 	cfgIndexPageTemplatePath = "index_page.template_path"

+	cfgWorkerPoolSize = "worker_pool_size"
+
 	// Web.
 	cfgWebReadBufferSize     = "web.read_buffer_size"
 	cfgWebWriteBufferSize    = "web.write_buffer_size"
@ -86,10 +88,11 @@ const (
 	cfgPprofAddress      = "pprof.address"

 	// Tracing ...
-	cfgTracingEnabled   = "tracing.enabled"
-	cfgTracingExporter  = "tracing.exporter"
-	cfgTracingEndpoint  = "tracing.endpoint"
-	cfgTracingTrustedCa = "tracing.trusted_ca"
+	cfgTracingEnabled    = "tracing.enabled"
+	cfgTracingExporter   = "tracing.exporter"
+	cfgTracingEndpoint   = "tracing.endpoint"
+	cfgTracingTrustedCa  = "tracing.trusted_ca"
+	cfgTracingAttributes = "tracing.attributes"

 	// Pool config.
 	cfgConTimeout         = "connect_timeout"
@ -227,9 +230,6 @@ func settings() *viper.Viper {
 	// pool:
 	v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold)

-	v.SetDefault(cfgIndexPageEnabled, false)
-	v.SetDefault(cfgIndexPageTemplatePath, "")
-
 	// frostfs:
 	v.SetDefault(cfgBufferMaxSizeForPut, defaultBufferMaxSizeForPut)

@ -241,6 +241,7 @@ func settings() *viper.Viper {
 	v.SetDefault(cfgWebStreamRequestBody, true)
 	v.SetDefault(cfgWebMaxRequestBodySize, fasthttp.DefaultMaxRequestBodySize)

+	v.SetDefault(cfgWorkerPoolSize, 1000)
 	// upload header
 	v.SetDefault(cfgUploaderHeaderEnableDefaultTimestamp, false)

@ -790,3 +791,27 @@ func fetchMultinetConfig(v *viper.Viper, l *zap.Logger) (cfg internalnet.Config)

 	return
 }
+
+func fetchTracingAttributes(v *viper.Viper) (map[string]string, error) {
+	attributes := make(map[string]string)
+	for i := 0; ; i++ {
+		key := cfgTracingAttributes + "." + strconv.Itoa(i) + "."
+		attrKey := v.GetString(key + "key")
+		attrValue := v.GetString(key + "value")
+		if attrKey == "" {
+			break
+		}
+
+		if _, ok := attributes[attrKey]; ok {
+			return nil, fmt.Errorf("tracing attribute key %s defined more than once", attrKey)
+		}
+
+		if attrValue == "" {
+			return nil, fmt.Errorf("empty tracing attribute value for key %s", attrKey)
+		}
+
+		attributes[attrKey] = attrValue
+	}
+
+	return attributes, nil
+}
--- a/config/config.env
+++ b/config/config.env
@ -104,6 +104,10 @@ HTTP_GW_TRACING_ENABLED=true
 HTTP_GW_TRACING_ENDPOINT="localhost:4317"
 HTTP_GW_TRACING_EXPORTER="otlp_grpc"
 HTTP_GW_TRACING_TRUSTED_CA=""
+HTTP_GW_TRACING_ATTRIBUTES_0_KEY=key0
+HTTP_GW_TRACING_ATTRIBUTES_0_VALUE=value
+HTTP_GW_TRACING_ATTRIBUTES_1_KEY=key1
+HTTP_GW_TRACING_ATTRIBUTES_1_VALUE=value

 HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824

@ -146,3 +150,12 @@ HTTP_GW_MULTINET_FALLBACK_DELAY=300ms
 # List of subnets and IP addresses to use as source for those subnets
 HTTP_GW_MULTINET_SUBNETS_1_MASK=1.2.3.4/24
 HTTP_GW_MULTINET_SUBNETS_1_SOURCE_IPS=1.2.3.4 1.2.3.5
+
+# Number of workers in handler's worker pool
+HTTP_GW_WORKER_POOL_SIZE=1000
+
+# Index page
+# Enable index page support
+HTTP_GW_INDEX_PAGE_ENABLED=false
+# Index page template path
+HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl
--- a/config/config.yaml
+++ b/config/config.yaml
@ -9,11 +9,17 @@ pprof:
 prometheus:
  enabled: false # Enable metrics.
  address: localhost:8084
+
 tracing:
  enabled: true
  exporter: "otlp_grpc"
  endpoint: "localhost:4317"
  trusted_ca: ""
+  attributes:
+    - key: key0
+      value: value
+    - key: key1
+      value: value

 logger:
  level: debug # Log level.
@ -107,6 +113,9 @@ request_timeout: 5s # Timeout to check node health during rebalance.
 rebalance_timer: 30s # Interval to check nodes health.
 pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy.

+# Number of workers in handler's worker pool
+worker_pool_size: 1000
+
 # Enable index page to see objects list for specified container and prefix
 index_page:
  enabled: false
--- a/docs/gate-configuration.md
+++ b/docs/gate-configuration.md
@ -75,18 +75,21 @@ request_timeout: 5s
 rebalance_timer: 30s
 pool_error_threshold: 100
 reconnect_interval: 1m
+worker_pool_size: 1000
+
 ```

-| Parameter              | Type       | SIGHUP reload | Default value | Description                                                                                     |
-|------------------------|------------|---------------|---------------|-------------------------------------------------------------------------------------------------|
-| `rpc_endpoint`         | `string`   | yes           |               | The address of the RPC host to which the gateway connects to resolve bucket names.              |
-| `resolve_order`        | `[]string` | yes           | `[nns, dns]`  | Order of bucket name resolvers to use.                                                          |
-| `connect_timeout`      | `duration` |               | `10s`         | Timeout to connect to a node.                                                                   |
-| `stream_timeout`       | `duration` |               | `10s`         | Timeout for individual operations in streaming RPC.                                             |
-| `request_timeout`      | `duration` |               | `15s`         | Timeout to check node health during rebalance.                                                  |
-| `rebalance_timer`      | `duration` |               | `60s`         | Interval to check node health.                                                                  |
-| `pool_error_threshold` | `uint32`   |               | `100`         | The number of errors on connection after which node is considered as unhealthy.                 |
-| `reconnect_interval`   | `duration` | no            | `1m`          | Listeners reconnection interval.                                                                |
+| Parameter              | Type       | SIGHUP reload | Default value | Description                                                                        |
+|------------------------|------------|---------------|---------------|------------------------------------------------------------------------------------|
+| `rpc_endpoint`         | `string`   | yes           |               | The address of the RPC host to which the gateway connects to resolve bucket names. |
+| `resolve_order`        | `[]string` | yes           | `[nns, dns]`  | Order of bucket name resolvers to use.                                             |
+| `connect_timeout`      | `duration` |               | `10s`         | Timeout to connect to a node.                                                      |
+| `stream_timeout`       | `duration` |               | `10s`         | Timeout for individual operations in streaming RPC.                                |
+| `request_timeout`      | `duration` |               | `15s`         | Timeout to check node health during rebalance.                                     |
+| `rebalance_timer`      | `duration` |               | `60s`         | Interval to check node health.                                                     |
+| `pool_error_threshold` | `uint32`   |               | `100`         | The number of errors on connection after which node is considered as unhealthy.    |
+| `reconnect_interval`   | `duration` | no            | `1m`          | Listeners reconnection interval.                                                   |
+| `worker_pool_size`     | `int`      | no            | `1000`        | Maximum worker count in handler's worker pool.                                     |

 # `wallet` section

@ -268,14 +271,36 @@ tracing:
  exporter: "otlp_grpc"
  endpoint: "localhost:4317"
  trusted_ca: "/etc/ssl/telemetry-trusted-ca.pem"
+  attributes:
+    - key: key0
+      value: value
+    - key: key1
+      value: value
 ```

-| Parameter    | Type     | SIGHUP reload | Default value | Description                                                                                                                     |
-|--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------|
-| `enabled`    | `bool`   | yes           | `false`       | Flag to enable the tracing.                                                                                                     |
-| `exporter`   | `string` | yes           |               | Trace collector type (`stdout` or `otlp_grpc` are supported).                                                                   |
-| `endpoint`   | `string` | yes           |               | Address of collector endpoint for OTLP exporters.                                                                               |
-| `trusted_ca` | `string` | yes           |               | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. |
+| Parameter    | Type                                   | SIGHUP reload | Default value | Description                                                                                                                     |
+| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- |
+| `enabled`    | `bool`                                 | yes           | `false`       | Flag to enable the tracing.                                                                                                     |
+| `exporter`   | `string`                               | yes           |               | Trace collector type (`stdout` or `otlp_grpc` are supported).                                                                   |
+| `endpoint`   | `string`                               | yes           |               | Address of collector endpoint for OTLP exporters.                                                                               |
+| `trusted_ca` | `string`                               | yes           |               | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. |
+| `attributes` | [[]Attributes](#attributes-subsection) | yes           |               | An array of configurable attributes in key-value format.                                                                        |
+
+
+#### `attributes` subsection
+
+```yaml
+  attributes:
+    - key: key0
+      value: value
+    - key: key1
+      value: value
+```
+
+| Parameter             | Type     | SIGHUP reload | Default value | Description                                              |
+|-----------------------|----------|---------------|---------------|----------------------------------------------------------|
+| `key`                 | `string` | yes           |               | Attribute key.                                           |
+| `value`               | `string` | yes           |               | Attribute value.                                         |

 # `runtime` section
 Contains runtime parameters.
@ -352,7 +377,12 @@ resolve_bucket:

 # `index_page` section

-Parameters for index HTML-page output with S3-bucket or S3-subdir content for `Get object` request
+Parameters for index HTML-page output. Activates if `GetObject` request returns `not found`. Two
+index page modes available:
+
+* `s3` mode uses tree service for listing objects,
+* `native` sends requests to nodes via native protocol.
+  If request pass S3-bucket name instead of CID, `s3` mode will be used, otherwise `native`.

 ```yaml
 index_page:
--- a/go.mod
+++ b/go.mod
@ -3,15 +3,16 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw
 go 1.22

 require (
-	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e
-	git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573
-	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98
+	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1
+	git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88
+	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3
 	git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972
 	git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
 	github.com/bluele/gcache v0.0.2
 	github.com/docker/go-units v0.4.0
 	github.com/fasthttp/router v1.4.1
 	github.com/nspcc-dev/neo-go v0.106.2
+	github.com/panjf2000/ants/v2 v2.5.0
 	github.com/prometheus/client_golang v1.19.0
 	github.com/prometheus/client_model v0.5.0
 	github.com/spf13/pflag v1.0.5
@ -24,7 +25,7 @@ require (
 	go.opentelemetry.io/otel v1.28.0
 	go.opentelemetry.io/otel/trace v1.28.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
+	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 	golang.org/x/net v0.26.0
 	golang.org/x/sys v0.22.0
 	google.golang.org/grpc v1.66.2
@ -41,7 +42,7 @@ require (
 	github.com/Microsoft/hcsshim v0.9.2 // indirect
 	github.com/VictoriaMetrics/easyproto v0.1.4 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
-	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
--- a/go.sum
+++ b/go.sum
@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0=
-git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU=
+git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 h1:ivcdxQeQDnx4srF2ezoaeVlF0FAycSAztwfIUJnUI4s=
+git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU=
 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4=
 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc=
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
-git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274=
-git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 h1:ijUci3thz0EwWkuRJDocW5D1RkVAJlt9xNG4CYepC90=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino=
+git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI=
+git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 h1:f7jan6eBDN88DKnKj8GKyWpfjBbSzjDALcDejYKRgCs=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3/go.mod h1:3txOjFJ8M/JFs01h7xOrnQHVn6hZgDNA16ivyUlu1iU=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8=
@ -114,8 +114,8 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
-github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
+github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
+github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
@ -682,6 +682,8 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q=
+github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
@ -928,8 +930,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ=
-golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
--- a/internal/handler/browse.go
+++ b/internal/handler/browse.go
@ -1,15 +1,21 @@
 package handler

 import (
+	"context"
 	"html/template"
 	"net/url"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/data"
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/utils"
+	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"github.com/docker/go-units"
 	"github.com/valyala/fasthttp"
 	"go.uber.org/zap"
@ -25,19 +31,68 @@ const (

 type (
 	BrowsePageData struct {
-		BucketName,
-		Prefix string
-		Objects []ResponseObject
+		HasErrors bool
+		Container string
+		Prefix    string
+		Protocol  string
+		Objects   []ResponseObject
 	}
 	ResponseObject struct {
 		OID      string
 		Created  string
 		FileName string
+		FilePath string
 		Size     string
 		IsDir    bool
+		GetURL   string
 	}
 )

+func newListObjectsResponseS3(attrs map[string]string) ResponseObject {
+	return ResponseObject{
+		Created:  formatTimestamp(attrs[attrCreated]),
+		OID:      attrs[attrOID],
+		FileName: attrs[attrFileName],
+		Size:     attrs[attrSize],
+		IsDir:    attrs[attrOID] == "",
+	}
+}
+
+func newListObjectsResponseNative(attrs map[string]string) ResponseObject {
+	filename := lastPathElement(attrs[object.AttributeFilePath])
+	if filename == "" {
+		filename = attrs[attrFileName]
+	}
+	return ResponseObject{
+		OID:      attrs[attrOID],
+		Created:  formatTimestamp(attrs[object.AttributeTimestamp] + "000"),
+		FileName: filename,
+		FilePath: attrs[object.AttributeFilePath],
+		Size:     attrs[attrSize],
+		IsDir:    false,
+	}
+}
+
+func getNextDir(filepath, prefix string) string {
+	restPath := strings.Replace(filepath, prefix, "", 1)
+	index := strings.Index(restPath, "/")
+	if index == -1 {
+		return ""
+	}
+	return restPath[:index]
+}
+
+func lastPathElement(path string) string {
+	if path == "" {
+		return path
+	}
+	index := strings.LastIndex(path, "/")
+	if index == len(path)-1 {
+		index = strings.LastIndex(path[:index], "/")
+	}
+	return path[index+1:]
+}
+
 func parseTimestamp(tstamp string) (time.Time, error) {
 	millis, err := strconv.ParseInt(tstamp, 10, 64)
 	if err != nil {
@ -47,16 +102,6 @@ func parseTimestamp(tstamp string) (time.Time, error) {
 	return time.UnixMilli(millis), nil
 }

-func NewResponseObject(nodes map[string]string) ResponseObject {
-	return ResponseObject{
-		OID:      nodes[attrOID],
-		Created:  nodes[attrCreated],
-		FileName: nodes[attrFileName],
-		Size:     nodes[attrSize],
-		IsDir:    nodes[attrOID] == "",
-	}
-}
-
 func formatTimestamp(strdate string) string {
 	date, err := parseTimestamp(strdate)
 	if err != nil || date.IsZero() {
@ -94,12 +139,9 @@ func trimPrefix(encPrefix string) string {
 	return prefix[:slashIndex]
 }

-func urlencode(prefix, filename string) string {
+func urlencode(path string) string {
 	var res strings.Builder
-	path := filename
-	if prefix != "" {
-		path = strings.Join([]string{prefix, filename}, "/")
-	}
+
 	prefixParts := strings.Split(path, "/")
 	for _, prefixPart := range prefixParts {
 		prefixPart = "/" + url.PathEscape(prefixPart)
@ -112,46 +154,220 @@ func urlencode(prefix, filename string) string {
 	return res.String()
 }

-func (h *Handler) browseObjects(c *fasthttp.RequestCtx, bucketInfo *data.BucketInfo, prefix string) {
+type GetObjectsResponse struct {
+	objects   []ResponseObject
+	hasErrors bool
+}
+
+func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
+	nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
+	if err != nil {
+		return nil, err
+	}
+
+	result := &GetObjectsResponse{
+		objects: make([]ResponseObject, 0, len(nodes)),
+	}
+	for _, node := range nodes {
+		meta := node.GetMeta()
+		if meta == nil {
+			continue
+		}
+		var attrs = make(map[string]string, len(meta))
+		for _, m := range meta {
+			attrs[m.GetKey()] = string(m.GetValue())
+		}
+		obj := newListObjectsResponseS3(attrs)
+		obj.FilePath = prefix + obj.FileName
+		obj.GetURL = "/get/" + bucketInfo.Name + urlencode(obj.FilePath)
+		result.objects = append(result.objects, obj)
+	}
+
+	return result, nil
+}
+
+func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) (*GetObjectsResponse, error) {
+	var basePath string
+	if ind := strings.LastIndex(prefix, "/"); ind != -1 {
+		basePath = prefix[:ind+1]
+	}
+
+	filters := object.NewSearchFilters()
+	filters.AddRootFilter()
+	if prefix != "" {
+		filters.AddFilter(object.AttributeFilePath, prefix, object.MatchCommonPrefix)
+	}
+
+	prm := PrmObjectSearch{
+		PrmAuth: PrmAuth{
+			BearerToken: bearerToken(ctx),
+		},
+		Container: bucketInfo.CID,
+		Filters:   filters,
+	}
+	objectIDs, err := h.frostfs.SearchObjects(ctx, prm)
+	if err != nil {
+		return nil, err
+	}
+	defer objectIDs.Close()
+
+	resp, err := h.headDirObjects(ctx, bucketInfo.CID, objectIDs, basePath)
+	if err != nil {
+		return nil, err
+	}
+
+	log := utils.GetReqLogOrDefault(ctx, h.log)
+	dirs := make(map[string]struct{})
+	result := &GetObjectsResponse{
+		objects: make([]ResponseObject, 0, 100),
+	}
+	for objExt := range resp {
+		if objExt.Error != nil {
+			log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error))
+			result.hasErrors = true
+			continue
+		}
+		if objExt.Object.IsDir {
+			if _, ok := dirs[objExt.Object.FileName]; ok {
+				continue
+			}
+			objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + urlencode(objExt.Object.FilePath)
+			dirs[objExt.Object.FileName] = struct{}{}
+		} else {
+			objExt.Object.GetURL = "/get/" + bucketInfo.CID.EncodeToString() + "/" + objExt.Object.OID
+		}
+		result.objects = append(result.objects, objExt.Object)
+	}
+	return result, nil
+}
+
+type ResponseObjectExtended struct {
+	Object ResponseObject
+	Error  error
+}
+
+func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs ResObjectSearch, basePath string) (<-chan ResponseObjectExtended, error) {
+	res := make(chan ResponseObjectExtended)
+
+	go func() {
+		defer close(res)
+		log := utils.GetReqLogOrDefault(ctx, h.log).With(
+			zap.String("cid", cnrID.EncodeToString()),
+			zap.String("path", basePath),
+		)
+		var wg sync.WaitGroup
+		err := objectIDs.Iterate(func(id oid.ID) bool {
+			wg.Add(1)
+			err := h.workerPool.Submit(func() {
+				defer wg.Done()
+				var obj ResponseObjectExtended
+				obj.Object, obj.Error = h.headDirObject(ctx, cnrID, id, basePath)
+				res <- obj
+			})
+			if err != nil {
+				wg.Done()
+				log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err))
+			}
+			select {
+			case <-ctx.Done():
+				return true
+			default:
+				return false
+			}
+		})
+		if err != nil {
+			log.Error(logs.FailedToIterateOverResponse, zap.Error(err))
+		}
+		wg.Wait()
+	}()
+
+	return res, nil
+}
+
+func (h *Handler) headDirObject(ctx context.Context, cnrID cid.ID, objID oid.ID, basePath string) (ResponseObject, error) {
+	addr := newAddress(cnrID, objID)
+	obj, err := h.frostfs.HeadObject(ctx, PrmObjectHead{
+		PrmAuth: PrmAuth{BearerToken: bearerToken(ctx)},
+		Address: addr,
+	})
+	if err != nil {
+		return ResponseObject{}, err
+	}
+
+	attrs := loadAttributes(obj.Attributes())
+	attrs[attrOID] = objID.EncodeToString()
+	if multipartSize, ok := attrs[attributeMultipartObjectSize]; ok {
+		attrs[attrSize] = multipartSize
+	} else {
+		attrs[attrSize] = strconv.FormatUint(obj.PayloadSize(), 10)
+	}
+
+	dirname := getNextDir(attrs[object.AttributeFilePath], basePath)
+	if dirname == "" {
+		return newListObjectsResponseNative(attrs), nil
+	}
+
+	return ResponseObject{
+		FileName: dirname,
+		FilePath: basePath + dirname,
+		IsDir:    true,
+	}, nil
+}
+
+type browseParams struct {
+	bucketInfo  *data.BucketInfo
+	prefix      string
+	isNative    bool
+	listObjects func(ctx context.Context, bucketName *data.BucketInfo, prefix string) (*GetObjectsResponse, error)
+}
+
+func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
+	const S3Protocol = "s3"
+	const FrostfsProtocol = "frostfs"
+
 	ctx := utils.GetContextFromRequest(c)
 	reqLog := utils.GetReqLogOrDefault(ctx, h.log)
-	log := reqLog.With(zap.String("bucket", bucketInfo.Name))
-
-	nodes, err := h.listObjects(ctx, bucketInfo, prefix)
+	log := reqLog.With(
+		zap.String("bucket", p.bucketInfo.Name),
+		zap.String("container", p.bucketInfo.CID.EncodeToString()),
+		zap.String("prefix", p.prefix),
+	)
+	resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
 	if err != nil {
 		logAndSendBucketError(c, log, err)
 		return
 	}

-	respObjects := make([]ResponseObject, len(nodes))
-
-	for i, node := range nodes {
-		respObjects[i] = NewResponseObject(node)
-	}
-
-	sort.Slice(respObjects, func(i, j int) bool {
-		if respObjects[i].IsDir == respObjects[j].IsDir {
-			return respObjects[i].FileName < respObjects[j].FileName
+	objects := resp.objects
+	sort.Slice(objects, func(i, j int) bool {
+		if objects[i].IsDir == objects[j].IsDir {
+			return objects[i].FileName < objects[j].FileName
 		}
-		return respObjects[i].IsDir
+		return objects[i].IsDir
 	})
-	indexTemplate := h.config.IndexPageTemplate()

 	tmpl, err := template.New("index").Funcs(template.FuncMap{
-		"formatTimestamp": formatTimestamp,
-		"formatSize":      formatSize,
-		"trimPrefix":      trimPrefix,
-		"urlencode":       urlencode,
-		"parentDir":       parentDir,
-	}).Parse(indexTemplate)
+		"formatSize": formatSize,
+		"trimPrefix": trimPrefix,
+		"urlencode":  urlencode,
+		"parentDir":  parentDir,
+	}).Parse(h.config.IndexPageTemplate())
 	if err != nil {
 		logAndSendBucketError(c, log, err)
 		return
 	}
+	bucketName := p.bucketInfo.Name
+	protocol := S3Protocol
+	if p.isNative {
+		bucketName = p.bucketInfo.CID.EncodeToString()
+		protocol = FrostfsProtocol
+	}
 	if err = tmpl.Execute(c, &BrowsePageData{
-		BucketName: bucketInfo.Name,
-		Prefix:     prefix,
-		Objects:    respObjects,
+		Container: bucketName,
+		Prefix:    p.prefix,
+		Objects:   objects,
+		Protocol:  protocol,
+		HasErrors: resp.hasErrors,
 	}); err != nil {
 		logAndSendBucketError(c, log, err)
 		return
--- a/internal/handler/download.go
+++ b/internal/handler/download.go
@ -23,13 +23,16 @@ import (

 // DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
 func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
-	test, _ := c.UserValue("oid").(string)
-	var id oid.ID
-	err := id.DecodeString(test)
-	if err != nil {
-		h.byObjectName(c, h.receiveFile)
-	} else {
-		h.byAddress(c, h.receiveFile)
+	oidURLParam := c.UserValue("oid").(string)
+	downloadQueryParam := c.QueryArgs().GetBool("download")
+
+	switch {
+	case isObjectID(oidURLParam):
+		h.byNativeAddress(c, h.receiveFile)
+	case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)):
+		h.byS3Path(c, h.receiveFile)
+	default:
+		h.browseIndex(c)
 	}
 }

@ -45,7 +48,7 @@ func (h *Handler) DownloadByAttribute(c *fasthttp.RequestCtx) {
 	h.byAttribute(c, h.receiveFile)
 }

-func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) {
+func (h *Handler) search(ctx context.Context, cnrID cid.ID, key, val string, op object.SearchMatchType) (ResObjectSearch, error) {
 	filters := object.NewSearchFilters()
 	filters.AddRootFilter()
 	filters.AddFilter(key, val, op)
@ -54,7 +57,7 @@ func (h *Handler) search(ctx context.Context, cnrID *cid.ID, key, val string, op
 		PrmAuth: PrmAuth{
 			BearerToken: bearerToken(ctx),
 		},
-		Container: *cnrID,
+		Container: cnrID,
 		Filters:   filters,
 	}

@ -102,7 +105,7 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
 		return
 	}

-	resSearch, err := h.search(ctx, &bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
+	resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
 	if err != nil {
 		log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
 		response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
--- a/internal/handler/handler.go
+++ b/internal/handler/handler.go
@ -22,6 +22,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
+	"github.com/panjf2000/ants/v2"
 	"github.com/valyala/fasthttp"
 	"go.uber.org/zap"
 )
@ -165,6 +166,7 @@ type Handler struct {
 	containerResolver ContainerResolver
 	tree              *tree.Tree
 	cache             *cache.BucketCache
+	workerPool        *ants.Pool
 }

 type AppParams struct {
@ -175,7 +177,7 @@ type AppParams struct {
 	Cache    *cache.BucketCache
 }

-func New(params *AppParams, config Config, tree *tree.Tree) *Handler {
+func New(params *AppParams, config Config, tree *tree.Tree, workerPool *ants.Pool) *Handler {
 	return &Handler{
 		log:               params.Logger,
 		frostfs:           params.FrostFS,
@ -184,14 +186,15 @@ func New(params *AppParams, config Config, tree *tree.Tree) *Handler {
 		containerResolver: params.Resolver,
 		tree:              tree,
 		cache:             params.Cache,
+		workerPool:        workerPool,
 	}
 }

-// byAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
+// byNativeAddress is a wrapper for function (e.g. request.headObject, request.receiveFile) that
 // prepares request and object address to it.
-func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
+func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
 	idCnr, _ := c.UserValue("cid").(string)
-	idObj, _ := c.UserValue("oid").(string)
+	idObj, _ := url.PathUnescape(c.UserValue("oid").(string))

 	ctx := utils.GetContextFromRequest(c)
 	reqLog := utils.GetReqLogOrDefault(ctx, h.log)
@ -215,12 +218,11 @@ func (h *Handler) byAddress(c *fasthttp.RequestCtx, f func(context.Context, requ
 	f(ctx, *h.newRequest(c, log), addr)
 }

-// byObjectName is a wrapper for function (e.g. request.headObject, request.receiveFile) that
-// prepares request and object address to it.
-func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
+// byS3Path is a wrapper for function (e.g. request.headObject, request.receiveFile) that
+// resolves object address from S3-like path <bucket name>/<object key>.
+func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
 	bucketname := c.UserValue("cid").(string)
 	key := c.UserValue("oid").(string)
-	download := c.QueryArgs().GetBool("download")

 	ctx := utils.GetContextFromRequest(c)
 	reqLog := utils.GetReqLogOrDefault(ctx, h.log)
@ -239,15 +241,6 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r
 	}

 	foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey)
-	if h.config.IndexPageEnabled() && !download && string(c.Method()) != fasthttp.MethodHead {
-		if isDir(unescapedKey) || isContainerRoot(unescapedKey) {
-			if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK {
-				c.SetStatusCode(code)
-				h.browseObjects(c, bktInfo, unescapedKey)
-				return
-			}
-		}
-	}
 	if err != nil {
 		if errors.Is(err, tree.ErrNodeAccessDenied) {
 			response.Error(c, "Access Denied", fasthttp.StatusForbidden)
@ -267,7 +260,7 @@ func (h *Handler) byObjectName(c *fasthttp.RequestCtx, f func(context.Context, r
 	f(ctx, *h.newRequest(c, log), addr)
 }

-// byAttribute is a wrapper similar to byAddress.
+// byAttribute is a wrapper similar to byNativeAddress.
 func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
 	scid, _ := c.UserValue("cid").(string)
 	key, _ := c.UserValue("attr_key").(string)
@ -298,7 +291,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, re
 		return
 	}

-	res, err := h.search(ctx, &bktInfo.CID, key, val, object.MatchStringEqual)
+	res, err := h.search(ctx, bktInfo.CID, key, val, object.MatchStringEqual)
 	if err != nil {
 		log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
 		response.Error(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
@ -395,24 +388,50 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
 	return bktInfo, err
 }

-func (h *Handler) listObjects(ctx context.Context, bucketInfo *data.BucketInfo, prefix string) ([]map[string]string, error) {
-	nodes, _, err := h.tree.GetSubTreeByPrefix(ctx, bucketInfo, prefix, true)
+func (h *Handler) browseIndex(c *fasthttp.RequestCtx) {
+	if !h.config.IndexPageEnabled() {
+		c.SetStatusCode(fasthttp.StatusNotFound)
+		return
+	}
+
+	cidURLParam := c.UserValue("cid").(string)
+	oidURLParam := c.UserValue("oid").(string)
+
+	ctx := utils.GetContextFromRequest(c)
+	reqLog := utils.GetReqLogOrDefault(ctx, h.log)
+	log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam))
+
+	unescapedKey, err := url.QueryUnescape(oidURLParam)
 	if err != nil {
-		return nil, err
+		logAndSendBucketError(c, log, err)
+		return
 	}

-	var objects = make([]map[string]string, 0, len(nodes))
-	for _, node := range nodes {
-		meta := node.GetMeta()
-		if meta == nil {
-			continue
-		}
-		var obj = make(map[string]string, len(meta))
-		for _, m := range meta {
-			obj[m.GetKey()] = string(m.GetValue())
-		}
-		objects = append(objects, obj)
+	bktInfo, err := h.getBucketInfo(ctx, cidURLParam, log)
+	if err != nil {
+		logAndSendBucketError(c, log, err)
+		return
 	}

-	return objects, nil
+	listFunc := h.getDirObjectsS3
+	isNativeList := false
+
+	err = h.tree.CheckSettingsNodeExist(ctx, bktInfo)
+	if err != nil {
+		if errors.Is(err, tree.ErrNodeNotFound) {
+			// tree probe failed, try to use native
+			listFunc = h.getDirObjectsNative
+			isNativeList = true
+		} else {
+			logAndSendBucketError(c, log, err)
+			return
+		}
+	}
+
+	h.browseObjects(c, browseParams{
+		bucketInfo:  bktInfo,
+		prefix:      unescapedKey,
+		listObjects: listFunc,
+		isNative:    isNativeList,
+	})
 }
--- a/internal/handler/handler_test.go
+++ b/internal/handler/handler_test.go
@ -26,6 +26,7 @@ import (
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
+	"github.com/panjf2000/ants/v2"
 	"github.com/stretchr/testify/require"
 	"github.com/valyala/fasthttp"
 	"go.uber.org/zap"
@ -57,10 +58,11 @@ func (c *configMock) IndexPageEnabled() bool {
 	return false
 }

-func (c *configMock) IndexPageTemplatePath() string {
+func (c *configMock) IndexPageTemplate() string {
 	return ""
 }
-func (c *configMock) IndexPageTemplate() string {
+
+func (c *configMock) IndexPageNativeTemplate() string {
 	return ""
 }

@ -126,7 +128,11 @@ func prepareHandlerContext() (*handlerContext, error) {
 	treeMock := &treeClientMock{}
 	cfgMock := &configMock{}

-	handler := New(params, cfgMock, tree.NewTree(treeMock))
+	workerPool, err := ants.NewPool(1000)
+	if err != nil {
+		return nil, err
+	}
+	handler := New(params, cfgMock, tree.NewTree(treeMock), workerPool)

 	return &handlerContext{
 		key:     key,
--- a/internal/handler/head.go
+++ b/internal/handler/head.go
@ -107,9 +107,9 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {

 	err := id.DecodeString(test)
 	if err != nil {
-		h.byObjectName(c, h.headObject)
+		h.byS3Path(c, h.headObject)
 	} else {
-		h.byAddress(c, h.headObject)
+		h.byNativeAddress(c, h.headObject)
 	}
 }

--- a/internal/handler/utils.go
+++ b/internal/handler/utils.go
@ -2,17 +2,16 @@ package handler

 import (
 	"context"
-	"errors"
 	"strings"
 	"time"

 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
-	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"github.com/valyala/fasthttp"
 	"go.uber.org/zap"
@ -46,19 +45,21 @@ func isDir(name string) bool {
 	return strings.HasSuffix(name, "/")
 }

+func isObjectID(s string) bool {
+	var objID oid.ID
+	return objID.DecodeString(s) == nil
+}
+
 func isContainerRoot(key string) bool {
 	return key == ""
 }

-func checkErrorType(err error) int {
-	switch {
-	case err == nil:
-		return fasthttp.StatusOK
-	case errors.Is(err, tree.ErrNodeAccessDenied):
-		return fasthttp.StatusForbidden
-	default:
-		return fasthttp.StatusNotFound
+func loadAttributes(attrs []object.Attribute) map[string]string {
+	result := make(map[string]string)
+	for _, attr := range attrs {
+		result[attr.Key()] = attr.Value()
 	}
+	return result
 }

 func isValidToken(s string) bool {
--- a/internal/logs/logs.go
+++ b/internal/logs/logs.go
@ -31,7 +31,8 @@ const (
 	CouldNotStoreFileInFrostfs                                            = "could not store file in frostfs"                                                        // Error in ../../uploader/upload.go
 	AddAttributeToResultObject                                            = "add attribute to result object"                                                         // Debug in ../../uploader/filter.go
 	FailedToCreateResolver                                                = "failed to create resolver"                                                              // Fatal in ../../app.go
-	FailedToReadIndexPageTemplate                                         = "failed to read index page template, set default"                                        // Warn in ../../app.go
+	FailedToCreateWorkerPool                                              = "failed to create worker pool"                                                           // Fatal in ../../app.go
+	FailedToReadIndexPageTemplate                                         = "failed to read index page template"                                                     // Error in ../../app.go
 	SetCustomIndexPageTemplate                                            = "set custom index page template"                                                         // Info in ../../app.go
 	ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty"     // Info in ../../app.go
 	MetricsAreDisabled                                                    = "metrics are disabled"                                                                   // Warn in ../../app.go
@ -71,6 +72,9 @@ const (
 	AddedStoragePeer                                                      = "added storage peer"                                                                     // Info in ../../settings.go
 	CouldntGetBucket                                                      = "could not get bucket"                                                                   // Error in ../handler/utils.go
 	CouldntPutBucketIntoCache                                             = "couldn't put bucket info into cache"                                                    // Warn in ../handler/handler.go
+	FailedToSumbitTaskToPool                                              = "failed to submit task to pool"                                                          // Error in ../handler/browse.go
+	FailedToHeadObject                                                    = "failed to head object"                                                                  // Error in ../handler/browse.go
+	FailedToIterateOverResponse                                           = "failed to iterate over search response"                                                 // Error in ../handler/browse.go
 	InvalidCacheEntryType                                                 = "invalid cache entry type"                                                               // Warn in ../cache/buckets.go
 	InvalidLifetimeUsingDefaultValue                                      = "invalid lifetime, using default value (in seconds)"                                     // Error in ../../cmd/http-gw/settings.go
 	InvalidCacheSizeUsingDefaultValue                                     = "invalid cache size, using default value"                                                // Error in ../../cmd/http-gw/settings.go
--- a/internal/templates/index.gotmpl
+++ b/internal/templates/index.gotmpl
@ -1,11 +1,20 @@
-{{$bucketName := .BucketName}}
+{{$container := .Container}}
 {{ $prefix := trimPrefix .Prefix }}
 <!DOCTYPE html>
 <html lang="en">
 <head>
    <meta charset="UTF-8"/>
-    <title>Index of s3://{{$bucketName}}/{{if $prefix}}/{{$prefix}}/{{end}}</title>
+    <title>Index of {{.Protocol}}://{{$container}}
+        /{{if $prefix}}/{{$prefix}}/{{end}}</title>
    <style>
+        .alert {
+            width: 80%;
+            box-sizing: border-box;
+            padding: 20px;
+            background-color: #f44336;
+            color: white;
+            margin-bottom: 15px;
+        }
        table {
            width: 80%;
            border-collapse: collapse;
@ -23,15 +32,25 @@
        th {
            background-color: #c3bcbc;
        }
+
+        h1 {
+            font-size: 1.5em;
+        }
        tr:nth-child(even) {background-color: #ebe7e7;}
    </style>
 </head>
 <body>
-<h1>Index of s3://{{$bucketName}}/{{if $prefix}}{{$prefix}}/{{end}}</h1>
+<h1>Index of {{.Protocol}}://{{$container}}/{{if $prefix}}{{$prefix}}/{{end}}</h1>
+{{ if .HasErrors }}
+    <div class="alert">
+        Errors occurred while processing the request. Perhaps some objects are missing
+    </div>
+{{ end }}
 <table>
    <thead>
    <tr>
        <th>Filename</th>
+        <th>OID</th>
        <th>Size</th>
        <th>Created</th>
        <th>Download</th>
@ -42,20 +61,22 @@
    {{if $trimmedPrefix }}
        <tr>
            <td>
-                ⮐<a href="/get/{{$bucketName}}{{ urlencode $trimmedPrefix "" }}">..</a>
+                ⮐<a href="/get/{{$container}}{{ urlencode $trimmedPrefix }}/">..</a>
            </td>
            <td></td>
            <td></td>
            <td></td>
+            <td></td>
        </tr>
    {{else}}
        <tr>
            <td>
-                ⮐<a href="/get/{{ $bucketName }}/">..</a>
+                ⮐<a href="/get/{{$container}}/">..</a>
            </td>
            <td></td>
            <td></td>
            <td></td>
+            <td></td>
        </tr>
    {{end}}
    {{range .Objects}}
@ -63,21 +84,22 @@
            <td>
                {{if .IsDir}}
                    🗀
-                    <a href="/get/{{ $bucketName }}{{ urlencode $prefix .FileName }}/">
+                    <a href="{{.GetURL}}/">
                        {{.FileName}}/
                    </a>
                {{else}}
                    🗎
-                    <a href="/get/{{ $bucketName }}{{ urlencode $prefix .FileName }}">
+                    <a href="{{ .GetURL }}">
                        {{.FileName}}
                    </a>
                {{end}}
            </td>
+            <td>{{.OID}}</td>
            <td>{{if not .IsDir}}{{ formatSize .Size }}{{end}}</td>
-            <td>{{if not .IsDir}}{{ formatTimestamp .Created }}{{end}}</td>
+            <td>{{ .Created }}</td>
            <td>
-                {{ if not .IsDir }}
-                    <a href="/get/{{ $bucketName}}{{ urlencode $prefix .FileName }}?download=true">
+                {{ if .OID }}
+                    <a href="{{ .GetURL }}?download=true">
                        Link
                    </a>
                {{ end }}
--- a/tree/tree.go
+++ b/tree/tree.go
@ -30,6 +30,11 @@ type (
 		Meta  map[string]string
 	}

+	multiSystemNode struct {
+		// the first element is latest
+		nodes []*treeNode
+	}
+
 	GetNodesParams struct {
 		CnrID      cid.ID
 		BktInfo    *data.BucketInfo
@ -50,18 +55,19 @@ var (
 )

 const (
-	FileNameKey = "FileName"
-)
+	FileNameKey      = "FileName"
+	settingsFileName = "bucket-settings"

-const (
-	oidKV = "OID"
+	oidKV      = "OID"
+	uploadIDKV = "UploadId"
+	sizeKV     = "Size"

 	// keys for delete marker nodes.
 	isDeleteMarkerKV = "IsDeleteMarker"
-	sizeKV           = "Size"

 	// versionTree -- ID of a tree with object versions.
 	versionTree = "version"
+	systemTree  = "system"

 	separator = "/"
 )
@ -135,6 +141,45 @@ func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion {
 	return version
 }

+func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) {
+	var (
+		err          error
+		index        int
+		maxTimestamp uint64
+	)
+
+	if len(nodes) == 0 {
+		return nil, errors.New("multi node must have at least one node")
+	}
+
+	treeNodes := make([]*treeNode, len(nodes))
+
+	for i, node := range nodes {
+		if treeNodes[i], err = newTreeNode(node); err != nil {
+			return nil, fmt.Errorf("parse system node response: %w", err)
+		}
+
+		if timestamp := getMaxTimestamp(node); timestamp > maxTimestamp {
+			index = i
+			maxTimestamp = timestamp
+		}
+	}
+
+	treeNodes[0], treeNodes[index] = treeNodes[index], treeNodes[0]
+
+	return &multiSystemNode{
+		nodes: treeNodes,
+	}, nil
+}
+
+func (m *multiSystemNode) Latest() *treeNode {
+	return m.nodes[0]
+}
+
+func (m *multiSystemNode) Old() []*treeNode {
+	return m.nodes[1:]
+}
+
 func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) {
 	nodes, err := c.GetVersions(ctx, cnrID, objectName)
 	if err != nil {
@ -165,6 +210,55 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
 	return c.service.GetNodes(ctx, p)
 }

+func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error {
+	_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) {
+	p := &GetNodesParams{
+		CnrID:      bktInfo.CID,
+		BktInfo:    bktInfo,
+		TreeID:     systemTree,
+		Path:       []string{name},
+		LatestOnly: false,
+		AllAttrs:   true,
+	}
+	nodes, err := c.service.GetNodes(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	nodes = filterMultipartNodes(nodes)
+
+	if len(nodes) == 0 {
+		return nil, ErrNodeNotFound
+	}
+
+	return newMultiNode(nodes)
+}
+
+func filterMultipartNodes(nodes []NodeResponse) []NodeResponse {
+	res := make([]NodeResponse, 0, len(nodes))
+
+LOOP:
+	for _, node := range nodes {
+		for _, meta := range node.GetMeta() {
+			if meta.GetKey() == uploadIDKV {
+				continue LOOP
+			}
+		}
+
+		res = append(res, node)
+	}
+
+	return res
+}
+
 func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
 	var (
 		maxCreationTime uint64
Author	SHA1	Message	Date
Alex Vanin	a2f8cb6735	Release v0.31.0 Signed-off-by: Alex Vanin <a.vanin@yadro.com>	2024-11-20 11:09:31 +03:00
Nikita Zinkevich	43764772aa	[#151 ] index page: Add browse via native protocol Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>	2024-11-19 17:33:21 +03:00
Roman Loginov	9c0b499ea6	[#164 ] Add tracing attributes Signed-off-by: Roman Loginov <r.loginov@yadro.com>	2024-11-18 12:48:04 +00:00
Vitaliy Potyarkin	22d905e51e	[#165 ] Execute CI on push to master Discussion: TrueCloudLab/frostfs-s3-gw#550 Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>	2024-11-15 11:41:13 +00:00
Vitaliy Potyarkin	d5b92446bd	[#162 ] Stop using obsolete .github directory This commit is a part of multi-repo cleanup effort: TrueCloudLab/frostfs-infra#136 Signed-off-by: Vitaliy Potyarkin <v.potyarkin@yadro.com>	2024-11-06 15:19:54 +03:00
Denis Kirillov	679731ee52	[#161 ] Update SDK Need fix TrueCloudLab/frostfs-sdk-go#282 Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>	2024-11-05 17:51:35 +03:00