Compare commits
2 commits
master
...
fix/index_
| Author | SHA1 | Date | |
|---|---|---|---|
| d2ab6bab23 | |||
181568f442
|
24 changed files with 140 additions and 322 deletions
|
|
@ -1,8 +1,4 @@
|
|||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
builds:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,4 @@
|
|||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,4 @@
|
|||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
vulncheck:
|
||||
|
|
|
|||
1
.github/CODEOWNERS
vendored
Normal file
1
.github/CODEOWNERS
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
* @alexvanin @dkirillov
|
||||
0
.forgejo/logo.svg → .github/logo.svg
vendored
0
.forgejo/logo.svg → .github/logo.svg
vendored
|
Before Width: | Height: | Size: 5.5 KiB After Width: | Height: | Size: 5.5 KiB |
50
CHANGELOG.md
50
CHANGELOG.md
|
|
@ -4,51 +4,13 @@ This document outlines major changes between releases.
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.31.0] - Rongbuk - 2024-11-20
|
||||
|
||||
### Fixed
|
||||
- Docker warnings during image build (#126)
|
||||
- `trace_id` parameter in logs (#148)
|
||||
- SIGHUP support for `tracing.enabled` config parameter (#157)
|
||||
|
||||
### Added
|
||||
- Vulnerability report document (#123)
|
||||
- Root CA configuration for tracing (#139)
|
||||
- Log sampling policy configuration (#147)
|
||||
- Index page support for buckets and containers (#137, #151)
|
||||
- CORS support (#158)
|
||||
- Source IP binding configuration for FrostFS requests (#160)
|
||||
- Tracing attributes (#164)
|
||||
|
||||
### Changed
|
||||
- Updated Go version to 1.22 (#132)
|
||||
|
||||
### Removed
|
||||
- Duplicated NNS Resolver code (#129)
|
||||
|
||||
## [0.30.3] - 2024-10-18
|
||||
|
||||
### Fixed
|
||||
- Get response on S3 multipart object (#142)
|
||||
|
||||
### Added
|
||||
- Support percent-encoding for GET queries (#134)
|
||||
- Add `trace_id` to logs (#148)
|
||||
- Add `cors` config params (#158)
|
||||
|
||||
### Changed
|
||||
- Split `FrostFS` interface into separate read methods (#127)
|
||||
|
||||
## [0.30.2] - 2024-09-03
|
||||
|
||||
### Added
|
||||
- Fuzzing tests (#135)
|
||||
|
||||
## [0.30.1] - 2024-08-20
|
||||
|
||||
### Fixed
|
||||
- Error counting in pool component before connection switch (#131)
|
||||
|
||||
### Added
|
||||
- Log of endpoint address during tree pool errors (#131)
|
||||
- Update go version to 1.22 (#132)
|
||||
|
||||
## [0.30.0] - Kangshung - 2024-07-22
|
||||
|
||||
|
|
@ -166,8 +128,4 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
|
|||
[0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.0...v0.28.1
|
||||
[0.29.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.28.1...v0.29.0
|
||||
[0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.29.0...v0.30.0
|
||||
[0.30.1]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...v0.30.1
|
||||
[0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.1...v0.30.2
|
||||
[0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.2...v0.30.3
|
||||
[0.31.0]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.3...v0.31.0
|
||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.31.0...master
|
||||
[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-http-gw/compare/v0.30.0...master
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
.* @alexvanin @dkirillov
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
<p align="center">
|
||||
<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS logo">
|
||||
<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://frostfs.info">FrostFS</a> is a decentralized distributed object storage integrated with the <a href="https://neo.org">NEO Blockchain</a>.
|
||||
|
|
|
|||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
v0.31.0
|
||||
v0.30.0
|
||||
|
|
|
|||
|
|
@ -90,7 +90,6 @@ type (
|
|||
appSettings struct {
|
||||
reconnectInterval time.Duration
|
||||
dialerSource *internalnet.DialerSource
|
||||
workerPoolSize int
|
||||
|
||||
mu sync.RWMutex
|
||||
defaultTimestamp bool
|
||||
|
|
@ -98,6 +97,7 @@ type (
|
|||
clientCut bool
|
||||
returnIndexPage bool
|
||||
indexPageTemplate string
|
||||
workerPoolSize int
|
||||
bufferMaxSizeForPut uint64
|
||||
namespaceHeader string
|
||||
defaultNamespaces []string
|
||||
|
|
@ -943,13 +943,6 @@ func (a *app) initTracing(ctx context.Context) {
|
|||
cfg.ServerCaCertPool = certPool
|
||||
}
|
||||
|
||||
attributes, err := fetchTracingAttributes(a.cfg)
|
||||
if err != nil {
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||
return
|
||||
}
|
||||
cfg.Attributes = attributes
|
||||
|
||||
updated, err := tracing.Setup(ctx, cfg)
|
||||
if err != nil {
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||
|
|
|
|||
|
|
@ -70,7 +70,6 @@ const (
|
|||
|
||||
cfgIndexPageEnabled = "index_page.enabled"
|
||||
cfgIndexPageTemplatePath = "index_page.template_path"
|
||||
|
||||
cfgWorkerPoolSize = "worker_pool_size"
|
||||
|
||||
// Web.
|
||||
|
|
@ -92,7 +91,6 @@ const (
|
|||
cfgTracingExporter = "tracing.exporter"
|
||||
cfgTracingEndpoint = "tracing.endpoint"
|
||||
cfgTracingTrustedCa = "tracing.trusted_ca"
|
||||
cfgTracingAttributes = "tracing.attributes"
|
||||
|
||||
// Pool config.
|
||||
cfgConTimeout = "connect_timeout"
|
||||
|
|
@ -791,27 +789,3 @@ func fetchMultinetConfig(v *viper.Viper, l *zap.Logger) (cfg internalnet.Config)
|
|||
|
||||
return
|
||||
}
|
||||
|
||||
func fetchTracingAttributes(v *viper.Viper) (map[string]string, error) {
|
||||
attributes := make(map[string]string)
|
||||
for i := 0; ; i++ {
|
||||
key := cfgTracingAttributes + "." + strconv.Itoa(i) + "."
|
||||
attrKey := v.GetString(key + "key")
|
||||
attrValue := v.GetString(key + "value")
|
||||
if attrKey == "" {
|
||||
break
|
||||
}
|
||||
|
||||
if _, ok := attributes[attrKey]; ok {
|
||||
return nil, fmt.Errorf("tracing attribute key %s defined more than once", attrKey)
|
||||
}
|
||||
|
||||
if attrValue == "" {
|
||||
return nil, fmt.Errorf("empty tracing attribute value for key %s", attrKey)
|
||||
}
|
||||
|
||||
attributes[attrKey] = attrValue
|
||||
}
|
||||
|
||||
return attributes, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -104,10 +104,6 @@ HTTP_GW_TRACING_ENABLED=true
|
|||
HTTP_GW_TRACING_ENDPOINT="localhost:4317"
|
||||
HTTP_GW_TRACING_EXPORTER="otlp_grpc"
|
||||
HTTP_GW_TRACING_TRUSTED_CA=""
|
||||
HTTP_GW_TRACING_ATTRIBUTES_0_KEY=key0
|
||||
HTTP_GW_TRACING_ATTRIBUTES_0_VALUE=value
|
||||
HTTP_GW_TRACING_ATTRIBUTES_1_KEY=key1
|
||||
HTTP_GW_TRACING_ATTRIBUTES_1_VALUE=value
|
||||
|
||||
HTTP_GW_RUNTIME_SOFT_MEMORY_LIMIT=1073741824
|
||||
|
||||
|
|
@ -150,12 +146,3 @@ HTTP_GW_MULTINET_FALLBACK_DELAY=300ms
|
|||
# List of subnets and IP addresses to use as source for those subnets
|
||||
HTTP_GW_MULTINET_SUBNETS_1_MASK=1.2.3.4/24
|
||||
HTTP_GW_MULTINET_SUBNETS_1_SOURCE_IPS=1.2.3.4 1.2.3.5
|
||||
|
||||
# Number of workers in handler's worker pool
|
||||
HTTP_GW_WORKER_POOL_SIZE=1000
|
||||
|
||||
# Index page
|
||||
# Enable index page support
|
||||
HTTP_GW_INDEX_PAGE_ENABLED=false
|
||||
# Index page template path
|
||||
HTTP_GW_INDEX_PAGE_TEMPLATE_PATH=internal/handler/templates/index.gotmpl
|
||||
|
|
@ -9,17 +9,11 @@ pprof:
|
|||
prometheus:
|
||||
enabled: false # Enable metrics.
|
||||
address: localhost:8084
|
||||
|
||||
tracing:
|
||||
enabled: true
|
||||
exporter: "otlp_grpc"
|
||||
endpoint: "localhost:4317"
|
||||
trusted_ca: ""
|
||||
attributes:
|
||||
- key: key0
|
||||
value: value
|
||||
- key: key1
|
||||
value: value
|
||||
|
||||
logger:
|
||||
level: debug # Log level.
|
||||
|
|
@ -113,9 +107,6 @@ request_timeout: 5s # Timeout to check node health during rebalance.
|
|||
rebalance_timer: 30s # Interval to check nodes health.
|
||||
pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy.
|
||||
|
||||
# Number of workers in handler's worker pool
|
||||
worker_pool_size: 1000
|
||||
|
||||
# Enable index page to see objects list for specified container and prefix
|
||||
index_page:
|
||||
enabled: false
|
||||
|
|
|
|||
|
|
@ -271,36 +271,14 @@ tracing:
|
|||
exporter: "otlp_grpc"
|
||||
endpoint: "localhost:4317"
|
||||
trusted_ca: "/etc/ssl/telemetry-trusted-ca.pem"
|
||||
attributes:
|
||||
- key: key0
|
||||
value: value
|
||||
- key: key1
|
||||
value: value
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
| ------------ | -------------------------------------- | ------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|
||||
|--------------|----------|---------------|---------------|---------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `enabled` | `bool` | yes | `false` | Flag to enable the tracing. |
|
||||
| `exporter` | `string` | yes | | Trace collector type (`stdout` or `otlp_grpc` are supported). |
|
||||
| `endpoint` | `string` | yes | | Address of collector endpoint for OTLP exporters. |
|
||||
| `trusted_ca` | `string` | yes | | Path to certificate of a certification authority in pem format, that issued the TLS certificate of the telemetry remote server. |
|
||||
| `attributes` | [[]Attributes](#attributes-subsection) | yes | | An array of configurable attributes in key-value format. |
|
||||
|
||||
|
||||
#### `attributes` subsection
|
||||
|
||||
```yaml
|
||||
attributes:
|
||||
- key: key0
|
||||
value: value
|
||||
- key: key1
|
||||
value: value
|
||||
```
|
||||
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|-----------------------|----------|---------------|---------------|----------------------------------------------------------|
|
||||
| `key` | `string` | yes | | Attribute key. |
|
||||
| `value` | `string` | yes | | Attribute value. |
|
||||
|
||||
# `runtime` section
|
||||
Contains runtime parameters.
|
||||
|
|
|
|||
10
go.mod
10
go.mod
|
|
@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-http-gw
|
|||
go 1.22
|
||||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98
|
||||
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972
|
||||
git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
|
||||
github.com/bluele/gcache v0.0.2
|
||||
|
|
@ -25,7 +25,7 @@ require (
|
|||
go.opentelemetry.io/otel v1.28.0
|
||||
go.opentelemetry.io/otel/trace v1.28.0
|
||||
go.uber.org/zap v1.27.0
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
|
||||
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
|
||||
golang.org/x/net v0.26.0
|
||||
golang.org/x/sys v0.22.0
|
||||
google.golang.org/grpc v1.66.2
|
||||
|
|
@ -42,7 +42,7 @@ require (
|
|||
github.com/Microsoft/hcsshim v0.9.2 // indirect
|
||||
github.com/VictoriaMetrics/easyproto v0.1.4 // indirect
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
|
|
|
|||
20
go.sum
20
go.sum
|
|
@ -37,16 +37,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
|
|||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1 h1:ivcdxQeQDnx4srF2ezoaeVlF0FAycSAztwfIUJnUI4s=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241011114054-f0fc40e116d1/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3 h1:f7jan6eBDN88DKnKj8GKyWpfjBbSzjDALcDejYKRgCs=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241022124111-5361f0ecebd3/go.mod h1:3txOjFJ8M/JFs01h7xOrnQHVn6hZgDNA16ivyUlu1iU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 h1:ijUci3thz0EwWkuRJDocW5D1RkVAJlt9xNG4CYepC90=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino=
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
|
||||
git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8=
|
||||
|
|
@ -114,8 +114,8 @@ github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu
|
|||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
|
||||
|
|
@ -930,8 +930,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
|
|||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
|
||||
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ=
|
||||
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
|
|
|||
|
|
@ -165,8 +165,9 @@ func (h *Handler) getDirObjectsS3(ctx context.Context, bucketInfo *data.BucketIn
|
|||
return nil, err
|
||||
}
|
||||
|
||||
result := &GetObjectsResponse{
|
||||
var result = &GetObjectsResponse{
|
||||
objects: make([]ResponseObject, 0, len(nodes)),
|
||||
hasErrors: false,
|
||||
}
|
||||
for _, node := range nodes {
|
||||
meta := node.GetMeta()
|
||||
|
|
|
|||
|
|
@ -24,12 +24,12 @@ import (
|
|||
// DownloadByAddressOrBucketName handles download requests using simple cid/oid or bucketname/key format.
|
||||
func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
|
||||
oidURLParam := c.UserValue("oid").(string)
|
||||
downloadQueryParam := c.QueryArgs().GetBool("download")
|
||||
cidURLParam := c.UserValue("cid").(string)
|
||||
|
||||
switch {
|
||||
case isObjectID(oidURLParam):
|
||||
case isContainerID(oidURLParam) && isObjectID(cidURLParam):
|
||||
h.byNativeAddress(c, h.receiveFile)
|
||||
case !isContainerRoot(oidURLParam) && (downloadQueryParam || !isDir(oidURLParam)):
|
||||
case !isContainerRoot(cidURLParam) && !isDir(cidURLParam):
|
||||
h.byS3Path(c, h.receiveFile)
|
||||
default:
|
||||
h.browseIndex(c)
|
||||
|
|
|
|||
|
|
@ -208,6 +208,16 @@ func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context
|
|||
|
||||
objID := new(oid.ID)
|
||||
if err = objID.DecodeString(idObj); err != nil {
|
||||
if h.config.IndexPageEnabled() {
|
||||
c.SetStatusCode(fasthttp.StatusNotFound)
|
||||
h.browseObjects(c, browseParams{
|
||||
bucketInfo: bktInfo,
|
||||
prefix: idObj,
|
||||
listObjects: h.getDirObjectsNative,
|
||||
isNative: true,
|
||||
})
|
||||
return
|
||||
}
|
||||
log.Error(logs.WrongObjectID, zap.Error(err))
|
||||
response.Error(c, "wrong object id", fasthttp.StatusBadRequest)
|
||||
return
|
||||
|
|
@ -223,6 +233,7 @@ func (h *Handler) byNativeAddress(c *fasthttp.RequestCtx, f func(context.Context
|
|||
func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
||||
bucketname := c.UserValue("cid").(string)
|
||||
key := c.UserValue("oid").(string)
|
||||
download := c.QueryArgs().GetBool("download")
|
||||
|
||||
ctx := utils.GetContextFromRequest(c)
|
||||
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
|
||||
|
|
@ -241,6 +252,20 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, reque
|
|||
}
|
||||
|
||||
foundOid, err := h.tree.GetLatestVersion(ctx, &bktInfo.CID, unescapedKey)
|
||||
if h.config.IndexPageEnabled() && !download && string(c.Method()) != fasthttp.MethodHead {
|
||||
if isDir(unescapedKey) || isContainerRoot(unescapedKey) {
|
||||
if code := checkErrorType(err); code == fasthttp.StatusNotFound || code == fasthttp.StatusOK {
|
||||
c.SetStatusCode(code)
|
||||
h.browseObjects(c, browseParams{
|
||||
bucketInfo: bktInfo,
|
||||
prefix: unescapedKey,
|
||||
listObjects: h.getDirObjectsS3,
|
||||
isNative: false,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, tree.ErrNodeAccessDenied) {
|
||||
response.Error(c, "Access Denied", fasthttp.StatusForbidden)
|
||||
|
|
@ -260,6 +285,53 @@ func (h *Handler) byS3Path(c *fasthttp.RequestCtx, f func(context.Context, reque
|
|||
f(ctx, *h.newRequest(c, log), addr)
|
||||
}
|
||||
|
||||
func (h *Handler) browseIndex(c *fasthttp.RequestCtx) {
|
||||
if !h.config.IndexPageEnabled() {
|
||||
c.SetStatusCode(fasthttp.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
cidURLParam := c.UserValue("cid").(string)
|
||||
oidURLParam := c.UserValue("oid").(string)
|
||||
|
||||
ctx := utils.GetContextFromRequest(c)
|
||||
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
|
||||
log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam))
|
||||
|
||||
unescapedKey, err := url.QueryUnescape(oidURLParam)
|
||||
if err != nil {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketInfo(ctx, oidURLParam, log)
|
||||
if err != nil {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
|
||||
c.SetStatusCode(fasthttp.StatusOK)
|
||||
|
||||
listFunc := h.getDirObjectsS3
|
||||
isNativeList := false
|
||||
|
||||
_, err = h.tree.GetLatestVersion(ctx, &bktInfo.CID, "")
|
||||
if err != nil {
|
||||
// tree probe failed, try to use native
|
||||
fmt.Println("!!! I failed tree probe")
|
||||
listFunc = h.getDirObjectsS3
|
||||
isNativeList = true
|
||||
}
|
||||
|
||||
c.SetStatusCode(fasthttp.StatusOK)
|
||||
h.browseObjects(c, browseParams{
|
||||
bucketInfo: bktInfo,
|
||||
prefix: unescapedKey,
|
||||
listObjects: listFunc,
|
||||
isNative: isNativeList,
|
||||
})
|
||||
}
|
||||
|
||||
// byAttribute is a wrapper similar to byNativeAddress.
|
||||
func (h *Handler) byAttribute(c *fasthttp.RequestCtx, f func(context.Context, request, oid.Address)) {
|
||||
scid, _ := c.UserValue("cid").(string)
|
||||
|
|
@ -387,51 +459,3 @@ func (h *Handler) readContainer(ctx context.Context, cnrID cid.ID) (*data.Bucket
|
|||
|
||||
return bktInfo, err
|
||||
}
|
||||
|
||||
func (h *Handler) browseIndex(c *fasthttp.RequestCtx) {
|
||||
if !h.config.IndexPageEnabled() {
|
||||
c.SetStatusCode(fasthttp.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
cidURLParam := c.UserValue("cid").(string)
|
||||
oidURLParam := c.UserValue("oid").(string)
|
||||
|
||||
ctx := utils.GetContextFromRequest(c)
|
||||
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
|
||||
log := reqLog.With(zap.String("cid", cidURLParam), zap.String("oid", oidURLParam))
|
||||
|
||||
unescapedKey, err := url.QueryUnescape(oidURLParam)
|
||||
if err != nil {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
|
||||
bktInfo, err := h.getBucketInfo(ctx, cidURLParam, log)
|
||||
if err != nil {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
|
||||
listFunc := h.getDirObjectsS3
|
||||
isNativeList := false
|
||||
|
||||
err = h.tree.CheckSettingsNodeExist(ctx, bktInfo)
|
||||
if err != nil {
|
||||
if errors.Is(err, tree.ErrNodeNotFound) {
|
||||
// tree probe failed, try to use native
|
||||
listFunc = h.getDirObjectsNative
|
||||
isNativeList = true
|
||||
} else {
|
||||
logAndSendBucketError(c, log, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
h.browseObjects(c, browseParams{
|
||||
bucketInfo: bktInfo,
|
||||
prefix: unescapedKey,
|
||||
listObjects: listFunc,
|
||||
isNative: isNativeList,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,12 +2,14 @@ package handler
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/response"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-http-gw/tree"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
|
|
@ -45,13 +47,29 @@ func isDir(name string) bool {
|
|||
return strings.HasSuffix(name, "/")
|
||||
}
|
||||
|
||||
func isContainerRoot(key string) bool {
|
||||
return key == ""
|
||||
}
|
||||
|
||||
func isContainerID(s string) bool {
|
||||
var cnrID cid.ID
|
||||
return cnrID.DecodeString(s) == nil
|
||||
}
|
||||
|
||||
func isObjectID(s string) bool {
|
||||
var objID oid.ID
|
||||
return objID.DecodeString(s) == nil
|
||||
}
|
||||
|
||||
func isContainerRoot(key string) bool {
|
||||
return key == ""
|
||||
func checkErrorType(err error) int {
|
||||
switch {
|
||||
case err == nil:
|
||||
return fasthttp.StatusOK
|
||||
case errors.Is(err, tree.ErrNodeAccessDenied):
|
||||
return fasthttp.StatusForbidden
|
||||
default:
|
||||
return fasthttp.StatusNotFound
|
||||
}
|
||||
}
|
||||
|
||||
func loadAttributes(attrs []object.Attribute) map[string]string {
|
||||
|
|
|
|||
100
tree/tree.go
100
tree/tree.go
|
|
@ -30,11 +30,6 @@ type (
|
|||
Meta map[string]string
|
||||
}
|
||||
|
||||
multiSystemNode struct {
|
||||
// the first element is latest
|
||||
nodes []*treeNode
|
||||
}
|
||||
|
||||
GetNodesParams struct {
|
||||
CnrID cid.ID
|
||||
BktInfo *data.BucketInfo
|
||||
|
|
@ -56,18 +51,17 @@ var (
|
|||
|
||||
const (
|
||||
FileNameKey = "FileName"
|
||||
settingsFileName = "bucket-settings"
|
||||
)
|
||||
|
||||
const (
|
||||
oidKV = "OID"
|
||||
uploadIDKV = "UploadId"
|
||||
sizeKV = "Size"
|
||||
|
||||
// keys for delete marker nodes.
|
||||
isDeleteMarkerKV = "IsDeleteMarker"
|
||||
sizeKV = "Size"
|
||||
|
||||
// versionTree -- ID of a tree with object versions.
|
||||
versionTree = "version"
|
||||
systemTree = "system"
|
||||
|
||||
separator = "/"
|
||||
)
|
||||
|
|
@ -141,45 +135,6 @@ func newNodeVersionFromTreeNode(treeNode *treeNode) *api.NodeVersion {
|
|||
return version
|
||||
}
|
||||
|
||||
func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) {
|
||||
var (
|
||||
err error
|
||||
index int
|
||||
maxTimestamp uint64
|
||||
)
|
||||
|
||||
if len(nodes) == 0 {
|
||||
return nil, errors.New("multi node must have at least one node")
|
||||
}
|
||||
|
||||
treeNodes := make([]*treeNode, len(nodes))
|
||||
|
||||
for i, node := range nodes {
|
||||
if treeNodes[i], err = newTreeNode(node); err != nil {
|
||||
return nil, fmt.Errorf("parse system node response: %w", err)
|
||||
}
|
||||
|
||||
if timestamp := getMaxTimestamp(node); timestamp > maxTimestamp {
|
||||
index = i
|
||||
maxTimestamp = timestamp
|
||||
}
|
||||
}
|
||||
|
||||
treeNodes[0], treeNodes[index] = treeNodes[index], treeNodes[0]
|
||||
|
||||
return &multiSystemNode{
|
||||
nodes: treeNodes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *multiSystemNode) Latest() *treeNode {
|
||||
return m.nodes[0]
|
||||
}
|
||||
|
||||
func (m *multiSystemNode) Old() []*treeNode {
|
||||
return m.nodes[1:]
|
||||
}
|
||||
|
||||
func (c *Tree) GetLatestVersion(ctx context.Context, cnrID *cid.ID, objectName string) (*api.NodeVersion, error) {
|
||||
nodes, err := c.GetVersions(ctx, cnrID, objectName)
|
||||
if err != nil {
|
||||
|
|
@ -210,55 +165,6 @@ func (c *Tree) GetVersions(ctx context.Context, cnrID *cid.ID, objectName string
|
|||
return c.service.GetNodes(ctx, p)
|
||||
}
|
||||
|
||||
func (c *Tree) CheckSettingsNodeExist(ctx context.Context, bktInfo *data.BucketInfo) error {
|
||||
_, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) {
|
||||
p := &GetNodesParams{
|
||||
CnrID: bktInfo.CID,
|
||||
BktInfo: bktInfo,
|
||||
TreeID: systemTree,
|
||||
Path: []string{name},
|
||||
LatestOnly: false,
|
||||
AllAttrs: true,
|
||||
}
|
||||
nodes, err := c.service.GetNodes(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodes = filterMultipartNodes(nodes)
|
||||
|
||||
if len(nodes) == 0 {
|
||||
return nil, ErrNodeNotFound
|
||||
}
|
||||
|
||||
return newMultiNode(nodes)
|
||||
}
|
||||
|
||||
func filterMultipartNodes(nodes []NodeResponse) []NodeResponse {
|
||||
res := make([]NodeResponse, 0, len(nodes))
|
||||
|
||||
LOOP:
|
||||
for _, node := range nodes {
|
||||
for _, meta := range node.GetMeta() {
|
||||
if meta.GetKey() == uploadIDKV {
|
||||
continue LOOP
|
||||
}
|
||||
}
|
||||
|
||||
res = append(res, node)
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
|
||||
var (
|
||||
maxCreationTime uint64
|
||||
|
|
|
|||
Loading…
Reference in a new issue