forked from TrueCloudLab/frostfs-s3-gw
Compare commits
12 commits
6a9d3261a7
...
9120e97ac5
Author | SHA1 | Date | |
---|---|---|---|
9120e97ac5 | |||
2fc328a6d2 | |||
b5fce5c8d2 | |||
41a128b1aa | |||
6617adc22b | |||
631d9d83b6 | |||
adec93af54 | |||
8898c2ec08 | |||
8efcc957ea | |||
6b728fef87 | |||
6b1f365e65 | |||
fcf1c45ad2 |
71 changed files with 909 additions and 359 deletions
|
@ -1,4 +1,4 @@
|
||||||
FROM golang:1.19 as builder
|
FROM golang:1.21 as builder
|
||||||
|
|
||||||
ARG BUILD=now
|
ARG BUILD=now
|
||||||
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||||
|
|
|
@ -6,7 +6,7 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go_versions: [ '1.19', '1.20' ]
|
go_versions: [ '1.20', '1.21' ]
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
|
|
|
@ -12,7 +12,7 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.20'
|
go-version: '1.21'
|
||||||
|
|
||||||
- name: Run commit format checker
|
- name: Run commit format checker
|
||||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v1
|
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v1
|
||||||
|
|
|
@ -7,17 +7,24 @@ jobs:
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
|
|
||||||
- name: golangci-lint
|
- name: Set up Go
|
||||||
uses: https://github.com/golangci/golangci-lint-action@v2
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
version: latest
|
go-version: '1.21'
|
||||||
|
cache: true
|
||||||
|
|
||||||
|
- name: Install linters
|
||||||
|
run: make lint-install
|
||||||
|
|
||||||
|
- name: Run linters
|
||||||
|
run: make lint
|
||||||
|
|
||||||
tests:
|
tests:
|
||||||
name: Tests
|
name: Tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go_versions: [ '1.19', '1.20' ]
|
go_versions: [ '1.20', '1.21' ]
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
|
|
|
@ -12,7 +12,7 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: '1.20'
|
go-version: '1.21'
|
||||||
|
|
||||||
- name: Install govulncheck
|
- name: Install govulncheck
|
||||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||||
|
|
|
@ -24,6 +24,16 @@ linters-settings:
|
||||||
govet:
|
govet:
|
||||||
# report about shadowed variables
|
# report about shadowed variables
|
||||||
check-shadowing: false
|
check-shadowing: false
|
||||||
|
custom:
|
||||||
|
truecloudlab-linters:
|
||||||
|
path: bin/external_linters.so
|
||||||
|
original-url: git.frostfs.info/TrueCloudLab/linters.git
|
||||||
|
settings:
|
||||||
|
noliteral:
|
||||||
|
enable: true
|
||||||
|
target-methods: ["Fatal"]
|
||||||
|
disable-packages: ["codes", "tc"]
|
||||||
|
constants-package: "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
|
|
||||||
linters:
|
linters:
|
||||||
enable:
|
enable:
|
||||||
|
@ -45,6 +55,7 @@ linters:
|
||||||
- gofmt
|
- gofmt
|
||||||
- whitespace
|
- whitespace
|
||||||
- goimports
|
- goimports
|
||||||
|
- truecloudlab-linters
|
||||||
disable-all: true
|
disable-all: true
|
||||||
fast: false
|
fast: false
|
||||||
|
|
||||||
|
|
|
@ -30,13 +30,20 @@ repos:
|
||||||
hooks:
|
hooks:
|
||||||
- id: shellcheck
|
- id: shellcheck
|
||||||
|
|
||||||
- repo: https://github.com/golangci/golangci-lint
|
|
||||||
rev: v1.51.2
|
|
||||||
hooks:
|
|
||||||
- id: golangci-lint
|
|
||||||
|
|
||||||
- repo: local
|
- repo: local
|
||||||
hooks:
|
hooks:
|
||||||
|
- id: make-lint-install
|
||||||
|
name: install linters
|
||||||
|
entry: make lint-install
|
||||||
|
language: system
|
||||||
|
pass_filenames: false
|
||||||
|
|
||||||
|
- id: make-lint
|
||||||
|
name: run linters
|
||||||
|
entry: make lint
|
||||||
|
language: system
|
||||||
|
pass_filenames: false
|
||||||
|
|
||||||
- id: go-unit-tests
|
- id: go-unit-tests
|
||||||
name: go unit tests
|
name: go unit tests
|
||||||
entry: make test
|
entry: make test
|
||||||
|
|
|
@ -13,6 +13,8 @@ This document outlines major changes between releases.
|
||||||
- Replace part on re-upload when use multipart upload (#176)
|
- Replace part on re-upload when use multipart upload (#176)
|
||||||
- Fix goroutine leak on put object error (#178)
|
- Fix goroutine leak on put object error (#178)
|
||||||
- Fix parsing signed headers in presigned urls (#182)
|
- Fix parsing signed headers in presigned urls (#182)
|
||||||
|
- Fix url escaping (#188)
|
||||||
|
- Use correct keys in `list-multipart-uploads` response (#185)
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
- Add a metric with addresses of nodes of the same and highest priority that are currently healthy (#51)
|
- Add a metric with addresses of nodes of the same and highest priority that are currently healthy (#51)
|
||||||
|
@ -25,6 +27,7 @@ This document outlines major changes between releases.
|
||||||
- Support multiple version credentials using GSet (#135)
|
- Support multiple version credentials using GSet (#135)
|
||||||
- Implement chunk uploading (#106)
|
- Implement chunk uploading (#106)
|
||||||
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
|
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
|
||||||
|
- Add new `frostfs.client_cut` config param (#192)
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
- Update prometheus to v1.15.0 (#94)
|
- Update prometheus to v1.15.0 (#94)
|
||||||
|
@ -38,6 +41,7 @@ This document outlines major changes between releases.
|
||||||
- Use chi router instead of archived gorlilla/mux (#149)
|
- Use chi router instead of archived gorlilla/mux (#149)
|
||||||
- Complete multipart upload doesn't unnecessary copy now. Thus, the total time of multipart upload was reduced by 2 times (#63)
|
- Complete multipart upload doesn't unnecessary copy now. Thus, the total time of multipart upload was reduced by 2 times (#63)
|
||||||
- Use gate key to form object owner (#175)
|
- Use gate key to form object owner (#175)
|
||||||
|
- Apply placement policies and copies if there is at least one valid value (#168)
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
- Drop `tree.service` param (now endpoints from `peers` section are used) (#133)
|
- Drop `tree.service` param (now endpoints from `peers` section are used) (#133)
|
||||||
|
|
25
Makefile
25
Makefile
|
@ -3,8 +3,9 @@
|
||||||
# Common variables
|
# Common variables
|
||||||
REPO ?= $(shell go list -m)
|
REPO ?= $(shell go list -m)
|
||||||
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
||||||
GO_VERSION ?= 1.19
|
GO_VERSION ?= 1.20
|
||||||
LINT_VERSION ?= 1.49.0
|
LINT_VERSION ?= 1.54.0
|
||||||
|
TRUECLOUDLAB_LINT_VERSION ?= 0.0.2
|
||||||
BINDIR = bin
|
BINDIR = bin
|
||||||
|
|
||||||
METRICS_DUMP_OUT ?= ./metrics-dump.json
|
METRICS_DUMP_OUT ?= ./metrics-dump.json
|
||||||
|
@ -18,6 +19,10 @@ REPO_BASENAME = $(shell basename `go list -m`)
|
||||||
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||||
|
|
||||||
|
OUTPUT_LINT_DIR ?= $(shell pwd)/bin
|
||||||
|
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
|
||||||
|
TMP_DIR := .cache
|
||||||
|
|
||||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
||||||
|
|
||||||
# .deb package versioning
|
# .deb package versioning
|
||||||
|
@ -101,9 +106,23 @@ dirty-image:
|
||||||
-f .docker/Dockerfile.dirty \
|
-f .docker/Dockerfile.dirty \
|
||||||
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
|
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
|
||||||
|
|
||||||
|
# Install linters
|
||||||
|
lint-install:
|
||||||
|
@mkdir -p $(TMP_DIR)
|
||||||
|
@rm -rf $(TMP_DIR)/linters
|
||||||
|
@git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters
|
||||||
|
@@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR)
|
||||||
|
@rm -rf $(TMP_DIR)/linters
|
||||||
|
@rmdir $(TMP_DIR) 2>/dev/null || true
|
||||||
|
@CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION)
|
||||||
|
|
||||||
# Run linters
|
# Run linters
|
||||||
lint:
|
lint:
|
||||||
@golangci-lint --timeout=5m run
|
@if [ ! -d "$(LINT_DIR)" ]; then \
|
||||||
|
echo "Run make lint-install"; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
$(LINT_DIR)/golangci-lint --timeout=5m run
|
||||||
|
|
||||||
# Run linters in Docker
|
# Run linters in Docker
|
||||||
docker/lint:
|
docker/lint:
|
||||||
|
|
3
api/cache/access_control.go
vendored
3
api/cache/access_control.go
vendored
|
@ -4,6 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -46,7 +47,7 @@ func (o *AccessControlCache) Get(owner user.ID, key string) bool {
|
||||||
|
|
||||||
result, ok := entry.(bool)
|
result, ok := entry.(bool)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
3
api/cache/accessbox.go
vendored
3
api/cache/accessbox.go
vendored
|
@ -5,6 +5,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -57,7 +58,7 @@ func (o *AccessBoxCache) Get(address oid.Address) *accessbox.Box {
|
||||||
|
|
||||||
result, ok := entry.(*accessbox.Box)
|
result, ok := entry.(*accessbox.Box)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
3
api/cache/buckets.go
vendored
3
api/cache/buckets.go
vendored
|
@ -5,6 +5,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -46,7 +47,7 @@ func (o *BucketCache) Get(key string) *data.BucketInfo {
|
||||||
|
|
||||||
result, ok := entry.(*data.BucketInfo)
|
result, ok := entry.(*data.BucketInfo)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
3
api/cache/names.go
vendored
3
api/cache/names.go
vendored
|
@ -4,6 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -48,7 +49,7 @@ func (o *ObjectsNameCache) Get(key string) *oid.Address {
|
||||||
|
|
||||||
result, ok := entry.(oid.Address)
|
result, ok := entry.(oid.Address)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
3
api/cache/objects.go
vendored
3
api/cache/objects.go
vendored
|
@ -5,6 +5,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -47,7 +48,7 @@ func (o *ObjectsCache) GetObject(address oid.Address) *data.ExtendedObjectInfo {
|
||||||
|
|
||||||
result, ok := entry.(*data.ExtendedObjectInfo)
|
result, ok := entry.(*data.ExtendedObjectInfo)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
5
api/cache/objectslist.go
vendored
5
api/cache/objectslist.go
vendored
|
@ -7,6 +7,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -75,7 +76,7 @@ func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion {
|
||||||
|
|
||||||
result, ok := entry.([]*data.NodeVersion)
|
result, ok := entry.([]*data.NodeVersion)
|
||||||
if !ok {
|
if !ok {
|
||||||
l.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -94,7 +95,7 @@ func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string,
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
k, ok := key.(ObjectsListKey)
|
k, ok := key.(ObjectsListKey)
|
||||||
if !ok {
|
if !ok {
|
||||||
l.logger.Warn("invalid cache key type", zap.String("actual", fmt.Sprintf("%T", key)),
|
l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", k)))
|
zap.String("expected", fmt.Sprintf("%T", k)))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
9
api/cache/system.go
vendored
9
api/cache/system.go
vendored
|
@ -5,6 +5,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"github.com/bluele/gcache"
|
"github.com/bluele/gcache"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -48,7 +49,7 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo {
|
||||||
|
|
||||||
result, ok := entry.(*data.ObjectInfo)
|
result, ok := entry.(*data.ObjectInfo)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -79,7 +80,7 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
||||||
|
|
||||||
result, ok := entry.(*data.CORSConfiguration)
|
result, ok := entry.(*data.CORSConfiguration)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -95,7 +96,7 @@ func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
||||||
|
|
||||||
result, ok := entry.(*data.BucketSettings)
|
result, ok := entry.(*data.BucketSettings)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -111,7 +112,7 @@ func (o *SystemCache) GetNotificationConfiguration(key string) *data.Notificatio
|
||||||
|
|
||||||
result, ok := entry.(*data.NotificationConfiguration)
|
result, ok := entry.(*data.NotificationConfiguration)
|
||||||
if !ok {
|
if !ok {
|
||||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -322,7 +322,7 @@ var errorCodes = errorCodeMap{
|
||||||
ErrInvalidMaxUploads: {
|
ErrInvalidMaxUploads: {
|
||||||
ErrCode: ErrInvalidMaxUploads,
|
ErrCode: ErrInvalidMaxUploads,
|
||||||
Code: "InvalidArgument",
|
Code: "InvalidArgument",
|
||||||
Description: "Argument max-uploads must be an integer between 0 and 2147483647",
|
Description: "Argument max-uploads must be an integer from 1 to 1000",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrInvalidMaxKeys: {
|
ErrInvalidMaxKeys: {
|
||||||
|
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
|
@ -470,7 +471,7 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
if err = h.sendNotifications(ctx, s); err != nil {
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
@ -1460,7 +1461,7 @@ func (h *handler) encodeObjectACL(ctx context.Context, bucketACL *layer.BucketAC
|
||||||
if read {
|
if read {
|
||||||
permission = aclFullControl
|
permission = aclFullControl
|
||||||
} else {
|
} else {
|
||||||
h.reqLogger(ctx).Warn("some acl not fully mapped")
|
h.reqLogger(ctx).Warn(logs.SomeACLNotFullyMapped)
|
||||||
}
|
}
|
||||||
|
|
||||||
var grantee *Grantee
|
var grantee *Grantee
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -56,13 +57,6 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
// DefaultPolicy is a default policy of placing containers in FrostFS if it's not set at the request.
|
|
||||||
DefaultPolicy = "REP 3"
|
|
||||||
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
|
|
||||||
DefaultCopiesNumber uint32 = 0
|
|
||||||
)
|
|
||||||
|
|
||||||
var _ api.Handler = (*handler)(nil)
|
var _ api.Handler = (*handler)(nil)
|
||||||
|
|
||||||
// New creates new api.Handler using given logger and client.
|
// New creates new api.Handler using given logger and client.
|
||||||
|
@ -75,7 +69,7 @@ func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config
|
||||||
}
|
}
|
||||||
|
|
||||||
if !cfg.NotificatorEnabled {
|
if !cfg.NotificatorEnabled {
|
||||||
log.Warn("notificator is disabled, s3 won't produce notification events")
|
log.Warn(logs.NotificatorIsDisabledS3WontProduceNotificationEvents)
|
||||||
} else if notificator == nil {
|
} else if notificator == nil {
|
||||||
return nil, errors.New("empty notificator")
|
return nil, errors.New("empty notificator")
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,7 @@ func TestGetObjectPartsAttributes(t *testing.T) {
|
||||||
|
|
||||||
createTestBucket(hc, bktName)
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
putObject(t, hc, bktName, objName)
|
putObject(hc, bktName, objName)
|
||||||
result := getObjectAttributes(hc, bktName, objName, objectParts)
|
result := getObjectAttributes(hc, bktName, objName, objectParts)
|
||||||
require.Nil(t, result.ObjectParts)
|
require.Nil(t, result.ObjectParts)
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -117,7 +118,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if srcSize, err := getObjectSize(extendedSrcObjInfo, encryptionParams); err != nil {
|
if srcSize, err := layer.GetObjectSize(srcObjInfo); err != nil {
|
||||||
h.logAndSendError(w, "failed to get source object size", reqInfo, err)
|
h.logAndSendError(w, "failed to get source object size", reqInfo, err)
|
||||||
return
|
return
|
||||||
} else if srcSize > layer.UploadMaxSize { //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
|
} else if srcSize > layer.UploadMaxSize { //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
|
||||||
|
@ -249,7 +250,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h.reqLogger(ctx).Info("object is copied", zap.Stringer("object_id", dstObjInfo.ID))
|
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
|
||||||
|
|
||||||
s := &SendNotificationParams{
|
s := &SendNotificationParams{
|
||||||
Event: EventObjectCreatedCopy,
|
Event: EventObjectCreatedCopy,
|
||||||
|
@ -258,7 +259,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
if err = h.sendNotifications(ctx, s); err != nil {
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if encryptionParams.Enabled() {
|
if encryptionParams.Enabled() {
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -99,13 +100,13 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName)
|
bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.reqLogger(ctx).Warn("get bucket info", zap.Error(err))
|
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
|
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.reqLogger(ctx).Warn("get bucket cors", zap.Error(err))
|
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
|
@ -114,7 +115,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
var objID oid.ID
|
var objID oid.ID
|
||||||
if len(versionID) != 0 {
|
if len(versionID) != 0 {
|
||||||
if err = objID.DecodeString(versionID); err != nil {
|
if err = objID.DecodeString(versionID); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -130,7 +131,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = h.sendNotifications(ctx, m); err != nil {
|
if err = h.sendNotifications(ctx, m); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if deletedObject.VersionID != "" {
|
if deletedObject.VersionID != "" {
|
||||||
|
@ -261,7 +262,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
||||||
zap.Array("objects", marshaler),
|
zap.Array("objects", marshaler),
|
||||||
zap.Errors("errors", errs),
|
zap.Errors("errors", errs),
|
||||||
}
|
}
|
||||||
h.reqLogger(ctx).Error("couldn't delete objects", fields...)
|
h.reqLogger(ctx).Error(logs.CouldntDeleteObjects, fields...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||||
|
|
|
@ -25,7 +25,7 @@ func TestDeleteBucketOnAlreadyRemovedError(t *testing.T) {
|
||||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
bktInfo := createTestBucket(hc, bktName)
|
||||||
|
|
||||||
putObject(t, hc, bktName, objName)
|
putObject(hc, bktName, objName)
|
||||||
|
|
||||||
addr := getAddressOfLastVersion(hc, bktInfo, objName)
|
addr := getAddressOfLastVersion(hc, bktInfo, objName)
|
||||||
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
|
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
|
||||||
|
@ -66,7 +66,7 @@ func TestDeleteBucketOnNotFoundError(t *testing.T) {
|
||||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||||
bktInfo := createTestBucket(hc, bktName)
|
bktInfo := createTestBucket(hc, bktName)
|
||||||
|
|
||||||
putObject(t, hc, bktName, objName)
|
putObject(hc, bktName, objName)
|
||||||
|
|
||||||
nodeVersion, err := hc.tree.GetUnversioned(hc.context, bktInfo, objName)
|
nodeVersion, err := hc.tree.GetUnversioned(hc.context, bktInfo, objName)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -98,7 +98,7 @@ func TestDeleteObjectFromSuspended(t *testing.T) {
|
||||||
bktName, objName := "bucket-versioned-for-removal", "object-to-delete"
|
bktName, objName := "bucket-versioned-for-removal", "object-to-delete"
|
||||||
|
|
||||||
createSuspendedBucket(t, tc, bktName)
|
createSuspendedBucket(t, tc, bktName)
|
||||||
putObject(t, tc, bktName, objName)
|
putObject(tc, bktName, objName)
|
||||||
|
|
||||||
versionID, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
|
versionID, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||||
require.True(t, isDeleteMarker)
|
require.True(t, isDeleteMarker)
|
||||||
|
@ -255,7 +255,7 @@ func TestDeleteMarkerSuspended(t *testing.T) {
|
||||||
|
|
||||||
t.Run("remove last unversioned non delete marker", func(t *testing.T) {
|
t.Run("remove last unversioned non delete marker", func(t *testing.T) {
|
||||||
objName := "obj3"
|
objName := "obj3"
|
||||||
putObject(t, tc, bktName, objName)
|
putObject(tc, bktName, objName)
|
||||||
|
|
||||||
nodeVersion, err := tc.tree.GetUnversioned(tc.Context(), bktInfo, objName)
|
nodeVersion, err := tc.tree.GetUnversioned(tc.Context(), bktInfo, objName)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -333,7 +333,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
|
||||||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||||
bktInfo, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
|
bktInfo, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
|
||||||
|
|
||||||
versions := listObjectsV1(t, tc, bktName, "", "", "", -1)
|
versions := listObjectsV1(tc, bktName, "", "", "", -1)
|
||||||
require.Len(t, versions.Contents, 1)
|
require.Len(t, versions.Contents, 1)
|
||||||
|
|
||||||
checkFound(t, tc, bktName, objName, objInfo.VersionID())
|
checkFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||||
|
@ -341,7 +341,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
|
||||||
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||||
|
|
||||||
// check cache is clean after object removal
|
// check cache is clean after object removal
|
||||||
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
|
versions = listObjectsV1(tc, bktName, "", "", "", -1)
|
||||||
require.Len(t, versions.Contents, 0)
|
require.Len(t, versions.Contents, 0)
|
||||||
|
|
||||||
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
|
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
|
||||||
|
@ -475,11 +475,11 @@ func getVersion(resp *ListObjectsVersionsResponse, objName string) []*ObjectVers
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func putObject(t *testing.T, tc *handlerContext, bktName, objName string) {
|
func putObject(hc *handlerContext, bktName, objName string) {
|
||||||
body := bytes.NewReader([]byte("content"))
|
body := bytes.NewReader([]byte("content"))
|
||||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, body)
|
w, r := prepareTestPayloadRequest(hc, bktName, objName, body)
|
||||||
tc.Handler().PutObjectHandler(w, r)
|
hc.Handler().PutObjectHandler(w, r)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createSuspendedBucket(t *testing.T, tc *handlerContext, bktName string) *data.BucketInfo {
|
func createSuspendedBucket(t *testing.T, tc *handlerContext, bktName string) *data.BucketInfo {
|
||||||
|
|
|
@ -12,7 +12,6 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -168,7 +167,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fullSize, err := getObjectSize(extendedInfo, encryptionParams)
|
fullSize, err := layer.GetObjectSize(info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logAndSendError(w, "invalid size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
|
h.logAndSendError(w, "invalid size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
|
||||||
return
|
return
|
||||||
|
@ -233,23 +232,6 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getObjectSize(extendedInfo *data.ExtendedObjectInfo, encryptionParams encryption.Params) (uint64, error) {
|
|
||||||
var err error
|
|
||||||
fullSize := extendedInfo.ObjectInfo.Size
|
|
||||||
|
|
||||||
if encryptionParams.Enabled() {
|
|
||||||
if fullSize, err = strconv.ParseUint(extendedInfo.ObjectInfo.Headers[layer.AttributeDecryptedSize], 10, 64); err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid decrypted size header: %w", err)
|
|
||||||
}
|
|
||||||
} else if extendedInfo.NodeVersion.IsCombined {
|
|
||||||
if fullSize, err = strconv.ParseUint(extendedInfo.ObjectInfo.Headers[layer.MultipartObjectSize], 10, 64); err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid multipart size header: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fullSize, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
|
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
|
||||||
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
|
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
|
||||||
return fmt.Errorf("%w: etag mismatched: '%s', '%s'", errors.GetAPIError(errors.ErrPreconditionFailed), args.IfMatch, info.HashSum)
|
return fmt.Errorf("%w: etag mismatched: '%s', '%s'", errors.GetAPIError(errors.ErrPreconditionFailed), args.IfMatch, info.HashSum)
|
||||||
|
|
|
@ -186,7 +186,7 @@ func TestGetObject(t *testing.T) {
|
||||||
bktName, objName := "bucket", "obj"
|
bktName, objName := "bucket", "obj"
|
||||||
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
||||||
|
|
||||||
putObject(hc.t, hc, bktName, objName)
|
putObject(hc, bktName, objName)
|
||||||
|
|
||||||
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
||||||
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
||||||
|
|
|
@ -3,9 +3,9 @@ package handler
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"io"
|
"io"
|
||||||
"math/rand"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
@ -38,6 +38,8 @@ type handlerContext struct {
|
||||||
tree *tree.Tree
|
tree *tree.Tree
|
||||||
context context.Context
|
context context.Context
|
||||||
kludge *kludgeSettingsMock
|
kludge *kludgeSettingsMock
|
||||||
|
|
||||||
|
layerFeatures *layer.FeatureSettingsMock
|
||||||
}
|
}
|
||||||
|
|
||||||
func (hc *handlerContext) Handler() *handler {
|
func (hc *handlerContext) Handler() *handler {
|
||||||
|
@ -123,11 +125,14 @@ func prepareHandlerContextBase(t *testing.T, minCache bool) *handlerContext {
|
||||||
cacheCfg = getMinCacheConfig(l)
|
cacheCfg = getMinCacheConfig(l)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
features := &layer.FeatureSettingsMock{}
|
||||||
|
|
||||||
layerCfg := &layer.Config{
|
layerCfg := &layer.Config{
|
||||||
Caches: cacheCfg,
|
Caches: cacheCfg,
|
||||||
AnonKey: layer.AnonymousKey{Key: key},
|
AnonKey: layer.AnonymousKey{Key: key},
|
||||||
Resolver: testResolver,
|
Resolver: testResolver,
|
||||||
TreeService: treeMock,
|
TreeService: treeMock,
|
||||||
|
Features: features,
|
||||||
}
|
}
|
||||||
|
|
||||||
var pp netmap.PlacementPolicy
|
var pp netmap.PlacementPolicy
|
||||||
|
@ -154,6 +159,8 @@ func prepareHandlerContextBase(t *testing.T, minCache bool) *handlerContext {
|
||||||
tree: treeMock,
|
tree: treeMock,
|
||||||
context: middleware.SetBoxData(context.Background(), newTestAccessBox(t, key)),
|
context: middleware.SetBoxData(context.Background(), newTestAccessBox(t, key)),
|
||||||
kludge: kludge,
|
kludge: kludge,
|
||||||
|
|
||||||
|
layerFeatures: features,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -114,7 +114,7 @@ func TestHeadObject(t *testing.T) {
|
||||||
bktName, objName := "bucket", "obj"
|
bktName, objName := "bucket", "obj"
|
||||||
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
||||||
|
|
||||||
putObject(hc.t, hc, bktName, objName)
|
putObject(hc, bktName, objName)
|
||||||
|
|
||||||
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
||||||
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
||||||
|
|
|
@ -536,7 +536,7 @@ func TestPutObjectWithLock(t *testing.T) {
|
||||||
createTestBucketWithLock(hc, bktName, lockConfig)
|
createTestBucketWithLock(hc, bktName, lockConfig)
|
||||||
|
|
||||||
objDefault := "obj-default-retention"
|
objDefault := "obj-default-retention"
|
||||||
putObject(t, hc, bktName, objDefault)
|
putObject(hc, bktName, objDefault)
|
||||||
|
|
||||||
getObjectRetentionApproximate(hc, bktName, objDefault, governanceMode, time.Now().Add(24*time.Hour))
|
getObjectRetentionApproximate(hc, bktName, objDefault, governanceMode, time.Now().Add(24*time.Hour))
|
||||||
getObjectLegalHold(hc, bktName, objDefault, legalHoldOff)
|
getObjectLegalHold(hc, bktName, objDefault, legalHoldOff)
|
||||||
|
@ -587,7 +587,7 @@ func TestPutLockErrors(t *testing.T) {
|
||||||
headers[api.AmzObjectLockRetainUntilDate] = "dummy"
|
headers[api.AmzObjectLockRetainUntilDate] = "dummy"
|
||||||
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrInvalidRetentionDate)
|
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrInvalidRetentionDate)
|
||||||
|
|
||||||
putObject(t, hc, bktName, objName)
|
putObject(hc, bktName, objName)
|
||||||
|
|
||||||
retention := &data.Retention{Mode: governanceMode}
|
retention := &data.Retention{Mode: governanceMode}
|
||||||
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrMalformedXML)
|
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrMalformedXML)
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -92,6 +93,13 @@ type (
|
||||||
const (
|
const (
|
||||||
uploadIDHeaderName = "uploadId"
|
uploadIDHeaderName = "uploadId"
|
||||||
partNumberHeaderName = "partNumber"
|
partNumberHeaderName = "partNumber"
|
||||||
|
|
||||||
|
prefixQueryName = "prefix"
|
||||||
|
delimiterQueryName = "delimiter"
|
||||||
|
maxUploadsQueryName = "max-uploads"
|
||||||
|
encodingTypeQueryName = "encoding-type"
|
||||||
|
keyMarkerQueryName = "key-marker"
|
||||||
|
uploadIDMarkerQueryName = "upload-id-marker"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
@ -511,7 +519,7 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
if err = h.sendNotifications(ctx, s); err != nil {
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return objInfo, nil
|
return objInfo, nil
|
||||||
|
@ -528,30 +536,27 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
|
||||||
|
|
||||||
var (
|
var (
|
||||||
queryValues = reqInfo.URL.Query()
|
queryValues = reqInfo.URL.Query()
|
||||||
delimiter = queryValues.Get("delimiter")
|
maxUploadsStr = queryValues.Get(maxUploadsQueryName)
|
||||||
prefix = queryValues.Get("prefix")
|
|
||||||
maxUploads = layer.MaxSizeUploadsList
|
maxUploads = layer.MaxSizeUploadsList
|
||||||
)
|
)
|
||||||
|
|
||||||
if queryValues.Get("max-uploads") != "" {
|
if maxUploadsStr != "" {
|
||||||
val, err := strconv.Atoi(queryValues.Get("max-uploads"))
|
val, err := strconv.Atoi(maxUploadsStr)
|
||||||
if err != nil || val < 0 {
|
if err != nil || val < 1 || val > 1000 {
|
||||||
h.logAndSendError(w, "invalid maxUploads", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxUploads))
|
h.logAndSendError(w, "invalid maxUploads", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxUploads))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if val < maxUploads {
|
|
||||||
maxUploads = val
|
maxUploads = val
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
p := &layer.ListMultipartUploadsParams{
|
p := &layer.ListMultipartUploadsParams{
|
||||||
Bkt: bktInfo,
|
Bkt: bktInfo,
|
||||||
Delimiter: delimiter,
|
Delimiter: queryValues.Get(delimiterQueryName),
|
||||||
EncodingType: queryValues.Get("encoding-type"),
|
EncodingType: queryValues.Get(encodingTypeQueryName),
|
||||||
KeyMarker: queryValues.Get("key-marker"),
|
KeyMarker: queryValues.Get(keyMarkerQueryName),
|
||||||
MaxUploads: maxUploads,
|
MaxUploads: maxUploads,
|
||||||
Prefix: prefix,
|
Prefix: queryValues.Get(prefixQueryName),
|
||||||
UploadIDMarker: queryValues.Get("upload-id-marker"),
|
UploadIDMarker: queryValues.Get(uploadIDMarkerQueryName),
|
||||||
}
|
}
|
||||||
|
|
||||||
list, err := h.obj.ListMultipartUploads(r.Context(), p)
|
list, err := h.obj.ListMultipartUploads(r.Context(), p)
|
||||||
|
|
|
@ -3,12 +3,16 @@ package handler
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -105,6 +109,164 @@ func TestMultipartReUploadPart(t *testing.T) {
|
||||||
equalDataSlices(t, append(data1, data2...), data)
|
equalDataSlices(t, append(data1, data2...), data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestListMultipartUploads(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-to-list-uploads"
|
||||||
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
|
objName1 := "/my/object/name"
|
||||||
|
uploadInfo1 := createMultipartUpload(hc, bktName, objName1, map[string]string{})
|
||||||
|
objName2 := "/my/object2"
|
||||||
|
uploadInfo2 := createMultipartUpload(hc, bktName, objName2, map[string]string{})
|
||||||
|
objName3 := "/zzz/object/name3"
|
||||||
|
uploadInfo3 := createMultipartUpload(hc, bktName, objName3, map[string]string{})
|
||||||
|
|
||||||
|
t.Run("check upload key", func(t *testing.T) {
|
||||||
|
listUploads := listAllMultipartUploads(hc, bktName)
|
||||||
|
require.Len(t, listUploads.Uploads, 3)
|
||||||
|
for i, upload := range []*InitiateMultipartUploadResponse{uploadInfo1, uploadInfo2, uploadInfo3} {
|
||||||
|
require.Equal(t, upload.UploadID, listUploads.Uploads[i].UploadID)
|
||||||
|
require.Equal(t, upload.Key, listUploads.Uploads[i].Key)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("check max uploads", func(t *testing.T) {
|
||||||
|
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", "", 2)
|
||||||
|
require.Len(t, listUploads.Uploads, 2)
|
||||||
|
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
|
||||||
|
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("check prefix", func(t *testing.T) {
|
||||||
|
listUploads := listMultipartUploadsBase(hc, bktName, "/my", "", "", "", -1)
|
||||||
|
require.Len(t, listUploads.Uploads, 2)
|
||||||
|
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
|
||||||
|
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("check markers", func(t *testing.T) {
|
||||||
|
t.Run("check only key-marker", func(t *testing.T) {
|
||||||
|
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", objName2, -1)
|
||||||
|
require.Len(t, listUploads.Uploads, 1)
|
||||||
|
// If upload-id-marker is not specified, only the keys lexicographically greater than the specified key-marker will be included in the list.
|
||||||
|
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("check only upload-id-marker", func(t *testing.T) {
|
||||||
|
uploadIDMarker := uploadInfo1.UploadID
|
||||||
|
if uploadIDMarker > uploadInfo2.UploadID {
|
||||||
|
uploadIDMarker = uploadInfo2.UploadID
|
||||||
|
}
|
||||||
|
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, "", -1)
|
||||||
|
// If key-marker is not specified, the upload-id-marker parameter is ignored.
|
||||||
|
require.Len(t, listUploads.Uploads, 3)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("check key-marker along with upload-id-marker", func(t *testing.T) {
|
||||||
|
uploadIDMarker := "00000000-0000-0000-0000-000000000000"
|
||||||
|
|
||||||
|
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, objName3, -1)
|
||||||
|
require.Len(t, listUploads.Uploads, 1)
|
||||||
|
// If upload-id-marker is specified, any multipart uploads for a key equal to the key-marker might also be included,
|
||||||
|
// provided those multipart uploads have upload IDs lexicographically greater than the specified upload-id-marker.
|
||||||
|
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartUploadSize(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName, objName := "bucket-for-test-multipart-size", "object-multipart"
|
||||||
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
|
partSize := layer.UploadMinSize
|
||||||
|
objLen := 2 * partSize
|
||||||
|
headers := map[string]string{}
|
||||||
|
|
||||||
|
data := multipartUpload(hc, bktName, objName, headers, objLen, partSize)
|
||||||
|
require.Equal(t, objLen, len(data))
|
||||||
|
|
||||||
|
t.Run("check correct size in list v1", func(t *testing.T) {
|
||||||
|
listV1 := listObjectsV1(hc, bktName, "", "", "", -1)
|
||||||
|
require.Len(t, listV1.Contents, 1)
|
||||||
|
require.Equal(t, objLen, int(listV1.Contents[0].Size))
|
||||||
|
require.Equal(t, objName, listV1.Contents[0].Key)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("check correct size in list v2", func(t *testing.T) {
|
||||||
|
listV2 := listObjectsV2(hc, bktName, "", "", "", "", -1)
|
||||||
|
require.Len(t, listV2.Contents, 1)
|
||||||
|
require.Equal(t, objLen, int(listV2.Contents[0].Size))
|
||||||
|
require.Equal(t, objName, listV2.Contents[0].Key)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("check correct get", func(t *testing.T) {
|
||||||
|
_, hdr := getObject(hc, bktName, objName)
|
||||||
|
require.Equal(t, strconv.Itoa(objLen), hdr.Get(api.ContentLength))
|
||||||
|
|
||||||
|
part := getObjectRange(t, hc, bktName, objName, partSize, objLen-1)
|
||||||
|
equalDataSlices(t, data[partSize:], part)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("check correct size when part copy", func(t *testing.T) {
|
||||||
|
objName2 := "obj2"
|
||||||
|
uploadInfo := createMultipartUpload(hc, bktName, objName2, headers)
|
||||||
|
sourceCopy := bktName + "/" + objName
|
||||||
|
uploadPartCopy(hc, bktName, objName2, uploadInfo.UploadID, 1, sourceCopy, 0, 0)
|
||||||
|
uploadPartCopy(hc, bktName, objName2, uploadInfo.UploadID, 2, sourceCopy, 0, partSize)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||||
|
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
|
||||||
|
}
|
||||||
|
|
||||||
|
func uploadPartCopyBase(hc *handlerContext, bktName, objName string, encrypted bool, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||||
|
query := make(url.Values)
|
||||||
|
query.Set(uploadIDQuery, uploadID)
|
||||||
|
query.Set(partNumberQuery, strconv.Itoa(num))
|
||||||
|
|
||||||
|
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, nil)
|
||||||
|
if encrypted {
|
||||||
|
setEncryptHeaders(r)
|
||||||
|
}
|
||||||
|
r.Header.Set(api.AmzCopySource, srcObj)
|
||||||
|
if start+end > 0 {
|
||||||
|
r.Header.Set(api.AmzCopySourceRange, fmt.Sprintf("bytes=%d-%d", start, end))
|
||||||
|
}
|
||||||
|
|
||||||
|
hc.Handler().UploadPartCopy(w, r)
|
||||||
|
uploadPartCopyResponse := &UploadPartCopyResponse{}
|
||||||
|
readResponse(hc.t, w, http.StatusOK, uploadPartCopyResponse)
|
||||||
|
|
||||||
|
return uploadPartCopyResponse
|
||||||
|
}
|
||||||
|
|
||||||
|
func listAllMultipartUploads(hc *handlerContext, bktName string) *ListMultipartUploadsResponse {
|
||||||
|
return listMultipartUploadsBase(hc, bktName, "", "", "", "", -1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func listMultipartUploadsBase(hc *handlerContext, bktName, prefix, delimiter, uploadIDMarker, keyMarker string, maxUploads int) *ListMultipartUploadsResponse {
|
||||||
|
query := make(url.Values)
|
||||||
|
query.Set(prefixQueryName, prefix)
|
||||||
|
query.Set(delimiterQueryName, delimiter)
|
||||||
|
query.Set(uploadIDMarkerQueryName, uploadIDMarker)
|
||||||
|
query.Set(keyMarkerQueryName, keyMarker)
|
||||||
|
if maxUploads != -1 {
|
||||||
|
query.Set(maxUploadsQueryName, strconv.Itoa(maxUploads))
|
||||||
|
}
|
||||||
|
|
||||||
|
w, r := prepareTestRequestWithQuery(hc, bktName, "", query, nil)
|
||||||
|
|
||||||
|
hc.Handler().ListMultipartUploadsHandler(w, r)
|
||||||
|
listPartsResponse := &ListMultipartUploadsResponse{}
|
||||||
|
readResponse(hc.t, w, http.StatusOK, listPartsResponse)
|
||||||
|
|
||||||
|
return listPartsResponse
|
||||||
|
}
|
||||||
|
|
||||||
func listParts(hc *handlerContext, bktName, objName string, uploadID string) *ListPartsResponse {
|
func listParts(hc *handlerContext, bktName, objName string, uploadID string) *ListPartsResponse {
|
||||||
return listPartsBase(hc, bktName, objName, false, uploadID)
|
return listPartsBase(hc, bktName, objName, false, uploadID)
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
@ -202,7 +203,7 @@ func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.Notif
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
h.reqLogger(ctx).Warn("failed to send test event because notifications is disabled")
|
h.reqLogger(ctx).Warn(logs.FailedToSendTestEventBecauseNotificationsIsDisabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
if q.ID == "" {
|
if q.ID == "" {
|
||||||
|
|
|
@ -198,6 +198,10 @@ func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Obje
|
||||||
ETag: obj.HashSum,
|
ETag: obj.HashSum,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if size, err := layer.GetObjectSize(obj); err == nil {
|
||||||
|
res.Size = size
|
||||||
|
}
|
||||||
|
|
||||||
if fetchOwner {
|
if fetchOwner {
|
||||||
res.Owner = &Owner{
|
res.Owner = &Owner{
|
||||||
ID: obj.Owner.String(),
|
ID: obj.Owner.String(),
|
||||||
|
|
|
@ -67,11 +67,11 @@ func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T
|
||||||
createTestObject(tc, bktInfo, objName)
|
createTestObject(tc, bktInfo, objName)
|
||||||
}
|
}
|
||||||
|
|
||||||
listV2Response1 := listObjectsV2(t, tc, bktName, "", "", "bar", "", 1)
|
listV2Response1 := listObjectsV2(tc, bktName, "", "", "bar", "", 1)
|
||||||
nextContinuationToken := listV2Response1.NextContinuationToken
|
nextContinuationToken := listV2Response1.NextContinuationToken
|
||||||
require.Equal(t, "baz", listV2Response1.Contents[0].Key)
|
require.Equal(t, "baz", listV2Response1.Contents[0].Key)
|
||||||
|
|
||||||
listV2Response2 := listObjectsV2(t, tc, bktName, "", "", "bar", nextContinuationToken, -1)
|
listV2Response2 := listObjectsV2(tc, bktName, "", "", "bar", nextContinuationToken, -1)
|
||||||
|
|
||||||
require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
|
require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
|
||||||
require.Equal(t, "bar", listV2Response2.StartAfter)
|
require.Equal(t, "bar", listV2Response2.StartAfter)
|
||||||
|
@ -92,7 +92,7 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
|
||||||
createTestObject(tc, bktInfo, objName)
|
createTestObject(tc, bktInfo, objName)
|
||||||
}
|
}
|
||||||
|
|
||||||
listV1Response := listObjectsV1(t, tc, bktName, "", "/", "", -1)
|
listV1Response := listObjectsV1(tc, bktName, "", "/", "", -1)
|
||||||
require.Equal(t, "/", listV1Response.Delimiter)
|
require.Equal(t, "/", listV1Response.Delimiter)
|
||||||
require.Equal(t, "asdf", listV1Response.Contents[0].Key)
|
require.Equal(t, "asdf", listV1Response.Contents[0].Key)
|
||||||
require.Len(t, listV1Response.CommonPrefixes, 2)
|
require.Len(t, listV1Response.CommonPrefixes, 2)
|
||||||
|
@ -100,6 +100,26 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
|
||||||
require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix)
|
require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestS3BucketListV2DelimiterPercentage(t *testing.T) {
|
||||||
|
tc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
bktName := "bucket-for-listing"
|
||||||
|
objects := []string{"b%ar", "b%az", "c%ab", "foo"}
|
||||||
|
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
||||||
|
|
||||||
|
for _, objName := range objects[1:] {
|
||||||
|
createTestObject(tc, bktInfo, objName)
|
||||||
|
}
|
||||||
|
|
||||||
|
listV2Response := listObjectsV2(tc, bktName, "", "%", "", "", -1)
|
||||||
|
require.Equal(t, "%", listV2Response.Delimiter)
|
||||||
|
require.Len(t, listV2Response.Contents, 1)
|
||||||
|
require.Equal(t, "foo", listV2Response.Contents[0].Key)
|
||||||
|
require.Len(t, listV2Response.CommonPrefixes, 2)
|
||||||
|
require.Equal(t, "b%", listV2Response.CommonPrefixes[0].Prefix)
|
||||||
|
require.Equal(t, "c%", listV2Response.CommonPrefixes[1].Prefix)
|
||||||
|
}
|
||||||
|
|
||||||
func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
||||||
tc := prepareHandlerContext(t)
|
tc := prepareHandlerContext(t)
|
||||||
|
|
||||||
|
@ -129,7 +149,7 @@ func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
||||||
validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"})
|
validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"})
|
||||||
}
|
}
|
||||||
|
|
||||||
func listObjectsV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
|
func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
|
||||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||||
if len(startAfter) != 0 {
|
if len(startAfter) != 0 {
|
||||||
query.Add("start-after", startAfter)
|
query.Add("start-after", startAfter)
|
||||||
|
@ -138,17 +158,17 @@ func listObjectsV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter,
|
||||||
query.Add("continuation-token", continuationToken)
|
query.Add("continuation-token", continuationToken)
|
||||||
}
|
}
|
||||||
|
|
||||||
w, r := prepareTestFullRequest(tc, bktName, "", query, nil)
|
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||||
tc.Handler().ListObjectsV2Handler(w, r)
|
hc.Handler().ListObjectsV2Handler(w, r)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
res := &ListObjectsV2Response{}
|
res := &ListObjectsV2Response{}
|
||||||
parseTestResponse(t, w, res)
|
parseTestResponse(hc.t, w, res)
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int,
|
func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int,
|
||||||
isTruncated, last bool, checkObjects, checkPrefixes []string) string {
|
isTruncated, last bool, checkObjects, checkPrefixes []string) string {
|
||||||
response := listObjectsV2(t, tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
|
response := listObjectsV2(tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
|
||||||
|
|
||||||
require.Equal(t, isTruncated, response.IsTruncated)
|
require.Equal(t, isTruncated, response.IsTruncated)
|
||||||
require.Equal(t, last, len(response.NextContinuationToken) == 0)
|
require.Equal(t, last, len(response.NextContinuationToken) == 0)
|
||||||
|
@ -182,16 +202,16 @@ func prepareCommonListObjectsQuery(prefix, delimiter string, maxKeys int) url.Va
|
||||||
return query
|
return query
|
||||||
}
|
}
|
||||||
|
|
||||||
func listObjectsV1(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
|
func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
|
||||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||||
if len(marker) != 0 {
|
if len(marker) != 0 {
|
||||||
query.Add("marker", marker)
|
query.Add("marker", marker)
|
||||||
}
|
}
|
||||||
|
|
||||||
w, r := prepareTestFullRequest(tc, bktName, "", query, nil)
|
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||||
tc.Handler().ListObjectsV1Handler(w, r)
|
hc.Handler().ListObjectsV1Handler(w, r)
|
||||||
assertStatus(t, w, http.StatusOK)
|
assertStatus(hc.t, w, http.StatusOK)
|
||||||
res := &ListObjectsV1Response{}
|
res := &ListObjectsV1Response{}
|
||||||
parseTestResponse(t, w, res)
|
parseTestResponse(hc.t, w, res)
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,6 +24,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -277,7 +278,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
if err = h.sendNotifications(ctx, s); err != nil {
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if containsACL {
|
if containsACL {
|
||||||
|
@ -494,7 +495,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
if err = h.sendNotifications(ctx, s); err != nil {
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if acl := auth.MultipartFormValue(r, "acl"); acl != "" {
|
if acl := auth.MultipartFormValue(r, "acl"); acl != "" {
|
||||||
|
@ -539,7 +540,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if settings, err := h.obj.GetBucketSettings(ctx, bktInfo); err != nil {
|
if settings, err := h.obj.GetBucketSettings(ctx, bktInfo); err != nil {
|
||||||
h.reqLogger(ctx).Warn("couldn't get bucket versioning", zap.String("bucket name", reqInfo.BucketName), zap.Error(err))
|
h.reqLogger(ctx).Warn(logs.CouldntGetBucketVersioning, zap.String("bucket name", reqInfo.BucketName), zap.Error(err))
|
||||||
} else if settings.VersioningEnabled() {
|
} else if settings.VersioningEnabled() {
|
||||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||||
}
|
}
|
||||||
|
@ -778,7 +779,7 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h.reqLogger(ctx).Info("bucket is created", zap.Stringer("container_id", bktInfo.CID))
|
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
||||||
|
|
||||||
if p.ObjectLockEnabled {
|
if p.ObjectLockEnabled {
|
||||||
sp := &layer.PutSettingsParams{
|
sp := &layer.PutSettingsParams{
|
||||||
|
|
|
@ -23,6 +23,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -309,3 +310,37 @@ func TestCreateBucket(t *testing.T) {
|
||||||
box2, _ := createAccessBox(t)
|
box2, _ := createAccessBox(t)
|
||||||
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
|
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPutObjectClientCut(t *testing.T) {
|
||||||
|
hc := prepareHandlerContext(t)
|
||||||
|
bktName, objName1, objName2 := "bkt-name", "obj-name1", "obj-name2"
|
||||||
|
createTestBucket(hc, bktName)
|
||||||
|
|
||||||
|
putObject(hc, bktName, objName1)
|
||||||
|
obj1 := getObjectFromLayer(hc, objName1)[0]
|
||||||
|
require.Empty(t, getObjectAttribute(obj1, "s3-client-cut"))
|
||||||
|
|
||||||
|
hc.layerFeatures.SetClientCut(true)
|
||||||
|
putObject(hc, bktName, objName2)
|
||||||
|
obj2 := getObjectFromLayer(hc, objName2)[0]
|
||||||
|
require.Equal(t, "true", getObjectAttribute(obj2, "s3-client-cut"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func getObjectFromLayer(hc *handlerContext, objName string) []*object.Object {
|
||||||
|
var res []*object.Object
|
||||||
|
for _, o := range hc.tp.Objects() {
|
||||||
|
if objName == getObjectAttribute(o, object.AttributeFilePath) {
|
||||||
|
res = append(res, o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func getObjectAttribute(obj *object.Object, attrName string) string {
|
||||||
|
for _, attr := range obj.Attributes() {
|
||||||
|
if attr.Key() == attrName {
|
||||||
|
return attr.Value()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -66,7 +67,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
if err = h.sendNotifications(ctx, s); err != nil {
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
@ -143,7 +144,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
|
||||||
ReqInfo: reqInfo,
|
ReqInfo: reqInfo,
|
||||||
}
|
}
|
||||||
if err = h.sendNotifications(ctx, s); err != nil {
|
if err = h.sendNotifications(ctx, s); err != nil {
|
||||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -36,7 +37,7 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
|
||||||
zap.String("description", logText),
|
zap.String("description", logText),
|
||||||
zap.Error(err)}
|
zap.Error(err)}
|
||||||
fields = append(fields, additional...)
|
fields = append(fields, additional...)
|
||||||
h.log.Error("request failed", fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
h.log.Error(logs.RequestFailed, fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
|
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
|
||||||
|
@ -49,7 +50,7 @@ func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string,
|
||||||
zap.String("description", logText),
|
zap.String("description", logText),
|
||||||
zap.Error(err)}
|
zap.Error(err)}
|
||||||
fields = append(fields, additional...)
|
fields = append(fields, additional...)
|
||||||
h.log.Error("request failed", fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
h.log.Error(logs.RequestFailed, fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
||||||
}
|
}
|
||||||
|
|
||||||
func transformToS3Error(err error) error {
|
func transformToS3Error(err error) error {
|
||||||
|
|
|
@ -3,6 +3,7 @@ package layer
|
||||||
import (
|
import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
|
@ -61,7 +62,7 @@ func (c *Cache) GetBucket(name string) *data.BucketInfo {
|
||||||
|
|
||||||
func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
|
func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
|
||||||
if err := c.bucketCache.Put(bktInfo); err != nil {
|
if err := c.bucketCache.Put(bktInfo); err != nil {
|
||||||
c.logger.Warn("couldn't put bucket info into cache",
|
c.logger.Warn(logs.CouldntPutBucketInfoIntoCache,
|
||||||
zap.String("bucket name", bktInfo.Name),
|
zap.String("bucket name", bktInfo.Name),
|
||||||
zap.Stringer("bucket cid", bktInfo.CID),
|
zap.Stringer("bucket cid", bktInfo.CID),
|
||||||
zap.Error(err))
|
zap.Error(err))
|
||||||
|
@ -104,13 +105,13 @@ func (c *Cache) GetLastObject(owner user.ID, bktName, objName string) *data.Exte
|
||||||
|
|
||||||
func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
|
func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
|
||||||
if err := c.objCache.PutObject(extObjInfo); err != nil {
|
if err := c.objCache.PutObject(extObjInfo); err != nil {
|
||||||
c.logger.Warn("couldn't add object to cache", zap.Error(err),
|
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
|
||||||
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
|
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
|
||||||
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
|
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
|
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
|
||||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -118,7 +119,7 @@ func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObject
|
||||||
c.PutObject(owner, extObjInfo)
|
c.PutObject(owner, extObjInfo)
|
||||||
|
|
||||||
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
|
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
|
||||||
c.logger.Warn("couldn't put obj address to name cache",
|
c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
|
||||||
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
|
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
|
||||||
zap.Error(err))
|
zap.Error(err))
|
||||||
}
|
}
|
||||||
|
@ -134,11 +135,11 @@ func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVer
|
||||||
|
|
||||||
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
|
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
|
||||||
if err := c.listsCache.PutVersions(key, list); err != nil {
|
if err := c.listsCache.PutVersions(key, list); err != nil {
|
||||||
c.logger.Warn("couldn't cache list of objects", zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -152,11 +153,11 @@ func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
|
||||||
|
|
||||||
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
|
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
|
||||||
if err := c.systemCache.PutTagging(key, tags); err != nil {
|
if err := c.systemCache.PutTagging(key, tags); err != nil {
|
||||||
c.logger.Error("couldn't cache tags", zap.Error(err))
|
c.logger.Error(logs.CouldntCacheTags, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -174,11 +175,11 @@ func (c *Cache) GetLockInfo(owner user.ID, key string) *data.LockInfo {
|
||||||
|
|
||||||
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
|
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
|
||||||
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
|
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
|
||||||
c.logger.Error("couldn't cache lock info", zap.Error(err))
|
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -195,11 +196,11 @@ func (c *Cache) GetSettings(owner user.ID, bktInfo *data.BucketInfo) *data.Bucke
|
||||||
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
||||||
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
||||||
if err := c.systemCache.PutSettings(key, settings); err != nil {
|
if err := c.systemCache.PutSettings(key, settings); err != nil {
|
||||||
c.logger.Warn("couldn't cache bucket settings", zap.String("bucket", bktInfo.Name), zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -217,11 +218,11 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
||||||
key := bkt.Name + bkt.CORSObjectName()
|
key := bkt.Name + bkt.CORSObjectName()
|
||||||
|
|
||||||
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
||||||
c.logger.Warn("couldn't cache cors", zap.String("bucket", bkt.Name), zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -242,10 +243,10 @@ func (c *Cache) GetNotificationConfiguration(owner user.ID, bktInfo *data.Bucket
|
||||||
func (c *Cache) PutNotificationConfiguration(owner user.ID, bktInfo *data.BucketInfo, configuration *data.NotificationConfiguration) {
|
func (c *Cache) PutNotificationConfiguration(owner user.ID, bktInfo *data.BucketInfo, configuration *data.NotificationConfiguration) {
|
||||||
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
|
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
|
||||||
if err := c.systemCache.PutNotificationConfiguration(key, configuration); err != nil {
|
if err := c.systemCache.PutNotificationConfiguration(key, configuration); err != nil {
|
||||||
c.logger.Warn("couldn't cache notification configuration", zap.String("bucket", bktInfo.Name), zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheNotificationConfiguration, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := c.accessCache.Put(owner, key); err != nil {
|
if err := c.accessCache.Put(owner, key); err != nil {
|
||||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
@ -63,7 +64,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
||||||
if len(attrLockEnabled) > 0 {
|
if len(attrLockEnabled) > 0 {
|
||||||
info.ObjectLockEnabled, err = strconv.ParseBool(attrLockEnabled)
|
info.ObjectLockEnabled, err = strconv.ParseBool(attrLockEnabled)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("could not parse container object lock enabled attribute",
|
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
|
||||||
zap.String("lock_enabled", attrLockEnabled),
|
zap.String("lock_enabled", attrLockEnabled),
|
||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
|
@ -78,7 +79,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
||||||
func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
res, err := n.frostFS.UserContainers(ctx, n.BearerOwner(ctx))
|
res, err := n.frostFS.UserContainers(ctx, n.BearerOwner(ctx))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Error("could not list user containers", zap.Error(err))
|
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err))
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -86,7 +87,7 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||||
for i := range res {
|
for i := range res {
|
||||||
info, err := n.containerInfo(ctx, res[i])
|
info, err := n.containerInfo(ctx, res[i])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Error("could not fetch container info", zap.Error(err))
|
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -57,7 +58,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
||||||
|
|
||||||
if !objIDToDeleteNotFound {
|
if !objIDToDeleteNotFound {
|
||||||
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
||||||
n.reqLogger(ctx).Error("couldn't delete cors object", zap.Error(err),
|
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
|
||||||
zap.String("cnrID", p.BktInfo.CID.EncodeToString()),
|
zap.String("cnrID", p.BktInfo.CID.EncodeToString()),
|
||||||
zap.String("objID", objIDToDelete.EncodeToString()))
|
zap.String("objID", objIDToDelete.EncodeToString()))
|
||||||
}
|
}
|
||||||
|
|
|
@ -111,6 +111,9 @@ type PrmObjectCreate struct {
|
||||||
|
|
||||||
// Number of object copies that is enough to consider put successful.
|
// Number of object copies that is enough to consider put successful.
|
||||||
CopiesNumber []uint32
|
CopiesNumber []uint32
|
||||||
|
|
||||||
|
// Enables client side object preparing.
|
||||||
|
ClientCut bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||||
|
|
|
@ -25,6 +25,18 @@ import (
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type FeatureSettingsMock struct {
|
||||||
|
clientCut bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *FeatureSettingsMock) ClientCut() bool {
|
||||||
|
return k.clientCut
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *FeatureSettingsMock) SetClientCut(clientCut bool) {
|
||||||
|
k.clientCut = clientCut
|
||||||
|
}
|
||||||
|
|
||||||
type TestFrostFS struct {
|
type TestFrostFS struct {
|
||||||
FrostFS
|
FrostFS
|
||||||
|
|
||||||
|
@ -222,6 +234,13 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
|
||||||
attrs = append(attrs, *a)
|
attrs = append(attrs, *a)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if prm.ClientCut {
|
||||||
|
a := object.NewAttribute()
|
||||||
|
a.SetKey("s3-client-cut")
|
||||||
|
a.SetValue("true")
|
||||||
|
attrs = append(attrs, *a)
|
||||||
|
}
|
||||||
|
|
||||||
for i := range prm.Attributes {
|
for i := range prm.Attributes {
|
||||||
a := object.NewAttribute()
|
a := object.NewAttribute()
|
||||||
a.SetKey(prm.Attributes[i][0])
|
a.SetKey(prm.Attributes[i][0])
|
||||||
|
|
|
@ -16,6 +16,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
|
@ -45,6 +46,10 @@ type (
|
||||||
Resolve(ctx context.Context, name string) (cid.ID, error)
|
Resolve(ctx context.Context, name string) (cid.ID, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FeatureSettings interface {
|
||||||
|
ClientCut() bool
|
||||||
|
}
|
||||||
|
|
||||||
layer struct {
|
layer struct {
|
||||||
frostFS FrostFS
|
frostFS FrostFS
|
||||||
gateOwner user.ID
|
gateOwner user.ID
|
||||||
|
@ -54,6 +59,7 @@ type (
|
||||||
ncontroller EventListener
|
ncontroller EventListener
|
||||||
cache *Cache
|
cache *Cache
|
||||||
treeService TreeService
|
treeService TreeService
|
||||||
|
features FeatureSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
Config struct {
|
Config struct {
|
||||||
|
@ -63,6 +69,7 @@ type (
|
||||||
AnonKey AnonymousKey
|
AnonKey AnonymousKey
|
||||||
Resolver BucketResolver
|
Resolver BucketResolver
|
||||||
TreeService TreeService
|
TreeService TreeService
|
||||||
|
Features FeatureSettings
|
||||||
}
|
}
|
||||||
|
|
||||||
// AnonymousKey contains data for anonymous requests.
|
// AnonymousKey contains data for anonymous requests.
|
||||||
|
@ -301,6 +308,7 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
|
||||||
resolver: config.Resolver,
|
resolver: config.Resolver,
|
||||||
cache: NewCache(config.Caches),
|
cache: NewCache(config.Caches),
|
||||||
treeService: config.TreeService,
|
treeService: config.TreeService,
|
||||||
|
features: config.Features,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -555,7 +563,7 @@ func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug("get object",
|
n.reqLogger(ctx).Debug(logs.GetObject,
|
||||||
zap.Stringer("cid", p.BktInfo.CID),
|
zap.Stringer("cid", p.BktInfo.CID),
|
||||||
zap.Stringer("oid", objInfo.ObjectInfo.ID))
|
zap.Stringer("oid", objInfo.ObjectInfo.ID))
|
||||||
|
|
||||||
|
@ -686,7 +694,7 @@ func (n *layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject)
|
||||||
|
|
||||||
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeID uint64) *VersionedObject {
|
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeID uint64) *VersionedObject {
|
||||||
if client.IsErrObjectAlreadyRemoved(obj.Error) {
|
if client.IsErrObjectAlreadyRemoved(obj.Error) {
|
||||||
n.reqLogger(ctx).Debug("object already removed",
|
n.reqLogger(ctx).Debug(logs.ObjectAlreadyRemoved,
|
||||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID))
|
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID))
|
||||||
|
|
||||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeID)
|
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeID)
|
||||||
|
@ -698,7 +706,7 @@ func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketIn
|
||||||
}
|
}
|
||||||
|
|
||||||
if client.IsErrObjectNotFound(obj.Error) {
|
if client.IsErrObjectNotFound(obj.Error) {
|
||||||
n.reqLogger(ctx).Debug("object not found",
|
n.reqLogger(ctx).Debug(logs.ObjectNotFound,
|
||||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID))
|
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID))
|
||||||
|
|
||||||
obj.Error = nil
|
obj.Error = nil
|
||||||
|
@ -776,7 +784,7 @@ func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error)
|
||||||
return cid.ID{}, err
|
return cid.ID{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Info("resolve bucket", zap.Stringer("cid", cnrID))
|
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID))
|
||||||
}
|
}
|
||||||
|
|
||||||
return cnrID, nil
|
return cnrID, nil
|
||||||
|
|
|
@ -16,6 +16,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"github.com/minio/sio"
|
"github.com/minio/sio"
|
||||||
|
@ -202,7 +203,7 @@ func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
|
||||||
func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
||||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||||
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||||
n.reqLogger(ctx).Warn("mismatched obj encryptionInfo", zap.Error(err))
|
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
|
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -237,7 +238,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
size = decSize
|
size = decSize
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug("upload part",
|
n.reqLogger(ctx).Debug(logs.UploadPart,
|
||||||
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
||||||
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
||||||
|
|
||||||
|
@ -258,7 +259,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
||||||
}
|
}
|
||||||
if !oldPartIDNotFound {
|
if !oldPartIDNotFound {
|
||||||
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
|
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
|
||||||
n.reqLogger(ctx).Error("couldn't delete old part object", zap.Error(err),
|
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
|
||||||
zap.String("cid", bktInfo.CID.EncodeToString()),
|
zap.String("cid", bktInfo.CID.EncodeToString()),
|
||||||
zap.String("oid", oldPartID.EncodeToString()))
|
zap.String("oid", oldPartID.EncodeToString()))
|
||||||
}
|
}
|
||||||
|
@ -288,10 +289,16 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
|
||||||
}
|
}
|
||||||
|
|
||||||
size := p.SrcObjInfo.Size
|
size := p.SrcObjInfo.Size
|
||||||
|
srcObjectSize := p.SrcObjInfo.Size
|
||||||
|
|
||||||
|
if objSize, err := GetObjectSize(p.SrcObjInfo); err == nil {
|
||||||
|
srcObjectSize = objSize
|
||||||
|
}
|
||||||
|
|
||||||
if p.Range != nil {
|
if p.Range != nil {
|
||||||
size = p.Range.End - p.Range.Start + 1
|
size = p.Range.End - p.Range.Start + 1
|
||||||
if p.Range.End > p.SrcObjInfo.Size {
|
if p.Range.End > srcObjectSize {
|
||||||
return nil, fmt.Errorf("%w: %d-%d/%d", s3errors.GetAPIError(s3errors.ErrInvalidCopyPartRangeSource), p.Range.Start, p.Range.End, p.SrcObjInfo.Size)
|
return nil, fmt.Errorf("%w: %d-%d/%d", s3errors.GetAPIError(s3errors.ErrInvalidCopyPartRangeSource), p.Range.Start, p.Range.End, srcObjectSize)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if size > UploadMaxSize {
|
if size > UploadMaxSize {
|
||||||
|
@ -412,7 +419,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
CopiesNumbers: multipartInfo.CopiesNumbers,
|
CopiesNumbers: multipartInfo.CopiesNumbers,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Error("could not put a completed object (multipart upload)",
|
n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject,
|
||||||
zap.String("uploadID", p.Info.UploadID),
|
zap.String("uploadID", p.Info.UploadID),
|
||||||
zap.String("uploadKey", p.Info.Key),
|
zap.String("uploadKey", p.Info.Key),
|
||||||
zap.Error(err))
|
zap.Error(err))
|
||||||
|
@ -424,7 +431,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
||||||
addr.SetContainer(p.Info.Bkt.CID)
|
addr.SetContainer(p.Info.Bkt.CID)
|
||||||
for _, partInfo := range partsInfo {
|
for _, partInfo := range partsInfo {
|
||||||
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
|
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
|
||||||
n.reqLogger(ctx).Warn("could not delete upload part",
|
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
|
||||||
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
|
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
|
||||||
zap.Error(err))
|
zap.Error(err))
|
||||||
}
|
}
|
||||||
|
@ -503,7 +510,7 @@ func (n *layer) AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) e
|
||||||
|
|
||||||
for _, info := range parts {
|
for _, info := range parts {
|
||||||
if err = n.objectDelete(ctx, p.Bkt, info.OID); err != nil {
|
if err = n.objectDelete(ctx, p.Bkt, info.OID); err != nil {
|
||||||
n.reqLogger(ctx).Warn("couldn't delete part", zap.String("cid", p.Bkt.CID.EncodeToString()),
|
n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", p.Bkt.CID.EncodeToString()),
|
||||||
zap.String("oid", info.OID.EncodeToString()), zap.Int("part number", info.Number), zap.Error(err))
|
zap.String("oid", info.OID.EncodeToString()), zap.Int("part number", info.Number), zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -520,7 +527,7 @@ func (n *layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
||||||
|
|
||||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||||
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||||
n.reqLogger(ctx).Warn("mismatched obj encryptionInfo", zap.Error(err))
|
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
|
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -584,7 +591,7 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
|
||||||
oids[i] = part.OID.EncodeToString()
|
oids[i] = part.OID.EncodeToString()
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug("part details",
|
n.reqLogger(ctx).Debug(logs.PartDetails,
|
||||||
zap.Stringer("cid", p.Bkt.CID),
|
zap.Stringer("cid", p.Bkt.CID),
|
||||||
zap.String("upload id", p.UploadID),
|
zap.String("upload id", p.UploadID),
|
||||||
zap.Ints("part numbers", partsNumbers),
|
zap.Ints("part numbers", partsNumbers),
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -46,7 +47,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
|
||||||
|
|
||||||
if !objIDToDeleteNotFound {
|
if !objIDToDeleteNotFound {
|
||||||
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
||||||
n.reqLogger(ctx).Error("couldn't delete notification configuration object", zap.Error(err),
|
n.reqLogger(ctx).Error(logs.CouldntDeleteNotificationConfigurationObject, zap.Error(err),
|
||||||
zap.String("cid", p.BktInfo.CID.EncodeToString()),
|
zap.String("cid", p.BktInfo.CID.EncodeToString()),
|
||||||
zap.String("oid", objIDToDelete.EncodeToString()))
|
zap.String("oid", objIDToDelete.EncodeToString()))
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,6 +19,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||||
|
@ -291,7 +292,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
n.reqLogger(ctx).Debug("put object", zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
||||||
|
|
||||||
newVersion := &data.NodeVersion{
|
newVersion := &data.NodeVersion{
|
||||||
BaseNodeVersion: data.BaseNodeVersion{
|
BaseNodeVersion: data.BaseNodeVersion{
|
||||||
|
@ -458,6 +459,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
|
||||||
// Returns object ID and payload sha256 hash.
|
// Returns object ID and payload sha256 hash.
|
||||||
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
|
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
|
||||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||||
|
prm.ClientCut = n.features.ClientCut()
|
||||||
var size uint64
|
var size uint64
|
||||||
hash := sha256.New()
|
hash := sha256.New()
|
||||||
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
||||||
|
@ -467,7 +469,7 @@ func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
|
||||||
id, err := n.frostFS.CreateObject(ctx, prm)
|
id, err := n.frostFS.CreateObject(ctx, prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
||||||
n.reqLogger(ctx).Warn("failed to discard put payload, probably goroutine leaks", zap.Error(errDiscard))
|
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0, oid.ID{}, nil, err
|
return 0, oid.ID{}, nil, err
|
||||||
|
@ -656,7 +658,7 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p allObjectParams,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
wg.Done()
|
wg.Done()
|
||||||
reqLog.Warn("failed to submit task to pool", zap.Error(err))
|
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
|
||||||
}
|
}
|
||||||
}(node)
|
}(node)
|
||||||
}
|
}
|
||||||
|
@ -798,7 +800,7 @@ func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo
|
||||||
|
|
||||||
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
n.reqLogger(ctx).Warn("could not fetch object meta", zap.Error(err))
|
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
|
@ -180,7 +181,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == nil && version != nil && !version.IsDeleteMarker() {
|
if err == nil && version != nil && !version.IsDeleteMarker() {
|
||||||
n.reqLogger(ctx).Debug("get tree node",
|
n.reqLogger(ctx).Debug(logs.GetTreeNode,
|
||||||
zap.Stringer("cid", objVersion.BktInfo.CID), zap.Stringer("oid", version.OID))
|
zap.Stringer("cid", objVersion.BktInfo.CID), zap.Stringer("oid", version.OID))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -97,6 +97,23 @@ func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectI
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func GetObjectSize(objInfo *data.ObjectInfo) (uint64, error) {
|
||||||
|
var err error
|
||||||
|
fullSize := objInfo.Size
|
||||||
|
|
||||||
|
if objInfo.Headers[AttributeDecryptedSize] != "" {
|
||||||
|
if fullSize, err = strconv.ParseUint(objInfo.Headers[AttributeDecryptedSize], 10, 64); err != nil {
|
||||||
|
return 0, fmt.Errorf("invalid decrypted size header: %w", err)
|
||||||
|
}
|
||||||
|
} else if objInfo.Headers[MultipartObjectSize] != "" {
|
||||||
|
if fullSize, err = strconv.ParseUint(objInfo.Headers[MultipartObjectSize], 10, 64); err != nil {
|
||||||
|
return 0, fmt.Errorf("invalid multipart size header: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fullSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
func FormEncryptionInfo(headers map[string]string) encryption.ObjectEncryption {
|
func FormEncryptionInfo(headers map[string]string) encryption.ObjectEncryption {
|
||||||
algorithm := headers[AttributeEncryptionAlgorithm]
|
algorithm := headers[AttributeEncryptionAlgorithm]
|
||||||
return encryption.ObjectEncryption{
|
return encryption.ObjectEncryption{
|
||||||
|
|
|
@ -170,6 +170,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
||||||
Caches: config,
|
Caches: config,
|
||||||
AnonKey: AnonymousKey{Key: key},
|
AnonKey: AnonymousKey{Key: key},
|
||||||
TreeService: NewTreeService(),
|
TreeService: NewTreeService(),
|
||||||
|
Features: &FeatureSettingsMock{},
|
||||||
}
|
}
|
||||||
|
|
||||||
return &testContext{
|
return &testContext{
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -15,9 +16,9 @@ func Auth(center auth.Center, log *zap.Logger) Func {
|
||||||
box, err := center.Authenticate(r)
|
box, err := center.Authenticate(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == auth.ErrNoAuthorizationHeader {
|
if err == auth.ErrNoAuthorizationHeader {
|
||||||
reqLogOrDefault(ctx, log).Debug("couldn't receive access box for gate key, random key will be used")
|
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed)
|
||||||
} else {
|
} else {
|
||||||
reqLogOrDefault(ctx, log).Error("failed to pass authentication", zap.Error(err))
|
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err))
|
||||||
if _, ok := err.(errors.Error); !ok {
|
if _, ok := err.(errors.Error); !ok {
|
||||||
err = errors.GetAPIError(errors.ErrAccessDenied)
|
err = errors.GetAPIError(errors.ErrAccessDenied)
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -141,7 +142,7 @@ func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) cidResolveFunc
|
||||||
|
|
||||||
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
|
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
reqLogOrDefault(ctx, log).Debug("failed to resolve CID", zap.Error(err))
|
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err))
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -207,7 +208,7 @@ func Request(log *zap.Logger) Func {
|
||||||
reqLogger := log.With(zap.String("request_id", reqInfo.RequestID))
|
reqLogger := log.With(zap.String("request_id", reqInfo.RequestID))
|
||||||
r = r.WithContext(SetReqLogger(r.Context(), reqLogger))
|
r = r.WithContext(SetReqLogger(r.Context(), reqLogger))
|
||||||
|
|
||||||
reqLogger.Info("request start", zap.String("host", r.Host),
|
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
|
||||||
zap.String("remote_host", reqInfo.RemoteHost))
|
zap.String("remote_host", reqInfo.RemoteHost))
|
||||||
|
|
||||||
// continue execution
|
// continue execution
|
||||||
|
@ -242,14 +243,7 @@ func AddObjectName(l *zap.Logger) Func {
|
||||||
|
|
||||||
rctx := chi.RouteContext(ctx)
|
rctx := chi.RouteContext(ctx)
|
||||||
// trim leading slash (always present)
|
// trim leading slash (always present)
|
||||||
obj := rctx.RoutePath[1:]
|
reqInfo.ObjectName = rctx.RoutePath[1:]
|
||||||
|
|
||||||
object, err := url.PathUnescape(obj)
|
|
||||||
if err != nil {
|
|
||||||
object = obj
|
|
||||||
}
|
|
||||||
|
|
||||||
reqInfo.ObjectName = object
|
|
||||||
|
|
||||||
reqLogger := reqLogOrDefault(ctx, l)
|
reqLogger := reqLogOrDefault(ctx, l)
|
||||||
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName))))
|
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName))))
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -319,7 +320,7 @@ func LogSuccessResponse(l *zap.Logger) Func {
|
||||||
reqLogger := reqLogOrDefault(ctx, l)
|
reqLogger := reqLogOrDefault(ctx, l)
|
||||||
reqInfo := GetReqInfo(ctx)
|
reqInfo := GetReqInfo(ctx)
|
||||||
|
|
||||||
reqLogger.Info("request end",
|
reqLogger.Info(logs.RequestEnd,
|
||||||
zap.String("method", reqInfo.API),
|
zap.String("method", reqInfo.API),
|
||||||
zap.String("bucket", reqInfo.BucketName),
|
zap.String("bucket", reqInfo.BucketName),
|
||||||
zap.String("object", reqInfo.ObjectName),
|
zap.String("object", reqInfo.ObjectName),
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -168,9 +169,9 @@ func (c *Controller) Listen(ctx context.Context) {
|
||||||
select {
|
select {
|
||||||
case msg := <-stream.ch:
|
case msg := <-stream.ch:
|
||||||
if err := stream.h.HandleMessage(ctx, msg); err != nil {
|
if err := stream.h.HandleMessage(ctx, msg); err != nil {
|
||||||
c.logger.Error("could not handle message", zap.Error(err))
|
c.logger.Error(logs.CouldNotHandleMessage, zap.Error(err))
|
||||||
} else if err = msg.Ack(); err != nil {
|
} else if err = msg.Ack(); err != nil {
|
||||||
c.logger.Error("could not ACK message", zap.Error(err))
|
c.logger.Error(logs.CouldNotACKMessage, zap.Error(err))
|
||||||
}
|
}
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
|
@ -187,10 +188,10 @@ func (c *Controller) SendNotifications(topics map[string]string, p *handler.Send
|
||||||
event.Records[0].S3.ConfigurationID = id
|
event.Records[0].S3.ConfigurationID = id
|
||||||
msg, err := json.Marshal(event)
|
msg, err := json.Marshal(event)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.logger.Error("couldn't marshal an event", zap.String("subject", topic), zap.Error(err))
|
c.logger.Error(logs.CouldntMarshalAnEvent, zap.String("subject", topic), zap.Error(err))
|
||||||
}
|
}
|
||||||
if err = c.publish(topic, msg); err != nil {
|
if err = c.publish(topic, msg); err != nil {
|
||||||
c.logger.Error("couldn't send an event to topic", zap.String("subject", topic), zap.Error(err))
|
c.logger.Error(logs.CouldntSendAnEventToTopic, zap.String("subject", topic), zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||||
s3middleware "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
s3middleware "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
|
@ -135,7 +136,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
})
|
})
|
||||||
|
|
||||||
if log := s3middleware.GetReqLog(ctx); log != nil {
|
if log := s3middleware.GetReqLog(ctx); log != nil {
|
||||||
log.Error("request unmatched", zap.String("method", reqInfo.API))
|
log.Error(logs.RequestUnmatched, zap.String("method", reqInfo.API))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -61,6 +61,51 @@ func TestRouterObjectWithSlashes(t *testing.T) {
|
||||||
require.Equal(t, objName, resp.ReqInfo.ObjectName)
|
require.Equal(t, objName, resp.ReqInfo.ObjectName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRouterObjectEscaping(t *testing.T) {
|
||||||
|
chiRouter := prepareRouter(t)
|
||||||
|
|
||||||
|
bktName := "dkirillov"
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
expectedObjName string
|
||||||
|
objName string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "simple",
|
||||||
|
expectedObjName: "object",
|
||||||
|
objName: "object",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "with slashes",
|
||||||
|
expectedObjName: "fix/object",
|
||||||
|
objName: "fix/object",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "with percentage",
|
||||||
|
expectedObjName: "fix/object%ac",
|
||||||
|
objName: "fix/object%ac",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "with percentage escaped",
|
||||||
|
expectedObjName: "fix/object%ac",
|
||||||
|
objName: "fix/object%25ac",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
target := fmt.Sprintf("/%s/%s", bktName, tc.objName)
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
r := httptest.NewRequest(http.MethodPut, target, nil)
|
||||||
|
|
||||||
|
chiRouter.ServeHTTP(w, r)
|
||||||
|
resp := readResponse(t, w)
|
||||||
|
require.Equal(t, "PutObject", resp.Method)
|
||||||
|
require.Equal(t, tc.expectedObjName, resp.ReqInfo.ObjectName)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func prepareRouter(t *testing.T) *chi.Mux {
|
func prepareRouter(t *testing.T) *chi.Mux {
|
||||||
throttleOps := middleware.ThrottleOpts{
|
throttleOps := middleware.ThrottleOpts{
|
||||||
Limit: 10,
|
Limit: 10,
|
||||||
|
|
|
@ -16,6 +16,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||||
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
||||||
|
@ -168,11 +169,11 @@ type (
|
||||||
|
|
||||||
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
|
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
|
||||||
if !opts.ID.Equals(cid.ID{}) {
|
if !opts.ID.Equals(cid.ID{}) {
|
||||||
a.log.Info("check container", zap.Stringer("cid", opts.ID))
|
a.log.Info(logs.CheckContainer, zap.Stringer("cid", opts.ID))
|
||||||
return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID)
|
return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.log.Info("create container",
|
a.log.Info(logs.CreateContainer,
|
||||||
zap.String("friendly_name", opts.FriendlyName),
|
zap.String("friendly_name", opts.FriendlyName),
|
||||||
zap.String("placement_policy", opts.PlacementPolicy))
|
zap.String("placement_policy", opts.PlacementPolicy))
|
||||||
|
|
||||||
|
@ -267,7 +268,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
||||||
return fmt.Errorf("check container: %w", err)
|
return fmt.Errorf("check container: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.log.Info("store bearer token into FrostFS",
|
a.log.Info(logs.StoreBearerTokenIntoFrostFS,
|
||||||
zap.Stringer("owner_tkn", idOwner))
|
zap.Stringer("owner_tkn", idOwner))
|
||||||
|
|
||||||
creds := tokens.New(a.frostFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log))
|
creds := tokens.New(a.frostFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log))
|
||||||
|
@ -345,7 +346,7 @@ func (a *Agent) UpdateSecret(ctx context.Context, w io.Writer, options *UpdateSe
|
||||||
|
|
||||||
var idOwner user.ID
|
var idOwner user.ID
|
||||||
user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey)
|
user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey)
|
||||||
a.log.Info("update access cred object into FrostFS",
|
a.log.Info(logs.UpdateAccessCredObjectIntoFrostFS,
|
||||||
zap.Stringer("owner_tkn", idOwner))
|
zap.Stringer("owner_tkn", idOwner))
|
||||||
|
|
||||||
oldAddr := options.Address
|
oldAddr := options.Address
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
@ -27,7 +28,7 @@ type PoolConfig struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func createFrostFS(ctx context.Context, log *zap.Logger, cfg PoolConfig) (authmate.FrostFS, error) {
|
func createFrostFS(ctx context.Context, log *zap.Logger, cfg PoolConfig) (authmate.FrostFS, error) {
|
||||||
log.Debug("prepare connection pool")
|
log.Debug(logs.PrepareConnectionPool)
|
||||||
|
|
||||||
var prm pool.InitParameters
|
var prm pool.InitParameters
|
||||||
prm.SetKey(&cfg.Key.PrivateKey)
|
prm.SetKey(&cfg.Key.PrivateKey)
|
||||||
|
|
178
cmd/s3-gw/app.go
178
cmd/s3-gw/app.go
|
@ -3,7 +3,6 @@ package main
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
@ -24,6 +23,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/xml"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/xml"
|
||||||
|
@ -71,6 +71,7 @@ type (
|
||||||
xmlDecoder *xml.DecoderProvider
|
xmlDecoder *xml.DecoderProvider
|
||||||
maxClient maxClientsConfig
|
maxClient maxClientsConfig
|
||||||
bypassContentEncodingInChunks atomic.Bool
|
bypassContentEncodingInChunks atomic.Bool
|
||||||
|
clientCut atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
maxClientsConfig struct {
|
maxClientsConfig struct {
|
||||||
|
@ -130,7 +131,7 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
// prepare random key for anonymous requests
|
// prepare random key for anonymous requests
|
||||||
randomKey, err := keys.NewPrivateKey()
|
randomKey, err := keys.NewPrivateKey()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal("couldn't generate random key", zap.Error(err))
|
a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
var gateOwner user.ID
|
var gateOwner user.ID
|
||||||
|
@ -144,6 +145,7 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
GateOwner: gateOwner,
|
GateOwner: gateOwner,
|
||||||
Resolver: a.bucketResolver,
|
Resolver: a.bucketResolver,
|
||||||
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
||||||
|
Features: a.settings,
|
||||||
}
|
}
|
||||||
|
|
||||||
// prepare object layer
|
// prepare object layer
|
||||||
|
@ -153,29 +155,25 @@ func (a *App) initLayer(ctx context.Context) {
|
||||||
nopts := getNotificationsOptions(a.cfg, a.log)
|
nopts := getNotificationsOptions(a.cfg, a.log)
|
||||||
a.nc, err = notifications.NewController(nopts, a.log)
|
a.nc, err = notifications.NewController(nopts, a.log)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal("failed to enable notifications", zap.Error(err))
|
a.log.Fatal(logs.FailedToEnableNotifications, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = a.obj.Initialize(ctx, a.nc); err != nil {
|
if err = a.obj.Initialize(ctx, a.nc); err != nil {
|
||||||
a.log.Fatal("couldn't initialize layer", zap.Error(err))
|
a.log.Fatal(logs.CouldntInitializeLayer, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
|
func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
|
||||||
policies, err := newPlacementPolicy(log.logger, v)
|
|
||||||
if err != nil {
|
|
||||||
log.logger.Fatal("failed to create new policy mapping", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
settings := &appSettings{
|
settings := &appSettings{
|
||||||
logLevel: log.lvl,
|
logLevel: log.lvl,
|
||||||
policies: policies,
|
policies: newPlacementPolicy(log.logger, v),
|
||||||
xmlDecoder: xml.NewDecoderProvider(v.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload)),
|
xmlDecoder: xml.NewDecoderProvider(v.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload)),
|
||||||
maxClient: newMaxClients(v),
|
maxClient: newMaxClients(v),
|
||||||
}
|
}
|
||||||
|
|
||||||
settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
||||||
|
settings.setClientCut(v.GetBool(cfgClientCut))
|
||||||
|
|
||||||
return settings
|
return settings
|
||||||
}
|
}
|
||||||
|
@ -188,6 +186,14 @@ func (s *appSettings) setBypassContentEncodingInChunks(bypass bool) {
|
||||||
s.bypassContentEncodingInChunks.Store(bypass)
|
s.bypassContentEncodingInChunks.Store(bypass)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) ClientCut() bool {
|
||||||
|
return s.clientCut.Load()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *appSettings) setClientCut(clientCut bool) {
|
||||||
|
s.clientCut.Store(clientCut)
|
||||||
|
}
|
||||||
|
|
||||||
func (a *App) initAPI(ctx context.Context) {
|
func (a *App) initAPI(ctx context.Context) {
|
||||||
a.initLayer(ctx)
|
a.initLayer(ctx)
|
||||||
a.initHandler()
|
a.initHandler()
|
||||||
|
@ -202,7 +208,7 @@ func (a *App) initResolver() {
|
||||||
var err error
|
var err error
|
||||||
a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverConfig())
|
a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverConfig())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal("failed to create resolver", zap.Error(err))
|
a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -215,11 +221,11 @@ func (a *App) getResolverConfig() ([]string, *resolver.Config) {
|
||||||
order := a.cfg.GetStringSlice(cfgResolveOrder)
|
order := a.cfg.GetStringSlice(cfgResolveOrder)
|
||||||
if resolveCfg.RPCAddress == "" {
|
if resolveCfg.RPCAddress == "" {
|
||||||
order = remove(order, resolver.NNSResolver)
|
order = remove(order, resolver.NNSResolver)
|
||||||
a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint))
|
a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(order) == 0 {
|
if len(order) == 0 {
|
||||||
a.log.Info("container resolver will be disabled because of resolvers 'resolver_order' is empty")
|
a.log.Info(logs.ContainerResolverWillBeDisabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
return order, resolveCfg
|
return order, resolveCfg
|
||||||
|
@ -240,10 +246,10 @@ func (a *App) initTracing(ctx context.Context) {
|
||||||
}
|
}
|
||||||
updated, err := tracing.Setup(ctx, cfg)
|
updated, err := tracing.Setup(ctx, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Warn("failed to initialize tracing", zap.Error(err))
|
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||||
}
|
}
|
||||||
if updated {
|
if updated {
|
||||||
a.log.Info("tracing config updated")
|
a.log.Info(logs.TracingConfigUpdated)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -253,7 +259,7 @@ func (a *App) shutdownTracing() {
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
if err := tracing.Shutdown(shdnCtx); err != nil {
|
if err := tracing.Shutdown(shdnCtx); err != nil {
|
||||||
a.log.Warn("failed to shutdown tracing", zap.Error(err))
|
a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -274,12 +280,12 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.
|
||||||
password := wallet.GetPassword(cfg, cfgWalletPassphrase)
|
password := wallet.GetPassword(cfg, cfgWalletPassphrase)
|
||||||
key, err := wallet.GetKeyFromPath(cfg.GetString(cfgWalletPath), cfg.GetString(cfgWalletAddress), password)
|
key, err := wallet.GetKeyFromPath(cfg.GetString(cfgWalletPath), cfg.GetString(cfgWalletAddress), password)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Fatal("could not load FrostFS private key", zap.Error(err))
|
logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
prm.SetKey(&key.PrivateKey)
|
prm.SetKey(&key.PrivateKey)
|
||||||
prmTree.SetKey(key)
|
prmTree.SetKey(key)
|
||||||
logger.Info("using credentials", zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
|
logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
|
||||||
|
|
||||||
for _, peer := range fetchPeers(logger, cfg) {
|
for _, peer := range fetchPeers(logger, cfg) {
|
||||||
prm.AddNode(peer)
|
prm.AddNode(peer)
|
||||||
|
@ -322,34 +328,28 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.
|
||||||
|
|
||||||
p, err := pool.NewPool(prm)
|
p, err := pool.NewPool(prm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Fatal("failed to create connection pool", zap.Error(err))
|
logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = p.Dial(ctx); err != nil {
|
if err = p.Dial(ctx); err != nil {
|
||||||
logger.Fatal("failed to dial connection pool", zap.Error(err))
|
logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
treePool, err := treepool.NewPool(prmTree)
|
treePool, err := treepool.NewPool(prmTree)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Fatal("failed to create tree pool", zap.Error(err))
|
logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err))
|
||||||
}
|
}
|
||||||
if err = treePool.Dial(ctx); err != nil {
|
if err = treePool.Dial(ctx); err != nil {
|
||||||
logger.Fatal("failed to dial tree pool", zap.Error(err))
|
logger.Fatal(logs.FailedToDialTreePool, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return p, treePool, key
|
return p, treePool, key
|
||||||
}
|
}
|
||||||
|
|
||||||
func newPlacementPolicy(l *zap.Logger, v *viper.Viper) (*placementPolicy, error) {
|
func newPlacementPolicy(l *zap.Logger, v *viper.Viper) *placementPolicy {
|
||||||
policies := &placementPolicy{
|
var policies placementPolicy
|
||||||
regionMap: make(map[string]netmap.PlacementPolicy),
|
policies.update(l, v)
|
||||||
defaultCopiesNumbers: []uint32{handler.DefaultCopiesNumber},
|
return &policies
|
||||||
}
|
|
||||||
|
|
||||||
policies.updateCopiesNumbers(l, v)
|
|
||||||
policies.updateDefaultCopiesNumbers(l, v)
|
|
||||||
|
|
||||||
return policies, policies.updatePolicy(v)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *placementPolicy) DefaultPlacementPolicy() netmap.PlacementPolicy {
|
func (p *placementPolicy) DefaultPlacementPolicy() netmap.PlacementPolicy {
|
||||||
|
@ -381,56 +381,18 @@ func (p *placementPolicy) DefaultCopiesNumbers() []uint32 {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *placementPolicy) update(l *zap.Logger, v *viper.Viper) {
|
func (p *placementPolicy) update(l *zap.Logger, v *viper.Viper) {
|
||||||
if err := p.updatePolicy(v); err != nil {
|
defaultPolicy := fetchDefaultPolicy(l, v)
|
||||||
l.Warn("policies won't be updated", zap.Error(err))
|
regionMap := fetchRegionMappingPolicies(l, v)
|
||||||
}
|
defaultCopies := fetchDefaultCopiesNumbers(l, v)
|
||||||
|
copiesNumbers := fetchCopiesNumbers(l, v)
|
||||||
p.updateCopiesNumbers(l, v)
|
|
||||||
p.updateDefaultCopiesNumbers(l, v)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *placementPolicy) updatePolicy(v *viper.Viper) error {
|
|
||||||
defaultPlacementPolicy, err := fetchDefaultPolicy(v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
regionMap, err := fetchRegionMappingPolicies(v)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
p.defaultPolicy = defaultPlacementPolicy
|
defer p.mu.Unlock()
|
||||||
|
|
||||||
|
p.defaultPolicy = defaultPolicy
|
||||||
p.regionMap = regionMap
|
p.regionMap = regionMap
|
||||||
p.mu.Unlock()
|
p.defaultCopiesNumbers = defaultCopies
|
||||||
|
p.copiesNumbers = copiesNumbers
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *placementPolicy) updateCopiesNumbers(l *zap.Logger, v *viper.Viper) {
|
|
||||||
if newCopiesNumbers, err := fetchCopiesNumbers(l, v); err != nil {
|
|
||||||
l.Warn("copies numbers won't be updated", zap.Error(err))
|
|
||||||
} else {
|
|
||||||
p.mu.Lock()
|
|
||||||
p.copiesNumbers = newCopiesNumbers
|
|
||||||
p.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *placementPolicy) updateDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) {
|
|
||||||
configuredValues, err := fetchDefaultCopiesNumbers(v)
|
|
||||||
|
|
||||||
if err == nil {
|
|
||||||
p.mu.Lock()
|
|
||||||
p.defaultCopiesNumbers = configuredValues
|
|
||||||
p.mu.Unlock()
|
|
||||||
l.Info("default copies numbers", zap.Uint32s("vector", p.defaultCopiesNumbers))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Error("cannot parse default copies numbers", zap.Error(err))
|
|
||||||
l.Warn("default copies numbers won't be updated", zap.Uint32s("current value", p.DefaultCopiesNumbers()))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func remove(list []string, element string) []string {
|
func remove(list []string, element string) []string {
|
||||||
|
@ -448,7 +410,7 @@ func remove(list []string, element string) []string {
|
||||||
// version (version.Version) and its name (frostfs-s3-gw). At the end, it writes
|
// version (version.Version) and its name (frostfs-s3-gw). At the end, it writes
|
||||||
// about the stop to the log.
|
// about the stop to the log.
|
||||||
func (a *App) Wait() {
|
func (a *App) Wait() {
|
||||||
a.log.Info("application started",
|
a.log.Info(logs.ApplicationStarted,
|
||||||
zap.String("name", "frostfs-s3-gw"),
|
zap.String("name", "frostfs-s3-gw"),
|
||||||
zap.String("version", version.Version),
|
zap.String("version", version.Version),
|
||||||
)
|
)
|
||||||
|
@ -458,7 +420,7 @@ func (a *App) Wait() {
|
||||||
|
|
||||||
<-a.webDone // wait for web-server to be stopped
|
<-a.webDone // wait for web-server to be stopped
|
||||||
|
|
||||||
a.log.Info("application finished")
|
a.log.Info(logs.ApplicationFinished)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) setHealthStatus() {
|
func (a *App) setHealthStatus() {
|
||||||
|
@ -469,7 +431,7 @@ func (a *App) setHealthStatus() {
|
||||||
func (a *App) Serve(ctx context.Context) {
|
func (a *App) Serve(ctx context.Context) {
|
||||||
// Attach S3 API:
|
// Attach S3 API:
|
||||||
domains := a.cfg.GetStringSlice(cfgListenDomains)
|
domains := a.cfg.GetStringSlice(cfgListenDomains)
|
||||||
a.log.Info("fetch domains, prepare to use API", zap.Strings("domains", domains))
|
a.log.Info(logs.FetchDomainsPrepareToUseAPI, zap.Strings("domains", domains))
|
||||||
|
|
||||||
throttleOps := middleware.ThrottleOpts{
|
throttleOps := middleware.ThrottleOpts{
|
||||||
Limit: a.settings.maxClient.count,
|
Limit: a.settings.maxClient.count,
|
||||||
|
@ -488,10 +450,10 @@ func (a *App) Serve(ctx context.Context) {
|
||||||
|
|
||||||
for i := range a.servers {
|
for i := range a.servers {
|
||||||
go func(i int) {
|
go func(i int) {
|
||||||
a.log.Info("starting server", zap.String("address", a.servers[i].Address()))
|
a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address()))
|
||||||
|
|
||||||
if err := srv.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed {
|
if err := srv.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed {
|
||||||
a.log.Fatal("listen and serve", zap.Error(err))
|
a.log.Fatal(logs.ListenAndServe, zap.Error(err))
|
||||||
}
|
}
|
||||||
}(i)
|
}(i)
|
||||||
}
|
}
|
||||||
|
@ -512,7 +474,7 @@ LOOP:
|
||||||
ctx, cancel := shutdownContext()
|
ctx, cancel := shutdownContext()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
a.log.Info("stopping server", zap.Error(srv.Shutdown(ctx)))
|
a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx)))
|
||||||
|
|
||||||
a.metrics.Shutdown()
|
a.metrics.Shutdown()
|
||||||
a.stopServices()
|
a.stopServices()
|
||||||
|
@ -526,23 +488,23 @@ func shutdownContext() (context.Context, context.CancelFunc) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) configReload(ctx context.Context) {
|
func (a *App) configReload(ctx context.Context) {
|
||||||
a.log.Info("SIGHUP config reload started")
|
a.log.Info(logs.SIGHUPConfigReloadStarted)
|
||||||
|
|
||||||
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
|
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
|
||||||
a.log.Warn("failed to reload config because it's missed")
|
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := readInConfig(a.cfg); err != nil {
|
if err := readInConfig(a.cfg); err != nil {
|
||||||
a.log.Warn("failed to reload config", zap.Error(err))
|
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := a.bucketResolver.UpdateResolvers(a.getResolverConfig()); err != nil {
|
if err := a.bucketResolver.UpdateResolvers(a.getResolverConfig()); err != nil {
|
||||||
a.log.Warn("failed to reload resolvers", zap.Error(err))
|
a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := a.updateServers(); err != nil {
|
if err := a.updateServers(); err != nil {
|
||||||
a.log.Warn("failed to reload server parameters", zap.Error(err))
|
a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
a.stopServices()
|
a.stopServices()
|
||||||
|
@ -554,12 +516,12 @@ func (a *App) configReload(ctx context.Context) {
|
||||||
a.initTracing(ctx)
|
a.initTracing(ctx)
|
||||||
a.setHealthStatus()
|
a.setHealthStatus()
|
||||||
|
|
||||||
a.log.Info("SIGHUP config reload completed")
|
a.log.Info(logs.SIGHUPConfigReloadCompleted)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) updateSettings() {
|
func (a *App) updateSettings() {
|
||||||
if lvl, err := getLogLevel(a.cfg); err != nil {
|
if lvl, err := getLogLevel(a.cfg); err != nil {
|
||||||
a.log.Warn("log level won't be updated", zap.Error(err))
|
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
a.settings.logLevel.SetLevel(lvl)
|
a.settings.logLevel.SetLevel(lvl)
|
||||||
}
|
}
|
||||||
|
@ -568,6 +530,7 @@ func (a *App) updateSettings() {
|
||||||
|
|
||||||
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
|
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
|
||||||
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
||||||
|
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) startServices() {
|
func (a *App) startServices() {
|
||||||
|
@ -593,16 +556,16 @@ func (a *App) initServers(ctx context.Context) {
|
||||||
}
|
}
|
||||||
srv, err := newServer(ctx, serverInfo)
|
srv, err := newServer(ctx, serverInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Warn("failed to add server", append(fields, zap.Error(err))...)
|
a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
a.servers = append(a.servers, srv)
|
a.servers = append(a.servers, srv)
|
||||||
a.log.Info("add server", fields...)
|
a.log.Info(logs.AddServer, fields...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(a.servers) == 0 {
|
if len(a.servers) == 0 {
|
||||||
a.log.Fatal("no healthy servers")
|
a.log.Fatal(logs.NoHealthyServers)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -713,29 +676,6 @@ func (a *App) initHandler() {
|
||||||
var err error
|
var err error
|
||||||
a.api, err = handler.New(a.log, a.obj, a.nc, cfg)
|
a.api, err = handler.New(a.log, a.obj, a.nc, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.log.Fatal("could not initialize API handler", zap.Error(err))
|
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func readRegionMap(filePath string) (map[string]string, error) {
|
|
||||||
regionMap := make(map[string]string)
|
|
||||||
|
|
||||||
if filePath == "" {
|
|
||||||
return regionMap, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := os.ReadFile(filePath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("coudln't read file '%s'", filePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = json.Unmarshal(data, ®ionMap); err != nil {
|
|
||||||
return nil, fmt.Errorf("unmarshal policies: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, ok := regionMap[api.DefaultLocationConstraint]; ok {
|
|
||||||
return nil, fmt.Errorf("config overrides %s location constraint", api.DefaultLocationConstraint)
|
|
||||||
}
|
|
||||||
|
|
||||||
return regionMap, nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
@ -10,9 +11,11 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/notifications"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/notifications"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||||
|
@ -30,11 +33,14 @@ const (
|
||||||
defaultShutdownTimeout = 15 * time.Second
|
defaultShutdownTimeout = 15 * time.Second
|
||||||
|
|
||||||
defaultPoolErrorThreshold uint32 = 100
|
defaultPoolErrorThreshold uint32 = 100
|
||||||
|
defaultPlacementPolicy = "REP 3"
|
||||||
|
|
||||||
defaultMaxClientsCount = 100
|
defaultMaxClientsCount = 100
|
||||||
defaultMaxClientsDeadline = time.Second * 30
|
defaultMaxClientsDeadline = time.Second * 30
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var defaultCopiesNumbers = []uint32{0}
|
||||||
|
|
||||||
const ( // Settings.
|
const ( // Settings.
|
||||||
// Logger.
|
// Logger.
|
||||||
cfgLoggerLevel = "logger.level"
|
cfgLoggerLevel = "logger.level"
|
||||||
|
@ -138,6 +144,8 @@ const ( // Settings.
|
||||||
// Configuration of parameters of requests to FrostFS.
|
// Configuration of parameters of requests to FrostFS.
|
||||||
// Number of the object copies to consider PUT to FrostFS successful.
|
// Number of the object copies to consider PUT to FrostFS successful.
|
||||||
cfgSetCopiesNumber = "frostfs.set_copies_number"
|
cfgSetCopiesNumber = "frostfs.set_copies_number"
|
||||||
|
// Enabling client side object preparing for PUT operations.
|
||||||
|
cfgClientCut = "frostfs.client_cut"
|
||||||
|
|
||||||
// List of allowed AccessKeyID prefixes.
|
// List of allowed AccessKeyID prefixes.
|
||||||
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
|
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
|
||||||
|
@ -222,24 +230,30 @@ func fetchMaxClientsDeadline(cfg *viper.Viper) time.Duration {
|
||||||
return maxClientsDeadline
|
return maxClientsDeadline
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchDefaultPolicy(cfg *viper.Viper) (netmap.PlacementPolicy, error) {
|
func fetchDefaultPolicy(l *zap.Logger, cfg *viper.Viper) netmap.PlacementPolicy {
|
||||||
defaultPolicyStr := handler.DefaultPolicy
|
var policy netmap.PlacementPolicy
|
||||||
|
|
||||||
if cfg.IsSet(cfgPolicyDefault) {
|
if cfg.IsSet(cfgPolicyDefault) {
|
||||||
defaultPolicyStr = cfg.GetString(cfgPolicyDefault)
|
policyStr := cfg.GetString(cfgPolicyDefault)
|
||||||
|
if err := policy.DecodeString(policyStr); err != nil {
|
||||||
|
l.Warn(logs.FailedToParseDefaultLocationConstraint,
|
||||||
|
zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy), zap.Error(err))
|
||||||
|
} else {
|
||||||
|
return policy
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var defaultPlacementPolicy netmap.PlacementPolicy
|
if err := policy.DecodeString(defaultPlacementPolicy); err != nil {
|
||||||
if err := defaultPlacementPolicy.DecodeString(defaultPolicyStr); err != nil {
|
l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint, zap.String("policy", defaultPlacementPolicy))
|
||||||
return netmap.PlacementPolicy{}, fmt.Errorf("parse default policy '%s': %w", defaultPolicyStr, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return defaultPlacementPolicy, nil
|
return policy
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchNATSTimeout(cfg *viper.Viper, l *zap.Logger) time.Duration {
|
func fetchNATSTimeout(cfg *viper.Viper, l *zap.Logger) time.Duration {
|
||||||
timeout := cfg.GetDuration(cfgNATSTimeout)
|
timeout := cfg.GetDuration(cfgNATSTimeout)
|
||||||
if timeout <= 0 {
|
if timeout <= 0 {
|
||||||
l.Error("invalid lifetime, using default value (in seconds)",
|
l.Error(logs.InvalidLifetimeUsingDefaultValue,
|
||||||
zap.String("parameter", cfgNATSTimeout),
|
zap.String("parameter", cfgNATSTimeout),
|
||||||
zap.Duration("value in config", timeout),
|
zap.Duration("value in config", timeout),
|
||||||
zap.Duration("default", notifications.DefaultTimeout))
|
zap.Duration("default", notifications.DefaultTimeout))
|
||||||
|
@ -253,7 +267,7 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV
|
||||||
if v.IsSet(cfgEntry) {
|
if v.IsSet(cfgEntry) {
|
||||||
lifetime := v.GetDuration(cfgEntry)
|
lifetime := v.GetDuration(cfgEntry)
|
||||||
if lifetime <= 0 {
|
if lifetime <= 0 {
|
||||||
l.Error("invalid lifetime, using default value (in seconds)",
|
l.Error(logs.InvalidLifetimeUsingDefaultValue,
|
||||||
zap.String("parameter", cfgEntry),
|
zap.String("parameter", cfgEntry),
|
||||||
zap.Duration("value in config", lifetime),
|
zap.Duration("value in config", lifetime),
|
||||||
zap.Duration("default", defaultValue))
|
zap.Duration("default", defaultValue))
|
||||||
|
@ -269,7 +283,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue
|
||||||
if v.IsSet(cfgEntry) {
|
if v.IsSet(cfgEntry) {
|
||||||
size := v.GetInt(cfgEntry)
|
size := v.GetInt(cfgEntry)
|
||||||
if size <= 0 {
|
if size <= 0 {
|
||||||
l.Error("invalid cache size, using default value",
|
l.Error(logs.InvalidCacheSizeUsingDefaultValue,
|
||||||
zap.String("parameter", cfgEntry),
|
zap.String("parameter", cfgEntry),
|
||||||
zap.Int("value in config", size),
|
zap.Int("value in config", size),
|
||||||
zap.Int("default", defaultValue))
|
zap.Int("default", defaultValue))
|
||||||
|
@ -288,7 +302,7 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
|
||||||
defaultMaxAge = cfg.GetInt(cfgDefaultMaxAge)
|
defaultMaxAge = cfg.GetInt(cfgDefaultMaxAge)
|
||||||
|
|
||||||
if defaultMaxAge <= 0 && defaultMaxAge != -1 {
|
if defaultMaxAge <= 0 && defaultMaxAge != -1 {
|
||||||
l.Fatal("invalid defaultMaxAge",
|
l.Fatal(logs.InvalidDefaultMaxAge,
|
||||||
zap.String("parameter", cfgDefaultMaxAge),
|
zap.String("parameter", cfgDefaultMaxAge),
|
||||||
zap.String("value in config", strconv.Itoa(defaultMaxAge)))
|
zap.String("value in config", strconv.Itoa(defaultMaxAge)))
|
||||||
}
|
}
|
||||||
|
@ -297,14 +311,21 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
|
||||||
return defaultMaxAge
|
return defaultMaxAge
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchRegionMappingPolicies(cfg *viper.Viper) (map[string]netmap.PlacementPolicy, error) {
|
func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netmap.PlacementPolicy {
|
||||||
regionPolicyMap, err := readRegionMap(cfg.GetString(cfgPolicyRegionMapFile))
|
filepath := cfg.GetString(cfgPolicyRegionMapFile)
|
||||||
|
regionPolicyMap, err := readRegionMap(filepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("read region map file: %w", err)
|
l.Warn(logs.FailedToReadRegionMapFilePolicies, zap.String("file", filepath), zap.Error(err))
|
||||||
|
return make(map[string]netmap.PlacementPolicy)
|
||||||
}
|
}
|
||||||
|
|
||||||
regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
|
regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
|
||||||
for region, policy := range regionPolicyMap {
|
for region, policy := range regionPolicyMap {
|
||||||
|
if region == api.DefaultLocationConstraint {
|
||||||
|
l.Warn(logs.DefaultLocationConstraintCantBeOverriden, zap.String("policy", policy))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
var pp netmap.PlacementPolicy
|
var pp netmap.PlacementPolicy
|
||||||
if err = pp.DecodeString(policy); err == nil {
|
if err = pp.DecodeString(policy); err == nil {
|
||||||
regionMap[region] = pp
|
regionMap[region] = pp
|
||||||
|
@ -316,29 +337,49 @@ func fetchRegionMappingPolicies(cfg *viper.Viper) (map[string]netmap.PlacementPo
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("parse region '%s' to policy mapping: %w", region, err)
|
l.Warn(logs.FailedToParseLocationConstraint, zap.String("region", region), zap.String("policy", policy))
|
||||||
|
}
|
||||||
|
|
||||||
|
return regionMap
|
||||||
|
}
|
||||||
|
|
||||||
|
func readRegionMap(filePath string) (map[string]string, error) {
|
||||||
|
regionMap := make(map[string]string)
|
||||||
|
|
||||||
|
if filePath == "" {
|
||||||
|
return regionMap, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("coudln't read file '%s'", filePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = json.Unmarshal(data, ®ionMap); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal policies: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return regionMap, nil
|
return regionMap, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchDefaultCopiesNumbers(v *viper.Viper) ([]uint32, error) {
|
func fetchDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) []uint32 {
|
||||||
unparsed := v.GetStringSlice(cfgSetCopiesNumber)
|
unparsed := v.GetStringSlice(cfgSetCopiesNumber)
|
||||||
var result []uint32
|
result := make([]uint32, len(unparsed))
|
||||||
|
|
||||||
for i := range unparsed {
|
for i := range unparsed {
|
||||||
parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32)
|
parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
l.Warn(logs.FailedToParseDefaultCopiesNumbers,
|
||||||
|
zap.Strings("copies numbers", unparsed), zap.Uint32s("default", defaultCopiesNumbers), zap.Error(err))
|
||||||
|
return defaultCopiesNumbers
|
||||||
}
|
}
|
||||||
result = append(result, uint32(parsedValue))
|
result[i] = uint32(parsedValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, nil
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) (map[string][]uint32, error) {
|
func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) map[string][]uint32 {
|
||||||
var copiesNums = make(map[string][]uint32)
|
copiesNums := make(map[string][]uint32)
|
||||||
for i := 0; ; i++ {
|
for i := 0; ; i++ {
|
||||||
key := cfgCopiesNumbers + "." + strconv.Itoa(i) + "."
|
key := cfgCopiesNumbers + "." + strconv.Itoa(i) + "."
|
||||||
constraint := v.GetString(key + "location_constraint")
|
constraint := v.GetString(key + "location_constraint")
|
||||||
|
@ -352,15 +393,17 @@ func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) (map[string][]uint32, err
|
||||||
for j := range vector {
|
for j := range vector {
|
||||||
parsedValue, err := strconv.ParseUint(vector[j], 10, 32)
|
parsedValue, err := strconv.ParseUint(vector[j], 10, 32)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
l.Warn(logs.FailedToParseCopiesNumbers, zap.String("location", constraint),
|
||||||
|
zap.Strings("copies numbers", vector), zap.Error(err))
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
vector32[j] = uint32(parsedValue)
|
vector32[j] = uint32(parsedValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
copiesNums[constraint] = vector32
|
copiesNums[constraint] = vector32
|
||||||
l.Info("constraint added", zap.String("location", constraint), zap.Strings("copies numbers", vector))
|
l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector))
|
||||||
}
|
}
|
||||||
return copiesNums, nil
|
return copiesNums
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
||||||
|
@ -372,7 +415,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
||||||
priority := v.GetInt(key + "priority")
|
priority := v.GetInt(key + "priority")
|
||||||
|
|
||||||
if address == "" {
|
if address == "" {
|
||||||
l.Warn("skip, empty address")
|
l.Warn(logs.SkipEmptyAddress)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if weight <= 0 { // unspecified or wrong
|
if weight <= 0 { // unspecified or wrong
|
||||||
|
@ -384,7 +427,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
||||||
|
|
||||||
nodes = append(nodes, pool.NewNodeParam(priority, address, weight))
|
nodes = append(nodes, pool.NewNodeParam(priority, address, weight))
|
||||||
|
|
||||||
l.Info("added storage peer",
|
l.Info(logs.AddedStoragePeer,
|
||||||
zap.Int("priority", priority),
|
zap.Int("priority", priority),
|
||||||
zap.String("address", address),
|
zap.String("address", address),
|
||||||
zap.Float64("weight", weight))
|
zap.Float64("weight", weight))
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -18,21 +19,21 @@ type Service struct {
|
||||||
// Start runs http service with the exposed endpoint on the configured port.
|
// Start runs http service with the exposed endpoint on the configured port.
|
||||||
func (ms *Service) Start() {
|
func (ms *Service) Start() {
|
||||||
if ms.enabled {
|
if ms.enabled {
|
||||||
ms.log.Info("service is running", zap.String("endpoint", ms.Addr))
|
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr))
|
||||||
err := ms.ListenAndServe()
|
err := ms.ListenAndServe()
|
||||||
if err != nil && err != http.ErrServerClosed {
|
if err != nil && err != http.ErrServerClosed {
|
||||||
ms.log.Warn("service couldn't start on configured port")
|
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ms.log.Info("service hasn't started since it's disabled")
|
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ShutDown stops the service.
|
// ShutDown stops the service.
|
||||||
func (ms *Service) ShutDown(ctx context.Context) {
|
func (ms *Service) ShutDown(ctx context.Context) {
|
||||||
ms.log.Info("shutting down service", zap.String("endpoint", ms.Addr))
|
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr))
|
||||||
err := ms.Shutdown(ctx)
|
err := ms.Shutdown(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ms.log.Panic("can't shut down service")
|
ms.log.Panic(logs.CantShutDownService)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -125,6 +125,8 @@ S3_GW_CORS_DEFAULT_MAX_AGE=600
|
||||||
# to consider PUT to FrostFS successful.
|
# to consider PUT to FrostFS successful.
|
||||||
# `0` or empty list means that object will be processed according to the container's placement policy
|
# `0` or empty list means that object will be processed according to the container's placement policy
|
||||||
S3_GW_FROSTFS_SET_COPIES_NUMBER=0
|
S3_GW_FROSTFS_SET_COPIES_NUMBER=0
|
||||||
|
# This flag enables client side object preparing.
|
||||||
|
S3_GW_FROSTFS_CLIENT_CUT=false
|
||||||
|
|
||||||
# List of allowed AccessKeyID prefixes
|
# List of allowed AccessKeyID prefixes
|
||||||
# If not set, S3 GW will accept all AccessKeyIDs
|
# If not set, S3 GW will accept all AccessKeyIDs
|
||||||
|
|
|
@ -150,6 +150,8 @@ frostfs:
|
||||||
# Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
|
# Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
|
||||||
# `[0]` or empty list means that object will be processed according to the container's placement policy
|
# `[0]` or empty list means that object will be processed according to the container's placement policy
|
||||||
set_copies_number: [0]
|
set_copies_number: [0]
|
||||||
|
# This flag enables client side object preparing.
|
||||||
|
client_cut: false
|
||||||
|
|
||||||
# List of allowed AccessKeyID prefixes
|
# List of allowed AccessKeyID prefixes
|
||||||
# If the parameter is omitted, S3 GW will accept all AccessKeyIDs
|
# If the parameter is omitted, S3 GW will accept all AccessKeyIDs
|
||||||
|
|
|
@ -500,17 +500,20 @@ tracing:
|
||||||
# `frostfs` section
|
# `frostfs` section
|
||||||
|
|
||||||
Contains parameters of requests to FrostFS.
|
Contains parameters of requests to FrostFS.
|
||||||
This value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
|
|
||||||
|
The `set_copies_number` value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
|
||||||
header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
|
header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
frostfs:
|
frostfs:
|
||||||
set_copies_number: [0]
|
set_copies_number: [0]
|
||||||
|
client_cut: false
|
||||||
```
|
```
|
||||||
|
|
||||||
| Parameter | Type | Default value | Description |
|
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||||
|---------------------|------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|---------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `set_copies_number` | `[]uint32` | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
|
| `set_copies_number` | `[]uint32` | yes | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
|
||||||
|
| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. |
|
||||||
|
|
||||||
# `resolve_bucket` section
|
# `resolve_bucket` section
|
||||||
|
|
||||||
|
|
2
go.mod
2
go.mod
|
@ -1,6 +1,6 @@
|
||||||
module git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
module git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||||
|
|
||||||
go 1.19
|
go 1.20
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44
|
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
|
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3 h1:GBRTOTRrtIvxi2TgxG7z/J7uRXiyb1SxR4247FaYCgU=
|
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
|
||||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
||||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
||||||
|
|
|
@ -243,6 +243,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
|
||||||
prmPut.SetHeader(*obj)
|
prmPut.SetHeader(*obj)
|
||||||
prmPut.SetPayload(prm.Payload)
|
prmPut.SetPayload(prm.Payload)
|
||||||
prmPut.SetCopiesNumberVector(prm.CopiesNumber)
|
prmPut.SetCopiesNumberVector(prm.CopiesNumber)
|
||||||
|
prmPut.SetClientCut(prm.ClientCut)
|
||||||
|
|
||||||
if prm.BearerToken != nil {
|
if prm.BearerToken != nil {
|
||||||
prmPut.UseBearer(*prm.BearerToken)
|
prmPut.UseBearer(*prm.BearerToken)
|
||||||
|
@ -262,6 +263,9 @@ type payloadReader struct {
|
||||||
|
|
||||||
func (x payloadReader) Read(p []byte) (int, error) {
|
func (x payloadReader) Read(p []byte) (int, error) {
|
||||||
n, err := x.ReadCloser.Read(p)
|
n, err := x.ReadCloser.Read(p)
|
||||||
|
if err != nil && errors.Is(err, io.EOF) {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
return n, handleObjectError("read payload", err)
|
return n, handleObjectError("read payload", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
115
internal/logs/logs.go
Normal file
115
internal/logs/logs.go
Normal file
|
@ -0,0 +1,115 @@
|
||||||
|
package logs
|
||||||
|
|
||||||
|
const (
|
||||||
|
RequestUnmatched = "request unmatched" // Error in ../../api/router.go
|
||||||
|
CheckContainer = "check container" // Info in ../../authmate/authmate.go
|
||||||
|
CreateContainer = "create container" // Info in ../../authmate/authmate.go
|
||||||
|
StoreBearerTokenIntoFrostFS = "store bearer token into FrostFS" // Info in ../../authmate/authmate.go
|
||||||
|
UpdateAccessCredObjectIntoFrostFS = "update access cred object into FrostFS" // Info in ../../authmate/authmate.go
|
||||||
|
MetricsAreDisabled = "metrics are disabled" // Warn in ../../metrics/app.go
|
||||||
|
FoundMoreThanOneUnversionedNode = "found more than one unversioned node" // Debug in ../../pkg/service/tree/tree.go
|
||||||
|
ServiceIsRunning = "service is running" // Info in ../../cmd/s3-gw/service.go
|
||||||
|
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../cmd/s3-gw/service.go
|
||||||
|
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../cmd/s3-gw/service.go
|
||||||
|
ShuttingDownService = "shutting down service" // Info in ../../cmd/s3-gw/service.go
|
||||||
|
ContainerResolverWillBeDisabled = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
TracingConfigUpdated = "tracing config updated" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
FetchDomainsPrepareToUseAPI = "fetch domains, prepare to use API" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToReloadConfig = "failed to reload config" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToReloadResolvers = "failed to reload resolvers" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToAddServer = "failed to add server" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
AddServer = "add server" // Info in ../../cmd/s3-gw/app.go
|
||||||
|
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver 'nns' won't be used since 'rpc_endpoint' isn't provided" // Warn in ../../cmd/s3-gw/app.go
|
||||||
|
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/s3-gw/app_settings.go
|
||||||
|
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/s3-gw/app_settings.go
|
||||||
|
FailedToParseDefaultLocationConstraint = "failed to parse 'default' location constraint, default one will be used" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
FailedToReadRegionMapFilePolicies = "failed to read region map file, policies will be empty" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
DefaultLocationConstraintCantBeOverriden = "'default' location constraint can't be overriden by custom policy, use 'placement_policy.default'" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
FailedToParseLocationConstraint = "failed to parse location constraint, it cannot be used" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
FailedToParseDefaultCopiesNumbers = "failed to parse 'default' copies numbers, default one will be used" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
FailedToParseCopiesNumbers = "failed to parse copies numbers, skip" // Warn in cmd/s3-gw/app_settings.go
|
||||||
|
FailedToParseDefaultDefaultLocationConstraint = "failed to parse default 'default' location constraint" // Fatal in cmd/s3-gw/app_settings.go
|
||||||
|
ConstraintAdded = "constraint added" // Info in ../../cmd/s3-gw/app_settings.go
|
||||||
|
SkipEmptyAddress = "skip, empty address" // Warn in ../../cmd/s3-gw/app_settings.go
|
||||||
|
AddedStoragePeer = "added storage peer" // Info in ../../cmd/s3-gw/app_settings.go
|
||||||
|
PrepareConnectionPool = "prepare connection pool" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||||
|
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../../api/cache/*
|
||||||
|
InvalidCacheKeyType = "invalid cache key type" // Warn in ../../api/cache/objectslist.go
|
||||||
|
ObjectIsCopied = "object is copied" // Info in ../../api/handler/copy.go
|
||||||
|
CouldntSendNotification = "couldn't send notification: %w" // Error in ../../api/handler/*
|
||||||
|
FailedToSendTestEventBecauseNotificationsIsDisabled = "failed to send test event because notifications is disabled" // Warn in ../../api/handler/notifications.go
|
||||||
|
RequestFailed = "request failed" // Error in ../../api/handler/util.go
|
||||||
|
GetBucketInfo = "get bucket info" // Warn in ../../api/handler/cors.go
|
||||||
|
GetBucketCors = "get bucket cors" // Warn in ../../api/handler/cors.go
|
||||||
|
SomeACLNotFullyMapped = "some acl not fully mapped" // Warn in ../../api/handler/acl.go
|
||||||
|
CouldntDeleteObjects = "couldn't delete objects" // Error in ../../api/handler/delete.go
|
||||||
|
NotificatorIsDisabledS3WontProduceNotificationEvents = "notificator is disabled, s3 won't produce notification events" // Warn in ../../api/handler/api.go
|
||||||
|
CouldntGetBucketVersioning = "couldn't get bucket versioning" // Warn in ../../api/handler/put.go
|
||||||
|
BucketIsCreated = "bucket is created" // Info in ../../api/handler/put.go
|
||||||
|
CouldntDeleteNotificationConfigurationObject = "couldn't delete notification configuration object" // Error in ../../api/layer/notifications.go
|
||||||
|
CouldNotParseContainerObjectLockEnabledAttribute = "could not parse container object lock enabled attribute" // Error in ../../api/layer/container.go
|
||||||
|
CouldNotListUserContainers = "could not list user containers" // Error in ../../api/layer/container.go
|
||||||
|
CouldNotFetchContainerInfo = "could not fetch container info" // Error in ../../api/layer/container.go
|
||||||
|
MismatchedObjEncryptionInfo = "mismatched obj encryptionInfo" // Warn in ../../api/layer/multipart_upload.go
|
||||||
|
UploadPart = "upload part" // Debug in ../../api/layer/multipart_upload.go
|
||||||
|
CouldntDeleteOldPartObject = "couldn't delete old part object" // Error in ../../api/layer/multipart_upload.go
|
||||||
|
CouldNotPutCompletedObject = "could not put a completed object (multipart upload)" // Error in ../../api/layer/multipart_upload.go
|
||||||
|
CouldNotDeleteUploadPart = "could not delete upload part" // Warn in ../../api/layer/multipart_upload.go
|
||||||
|
CouldntDeletePart = "couldn't delete part" // Warn in ../../api/layer/multipart_upload.go
|
||||||
|
PartDetails = "part details" // Debug in ../../api/layer/multipart_upload.go
|
||||||
|
GetObject = "get object" // Debug in ../../api/layer/layer.go
|
||||||
|
ObjectAlreadyRemoved = "object already removed" // Debug in ../../api/layer/layer.go
|
||||||
|
ObjectNotFound = "object not found" // Debug in ../../api/layer/layer.go
|
||||||
|
ResolveBucket = "resolve bucket" // Info in ../../api/layer/layer.go
|
||||||
|
CouldntDeleteCorsObject = "couldn't delete cors object" // Error in ../../api/layer/cors.go
|
||||||
|
PutObject = "put object" // Debug in ../../api/layer/object.go
|
||||||
|
FailedToDiscardPutPayloadProbablyGoroutineLeaks = "failed to discard put payload, probably goroutine leaks" // Warn in ../../api/layer/object.go
|
||||||
|
FailedToSubmitTaskToPool = "failed to submit task to pool" // Warn in ../../api/layer/object.go
|
||||||
|
CouldNotFetchObjectMeta = "could not fetch object meta" // Warn in ../../api/layer/object.go
|
||||||
|
GetTreeNode = "get tree node" // Debug in ../../api/layer/tagging.go
|
||||||
|
CouldntPutBucketInfoIntoCache = "couldn't put bucket info into cache" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntAddObjectToCache = "couldn't add object to cache" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheAccessControlOperation = "couldn't cache access control operation" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntPutObjAddressToNameCache = "couldn't put obj address to name cache" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheListOfObjects = "couldn't cache list of objects" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheTags = "couldn't cache tags" // Error in ../../api/layer/cache.go
|
||||||
|
CouldntCacheLockInfo = "couldn't cache lock info" // Error in ../../api/layer/cache.go
|
||||||
|
CouldntCacheBucketSettings = "couldn't cache bucket settings" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheCors = "couldn't cache cors" // Warn in ../../api/layer/cache.go
|
||||||
|
CouldntCacheNotificationConfiguration = "couldn't cache notification configuration" // Warn in ../../api/layer/cache.go
|
||||||
|
RequestEnd = "request end" // Info in ../../api/middleware/response.go
|
||||||
|
CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed = "couldn't receive access box for gate key, random key will be used" // Debug in ../../api/middleware/auth.go
|
||||||
|
FailedToPassAuthentication = "failed to pass authentication" // Error in ../../api/middleware/auth.go
|
||||||
|
FailedToResolveCID = "failed to resolve CID" // Debug in ../../api/middleware/metrics.go
|
||||||
|
RequestStart = "request start" // Info in ../../api/middleware/reqinfo.go
|
||||||
|
CouldNotHandleMessage = "could not handle message" // Error in ../../api/notifications/controller.go
|
||||||
|
CouldNotACKMessage = "could not ACK message" // Error in ../../api/notifications/controller.go
|
||||||
|
CouldntMarshalAnEvent = "couldn't marshal an event" // Error in ../../api/notifications/controller.go
|
||||||
|
CouldntSendAnEventToTopic = "couldn't send an event to topic" // Error in ../../api/notifications/controller.go
|
||||||
|
InvalidDefaultMaxAge = "invalid defaultMaxAge" // Fatal in ../../cmd/s3-gw/app_settings.go
|
||||||
|
CantShutDownService = "can't shut down service" // Panic in ../../cmd/s3-gw/service.go
|
||||||
|
CouldntGenerateRandomKey = "couldn't generate random key" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToEnableNotifications = "failed to enable notifications" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
CouldntInitializeLayer = "couldn't initialize layer" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToCreateResolver = "failed to create resolver" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
ListenAndServe = "listen and serve" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
NoHealthyServers = "no healthy servers" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
CouldNotInitializeAPIHandler = "could not initialize API handler" // Fatal in ../../cmd/s3-gw/app.go
|
||||||
|
)
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
dto "github.com/prometheus/client_model/go"
|
dto "github.com/prometheus/client_model/go"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
@ -17,7 +18,7 @@ type AppMetrics struct {
|
||||||
|
|
||||||
func NewAppMetrics(logger *zap.Logger, poolStatistics StatisticScraper, enabled bool) *AppMetrics {
|
func NewAppMetrics(logger *zap.Logger, poolStatistics StatisticScraper, enabled bool) *AppMetrics {
|
||||||
if !enabled {
|
if !enabled {
|
||||||
logger.Warn("metrics are disabled")
|
logger.Warn(logs.MetricsAreDisabled)
|
||||||
}
|
}
|
||||||
return &AppMetrics{
|
return &AppMetrics{
|
||||||
logger: logger,
|
logger: logger,
|
||||||
|
@ -28,7 +29,7 @@ func NewAppMetrics(logger *zap.Logger, poolStatistics StatisticScraper, enabled
|
||||||
|
|
||||||
func (m *AppMetrics) SetEnabled(enabled bool) {
|
func (m *AppMetrics) SetEnabled(enabled bool) {
|
||||||
if !enabled {
|
if !enabled {
|
||||||
m.logger.Warn("metrics are disabled")
|
m.logger.Warn(logs.MetricsAreDisabled)
|
||||||
}
|
}
|
||||||
|
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
|
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -220,6 +221,30 @@ func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeV
|
||||||
return version
|
return version
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newMultipartInfoFromTreeNode(filePath string, treeNode *treeNode) (*data.MultipartInfo, error) {
|
||||||
|
uploadID, _ := treeNode.Get(uploadIDKV)
|
||||||
|
if uploadID == "" {
|
||||||
|
return nil, fmt.Errorf("it's not a multipart node")
|
||||||
|
}
|
||||||
|
|
||||||
|
multipartInfo := &data.MultipartInfo{
|
||||||
|
ID: treeNode.ID,
|
||||||
|
Key: filePath,
|
||||||
|
UploadID: uploadID,
|
||||||
|
Meta: treeNode.Meta,
|
||||||
|
}
|
||||||
|
|
||||||
|
ownerID, _ := treeNode.Get(ownerKV)
|
||||||
|
_ = multipartInfo.Owner.DecodeString(ownerID)
|
||||||
|
|
||||||
|
created, _ := treeNode.Get(createdKV)
|
||||||
|
if utcMilli, err := strconv.ParseInt(created, 10, 64); err == nil {
|
||||||
|
multipartInfo.Created = time.UnixMilli(utcMilli)
|
||||||
|
}
|
||||||
|
|
||||||
|
return multipartInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
func newMultipartInfo(node NodeResponse) (*data.MultipartInfo, error) {
|
func newMultipartInfo(node NodeResponse) (*data.MultipartInfo, error) {
|
||||||
multipartInfo := &data.MultipartInfo{
|
multipartInfo := &data.MultipartInfo{
|
||||||
ID: node.GetNodeID(),
|
ID: node.GetNodeID(),
|
||||||
|
@ -829,7 +854,7 @@ func (c *Tree) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, tre
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(nodes) > 1 {
|
if len(nodes) > 1 {
|
||||||
c.reqLogger(ctx).Debug("found more than one unversioned node",
|
c.reqLogger(ctx).Debug(logs.FoundMoreThanOneUnversionedNode,
|
||||||
zap.String("treeID", treeID), zap.String("filepath", filepath))
|
zap.String("treeID", treeID), zap.String("filepath", filepath))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -857,14 +882,14 @@ func (c *Tree) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketIn
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) {
|
func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) {
|
||||||
subTreeNodes, _, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false)
|
subTreeNodes, headPrefix, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var result []*data.MultipartInfo
|
var result []*data.MultipartInfo
|
||||||
for _, node := range subTreeNodes {
|
for _, node := range subTreeNodes {
|
||||||
multipartUploads, err := c.getSubTreeMultipartUploads(ctx, bktInfo, node.GetNodeID())
|
multipartUploads, err := c.getSubTreeMultipartUploads(ctx, bktInfo, node.GetNodeID(), headPrefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -874,19 +899,55 @@ func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.Bu
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) ([]*data.MultipartInfo, error) {
|
func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, parentFilePath string) ([]*data.MultipartInfo, error) {
|
||||||
subTree, err := c.service.GetSubTree(ctx, bktInfo, systemTree, nodeID, maxGetSubTreeDepth)
|
subTree, err := c.service.GetSubTree(ctx, bktInfo, systemTree, nodeID, maxGetSubTreeDepth)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
result := make([]*data.MultipartInfo, 0, len(subTree))
|
var parentPrefix string
|
||||||
for _, node := range subTree {
|
if parentFilePath != "" { // The root of subTree can also have a parent
|
||||||
multipartInfo, err := newMultipartInfo(node)
|
parentPrefix = strings.TrimSuffix(parentFilePath, separator) + separator // To avoid 'foo//bar'
|
||||||
if err != nil { // missed uploadID (it's a part node)
|
}
|
||||||
|
|
||||||
|
var filepath string
|
||||||
|
namesMap := make(map[uint64]string, len(subTree))
|
||||||
|
multiparts := make(map[string][]*data.MultipartInfo, len(subTree))
|
||||||
|
|
||||||
|
for i, node := range subTree {
|
||||||
|
treeNode, fileName, err := parseTreeNode(node)
|
||||||
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
result = append(result, multipartInfo)
|
|
||||||
|
if i != 0 {
|
||||||
|
if filepath, err = formFilePath(node, fileName, namesMap); err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid node order: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
filepath = parentPrefix + fileName
|
||||||
|
namesMap[treeNode.ID] = filepath
|
||||||
|
}
|
||||||
|
|
||||||
|
multipartInfo, err := newMultipartInfoFromTreeNode(filepath, treeNode)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
key := formLatestNodeKey(node.GetParentID(), fileName)
|
||||||
|
multipartInfos, ok := multiparts[key]
|
||||||
|
if !ok {
|
||||||
|
multipartInfos = []*data.MultipartInfo{multipartInfo}
|
||||||
|
} else {
|
||||||
|
multipartInfos = append(multipartInfos, multipartInfo)
|
||||||
|
}
|
||||||
|
|
||||||
|
multiparts[key] = multipartInfos
|
||||||
|
}
|
||||||
|
|
||||||
|
result := make([]*data.MultipartInfo, 0, len(multiparts))
|
||||||
|
for _, multipartInfo := range multiparts {
|
||||||
|
result = append(result, multipartInfo...)
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
|
|
Loading…
Reference in a new issue