forked from TrueCloudLab/frostfs-s3-gw
Compare commits
12 commits
6a9d3261a7
...
9120e97ac5
Author | SHA1 | Date | |
---|---|---|---|
9120e97ac5 | |||
2fc328a6d2 | |||
b5fce5c8d2 | |||
41a128b1aa | |||
6617adc22b | |||
631d9d83b6 | |||
adec93af54 | |||
8898c2ec08 | |||
8efcc957ea | |||
6b728fef87 | |||
6b1f365e65 | |||
fcf1c45ad2 |
71 changed files with 909 additions and 359 deletions
|
@ -1,4 +1,4 @@
|
|||
FROM golang:1.19 as builder
|
||||
FROM golang:1.21 as builder
|
||||
|
||||
ARG BUILD=now
|
||||
ARG REPO=git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||
|
|
|
@ -6,7 +6,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.19', '1.20' ]
|
||||
go_versions: [ '1.20', '1.21' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
|
@ -12,7 +12,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Run commit format checker
|
||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v1
|
||||
|
|
|
@ -7,17 +7,24 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: golangci-lint
|
||||
uses: https://github.com/golangci/golangci-lint-action@v2
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
version: latest
|
||||
go-version: '1.21'
|
||||
cache: true
|
||||
|
||||
- name: Install linters
|
||||
run: make lint-install
|
||||
|
||||
- name: Run linters
|
||||
run: make lint
|
||||
|
||||
tests:
|
||||
name: Tests
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.19', '1.20' ]
|
||||
go_versions: [ '1.20', '1.21' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
|
@ -12,7 +12,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Install govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
|
|
@ -24,6 +24,16 @@ linters-settings:
|
|||
govet:
|
||||
# report about shadowed variables
|
||||
check-shadowing: false
|
||||
custom:
|
||||
truecloudlab-linters:
|
||||
path: bin/external_linters.so
|
||||
original-url: git.frostfs.info/TrueCloudLab/linters.git
|
||||
settings:
|
||||
noliteral:
|
||||
enable: true
|
||||
target-methods: ["Fatal"]
|
||||
disable-packages: ["codes", "tc"]
|
||||
constants-package: "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
|
||||
linters:
|
||||
enable:
|
||||
|
@ -45,6 +55,7 @@ linters:
|
|||
- gofmt
|
||||
- whitespace
|
||||
- goimports
|
||||
- truecloudlab-linters
|
||||
disable-all: true
|
||||
fast: false
|
||||
|
||||
|
|
|
@ -30,16 +30,23 @@ repos:
|
|||
hooks:
|
||||
- id: shellcheck
|
||||
|
||||
- repo: https://github.com/golangci/golangci-lint
|
||||
rev: v1.51.2
|
||||
hooks:
|
||||
- id: golangci-lint
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-unit-tests
|
||||
name: go unit tests
|
||||
entry: make test
|
||||
pass_filenames: false
|
||||
types: [go]
|
||||
language: system
|
||||
- id: make-lint-install
|
||||
name: install linters
|
||||
entry: make lint-install
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
||||
- id: make-lint
|
||||
name: run linters
|
||||
entry: make lint
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
||||
- id: go-unit-tests
|
||||
name: go unit tests
|
||||
entry: make test
|
||||
pass_filenames: false
|
||||
types: [go]
|
||||
language: system
|
||||
|
|
|
@ -13,6 +13,8 @@ This document outlines major changes between releases.
|
|||
- Replace part on re-upload when use multipart upload (#176)
|
||||
- Fix goroutine leak on put object error (#178)
|
||||
- Fix parsing signed headers in presigned urls (#182)
|
||||
- Fix url escaping (#188)
|
||||
- Use correct keys in `list-multipart-uploads` response (#185)
|
||||
|
||||
### Added
|
||||
- Add a metric with addresses of nodes of the same and highest priority that are currently healthy (#51)
|
||||
|
@ -25,6 +27,7 @@ This document outlines major changes between releases.
|
|||
- Support multiple version credentials using GSet (#135)
|
||||
- Implement chunk uploading (#106)
|
||||
- Add new `kludge.bypass_content_encoding_check_in_chunks` config param (#146)
|
||||
- Add new `frostfs.client_cut` config param (#192)
|
||||
|
||||
### Changed
|
||||
- Update prometheus to v1.15.0 (#94)
|
||||
|
@ -38,6 +41,7 @@ This document outlines major changes between releases.
|
|||
- Use chi router instead of archived gorlilla/mux (#149)
|
||||
- Complete multipart upload doesn't unnecessary copy now. Thus, the total time of multipart upload was reduced by 2 times (#63)
|
||||
- Use gate key to form object owner (#175)
|
||||
- Apply placement policies and copies if there is at least one valid value (#168)
|
||||
|
||||
### Removed
|
||||
- Drop `tree.service` param (now endpoints from `peers` section are used) (#133)
|
||||
|
|
25
Makefile
25
Makefile
|
@ -3,8 +3,9 @@
|
|||
# Common variables
|
||||
REPO ?= $(shell go list -m)
|
||||
VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop")
|
||||
GO_VERSION ?= 1.19
|
||||
LINT_VERSION ?= 1.49.0
|
||||
GO_VERSION ?= 1.20
|
||||
LINT_VERSION ?= 1.54.0
|
||||
TRUECLOUDLAB_LINT_VERSION ?= 0.0.2
|
||||
BINDIR = bin
|
||||
|
||||
METRICS_DUMP_OUT ?= ./metrics-dump.json
|
||||
|
@ -18,6 +19,10 @@ REPO_BASENAME = $(shell basename `go list -m`)
|
|||
HUB_IMAGE ?= "truecloudlab/$(REPO_BASENAME)"
|
||||
HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')"
|
||||
|
||||
OUTPUT_LINT_DIR ?= $(shell pwd)/bin
|
||||
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
|
||||
TMP_DIR := .cache
|
||||
|
||||
.PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
|
||||
|
||||
# .deb package versioning
|
||||
|
@ -101,9 +106,23 @@ dirty-image:
|
|||
-f .docker/Dockerfile.dirty \
|
||||
-t $(HUB_IMAGE)-dirty:$(HUB_TAG) .
|
||||
|
||||
# Install linters
|
||||
lint-install:
|
||||
@mkdir -p $(TMP_DIR)
|
||||
@rm -rf $(TMP_DIR)/linters
|
||||
@git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters
|
||||
@@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR)
|
||||
@rm -rf $(TMP_DIR)/linters
|
||||
@rmdir $(TMP_DIR) 2>/dev/null || true
|
||||
@CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION)
|
||||
|
||||
# Run linters
|
||||
lint:
|
||||
@golangci-lint --timeout=5m run
|
||||
@if [ ! -d "$(LINT_DIR)" ]; then \
|
||||
echo "Run make lint-install"; \
|
||||
exit 1; \
|
||||
fi
|
||||
$(LINT_DIR)/golangci-lint --timeout=5m run
|
||||
|
||||
# Run linters in Docker
|
||||
docker/lint:
|
||||
|
|
3
api/cache/access_control.go
vendored
3
api/cache/access_control.go
vendored
|
@ -4,6 +4,7 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
|
@ -46,7 +47,7 @@ func (o *AccessControlCache) Get(owner user.ID, key string) bool {
|
|||
|
||||
result, ok := entry.(bool)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return false
|
||||
}
|
||||
|
|
3
api/cache/accessbox.go
vendored
3
api/cache/accessbox.go
vendored
|
@ -5,6 +5,7 @@ import (
|
|||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
|
@ -57,7 +58,7 @@ func (o *AccessBoxCache) Get(address oid.Address) *accessbox.Box {
|
|||
|
||||
result, ok := entry.(*accessbox.Box)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
|
3
api/cache/buckets.go
vendored
3
api/cache/buckets.go
vendored
|
@ -5,6 +5,7 @@ import (
|
|||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -46,7 +47,7 @@ func (o *BucketCache) Get(key string) *data.BucketInfo {
|
|||
|
||||
result, ok := entry.(*data.BucketInfo)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
|
3
api/cache/names.go
vendored
3
api/cache/names.go
vendored
|
@ -4,6 +4,7 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
|
@ -48,7 +49,7 @@ func (o *ObjectsNameCache) Get(key string) *oid.Address {
|
|||
|
||||
result, ok := entry.(oid.Address)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
|
3
api/cache/objects.go
vendored
3
api/cache/objects.go
vendored
|
@ -5,6 +5,7 @@ import (
|
|||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
|
@ -47,7 +48,7 @@ func (o *ObjectsCache) GetObject(address oid.Address) *data.ExtendedObjectInfo {
|
|||
|
||||
result, ok := entry.(*data.ExtendedObjectInfo)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
|
5
api/cache/objectslist.go
vendored
5
api/cache/objectslist.go
vendored
|
@ -7,6 +7,7 @@ import (
|
|||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
|
@ -75,7 +76,7 @@ func (l *ObjectsListCache) GetVersions(key ObjectsListKey) []*data.NodeVersion {
|
|||
|
||||
result, ok := entry.([]*data.NodeVersion)
|
||||
if !ok {
|
||||
l.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
l.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -94,7 +95,7 @@ func (l *ObjectsListCache) CleanCacheEntriesContainingObject(objectName string,
|
|||
for _, key := range keys {
|
||||
k, ok := key.(ObjectsListKey)
|
||||
if !ok {
|
||||
l.logger.Warn("invalid cache key type", zap.String("actual", fmt.Sprintf("%T", key)),
|
||||
l.logger.Warn(logs.InvalidCacheKeyType, zap.String("actual", fmt.Sprintf("%T", key)),
|
||||
zap.String("expected", fmt.Sprintf("%T", k)))
|
||||
continue
|
||||
}
|
||||
|
|
9
api/cache/system.go
vendored
9
api/cache/system.go
vendored
|
@ -5,6 +5,7 @@ import (
|
|||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/bluele/gcache"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -48,7 +49,7 @@ func (o *SystemCache) GetObject(key string) *data.ObjectInfo {
|
|||
|
||||
result, ok := entry.(*data.ObjectInfo)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -79,7 +80,7 @@ func (o *SystemCache) GetCORS(key string) *data.CORSConfiguration {
|
|||
|
||||
result, ok := entry.(*data.CORSConfiguration)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -95,7 +96,7 @@ func (o *SystemCache) GetSettings(key string) *data.BucketSettings {
|
|||
|
||||
result, ok := entry.(*data.BucketSettings)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
@ -111,7 +112,7 @@ func (o *SystemCache) GetNotificationConfiguration(key string) *data.Notificatio
|
|||
|
||||
result, ok := entry.(*data.NotificationConfiguration)
|
||||
if !ok {
|
||||
o.logger.Warn("invalid cache entry type", zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
|
||||
zap.String("expected", fmt.Sprintf("%T", result)))
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -322,7 +322,7 @@ var errorCodes = errorCodeMap{
|
|||
ErrInvalidMaxUploads: {
|
||||
ErrCode: ErrInvalidMaxUploads,
|
||||
Code: "InvalidArgument",
|
||||
Description: "Argument max-uploads must be an integer between 0 and 2147483647",
|
||||
Description: "Argument max-uploads must be an integer from 1 to 1000",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidMaxKeys: {
|
||||
|
|
|
@ -20,6 +20,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
@ -470,7 +471,7 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
|||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
@ -1460,7 +1461,7 @@ func (h *handler) encodeObjectACL(ctx context.Context, bucketACL *layer.BucketAC
|
|||
if read {
|
||||
permission = aclFullControl
|
||||
} else {
|
||||
h.reqLogger(ctx).Warn("some acl not fully mapped")
|
||||
h.reqLogger(ctx).Warn(logs.SomeACLNotFullyMapped)
|
||||
}
|
||||
|
||||
var grantee *Grantee
|
||||
|
|
|
@ -11,6 +11,7 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -56,13 +57,6 @@ type (
|
|||
}
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPolicy is a default policy of placing containers in FrostFS if it's not set at the request.
|
||||
DefaultPolicy = "REP 3"
|
||||
// DefaultCopiesNumber is a default number of object copies that is enough to consider put successful if it's not set in config.
|
||||
DefaultCopiesNumber uint32 = 0
|
||||
)
|
||||
|
||||
var _ api.Handler = (*handler)(nil)
|
||||
|
||||
// New creates new api.Handler using given logger and client.
|
||||
|
@ -75,7 +69,7 @@ func New(log *zap.Logger, obj layer.Client, notificator Notificator, cfg *Config
|
|||
}
|
||||
|
||||
if !cfg.NotificatorEnabled {
|
||||
log.Warn("notificator is disabled, s3 won't produce notification events")
|
||||
log.Warn(logs.NotificatorIsDisabledS3WontProduceNotificationEvents)
|
||||
} else if notificator == nil {
|
||||
return nil, errors.New("empty notificator")
|
||||
}
|
||||
|
|
|
@ -17,7 +17,7 @@ func TestGetObjectPartsAttributes(t *testing.T) {
|
|||
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
putObject(t, hc, bktName, objName)
|
||||
putObject(hc, bktName, objName)
|
||||
result := getObjectAttributes(hc, bktName, objName, objectParts)
|
||||
require.Nil(t, result.ObjectParts)
|
||||
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -117,7 +118,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
if srcSize, err := getObjectSize(extendedSrcObjInfo, encryptionParams); err != nil {
|
||||
if srcSize, err := layer.GetObjectSize(srcObjInfo); err != nil {
|
||||
h.logAndSendError(w, "failed to get source object size", reqInfo, err)
|
||||
return
|
||||
} else if srcSize > layer.UploadMaxSize { //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
|
||||
|
@ -249,7 +250,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
h.reqLogger(ctx).Info("object is copied", zap.Stringer("object_id", dstObjInfo.ID))
|
||||
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
|
||||
|
||||
s := &SendNotificationParams{
|
||||
Event: EventObjectCreatedCopy,
|
||||
|
@ -258,7 +259,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
|
||||
if encryptionParams.Enabled() {
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -99,13 +100,13 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
bktInfo, err := h.obj.GetBucketInfo(ctx, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
h.reqLogger(ctx).Warn("get bucket info", zap.Error(err))
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
cors, err := h.obj.GetBucketCORS(ctx, bktInfo)
|
||||
if err != nil {
|
||||
h.reqLogger(ctx).Warn("get bucket cors", zap.Error(err))
|
||||
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
@ -11,6 +11,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
|
@ -114,7 +115,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
var objID oid.ID
|
||||
if len(versionID) != 0 {
|
||||
if err = objID.DecodeString(versionID); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -130,7 +131,7 @@ func (h *handler) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
if err = h.sendNotifications(ctx, m); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
|
||||
if deletedObject.VersionID != "" {
|
||||
|
@ -261,7 +262,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
|
|||
zap.Array("objects", marshaler),
|
||||
zap.Errors("errors", errs),
|
||||
}
|
||||
h.reqLogger(ctx).Error("couldn't delete objects", fields...)
|
||||
h.reqLogger(ctx).Error(logs.CouldntDeleteObjects, fields...)
|
||||
}
|
||||
|
||||
if err = middleware.EncodeToResponse(w, response); err != nil {
|
||||
|
|
|
@ -25,7 +25,7 @@ func TestDeleteBucketOnAlreadyRemovedError(t *testing.T) {
|
|||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
putObject(t, hc, bktName, objName)
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
addr := getAddressOfLastVersion(hc, bktInfo, objName)
|
||||
hc.tp.SetObjectError(addr, &apistatus.ObjectAlreadyRemoved{})
|
||||
|
@ -66,7 +66,7 @@ func TestDeleteBucketOnNotFoundError(t *testing.T) {
|
|||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
bktInfo := createTestBucket(hc, bktName)
|
||||
|
||||
putObject(t, hc, bktName, objName)
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
nodeVersion, err := hc.tree.GetUnversioned(hc.context, bktInfo, objName)
|
||||
require.NoError(t, err)
|
||||
|
@ -98,7 +98,7 @@ func TestDeleteObjectFromSuspended(t *testing.T) {
|
|||
bktName, objName := "bucket-versioned-for-removal", "object-to-delete"
|
||||
|
||||
createSuspendedBucket(t, tc, bktName)
|
||||
putObject(t, tc, bktName, objName)
|
||||
putObject(tc, bktName, objName)
|
||||
|
||||
versionID, isDeleteMarker := deleteObject(t, tc, bktName, objName, emptyVersion)
|
||||
require.True(t, isDeleteMarker)
|
||||
|
@ -255,7 +255,7 @@ func TestDeleteMarkerSuspended(t *testing.T) {
|
|||
|
||||
t.Run("remove last unversioned non delete marker", func(t *testing.T) {
|
||||
objName := "obj3"
|
||||
putObject(t, tc, bktName, objName)
|
||||
putObject(tc, bktName, objName)
|
||||
|
||||
nodeVersion, err := tc.tree.GetUnversioned(tc.Context(), bktInfo, objName)
|
||||
require.NoError(t, err)
|
||||
|
@ -333,7 +333,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
|
|||
bktName, objName := "bucket-for-removal", "object-to-delete"
|
||||
bktInfo, objInfo := createVersionedBucketAndObject(t, tc, bktName, objName)
|
||||
|
||||
versions := listObjectsV1(t, tc, bktName, "", "", "", -1)
|
||||
versions := listObjectsV1(tc, bktName, "", "", "", -1)
|
||||
require.Len(t, versions.Contents, 1)
|
||||
|
||||
checkFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||
|
@ -341,7 +341,7 @@ func TestDeleteObjectFromListCache(t *testing.T) {
|
|||
checkNotFound(t, tc, bktName, objName, objInfo.VersionID())
|
||||
|
||||
// check cache is clean after object removal
|
||||
versions = listObjectsV1(t, tc, bktName, "", "", "", -1)
|
||||
versions = listObjectsV1(tc, bktName, "", "", "", -1)
|
||||
require.Len(t, versions.Contents, 0)
|
||||
|
||||
require.False(t, existInMockedFrostFS(tc, bktInfo, objInfo))
|
||||
|
@ -475,11 +475,11 @@ func getVersion(resp *ListObjectsVersionsResponse, objName string) []*ObjectVers
|
|||
return res
|
||||
}
|
||||
|
||||
func putObject(t *testing.T, tc *handlerContext, bktName, objName string) {
|
||||
func putObject(hc *handlerContext, bktName, objName string) {
|
||||
body := bytes.NewReader([]byte("content"))
|
||||
w, r := prepareTestPayloadRequest(tc, bktName, objName, body)
|
||||
tc.Handler().PutObjectHandler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, objName, body)
|
||||
hc.Handler().PutObjectHandler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
}
|
||||
|
||||
func createSuspendedBucket(t *testing.T, tc *handlerContext, bktName string) *data.BucketInfo {
|
||||
|
|
|
@ -12,7 +12,6 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -168,7 +167,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
fullSize, err := getObjectSize(extendedInfo, encryptionParams)
|
||||
fullSize, err := layer.GetObjectSize(info)
|
||||
if err != nil {
|
||||
h.logAndSendError(w, "invalid size header", reqInfo, errors.GetAPIError(errors.ErrBadRequest))
|
||||
return
|
||||
|
@ -233,23 +232,6 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
}
|
||||
|
||||
func getObjectSize(extendedInfo *data.ExtendedObjectInfo, encryptionParams encryption.Params) (uint64, error) {
|
||||
var err error
|
||||
fullSize := extendedInfo.ObjectInfo.Size
|
||||
|
||||
if encryptionParams.Enabled() {
|
||||
if fullSize, err = strconv.ParseUint(extendedInfo.ObjectInfo.Headers[layer.AttributeDecryptedSize], 10, 64); err != nil {
|
||||
return 0, fmt.Errorf("invalid decrypted size header: %w", err)
|
||||
}
|
||||
} else if extendedInfo.NodeVersion.IsCombined {
|
||||
if fullSize, err = strconv.ParseUint(extendedInfo.ObjectInfo.Headers[layer.MultipartObjectSize], 10, 64); err != nil {
|
||||
return 0, fmt.Errorf("invalid multipart size header: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return fullSize, nil
|
||||
}
|
||||
|
||||
func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs) error {
|
||||
if len(args.IfMatch) > 0 && args.IfMatch != info.HashSum {
|
||||
return fmt.Errorf("%w: etag mismatched: '%s', '%s'", errors.GetAPIError(errors.ErrPreconditionFailed), args.IfMatch, info.HashSum)
|
||||
|
|
|
@ -186,7 +186,7 @@ func TestGetObject(t *testing.T) {
|
|||
bktName, objName := "bucket", "obj"
|
||||
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
||||
|
||||
putObject(hc.t, hc, bktName, objName)
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
||||
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
||||
|
|
|
@ -3,9 +3,9 @@ package handler
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
|
@ -38,6 +38,8 @@ type handlerContext struct {
|
|||
tree *tree.Tree
|
||||
context context.Context
|
||||
kludge *kludgeSettingsMock
|
||||
|
||||
layerFeatures *layer.FeatureSettingsMock
|
||||
}
|
||||
|
||||
func (hc *handlerContext) Handler() *handler {
|
||||
|
@ -123,11 +125,14 @@ func prepareHandlerContextBase(t *testing.T, minCache bool) *handlerContext {
|
|||
cacheCfg = getMinCacheConfig(l)
|
||||
}
|
||||
|
||||
features := &layer.FeatureSettingsMock{}
|
||||
|
||||
layerCfg := &layer.Config{
|
||||
Caches: cacheCfg,
|
||||
AnonKey: layer.AnonymousKey{Key: key},
|
||||
Resolver: testResolver,
|
||||
TreeService: treeMock,
|
||||
Features: features,
|
||||
}
|
||||
|
||||
var pp netmap.PlacementPolicy
|
||||
|
@ -154,6 +159,8 @@ func prepareHandlerContextBase(t *testing.T, minCache bool) *handlerContext {
|
|||
tree: treeMock,
|
||||
context: middleware.SetBoxData(context.Background(), newTestAccessBox(t, key)),
|
||||
kludge: kludge,
|
||||
|
||||
layerFeatures: features,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -114,7 +114,7 @@ func TestHeadObject(t *testing.T) {
|
|||
bktName, objName := "bucket", "obj"
|
||||
bktInfo, objInfo := createVersionedBucketAndObject(hc.t, hc, bktName, objName)
|
||||
|
||||
putObject(hc.t, hc, bktName, objName)
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
checkFound(hc.t, hc, bktName, objName, objInfo.VersionID())
|
||||
checkFound(hc.t, hc, bktName, objName, emptyVersion)
|
||||
|
|
|
@ -536,7 +536,7 @@ func TestPutObjectWithLock(t *testing.T) {
|
|||
createTestBucketWithLock(hc, bktName, lockConfig)
|
||||
|
||||
objDefault := "obj-default-retention"
|
||||
putObject(t, hc, bktName, objDefault)
|
||||
putObject(hc, bktName, objDefault)
|
||||
|
||||
getObjectRetentionApproximate(hc, bktName, objDefault, governanceMode, time.Now().Add(24*time.Hour))
|
||||
getObjectLegalHold(hc, bktName, objDefault, legalHoldOff)
|
||||
|
@ -587,7 +587,7 @@ func TestPutLockErrors(t *testing.T) {
|
|||
headers[api.AmzObjectLockRetainUntilDate] = "dummy"
|
||||
putObjectWithLockFailed(t, hc, bktName, objName, headers, apiErrors.ErrInvalidRetentionDate)
|
||||
|
||||
putObject(t, hc, bktName, objName)
|
||||
putObject(hc, bktName, objName)
|
||||
|
||||
retention := &data.Retention{Mode: governanceMode}
|
||||
putObjectRetentionFailed(t, hc, bktName, objName, retention, apiErrors.ErrMalformedXML)
|
||||
|
|
|
@ -14,6 +14,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -92,6 +93,13 @@ type (
|
|||
const (
|
||||
uploadIDHeaderName = "uploadId"
|
||||
partNumberHeaderName = "partNumber"
|
||||
|
||||
prefixQueryName = "prefix"
|
||||
delimiterQueryName = "delimiter"
|
||||
maxUploadsQueryName = "max-uploads"
|
||||
encodingTypeQueryName = "encoding-type"
|
||||
keyMarkerQueryName = "key-marker"
|
||||
uploadIDMarkerQueryName = "upload-id-marker"
|
||||
)
|
||||
|
||||
func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
@ -511,7 +519,7 @@ func (h *handler) completeMultipartUpload(r *http.Request, c *layer.CompleteMult
|
|||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
|
||||
return objInfo, nil
|
||||
|
@ -527,31 +535,28 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req
|
|||
}
|
||||
|
||||
var (
|
||||
queryValues = reqInfo.URL.Query()
|
||||
delimiter = queryValues.Get("delimiter")
|
||||
prefix = queryValues.Get("prefix")
|
||||
maxUploads = layer.MaxSizeUploadsList
|
||||
queryValues = reqInfo.URL.Query()
|
||||
maxUploadsStr = queryValues.Get(maxUploadsQueryName)
|
||||
maxUploads = layer.MaxSizeUploadsList
|
||||
)
|
||||
|
||||
if queryValues.Get("max-uploads") != "" {
|
||||
val, err := strconv.Atoi(queryValues.Get("max-uploads"))
|
||||
if err != nil || val < 0 {
|
||||
if maxUploadsStr != "" {
|
||||
val, err := strconv.Atoi(maxUploadsStr)
|
||||
if err != nil || val < 1 || val > 1000 {
|
||||
h.logAndSendError(w, "invalid maxUploads", reqInfo, errors.GetAPIError(errors.ErrInvalidMaxUploads))
|
||||
return
|
||||
}
|
||||
if val < maxUploads {
|
||||
maxUploads = val
|
||||
}
|
||||
maxUploads = val
|
||||
}
|
||||
|
||||
p := &layer.ListMultipartUploadsParams{
|
||||
Bkt: bktInfo,
|
||||
Delimiter: delimiter,
|
||||
EncodingType: queryValues.Get("encoding-type"),
|
||||
KeyMarker: queryValues.Get("key-marker"),
|
||||
Delimiter: queryValues.Get(delimiterQueryName),
|
||||
EncodingType: queryValues.Get(encodingTypeQueryName),
|
||||
KeyMarker: queryValues.Get(keyMarkerQueryName),
|
||||
MaxUploads: maxUploads,
|
||||
Prefix: prefix,
|
||||
UploadIDMarker: queryValues.Get("upload-id-marker"),
|
||||
Prefix: queryValues.Get(prefixQueryName),
|
||||
UploadIDMarker: queryValues.Get(uploadIDMarkerQueryName),
|
||||
}
|
||||
|
||||
list, err := h.obj.ListMultipartUploads(r.Context(), p)
|
||||
|
|
|
@ -3,12 +3,16 @@ package handler
|
|||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
s3Errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -105,6 +109,164 @@ func TestMultipartReUploadPart(t *testing.T) {
|
|||
equalDataSlices(t, append(data1, data2...), data)
|
||||
}
|
||||
|
||||
func TestListMultipartUploads(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-to-list-uploads"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
objName1 := "/my/object/name"
|
||||
uploadInfo1 := createMultipartUpload(hc, bktName, objName1, map[string]string{})
|
||||
objName2 := "/my/object2"
|
||||
uploadInfo2 := createMultipartUpload(hc, bktName, objName2, map[string]string{})
|
||||
objName3 := "/zzz/object/name3"
|
||||
uploadInfo3 := createMultipartUpload(hc, bktName, objName3, map[string]string{})
|
||||
|
||||
t.Run("check upload key", func(t *testing.T) {
|
||||
listUploads := listAllMultipartUploads(hc, bktName)
|
||||
require.Len(t, listUploads.Uploads, 3)
|
||||
for i, upload := range []*InitiateMultipartUploadResponse{uploadInfo1, uploadInfo2, uploadInfo3} {
|
||||
require.Equal(t, upload.UploadID, listUploads.Uploads[i].UploadID)
|
||||
require.Equal(t, upload.Key, listUploads.Uploads[i].Key)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("check max uploads", func(t *testing.T) {
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", "", 2)
|
||||
require.Len(t, listUploads.Uploads, 2)
|
||||
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
|
||||
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
|
||||
})
|
||||
|
||||
t.Run("check prefix", func(t *testing.T) {
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "/my", "", "", "", -1)
|
||||
require.Len(t, listUploads.Uploads, 2)
|
||||
require.Equal(t, uploadInfo1.UploadID, listUploads.Uploads[0].UploadID)
|
||||
require.Equal(t, uploadInfo2.UploadID, listUploads.Uploads[1].UploadID)
|
||||
})
|
||||
|
||||
t.Run("check markers", func(t *testing.T) {
|
||||
t.Run("check only key-marker", func(t *testing.T) {
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "", "", "", objName2, -1)
|
||||
require.Len(t, listUploads.Uploads, 1)
|
||||
// If upload-id-marker is not specified, only the keys lexicographically greater than the specified key-marker will be included in the list.
|
||||
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
|
||||
})
|
||||
|
||||
t.Run("check only upload-id-marker", func(t *testing.T) {
|
||||
uploadIDMarker := uploadInfo1.UploadID
|
||||
if uploadIDMarker > uploadInfo2.UploadID {
|
||||
uploadIDMarker = uploadInfo2.UploadID
|
||||
}
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, "", -1)
|
||||
// If key-marker is not specified, the upload-id-marker parameter is ignored.
|
||||
require.Len(t, listUploads.Uploads, 3)
|
||||
})
|
||||
|
||||
t.Run("check key-marker along with upload-id-marker", func(t *testing.T) {
|
||||
uploadIDMarker := "00000000-0000-0000-0000-000000000000"
|
||||
|
||||
listUploads := listMultipartUploadsBase(hc, bktName, "", "", uploadIDMarker, objName3, -1)
|
||||
require.Len(t, listUploads.Uploads, 1)
|
||||
// If upload-id-marker is specified, any multipart uploads for a key equal to the key-marker might also be included,
|
||||
// provided those multipart uploads have upload IDs lexicographically greater than the specified upload-id-marker.
|
||||
require.Equal(t, uploadInfo3.UploadID, listUploads.Uploads[0].UploadID)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipartUploadSize(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-test-multipart-size", "object-multipart"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
partSize := layer.UploadMinSize
|
||||
objLen := 2 * partSize
|
||||
headers := map[string]string{}
|
||||
|
||||
data := multipartUpload(hc, bktName, objName, headers, objLen, partSize)
|
||||
require.Equal(t, objLen, len(data))
|
||||
|
||||
t.Run("check correct size in list v1", func(t *testing.T) {
|
||||
listV1 := listObjectsV1(hc, bktName, "", "", "", -1)
|
||||
require.Len(t, listV1.Contents, 1)
|
||||
require.Equal(t, objLen, int(listV1.Contents[0].Size))
|
||||
require.Equal(t, objName, listV1.Contents[0].Key)
|
||||
})
|
||||
|
||||
t.Run("check correct size in list v2", func(t *testing.T) {
|
||||
listV2 := listObjectsV2(hc, bktName, "", "", "", "", -1)
|
||||
require.Len(t, listV2.Contents, 1)
|
||||
require.Equal(t, objLen, int(listV2.Contents[0].Size))
|
||||
require.Equal(t, objName, listV2.Contents[0].Key)
|
||||
})
|
||||
|
||||
t.Run("check correct get", func(t *testing.T) {
|
||||
_, hdr := getObject(hc, bktName, objName)
|
||||
require.Equal(t, strconv.Itoa(objLen), hdr.Get(api.ContentLength))
|
||||
|
||||
part := getObjectRange(t, hc, bktName, objName, partSize, objLen-1)
|
||||
equalDataSlices(t, data[partSize:], part)
|
||||
})
|
||||
|
||||
t.Run("check correct size when part copy", func(t *testing.T) {
|
||||
objName2 := "obj2"
|
||||
uploadInfo := createMultipartUpload(hc, bktName, objName2, headers)
|
||||
sourceCopy := bktName + "/" + objName
|
||||
uploadPartCopy(hc, bktName, objName2, uploadInfo.UploadID, 1, sourceCopy, 0, 0)
|
||||
uploadPartCopy(hc, bktName, objName2, uploadInfo.UploadID, 2, sourceCopy, 0, partSize)
|
||||
})
|
||||
}
|
||||
|
||||
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
|
||||
}
|
||||
|
||||
func uploadPartCopyBase(hc *handlerContext, bktName, objName string, encrypted bool, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||
query := make(url.Values)
|
||||
query.Set(uploadIDQuery, uploadID)
|
||||
query.Set(partNumberQuery, strconv.Itoa(num))
|
||||
|
||||
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, nil)
|
||||
if encrypted {
|
||||
setEncryptHeaders(r)
|
||||
}
|
||||
r.Header.Set(api.AmzCopySource, srcObj)
|
||||
if start+end > 0 {
|
||||
r.Header.Set(api.AmzCopySourceRange, fmt.Sprintf("bytes=%d-%d", start, end))
|
||||
}
|
||||
|
||||
hc.Handler().UploadPartCopy(w, r)
|
||||
uploadPartCopyResponse := &UploadPartCopyResponse{}
|
||||
readResponse(hc.t, w, http.StatusOK, uploadPartCopyResponse)
|
||||
|
||||
return uploadPartCopyResponse
|
||||
}
|
||||
|
||||
func listAllMultipartUploads(hc *handlerContext, bktName string) *ListMultipartUploadsResponse {
|
||||
return listMultipartUploadsBase(hc, bktName, "", "", "", "", -1)
|
||||
}
|
||||
|
||||
func listMultipartUploadsBase(hc *handlerContext, bktName, prefix, delimiter, uploadIDMarker, keyMarker string, maxUploads int) *ListMultipartUploadsResponse {
|
||||
query := make(url.Values)
|
||||
query.Set(prefixQueryName, prefix)
|
||||
query.Set(delimiterQueryName, delimiter)
|
||||
query.Set(uploadIDMarkerQueryName, uploadIDMarker)
|
||||
query.Set(keyMarkerQueryName, keyMarker)
|
||||
if maxUploads != -1 {
|
||||
query.Set(maxUploadsQueryName, strconv.Itoa(maxUploads))
|
||||
}
|
||||
|
||||
w, r := prepareTestRequestWithQuery(hc, bktName, "", query, nil)
|
||||
|
||||
hc.Handler().ListMultipartUploadsHandler(w, r)
|
||||
listPartsResponse := &ListMultipartUploadsResponse{}
|
||||
readResponse(hc.t, w, http.StatusOK, listPartsResponse)
|
||||
|
||||
return listPartsResponse
|
||||
}
|
||||
|
||||
func listParts(hc *handlerContext, bktName, objName string, uploadID string) *ListPartsResponse {
|
||||
return listPartsBase(hc, bktName, objName, false, uploadID)
|
||||
}
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
@ -202,7 +203,7 @@ func (h *handler) checkBucketConfiguration(ctx context.Context, conf *data.Notif
|
|||
return
|
||||
}
|
||||
} else {
|
||||
h.reqLogger(ctx).Warn("failed to send test event because notifications is disabled")
|
||||
h.reqLogger(ctx).Warn(logs.FailedToSendTestEventBecauseNotificationsIsDisabled)
|
||||
}
|
||||
|
||||
if q.ID == "" {
|
||||
|
|
|
@ -198,6 +198,10 @@ func fillContents(src []*data.ObjectInfo, encode string, fetchOwner bool) []Obje
|
|||
ETag: obj.HashSum,
|
||||
}
|
||||
|
||||
if size, err := layer.GetObjectSize(obj); err == nil {
|
||||
res.Size = size
|
||||
}
|
||||
|
||||
if fetchOwner {
|
||||
res.Owner = &Owner{
|
||||
ID: obj.Owner.String(),
|
||||
|
|
|
@ -67,11 +67,11 @@ func TestS3CompatibilityBucketListV2BothContinuationTokenStartAfter(t *testing.T
|
|||
createTestObject(tc, bktInfo, objName)
|
||||
}
|
||||
|
||||
listV2Response1 := listObjectsV2(t, tc, bktName, "", "", "bar", "", 1)
|
||||
listV2Response1 := listObjectsV2(tc, bktName, "", "", "bar", "", 1)
|
||||
nextContinuationToken := listV2Response1.NextContinuationToken
|
||||
require.Equal(t, "baz", listV2Response1.Contents[0].Key)
|
||||
|
||||
listV2Response2 := listObjectsV2(t, tc, bktName, "", "", "bar", nextContinuationToken, -1)
|
||||
listV2Response2 := listObjectsV2(tc, bktName, "", "", "bar", nextContinuationToken, -1)
|
||||
|
||||
require.Equal(t, nextContinuationToken, listV2Response2.ContinuationToken)
|
||||
require.Equal(t, "bar", listV2Response2.StartAfter)
|
||||
|
@ -92,7 +92,7 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
|
|||
createTestObject(tc, bktInfo, objName)
|
||||
}
|
||||
|
||||
listV1Response := listObjectsV1(t, tc, bktName, "", "/", "", -1)
|
||||
listV1Response := listObjectsV1(tc, bktName, "", "/", "", -1)
|
||||
require.Equal(t, "/", listV1Response.Delimiter)
|
||||
require.Equal(t, "asdf", listV1Response.Contents[0].Key)
|
||||
require.Len(t, listV1Response.CommonPrefixes, 2)
|
||||
|
@ -100,6 +100,26 @@ func TestS3BucketListDelimiterBasic(t *testing.T) {
|
|||
require.Equal(t, "quux/", listV1Response.CommonPrefixes[1].Prefix)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2DelimiterPercentage(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
bktName := "bucket-for-listing"
|
||||
objects := []string{"b%ar", "b%az", "c%ab", "foo"}
|
||||
bktInfo, _ := createBucketAndObject(tc, bktName, objects[0])
|
||||
|
||||
for _, objName := range objects[1:] {
|
||||
createTestObject(tc, bktInfo, objName)
|
||||
}
|
||||
|
||||
listV2Response := listObjectsV2(tc, bktName, "", "%", "", "", -1)
|
||||
require.Equal(t, "%", listV2Response.Delimiter)
|
||||
require.Len(t, listV2Response.Contents, 1)
|
||||
require.Equal(t, "foo", listV2Response.Contents[0].Key)
|
||||
require.Len(t, listV2Response.CommonPrefixes, 2)
|
||||
require.Equal(t, "b%", listV2Response.CommonPrefixes[0].Prefix)
|
||||
require.Equal(t, "c%", listV2Response.CommonPrefixes[1].Prefix)
|
||||
}
|
||||
|
||||
func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
||||
tc := prepareHandlerContext(t)
|
||||
|
||||
|
@ -129,7 +149,7 @@ func TestS3BucketListV2DelimiterPrefix(t *testing.T) {
|
|||
validateListV2(t, tc, bktName, prefix, delim, "", 2, false, true, []string{"boo/bar"}, []string{"boo/baz/"})
|
||||
}
|
||||
|
||||
func listObjectsV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
|
||||
func listObjectsV2(hc *handlerContext, bktName, prefix, delimiter, startAfter, continuationToken string, maxKeys int) *ListObjectsV2Response {
|
||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||
if len(startAfter) != 0 {
|
||||
query.Add("start-after", startAfter)
|
||||
|
@ -138,17 +158,17 @@ func listObjectsV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter,
|
|||
query.Add("continuation-token", continuationToken)
|
||||
}
|
||||
|
||||
w, r := prepareTestFullRequest(tc, bktName, "", query, nil)
|
||||
tc.Handler().ListObjectsV2Handler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||
hc.Handler().ListObjectsV2Handler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
res := &ListObjectsV2Response{}
|
||||
parseTestResponse(t, w, res)
|
||||
parseTestResponse(hc.t, w, res)
|
||||
return res
|
||||
}
|
||||
|
||||
func validateListV2(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, continuationToken string, maxKeys int,
|
||||
isTruncated, last bool, checkObjects, checkPrefixes []string) string {
|
||||
response := listObjectsV2(t, tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
|
||||
response := listObjectsV2(tc, bktName, prefix, delimiter, "", continuationToken, maxKeys)
|
||||
|
||||
require.Equal(t, isTruncated, response.IsTruncated)
|
||||
require.Equal(t, last, len(response.NextContinuationToken) == 0)
|
||||
|
@ -182,16 +202,16 @@ func prepareCommonListObjectsQuery(prefix, delimiter string, maxKeys int) url.Va
|
|||
return query
|
||||
}
|
||||
|
||||
func listObjectsV1(t *testing.T, tc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
|
||||
func listObjectsV1(hc *handlerContext, bktName, prefix, delimiter, marker string, maxKeys int) *ListObjectsV1Response {
|
||||
query := prepareCommonListObjectsQuery(prefix, delimiter, maxKeys)
|
||||
if len(marker) != 0 {
|
||||
query.Add("marker", marker)
|
||||
}
|
||||
|
||||
w, r := prepareTestFullRequest(tc, bktName, "", query, nil)
|
||||
tc.Handler().ListObjectsV1Handler(w, r)
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
w, r := prepareTestFullRequest(hc, bktName, "", query, nil)
|
||||
hc.Handler().ListObjectsV1Handler(w, r)
|
||||
assertStatus(hc.t, w, http.StatusOK)
|
||||
res := &ListObjectsV1Response{}
|
||||
parseTestResponse(t, w, res)
|
||||
parseTestResponse(hc.t, w, res)
|
||||
return res
|
||||
}
|
||||
|
|
|
@ -24,6 +24,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
|
@ -277,7 +278,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
|
||||
if containsACL {
|
||||
|
@ -494,7 +495,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
|
||||
if acl := auth.MultipartFormValue(r, "acl"); acl != "" {
|
||||
|
@ -539,7 +540,7 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
if settings, err := h.obj.GetBucketSettings(ctx, bktInfo); err != nil {
|
||||
h.reqLogger(ctx).Warn("couldn't get bucket versioning", zap.String("bucket name", reqInfo.BucketName), zap.Error(err))
|
||||
h.reqLogger(ctx).Warn(logs.CouldntGetBucketVersioning, zap.String("bucket name", reqInfo.BucketName), zap.Error(err))
|
||||
} else if settings.VersioningEnabled() {
|
||||
w.Header().Set(api.AmzVersionID, objInfo.VersionID())
|
||||
}
|
||||
|
@ -778,7 +779,7 @@ func (h *handler) CreateBucketHandler(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
h.reqLogger(ctx).Info("bucket is created", zap.Stringer("container_id", bktInfo.CID))
|
||||
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
|
||||
|
||||
if p.ObjectLockEnabled {
|
||||
sp := &layer.PutSettingsParams{
|
||||
|
|
|
@ -23,6 +23,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
@ -309,3 +310,37 @@ func TestCreateBucket(t *testing.T) {
|
|||
box2, _ := createAccessBox(t)
|
||||
createBucketAssertS3Error(hc, bktName, box2, s3errors.ErrBucketAlreadyExists)
|
||||
}
|
||||
|
||||
func TestPutObjectClientCut(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
bktName, objName1, objName2 := "bkt-name", "obj-name1", "obj-name2"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
putObject(hc, bktName, objName1)
|
||||
obj1 := getObjectFromLayer(hc, objName1)[0]
|
||||
require.Empty(t, getObjectAttribute(obj1, "s3-client-cut"))
|
||||
|
||||
hc.layerFeatures.SetClientCut(true)
|
||||
putObject(hc, bktName, objName2)
|
||||
obj2 := getObjectFromLayer(hc, objName2)[0]
|
||||
require.Equal(t, "true", getObjectAttribute(obj2, "s3-client-cut"))
|
||||
}
|
||||
|
||||
func getObjectFromLayer(hc *handlerContext, objName string) []*object.Object {
|
||||
var res []*object.Object
|
||||
for _, o := range hc.tp.Objects() {
|
||||
if objName == getObjectAttribute(o, object.AttributeFilePath) {
|
||||
res = append(res, o)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func getObjectAttribute(obj *object.Object, attrName string) string {
|
||||
for _, attr := range obj.Attributes() {
|
||||
if attr.Key() == attrName {
|
||||
return attr.Value()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
|
@ -13,6 +13,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -66,7 +67,7 @@ func (h *handler) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request
|
|||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
@ -143,7 +144,7 @@ func (h *handler) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Requ
|
|||
ReqInfo: reqInfo,
|
||||
}
|
||||
if err = h.sendNotifications(ctx, s); err != nil {
|
||||
h.reqLogger(ctx).Error("couldn't send notification: %w", zap.Error(err))
|
||||
h.reqLogger(ctx).Error(logs.CouldntSendNotification, zap.Error(err))
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
|
|
@ -13,6 +13,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
frosterrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -36,7 +37,7 @@ func (h *handler) logAndSendError(w http.ResponseWriter, logText string, reqInfo
|
|||
zap.String("description", logText),
|
||||
zap.Error(err)}
|
||||
fields = append(fields, additional...)
|
||||
h.log.Error("request failed", fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
||||
h.log.Error(logs.RequestFailed, fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
||||
}
|
||||
|
||||
func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string, reqInfo *middleware.ReqInfo, err error, additional ...zap.Field) {
|
||||
|
@ -49,7 +50,7 @@ func (h *handler) logAndSendErrorNoHeader(w http.ResponseWriter, logText string,
|
|||
zap.String("description", logText),
|
||||
zap.Error(err)}
|
||||
fields = append(fields, additional...)
|
||||
h.log.Error("request failed", fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
||||
h.log.Error(logs.RequestFailed, fields...) // consider using h.reqLogger (it requires accept context.Context or http.Request)
|
||||
}
|
||||
|
||||
func transformToS3Error(err error) error {
|
||||
|
|
|
@ -3,6 +3,7 @@ package layer
|
|||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
|
@ -61,7 +62,7 @@ func (c *Cache) GetBucket(name string) *data.BucketInfo {
|
|||
|
||||
func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
|
||||
if err := c.bucketCache.Put(bktInfo); err != nil {
|
||||
c.logger.Warn("couldn't put bucket info into cache",
|
||||
c.logger.Warn(logs.CouldntPutBucketInfoIntoCache,
|
||||
zap.String("bucket name", bktInfo.Name),
|
||||
zap.Stringer("bucket cid", bktInfo.CID),
|
||||
zap.Error(err))
|
||||
|
@ -104,13 +105,13 @@ func (c *Cache) GetLastObject(owner user.ID, bktName, objName string) *data.Exte
|
|||
|
||||
func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
|
||||
if err := c.objCache.PutObject(extObjInfo); err != nil {
|
||||
c.logger.Warn("couldn't add object to cache", zap.Error(err),
|
||||
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
|
||||
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
|
||||
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -118,7 +119,7 @@ func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObject
|
|||
c.PutObject(owner, extObjInfo)
|
||||
|
||||
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
|
||||
c.logger.Warn("couldn't put obj address to name cache",
|
||||
c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
|
||||
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
|
||||
zap.Error(err))
|
||||
}
|
||||
|
@ -134,11 +135,11 @@ func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVer
|
|||
|
||||
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
|
||||
if err := c.listsCache.PutVersions(key, list); err != nil {
|
||||
c.logger.Warn("couldn't cache list of objects", zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key.String()); err != nil {
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -152,11 +153,11 @@ func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
|
|||
|
||||
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
|
||||
if err := c.systemCache.PutTagging(key, tags); err != nil {
|
||||
c.logger.Error("couldn't cache tags", zap.Error(err))
|
||||
c.logger.Error(logs.CouldntCacheTags, zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -174,11 +175,11 @@ func (c *Cache) GetLockInfo(owner user.ID, key string) *data.LockInfo {
|
|||
|
||||
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
|
||||
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
|
||||
c.logger.Error("couldn't cache lock info", zap.Error(err))
|
||||
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -195,11 +196,11 @@ func (c *Cache) GetSettings(owner user.ID, bktInfo *data.BucketInfo) *data.Bucke
|
|||
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
|
||||
key := bktInfo.Name + bktInfo.SettingsObjectName()
|
||||
if err := c.systemCache.PutSettings(key, settings); err != nil {
|
||||
c.logger.Warn("couldn't cache bucket settings", zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -217,11 +218,11 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
|
|||
key := bkt.Name + bkt.CORSObjectName()
|
||||
|
||||
if err := c.systemCache.PutCORS(key, cors); err != nil {
|
||||
c.logger.Warn("couldn't cache cors", zap.String("bucket", bkt.Name), zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -242,10 +243,10 @@ func (c *Cache) GetNotificationConfiguration(owner user.ID, bktInfo *data.Bucket
|
|||
func (c *Cache) PutNotificationConfiguration(owner user.ID, bktInfo *data.BucketInfo, configuration *data.NotificationConfiguration) {
|
||||
key := bktInfo.Name + bktInfo.NotificationConfigurationObjectName()
|
||||
if err := c.systemCache.PutNotificationConfiguration(key, configuration); err != nil {
|
||||
c.logger.Warn("couldn't cache notification configuration", zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheNotificationConfiguration, zap.String("bucket", bktInfo.Name), zap.Error(err))
|
||||
}
|
||||
|
||||
if err := c.accessCache.Put(owner, key); err != nil {
|
||||
c.logger.Warn("couldn't cache access control operation", zap.Error(err))
|
||||
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
|
@ -63,7 +64,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
|||
if len(attrLockEnabled) > 0 {
|
||||
info.ObjectLockEnabled, err = strconv.ParseBool(attrLockEnabled)
|
||||
if err != nil {
|
||||
log.Error("could not parse container object lock enabled attribute",
|
||||
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
|
||||
zap.String("lock_enabled", attrLockEnabled),
|
||||
zap.Error(err),
|
||||
)
|
||||
|
@ -78,7 +79,7 @@ func (n *layer) containerInfo(ctx context.Context, idCnr cid.ID) (*data.BucketIn
|
|||
func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
||||
res, err := n.frostFS.UserContainers(ctx, n.BearerOwner(ctx))
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error("could not list user containers", zap.Error(err))
|
||||
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
@ -86,7 +87,7 @@ func (n *layer) containerList(ctx context.Context) ([]*data.BucketInfo, error) {
|
|||
for i := range res {
|
||||
info, err := n.containerInfo(ctx, res[i])
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error("could not fetch container info", zap.Error(err))
|
||||
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -57,7 +58,7 @@ func (n *layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
|
|||
|
||||
if !objIDToDeleteNotFound {
|
||||
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
||||
n.reqLogger(ctx).Error("couldn't delete cors object", zap.Error(err),
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
|
||||
zap.String("cnrID", p.BktInfo.CID.EncodeToString()),
|
||||
zap.String("objID", objIDToDelete.EncodeToString()))
|
||||
}
|
||||
|
|
|
@ -111,6 +111,9 @@ type PrmObjectCreate struct {
|
|||
|
||||
// Number of object copies that is enough to consider put successful.
|
||||
CopiesNumber []uint32
|
||||
|
||||
// Enables client side object preparing.
|
||||
ClientCut bool
|
||||
}
|
||||
|
||||
// PrmObjectDelete groups parameters of FrostFS.DeleteObject operation.
|
||||
|
|
|
@ -25,6 +25,18 @@ import (
|
|||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
)
|
||||
|
||||
type FeatureSettingsMock struct {
|
||||
clientCut bool
|
||||
}
|
||||
|
||||
func (k *FeatureSettingsMock) ClientCut() bool {
|
||||
return k.clientCut
|
||||
}
|
||||
|
||||
func (k *FeatureSettingsMock) SetClientCut(clientCut bool) {
|
||||
k.clientCut = clientCut
|
||||
}
|
||||
|
||||
type TestFrostFS struct {
|
||||
FrostFS
|
||||
|
||||
|
@ -222,6 +234,13 @@ func (t *TestFrostFS) CreateObject(_ context.Context, prm PrmObjectCreate) (oid.
|
|||
attrs = append(attrs, *a)
|
||||
}
|
||||
|
||||
if prm.ClientCut {
|
||||
a := object.NewAttribute()
|
||||
a.SetKey("s3-client-cut")
|
||||
a.SetValue("true")
|
||||
attrs = append(attrs, *a)
|
||||
}
|
||||
|
||||
for i := range prm.Attributes {
|
||||
a := object.NewAttribute()
|
||||
a.SetKey(prm.Attributes[i][0])
|
||||
|
|
|
@ -16,6 +16,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
|
@ -45,6 +46,10 @@ type (
|
|||
Resolve(ctx context.Context, name string) (cid.ID, error)
|
||||
}
|
||||
|
||||
FeatureSettings interface {
|
||||
ClientCut() bool
|
||||
}
|
||||
|
||||
layer struct {
|
||||
frostFS FrostFS
|
||||
gateOwner user.ID
|
||||
|
@ -54,6 +59,7 @@ type (
|
|||
ncontroller EventListener
|
||||
cache *Cache
|
||||
treeService TreeService
|
||||
features FeatureSettings
|
||||
}
|
||||
|
||||
Config struct {
|
||||
|
@ -63,6 +69,7 @@ type (
|
|||
AnonKey AnonymousKey
|
||||
Resolver BucketResolver
|
||||
TreeService TreeService
|
||||
Features FeatureSettings
|
||||
}
|
||||
|
||||
// AnonymousKey contains data for anonymous requests.
|
||||
|
@ -301,6 +308,7 @@ func NewLayer(log *zap.Logger, frostFS FrostFS, config *Config) Client {
|
|||
resolver: config.Resolver,
|
||||
cache: NewCache(config.Caches),
|
||||
treeService: config.TreeService,
|
||||
features: config.Features,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -555,7 +563,7 @@ func (n *layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
|
|||
return nil, err
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug("get object",
|
||||
n.reqLogger(ctx).Debug(logs.GetObject,
|
||||
zap.Stringer("cid", p.BktInfo.CID),
|
||||
zap.Stringer("oid", objInfo.ObjectInfo.ID))
|
||||
|
||||
|
@ -686,7 +694,7 @@ func (n *layer) handleNotFoundError(bkt *data.BucketInfo, obj *VersionedObject)
|
|||
|
||||
func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketInfo, obj *VersionedObject, nodeID uint64) *VersionedObject {
|
||||
if client.IsErrObjectAlreadyRemoved(obj.Error) {
|
||||
n.reqLogger(ctx).Debug("object already removed",
|
||||
n.reqLogger(ctx).Debug(logs.ObjectAlreadyRemoved,
|
||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID))
|
||||
|
||||
obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeID)
|
||||
|
@ -698,7 +706,7 @@ func (n *layer) handleObjectDeleteErrors(ctx context.Context, bkt *data.BucketIn
|
|||
}
|
||||
|
||||
if client.IsErrObjectNotFound(obj.Error) {
|
||||
n.reqLogger(ctx).Debug("object not found",
|
||||
n.reqLogger(ctx).Debug(logs.ObjectNotFound,
|
||||
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID))
|
||||
|
||||
obj.Error = nil
|
||||
|
@ -776,7 +784,7 @@ func (n *layer) ResolveBucket(ctx context.Context, name string) (cid.ID, error)
|
|||
return cid.ID{}, err
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Info("resolve bucket", zap.Stringer("cid", cnrID))
|
||||
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID))
|
||||
}
|
||||
|
||||
return cnrID, nil
|
||||
|
|
|
@ -16,6 +16,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"github.com/minio/sio"
|
||||
|
@ -202,7 +203,7 @@ func (n *layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
|
|||
func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
|
||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||
n.reqLogger(ctx).Warn("mismatched obj encryptionInfo", zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
|
||||
}
|
||||
|
||||
|
@ -237,7 +238,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
size = decSize
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug("upload part",
|
||||
n.reqLogger(ctx).Debug(logs.UploadPart,
|
||||
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
||||
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
||||
|
||||
|
@ -258,7 +259,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
}
|
||||
if !oldPartIDNotFound {
|
||||
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
|
||||
n.reqLogger(ctx).Error("couldn't delete old part object", zap.Error(err),
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
|
||||
zap.String("cid", bktInfo.CID.EncodeToString()),
|
||||
zap.String("oid", oldPartID.EncodeToString()))
|
||||
}
|
||||
|
@ -288,10 +289,16 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
|
|||
}
|
||||
|
||||
size := p.SrcObjInfo.Size
|
||||
srcObjectSize := p.SrcObjInfo.Size
|
||||
|
||||
if objSize, err := GetObjectSize(p.SrcObjInfo); err == nil {
|
||||
srcObjectSize = objSize
|
||||
}
|
||||
|
||||
if p.Range != nil {
|
||||
size = p.Range.End - p.Range.Start + 1
|
||||
if p.Range.End > p.SrcObjInfo.Size {
|
||||
return nil, fmt.Errorf("%w: %d-%d/%d", s3errors.GetAPIError(s3errors.ErrInvalidCopyPartRangeSource), p.Range.Start, p.Range.End, p.SrcObjInfo.Size)
|
||||
if p.Range.End > srcObjectSize {
|
||||
return nil, fmt.Errorf("%w: %d-%d/%d", s3errors.GetAPIError(s3errors.ErrInvalidCopyPartRangeSource), p.Range.Start, p.Range.End, srcObjectSize)
|
||||
}
|
||||
}
|
||||
if size > UploadMaxSize {
|
||||
|
@ -412,7 +419,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
|||
CopiesNumbers: multipartInfo.CopiesNumbers,
|
||||
})
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Error("could not put a completed object (multipart upload)",
|
||||
n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject,
|
||||
zap.String("uploadID", p.Info.UploadID),
|
||||
zap.String("uploadKey", p.Info.Key),
|
||||
zap.Error(err))
|
||||
|
@ -424,7 +431,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
|
|||
addr.SetContainer(p.Info.Bkt.CID)
|
||||
for _, partInfo := range partsInfo {
|
||||
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
|
||||
n.reqLogger(ctx).Warn("could not delete upload part",
|
||||
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
|
||||
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
|
||||
zap.Error(err))
|
||||
}
|
||||
|
@ -503,7 +510,7 @@ func (n *layer) AbortMultipartUpload(ctx context.Context, p *UploadInfoParams) e
|
|||
|
||||
for _, info := range parts {
|
||||
if err = n.objectDelete(ctx, p.Bkt, info.OID); err != nil {
|
||||
n.reqLogger(ctx).Warn("couldn't delete part", zap.String("cid", p.Bkt.CID.EncodeToString()),
|
||||
n.reqLogger(ctx).Warn(logs.CouldntDeletePart, zap.String("cid", p.Bkt.CID.EncodeToString()),
|
||||
zap.String("oid", info.OID.EncodeToString()), zap.Int("part number", info.Number), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
@ -520,7 +527,7 @@ func (n *layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
|
|||
|
||||
encInfo := FormEncryptionInfo(multipartInfo.Meta)
|
||||
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
|
||||
n.reqLogger(ctx).Warn("mismatched obj encryptionInfo", zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrInvalidEncryptionParameters)
|
||||
}
|
||||
|
||||
|
@ -584,7 +591,7 @@ func (n *layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
|
|||
oids[i] = part.OID.EncodeToString()
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug("part details",
|
||||
n.reqLogger(ctx).Debug(logs.PartDetails,
|
||||
zap.Stringer("cid", p.Bkt.CID),
|
||||
zap.String("upload id", p.UploadID),
|
||||
zap.Ints("part numbers", partsNumbers),
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -46,7 +47,7 @@ func (n *layer) PutBucketNotificationConfiguration(ctx context.Context, p *PutBu
|
|||
|
||||
if !objIDToDeleteNotFound {
|
||||
if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
|
||||
n.reqLogger(ctx).Error("couldn't delete notification configuration object", zap.Error(err),
|
||||
n.reqLogger(ctx).Error(logs.CouldntDeleteNotificationConfigurationObject, zap.Error(err),
|
||||
zap.String("cid", p.BktInfo.CID.EncodeToString()),
|
||||
zap.String("oid", objIDToDelete.EncodeToString()))
|
||||
}
|
||||
|
|
|
@ -19,6 +19,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
|
@ -291,7 +292,7 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
|||
return nil, err
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug("put object", zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
||||
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
||||
|
||||
newVersion := &data.NodeVersion{
|
||||
BaseNodeVersion: data.BaseNodeVersion{
|
||||
|
@ -458,6 +459,7 @@ func (n *layer) objectDelete(ctx context.Context, bktInfo *data.BucketInfo, idOb
|
|||
// Returns object ID and payload sha256 hash.
|
||||
func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktInfo *data.BucketInfo) (uint64, oid.ID, []byte, error) {
|
||||
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
|
||||
prm.ClientCut = n.features.ClientCut()
|
||||
var size uint64
|
||||
hash := sha256.New()
|
||||
prm.Payload = wrapReader(prm.Payload, 64*1024, func(buf []byte) {
|
||||
|
@ -467,7 +469,7 @@ func (n *layer) objectPutAndHash(ctx context.Context, prm PrmObjectCreate, bktIn
|
|||
id, err := n.frostFS.CreateObject(ctx, prm)
|
||||
if err != nil {
|
||||
if _, errDiscard := io.Copy(io.Discard, prm.Payload); errDiscard != nil {
|
||||
n.reqLogger(ctx).Warn("failed to discard put payload, probably goroutine leaks", zap.Error(errDiscard))
|
||||
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
|
||||
}
|
||||
|
||||
return 0, oid.ID{}, nil, err
|
||||
|
@ -656,7 +658,7 @@ func (n *layer) initWorkerPool(ctx context.Context, size int, p allObjectParams,
|
|||
})
|
||||
if err != nil {
|
||||
wg.Done()
|
||||
reqLog.Warn("failed to submit task to pool", zap.Error(err))
|
||||
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
|
||||
}
|
||||
}(node)
|
||||
}
|
||||
|
@ -798,7 +800,7 @@ func (n *layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo
|
|||
|
||||
meta, err := n.objectHead(ctx, bktInfo, node.OID)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Warn("could not fetch object meta", zap.Error(err))
|
||||
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
|
@ -180,7 +181,7 @@ func (n *layer) getNodeVersion(ctx context.Context, objVersion *ObjectVersion) (
|
|||
}
|
||||
|
||||
if err == nil && version != nil && !version.IsDeleteMarker() {
|
||||
n.reqLogger(ctx).Debug("get tree node",
|
||||
n.reqLogger(ctx).Debug(logs.GetTreeNode,
|
||||
zap.Stringer("cid", objVersion.BktInfo.CID), zap.Stringer("oid", version.OID))
|
||||
}
|
||||
|
||||
|
|
|
@ -97,6 +97,23 @@ func objectInfoFromMeta(bkt *data.BucketInfo, meta *object.Object) *data.ObjectI
|
|||
}
|
||||
}
|
||||
|
||||
func GetObjectSize(objInfo *data.ObjectInfo) (uint64, error) {
|
||||
var err error
|
||||
fullSize := objInfo.Size
|
||||
|
||||
if objInfo.Headers[AttributeDecryptedSize] != "" {
|
||||
if fullSize, err = strconv.ParseUint(objInfo.Headers[AttributeDecryptedSize], 10, 64); err != nil {
|
||||
return 0, fmt.Errorf("invalid decrypted size header: %w", err)
|
||||
}
|
||||
} else if objInfo.Headers[MultipartObjectSize] != "" {
|
||||
if fullSize, err = strconv.ParseUint(objInfo.Headers[MultipartObjectSize], 10, 64); err != nil {
|
||||
return 0, fmt.Errorf("invalid multipart size header: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return fullSize, nil
|
||||
}
|
||||
|
||||
func FormEncryptionInfo(headers map[string]string) encryption.ObjectEncryption {
|
||||
algorithm := headers[AttributeEncryptionAlgorithm]
|
||||
return encryption.ObjectEncryption{
|
||||
|
|
|
@ -170,6 +170,7 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext {
|
|||
Caches: config,
|
||||
AnonKey: AnonymousKey{Key: key},
|
||||
TreeService: NewTreeService(),
|
||||
Features: &FeatureSettingsMock{},
|
||||
}
|
||||
|
||||
return &testContext{
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -15,9 +16,9 @@ func Auth(center auth.Center, log *zap.Logger) Func {
|
|||
box, err := center.Authenticate(r)
|
||||
if err != nil {
|
||||
if err == auth.ErrNoAuthorizationHeader {
|
||||
reqLogOrDefault(ctx, log).Debug("couldn't receive access box for gate key, random key will be used")
|
||||
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed)
|
||||
} else {
|
||||
reqLogOrDefault(ctx, log).Error("failed to pass authentication", zap.Error(err))
|
||||
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err))
|
||||
if _, ok := err.(errors.Error); !ok {
|
||||
err = errors.GetAPIError(errors.ErrAccessDenied)
|
||||
}
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"go.uber.org/zap"
|
||||
|
@ -141,7 +142,7 @@ func resolveCID(log *zap.Logger, resolveBucket BucketResolveFunc) cidResolveFunc
|
|||
|
||||
bktInfo, err := resolveBucket(ctx, reqInfo.BucketName)
|
||||
if err != nil {
|
||||
reqLogOrDefault(ctx, log).Debug("failed to resolve CID", zap.Error(err))
|
||||
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err))
|
||||
return ""
|
||||
}
|
||||
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
|
@ -207,7 +208,7 @@ func Request(log *zap.Logger) Func {
|
|||
reqLogger := log.With(zap.String("request_id", reqInfo.RequestID))
|
||||
r = r.WithContext(SetReqLogger(r.Context(), reqLogger))
|
||||
|
||||
reqLogger.Info("request start", zap.String("host", r.Host),
|
||||
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
|
||||
zap.String("remote_host", reqInfo.RemoteHost))
|
||||
|
||||
// continue execution
|
||||
|
@ -242,14 +243,7 @@ func AddObjectName(l *zap.Logger) Func {
|
|||
|
||||
rctx := chi.RouteContext(ctx)
|
||||
// trim leading slash (always present)
|
||||
obj := rctx.RoutePath[1:]
|
||||
|
||||
object, err := url.PathUnescape(obj)
|
||||
if err != nil {
|
||||
object = obj
|
||||
}
|
||||
|
||||
reqInfo.ObjectName = object
|
||||
reqInfo.ObjectName = rctx.RoutePath[1:]
|
||||
|
||||
reqLogger := reqLogOrDefault(ctx, l)
|
||||
r = r.WithContext(SetReqLogger(ctx, reqLogger.With(zap.String("object", reqInfo.ObjectName))))
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -319,7 +320,7 @@ func LogSuccessResponse(l *zap.Logger) Func {
|
|||
reqLogger := reqLogOrDefault(ctx, l)
|
||||
reqInfo := GetReqInfo(ctx)
|
||||
|
||||
reqLogger.Info("request end",
|
||||
reqLogger.Info(logs.RequestEnd,
|
||||
zap.String("method", reqInfo.API),
|
||||
zap.String("bucket", reqInfo.BucketName),
|
||||
zap.String("object", reqInfo.ObjectName),
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -168,9 +169,9 @@ func (c *Controller) Listen(ctx context.Context) {
|
|||
select {
|
||||
case msg := <-stream.ch:
|
||||
if err := stream.h.HandleMessage(ctx, msg); err != nil {
|
||||
c.logger.Error("could not handle message", zap.Error(err))
|
||||
c.logger.Error(logs.CouldNotHandleMessage, zap.Error(err))
|
||||
} else if err = msg.Ack(); err != nil {
|
||||
c.logger.Error("could not ACK message", zap.Error(err))
|
||||
c.logger.Error(logs.CouldNotACKMessage, zap.Error(err))
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
@ -187,10 +188,10 @@ func (c *Controller) SendNotifications(topics map[string]string, p *handler.Send
|
|||
event.Records[0].S3.ConfigurationID = id
|
||||
msg, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
c.logger.Error("couldn't marshal an event", zap.String("subject", topic), zap.Error(err))
|
||||
c.logger.Error(logs.CouldntMarshalAnEvent, zap.String("subject", topic), zap.Error(err))
|
||||
}
|
||||
if err = c.publish(topic, msg); err != nil {
|
||||
c.logger.Error("couldn't send an event to topic", zap.String("subject", topic), zap.Error(err))
|
||||
c.logger.Error(logs.CouldntSendAnEventToTopic, zap.String("subject", topic), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
s3middleware "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
@ -135,7 +136,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
|
|||
})
|
||||
|
||||
if log := s3middleware.GetReqLog(ctx); log != nil {
|
||||
log.Error("request unmatched", zap.String("method", reqInfo.API))
|
||||
log.Error(logs.RequestUnmatched, zap.String("method", reqInfo.API))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -61,6 +61,51 @@ func TestRouterObjectWithSlashes(t *testing.T) {
|
|||
require.Equal(t, objName, resp.ReqInfo.ObjectName)
|
||||
}
|
||||
|
||||
func TestRouterObjectEscaping(t *testing.T) {
|
||||
chiRouter := prepareRouter(t)
|
||||
|
||||
bktName := "dkirillov"
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
expectedObjName string
|
||||
objName string
|
||||
}{
|
||||
{
|
||||
name: "simple",
|
||||
expectedObjName: "object",
|
||||
objName: "object",
|
||||
},
|
||||
{
|
||||
name: "with slashes",
|
||||
expectedObjName: "fix/object",
|
||||
objName: "fix/object",
|
||||
},
|
||||
{
|
||||
name: "with percentage",
|
||||
expectedObjName: "fix/object%ac",
|
||||
objName: "fix/object%ac",
|
||||
},
|
||||
{
|
||||
name: "with percentage escaped",
|
||||
expectedObjName: "fix/object%ac",
|
||||
objName: "fix/object%25ac",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
target := fmt.Sprintf("/%s/%s", bktName, tc.objName)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodPut, target, nil)
|
||||
|
||||
chiRouter.ServeHTTP(w, r)
|
||||
resp := readResponse(t, w)
|
||||
require.Equal(t, "PutObject", resp.Method)
|
||||
require.Equal(t, tc.expectedObjName, resp.ReqInfo.ObjectName)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func prepareRouter(t *testing.T) *chi.Mux {
|
||||
throttleOps := middleware.ThrottleOpts{
|
||||
Limit: 10,
|
||||
|
|
|
@ -16,6 +16,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
|
||||
|
@ -168,11 +169,11 @@ type (
|
|||
|
||||
func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) {
|
||||
if !opts.ID.Equals(cid.ID{}) {
|
||||
a.log.Info("check container", zap.Stringer("cid", opts.ID))
|
||||
a.log.Info(logs.CheckContainer, zap.Stringer("cid", opts.ID))
|
||||
return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID)
|
||||
}
|
||||
|
||||
a.log.Info("create container",
|
||||
a.log.Info(logs.CreateContainer,
|
||||
zap.String("friendly_name", opts.FriendlyName),
|
||||
zap.String("placement_policy", opts.PlacementPolicy))
|
||||
|
||||
|
@ -267,7 +268,7 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
|
|||
return fmt.Errorf("check container: %w", err)
|
||||
}
|
||||
|
||||
a.log.Info("store bearer token into FrostFS",
|
||||
a.log.Info(logs.StoreBearerTokenIntoFrostFS,
|
||||
zap.Stringer("owner_tkn", idOwner))
|
||||
|
||||
creds := tokens.New(a.frostFS, secrets.EphemeralKey, cache.DefaultAccessBoxConfig(a.log))
|
||||
|
@ -345,7 +346,7 @@ func (a *Agent) UpdateSecret(ctx context.Context, w io.Writer, options *UpdateSe
|
|||
|
||||
var idOwner user.ID
|
||||
user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey)
|
||||
a.log.Info("update access cred object into FrostFS",
|
||||
a.log.Info(logs.UpdateAccessCredObjectIntoFrostFS,
|
||||
zap.Stringer("owner_tkn", idOwner))
|
||||
|
||||
oldAddr := options.Address
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/spf13/viper"
|
||||
|
@ -27,7 +28,7 @@ type PoolConfig struct {
|
|||
}
|
||||
|
||||
func createFrostFS(ctx context.Context, log *zap.Logger, cfg PoolConfig) (authmate.FrostFS, error) {
|
||||
log.Debug("prepare connection pool")
|
||||
log.Debug(logs.PrepareConnectionPool)
|
||||
|
||||
var prm pool.InitParameters
|
||||
prm.SetKey(&cfg.Key.PrivateKey)
|
||||
|
|
178
cmd/s3-gw/app.go
178
cmd/s3-gw/app.go
|
@ -3,7 +3,6 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
|
@ -24,6 +23,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/services"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/xml"
|
||||
|
@ -71,6 +71,7 @@ type (
|
|||
xmlDecoder *xml.DecoderProvider
|
||||
maxClient maxClientsConfig
|
||||
bypassContentEncodingInChunks atomic.Bool
|
||||
clientCut atomic.Bool
|
||||
}
|
||||
|
||||
maxClientsConfig struct {
|
||||
|
@ -130,7 +131,7 @@ func (a *App) initLayer(ctx context.Context) {
|
|||
// prepare random key for anonymous requests
|
||||
randomKey, err := keys.NewPrivateKey()
|
||||
if err != nil {
|
||||
a.log.Fatal("couldn't generate random key", zap.Error(err))
|
||||
a.log.Fatal(logs.CouldntGenerateRandomKey, zap.Error(err))
|
||||
}
|
||||
|
||||
var gateOwner user.ID
|
||||
|
@ -144,6 +145,7 @@ func (a *App) initLayer(ctx context.Context) {
|
|||
GateOwner: gateOwner,
|
||||
Resolver: a.bucketResolver,
|
||||
TreeService: tree.NewTree(services.NewPoolWrapper(a.treePool), a.log),
|
||||
Features: a.settings,
|
||||
}
|
||||
|
||||
// prepare object layer
|
||||
|
@ -153,29 +155,25 @@ func (a *App) initLayer(ctx context.Context) {
|
|||
nopts := getNotificationsOptions(a.cfg, a.log)
|
||||
a.nc, err = notifications.NewController(nopts, a.log)
|
||||
if err != nil {
|
||||
a.log.Fatal("failed to enable notifications", zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToEnableNotifications, zap.Error(err))
|
||||
}
|
||||
|
||||
if err = a.obj.Initialize(ctx, a.nc); err != nil {
|
||||
a.log.Fatal("couldn't initialize layer", zap.Error(err))
|
||||
a.log.Fatal(logs.CouldntInitializeLayer, zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
|
||||
policies, err := newPlacementPolicy(log.logger, v)
|
||||
if err != nil {
|
||||
log.logger.Fatal("failed to create new policy mapping", zap.Error(err))
|
||||
}
|
||||
|
||||
settings := &appSettings{
|
||||
logLevel: log.lvl,
|
||||
policies: policies,
|
||||
policies: newPlacementPolicy(log.logger, v),
|
||||
xmlDecoder: xml.NewDecoderProvider(v.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload)),
|
||||
maxClient: newMaxClients(v),
|
||||
}
|
||||
|
||||
settings.setBypassContentEncodingInChunks(v.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
||||
settings.setClientCut(v.GetBool(cfgClientCut))
|
||||
|
||||
return settings
|
||||
}
|
||||
|
@ -188,6 +186,14 @@ func (s *appSettings) setBypassContentEncodingInChunks(bypass bool) {
|
|||
s.bypassContentEncodingInChunks.Store(bypass)
|
||||
}
|
||||
|
||||
func (s *appSettings) ClientCut() bool {
|
||||
return s.clientCut.Load()
|
||||
}
|
||||
|
||||
func (s *appSettings) setClientCut(clientCut bool) {
|
||||
s.clientCut.Store(clientCut)
|
||||
}
|
||||
|
||||
func (a *App) initAPI(ctx context.Context) {
|
||||
a.initLayer(ctx)
|
||||
a.initHandler()
|
||||
|
@ -202,7 +208,7 @@ func (a *App) initResolver() {
|
|||
var err error
|
||||
a.bucketResolver, err = resolver.NewBucketResolver(a.getResolverConfig())
|
||||
if err != nil {
|
||||
a.log.Fatal("failed to create resolver", zap.Error(err))
|
||||
a.log.Fatal(logs.FailedToCreateResolver, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -215,11 +221,11 @@ func (a *App) getResolverConfig() ([]string, *resolver.Config) {
|
|||
order := a.cfg.GetStringSlice(cfgResolveOrder)
|
||||
if resolveCfg.RPCAddress == "" {
|
||||
order = remove(order, resolver.NNSResolver)
|
||||
a.log.Warn(fmt.Sprintf("resolver '%s' won't be used since '%s' isn't provided", resolver.NNSResolver, cfgRPCEndpoint))
|
||||
a.log.Warn(logs.ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided)
|
||||
}
|
||||
|
||||
if len(order) == 0 {
|
||||
a.log.Info("container resolver will be disabled because of resolvers 'resolver_order' is empty")
|
||||
a.log.Info(logs.ContainerResolverWillBeDisabled)
|
||||
}
|
||||
|
||||
return order, resolveCfg
|
||||
|
@ -240,10 +246,10 @@ func (a *App) initTracing(ctx context.Context) {
|
|||
}
|
||||
updated, err := tracing.Setup(ctx, cfg)
|
||||
if err != nil {
|
||||
a.log.Warn("failed to initialize tracing", zap.Error(err))
|
||||
a.log.Warn(logs.FailedToInitializeTracing, zap.Error(err))
|
||||
}
|
||||
if updated {
|
||||
a.log.Info("tracing config updated")
|
||||
a.log.Info(logs.TracingConfigUpdated)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -253,7 +259,7 @@ func (a *App) shutdownTracing() {
|
|||
defer cancel()
|
||||
|
||||
if err := tracing.Shutdown(shdnCtx); err != nil {
|
||||
a.log.Warn("failed to shutdown tracing", zap.Error(err))
|
||||
a.log.Warn(logs.FailedToShutdownTracing, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -274,12 +280,12 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.
|
|||
password := wallet.GetPassword(cfg, cfgWalletPassphrase)
|
||||
key, err := wallet.GetKeyFromPath(cfg.GetString(cfgWalletPath), cfg.GetString(cfgWalletAddress), password)
|
||||
if err != nil {
|
||||
logger.Fatal("could not load FrostFS private key", zap.Error(err))
|
||||
logger.Fatal(logs.CouldNotLoadFrostFSPrivateKey, zap.Error(err))
|
||||
}
|
||||
|
||||
prm.SetKey(&key.PrivateKey)
|
||||
prmTree.SetKey(key)
|
||||
logger.Info("using credentials", zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
|
||||
logger.Info(logs.UsingCredentials, zap.String("FrostFS", hex.EncodeToString(key.PublicKey().Bytes())))
|
||||
|
||||
for _, peer := range fetchPeers(logger, cfg) {
|
||||
prm.AddNode(peer)
|
||||
|
@ -322,34 +328,28 @@ func getPools(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.
|
|||
|
||||
p, err := pool.NewPool(prm)
|
||||
if err != nil {
|
||||
logger.Fatal("failed to create connection pool", zap.Error(err))
|
||||
logger.Fatal(logs.FailedToCreateConnectionPool, zap.Error(err))
|
||||
}
|
||||
|
||||
if err = p.Dial(ctx); err != nil {
|
||||
logger.Fatal("failed to dial connection pool", zap.Error(err))
|
||||
logger.Fatal(logs.FailedToDialConnectionPool, zap.Error(err))
|
||||
}
|
||||
|
||||
treePool, err := treepool.NewPool(prmTree)
|
||||
if err != nil {
|
||||
logger.Fatal("failed to create tree pool", zap.Error(err))
|
||||
logger.Fatal(logs.FailedToCreateTreePool, zap.Error(err))
|
||||
}
|
||||
if err = treePool.Dial(ctx); err != nil {
|
||||
logger.Fatal("failed to dial tree pool", zap.Error(err))
|
||||
logger.Fatal(logs.FailedToDialTreePool, zap.Error(err))
|
||||
}
|
||||
|
||||
return p, treePool, key
|
||||
}
|
||||
|
||||
func newPlacementPolicy(l *zap.Logger, v *viper.Viper) (*placementPolicy, error) {
|
||||
policies := &placementPolicy{
|
||||
regionMap: make(map[string]netmap.PlacementPolicy),
|
||||
defaultCopiesNumbers: []uint32{handler.DefaultCopiesNumber},
|
||||
}
|
||||
|
||||
policies.updateCopiesNumbers(l, v)
|
||||
policies.updateDefaultCopiesNumbers(l, v)
|
||||
|
||||
return policies, policies.updatePolicy(v)
|
||||
func newPlacementPolicy(l *zap.Logger, v *viper.Viper) *placementPolicy {
|
||||
var policies placementPolicy
|
||||
policies.update(l, v)
|
||||
return &policies
|
||||
}
|
||||
|
||||
func (p *placementPolicy) DefaultPlacementPolicy() netmap.PlacementPolicy {
|
||||
|
@ -381,56 +381,18 @@ func (p *placementPolicy) DefaultCopiesNumbers() []uint32 {
|
|||
}
|
||||
|
||||
func (p *placementPolicy) update(l *zap.Logger, v *viper.Viper) {
|
||||
if err := p.updatePolicy(v); err != nil {
|
||||
l.Warn("policies won't be updated", zap.Error(err))
|
||||
}
|
||||
|
||||
p.updateCopiesNumbers(l, v)
|
||||
p.updateDefaultCopiesNumbers(l, v)
|
||||
}
|
||||
|
||||
func (p *placementPolicy) updatePolicy(v *viper.Viper) error {
|
||||
defaultPlacementPolicy, err := fetchDefaultPolicy(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
regionMap, err := fetchRegionMappingPolicies(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaultPolicy := fetchDefaultPolicy(l, v)
|
||||
regionMap := fetchRegionMappingPolicies(l, v)
|
||||
defaultCopies := fetchDefaultCopiesNumbers(l, v)
|
||||
copiesNumbers := fetchCopiesNumbers(l, v)
|
||||
|
||||
p.mu.Lock()
|
||||
p.defaultPolicy = defaultPlacementPolicy
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.defaultPolicy = defaultPolicy
|
||||
p.regionMap = regionMap
|
||||
p.mu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *placementPolicy) updateCopiesNumbers(l *zap.Logger, v *viper.Viper) {
|
||||
if newCopiesNumbers, err := fetchCopiesNumbers(l, v); err != nil {
|
||||
l.Warn("copies numbers won't be updated", zap.Error(err))
|
||||
} else {
|
||||
p.mu.Lock()
|
||||
p.copiesNumbers = newCopiesNumbers
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *placementPolicy) updateDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) {
|
||||
configuredValues, err := fetchDefaultCopiesNumbers(v)
|
||||
|
||||
if err == nil {
|
||||
p.mu.Lock()
|
||||
p.defaultCopiesNumbers = configuredValues
|
||||
p.mu.Unlock()
|
||||
l.Info("default copies numbers", zap.Uint32s("vector", p.defaultCopiesNumbers))
|
||||
return
|
||||
}
|
||||
|
||||
l.Error("cannot parse default copies numbers", zap.Error(err))
|
||||
l.Warn("default copies numbers won't be updated", zap.Uint32s("current value", p.DefaultCopiesNumbers()))
|
||||
p.defaultCopiesNumbers = defaultCopies
|
||||
p.copiesNumbers = copiesNumbers
|
||||
}
|
||||
|
||||
func remove(list []string, element string) []string {
|
||||
|
@ -448,7 +410,7 @@ func remove(list []string, element string) []string {
|
|||
// version (version.Version) and its name (frostfs-s3-gw). At the end, it writes
|
||||
// about the stop to the log.
|
||||
func (a *App) Wait() {
|
||||
a.log.Info("application started",
|
||||
a.log.Info(logs.ApplicationStarted,
|
||||
zap.String("name", "frostfs-s3-gw"),
|
||||
zap.String("version", version.Version),
|
||||
)
|
||||
|
@ -458,7 +420,7 @@ func (a *App) Wait() {
|
|||
|
||||
<-a.webDone // wait for web-server to be stopped
|
||||
|
||||
a.log.Info("application finished")
|
||||
a.log.Info(logs.ApplicationFinished)
|
||||
}
|
||||
|
||||
func (a *App) setHealthStatus() {
|
||||
|
@ -469,7 +431,7 @@ func (a *App) setHealthStatus() {
|
|||
func (a *App) Serve(ctx context.Context) {
|
||||
// Attach S3 API:
|
||||
domains := a.cfg.GetStringSlice(cfgListenDomains)
|
||||
a.log.Info("fetch domains, prepare to use API", zap.Strings("domains", domains))
|
||||
a.log.Info(logs.FetchDomainsPrepareToUseAPI, zap.Strings("domains", domains))
|
||||
|
||||
throttleOps := middleware.ThrottleOpts{
|
||||
Limit: a.settings.maxClient.count,
|
||||
|
@ -488,10 +450,10 @@ func (a *App) Serve(ctx context.Context) {
|
|||
|
||||
for i := range a.servers {
|
||||
go func(i int) {
|
||||
a.log.Info("starting server", zap.String("address", a.servers[i].Address()))
|
||||
a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address()))
|
||||
|
||||
if err := srv.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed {
|
||||
a.log.Fatal("listen and serve", zap.Error(err))
|
||||
a.log.Fatal(logs.ListenAndServe, zap.Error(err))
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
@ -512,7 +474,7 @@ LOOP:
|
|||
ctx, cancel := shutdownContext()
|
||||
defer cancel()
|
||||
|
||||
a.log.Info("stopping server", zap.Error(srv.Shutdown(ctx)))
|
||||
a.log.Info(logs.StoppingServer, zap.Error(srv.Shutdown(ctx)))
|
||||
|
||||
a.metrics.Shutdown()
|
||||
a.stopServices()
|
||||
|
@ -526,23 +488,23 @@ func shutdownContext() (context.Context, context.CancelFunc) {
|
|||
}
|
||||
|
||||
func (a *App) configReload(ctx context.Context) {
|
||||
a.log.Info("SIGHUP config reload started")
|
||||
a.log.Info(logs.SIGHUPConfigReloadStarted)
|
||||
|
||||
if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
|
||||
a.log.Warn("failed to reload config because it's missed")
|
||||
a.log.Warn(logs.FailedToReloadConfigBecauseItsMissed)
|
||||
return
|
||||
}
|
||||
if err := readInConfig(a.cfg); err != nil {
|
||||
a.log.Warn("failed to reload config", zap.Error(err))
|
||||
a.log.Warn(logs.FailedToReloadConfig, zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
if err := a.bucketResolver.UpdateResolvers(a.getResolverConfig()); err != nil {
|
||||
a.log.Warn("failed to reload resolvers", zap.Error(err))
|
||||
a.log.Warn(logs.FailedToReloadResolvers, zap.Error(err))
|
||||
}
|
||||
|
||||
if err := a.updateServers(); err != nil {
|
||||
a.log.Warn("failed to reload server parameters", zap.Error(err))
|
||||
a.log.Warn(logs.FailedToReloadServerParameters, zap.Error(err))
|
||||
}
|
||||
|
||||
a.stopServices()
|
||||
|
@ -554,12 +516,12 @@ func (a *App) configReload(ctx context.Context) {
|
|||
a.initTracing(ctx)
|
||||
a.setHealthStatus()
|
||||
|
||||
a.log.Info("SIGHUP config reload completed")
|
||||
a.log.Info(logs.SIGHUPConfigReloadCompleted)
|
||||
}
|
||||
|
||||
func (a *App) updateSettings() {
|
||||
if lvl, err := getLogLevel(a.cfg); err != nil {
|
||||
a.log.Warn("log level won't be updated", zap.Error(err))
|
||||
a.log.Warn(logs.LogLevelWontBeUpdated, zap.Error(err))
|
||||
} else {
|
||||
a.settings.logLevel.SetLevel(lvl)
|
||||
}
|
||||
|
@ -568,6 +530,7 @@ func (a *App) updateSettings() {
|
|||
|
||||
a.settings.xmlDecoder.UseDefaultNamespaceForCompleteMultipart(a.cfg.GetBool(cfgKludgeUseDefaultXMLNSForCompleteMultipartUpload))
|
||||
a.settings.setBypassContentEncodingInChunks(a.cfg.GetBool(cfgKludgeBypassContentEncodingCheckInChunks))
|
||||
a.settings.setClientCut(a.cfg.GetBool(cfgClientCut))
|
||||
}
|
||||
|
||||
func (a *App) startServices() {
|
||||
|
@ -593,16 +556,16 @@ func (a *App) initServers(ctx context.Context) {
|
|||
}
|
||||
srv, err := newServer(ctx, serverInfo)
|
||||
if err != nil {
|
||||
a.log.Warn("failed to add server", append(fields, zap.Error(err))...)
|
||||
a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...)
|
||||
continue
|
||||
}
|
||||
|
||||
a.servers = append(a.servers, srv)
|
||||
a.log.Info("add server", fields...)
|
||||
a.log.Info(logs.AddServer, fields...)
|
||||
}
|
||||
|
||||
if len(a.servers) == 0 {
|
||||
a.log.Fatal("no healthy servers")
|
||||
a.log.Fatal(logs.NoHealthyServers)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -713,29 +676,6 @@ func (a *App) initHandler() {
|
|||
var err error
|
||||
a.api, err = handler.New(a.log, a.obj, a.nc, cfg)
|
||||
if err != nil {
|
||||
a.log.Fatal("could not initialize API handler", zap.Error(err))
|
||||
a.log.Fatal(logs.CouldNotInitializeAPIHandler, zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func readRegionMap(filePath string) (map[string]string, error) {
|
||||
regionMap := make(map[string]string)
|
||||
|
||||
if filePath == "" {
|
||||
return regionMap, nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("coudln't read file '%s'", filePath)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(data, ®ionMap); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal policies: %w", err)
|
||||
}
|
||||
|
||||
if _, ok := regionMap[api.DefaultLocationConstraint]; ok {
|
||||
return nil, fmt.Errorf("config overrides %s location constraint", api.DefaultLocationConstraint)
|
||||
}
|
||||
|
||||
return regionMap, nil
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
|
@ -10,9 +11,11 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/notifications"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
|
||||
|
@ -30,11 +33,14 @@ const (
|
|||
defaultShutdownTimeout = 15 * time.Second
|
||||
|
||||
defaultPoolErrorThreshold uint32 = 100
|
||||
defaultPlacementPolicy = "REP 3"
|
||||
|
||||
defaultMaxClientsCount = 100
|
||||
defaultMaxClientsDeadline = time.Second * 30
|
||||
)
|
||||
|
||||
var defaultCopiesNumbers = []uint32{0}
|
||||
|
||||
const ( // Settings.
|
||||
// Logger.
|
||||
cfgLoggerLevel = "logger.level"
|
||||
|
@ -138,6 +144,8 @@ const ( // Settings.
|
|||
// Configuration of parameters of requests to FrostFS.
|
||||
// Number of the object copies to consider PUT to FrostFS successful.
|
||||
cfgSetCopiesNumber = "frostfs.set_copies_number"
|
||||
// Enabling client side object preparing for PUT operations.
|
||||
cfgClientCut = "frostfs.client_cut"
|
||||
|
||||
// List of allowed AccessKeyID prefixes.
|
||||
cfgAllowedAccessKeyIDPrefixes = "allowed_access_key_id_prefixes"
|
||||
|
@ -222,24 +230,30 @@ func fetchMaxClientsDeadline(cfg *viper.Viper) time.Duration {
|
|||
return maxClientsDeadline
|
||||
}
|
||||
|
||||
func fetchDefaultPolicy(cfg *viper.Viper) (netmap.PlacementPolicy, error) {
|
||||
defaultPolicyStr := handler.DefaultPolicy
|
||||
func fetchDefaultPolicy(l *zap.Logger, cfg *viper.Viper) netmap.PlacementPolicy {
|
||||
var policy netmap.PlacementPolicy
|
||||
|
||||
if cfg.IsSet(cfgPolicyDefault) {
|
||||
defaultPolicyStr = cfg.GetString(cfgPolicyDefault)
|
||||
policyStr := cfg.GetString(cfgPolicyDefault)
|
||||
if err := policy.DecodeString(policyStr); err != nil {
|
||||
l.Warn(logs.FailedToParseDefaultLocationConstraint,
|
||||
zap.String("policy", policyStr), zap.String("default", defaultPlacementPolicy), zap.Error(err))
|
||||
} else {
|
||||
return policy
|
||||
}
|
||||
}
|
||||
|
||||
var defaultPlacementPolicy netmap.PlacementPolicy
|
||||
if err := defaultPlacementPolicy.DecodeString(defaultPolicyStr); err != nil {
|
||||
return netmap.PlacementPolicy{}, fmt.Errorf("parse default policy '%s': %w", defaultPolicyStr, err)
|
||||
if err := policy.DecodeString(defaultPlacementPolicy); err != nil {
|
||||
l.Fatal(logs.FailedToParseDefaultDefaultLocationConstraint, zap.String("policy", defaultPlacementPolicy))
|
||||
}
|
||||
|
||||
return defaultPlacementPolicy, nil
|
||||
return policy
|
||||
}
|
||||
|
||||
func fetchNATSTimeout(cfg *viper.Viper, l *zap.Logger) time.Duration {
|
||||
timeout := cfg.GetDuration(cfgNATSTimeout)
|
||||
if timeout <= 0 {
|
||||
l.Error("invalid lifetime, using default value (in seconds)",
|
||||
l.Error(logs.InvalidLifetimeUsingDefaultValue,
|
||||
zap.String("parameter", cfgNATSTimeout),
|
||||
zap.Duration("value in config", timeout),
|
||||
zap.Duration("default", notifications.DefaultTimeout))
|
||||
|
@ -253,7 +267,7 @@ func fetchCacheLifetime(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultV
|
|||
if v.IsSet(cfgEntry) {
|
||||
lifetime := v.GetDuration(cfgEntry)
|
||||
if lifetime <= 0 {
|
||||
l.Error("invalid lifetime, using default value (in seconds)",
|
||||
l.Error(logs.InvalidLifetimeUsingDefaultValue,
|
||||
zap.String("parameter", cfgEntry),
|
||||
zap.Duration("value in config", lifetime),
|
||||
zap.Duration("default", defaultValue))
|
||||
|
@ -269,7 +283,7 @@ func fetchCacheSize(v *viper.Viper, l *zap.Logger, cfgEntry string, defaultValue
|
|||
if v.IsSet(cfgEntry) {
|
||||
size := v.GetInt(cfgEntry)
|
||||
if size <= 0 {
|
||||
l.Error("invalid cache size, using default value",
|
||||
l.Error(logs.InvalidCacheSizeUsingDefaultValue,
|
||||
zap.String("parameter", cfgEntry),
|
||||
zap.Int("value in config", size),
|
||||
zap.Int("default", defaultValue))
|
||||
|
@ -288,7 +302,7 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
|
|||
defaultMaxAge = cfg.GetInt(cfgDefaultMaxAge)
|
||||
|
||||
if defaultMaxAge <= 0 && defaultMaxAge != -1 {
|
||||
l.Fatal("invalid defaultMaxAge",
|
||||
l.Fatal(logs.InvalidDefaultMaxAge,
|
||||
zap.String("parameter", cfgDefaultMaxAge),
|
||||
zap.String("value in config", strconv.Itoa(defaultMaxAge)))
|
||||
}
|
||||
|
@ -297,14 +311,21 @@ func fetchDefaultMaxAge(cfg *viper.Viper, l *zap.Logger) int {
|
|||
return defaultMaxAge
|
||||
}
|
||||
|
||||
func fetchRegionMappingPolicies(cfg *viper.Viper) (map[string]netmap.PlacementPolicy, error) {
|
||||
regionPolicyMap, err := readRegionMap(cfg.GetString(cfgPolicyRegionMapFile))
|
||||
func fetchRegionMappingPolicies(l *zap.Logger, cfg *viper.Viper) map[string]netmap.PlacementPolicy {
|
||||
filepath := cfg.GetString(cfgPolicyRegionMapFile)
|
||||
regionPolicyMap, err := readRegionMap(filepath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read region map file: %w", err)
|
||||
l.Warn(logs.FailedToReadRegionMapFilePolicies, zap.String("file", filepath), zap.Error(err))
|
||||
return make(map[string]netmap.PlacementPolicy)
|
||||
}
|
||||
|
||||
regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
|
||||
for region, policy := range regionPolicyMap {
|
||||
if region == api.DefaultLocationConstraint {
|
||||
l.Warn(logs.DefaultLocationConstraintCantBeOverriden, zap.String("policy", policy))
|
||||
continue
|
||||
}
|
||||
|
||||
var pp netmap.PlacementPolicy
|
||||
if err = pp.DecodeString(policy); err == nil {
|
||||
regionMap[region] = pp
|
||||
|
@ -316,29 +337,49 @@ func fetchRegionMappingPolicies(cfg *viper.Viper) (map[string]netmap.PlacementPo
|
|||
continue
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("parse region '%s' to policy mapping: %w", region, err)
|
||||
l.Warn(logs.FailedToParseLocationConstraint, zap.String("region", region), zap.String("policy", policy))
|
||||
}
|
||||
|
||||
return regionMap
|
||||
}
|
||||
|
||||
func readRegionMap(filePath string) (map[string]string, error) {
|
||||
regionMap := make(map[string]string)
|
||||
|
||||
if filePath == "" {
|
||||
return regionMap, nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("coudln't read file '%s'", filePath)
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(data, ®ionMap); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal policies: %w", err)
|
||||
}
|
||||
|
||||
return regionMap, nil
|
||||
}
|
||||
|
||||
func fetchDefaultCopiesNumbers(v *viper.Viper) ([]uint32, error) {
|
||||
func fetchDefaultCopiesNumbers(l *zap.Logger, v *viper.Viper) []uint32 {
|
||||
unparsed := v.GetStringSlice(cfgSetCopiesNumber)
|
||||
var result []uint32
|
||||
|
||||
result := make([]uint32, len(unparsed))
|
||||
for i := range unparsed {
|
||||
parsedValue, err := strconv.ParseUint(unparsed[i], 10, 32)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
l.Warn(logs.FailedToParseDefaultCopiesNumbers,
|
||||
zap.Strings("copies numbers", unparsed), zap.Uint32s("default", defaultCopiesNumbers), zap.Error(err))
|
||||
return defaultCopiesNumbers
|
||||
}
|
||||
result = append(result, uint32(parsedValue))
|
||||
result[i] = uint32(parsedValue)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
return result
|
||||
}
|
||||
|
||||
func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) (map[string][]uint32, error) {
|
||||
var copiesNums = make(map[string][]uint32)
|
||||
func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) map[string][]uint32 {
|
||||
copiesNums := make(map[string][]uint32)
|
||||
for i := 0; ; i++ {
|
||||
key := cfgCopiesNumbers + "." + strconv.Itoa(i) + "."
|
||||
constraint := v.GetString(key + "location_constraint")
|
||||
|
@ -352,15 +393,17 @@ func fetchCopiesNumbers(l *zap.Logger, v *viper.Viper) (map[string][]uint32, err
|
|||
for j := range vector {
|
||||
parsedValue, err := strconv.ParseUint(vector[j], 10, 32)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
l.Warn(logs.FailedToParseCopiesNumbers, zap.String("location", constraint),
|
||||
zap.Strings("copies numbers", vector), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
vector32[j] = uint32(parsedValue)
|
||||
}
|
||||
|
||||
copiesNums[constraint] = vector32
|
||||
l.Info("constraint added", zap.String("location", constraint), zap.Strings("copies numbers", vector))
|
||||
l.Info(logs.ConstraintAdded, zap.String("location", constraint), zap.Strings("copies numbers", vector))
|
||||
}
|
||||
return copiesNums, nil
|
||||
return copiesNums
|
||||
}
|
||||
|
||||
func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
||||
|
@ -372,7 +415,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
|||
priority := v.GetInt(key + "priority")
|
||||
|
||||
if address == "" {
|
||||
l.Warn("skip, empty address")
|
||||
l.Warn(logs.SkipEmptyAddress)
|
||||
break
|
||||
}
|
||||
if weight <= 0 { // unspecified or wrong
|
||||
|
@ -384,7 +427,7 @@ func fetchPeers(l *zap.Logger, v *viper.Viper) []pool.NodeParam {
|
|||
|
||||
nodes = append(nodes, pool.NewNodeParam(priority, address, weight))
|
||||
|
||||
l.Info("added storage peer",
|
||||
l.Info(logs.AddedStoragePeer,
|
||||
zap.Int("priority", priority),
|
||||
zap.String("address", address),
|
||||
zap.Float64("weight", weight))
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"net/http"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -18,21 +19,21 @@ type Service struct {
|
|||
// Start runs http service with the exposed endpoint on the configured port.
|
||||
func (ms *Service) Start() {
|
||||
if ms.enabled {
|
||||
ms.log.Info("service is running", zap.String("endpoint", ms.Addr))
|
||||
ms.log.Info(logs.ServiceIsRunning, zap.String("endpoint", ms.Addr))
|
||||
err := ms.ListenAndServe()
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
ms.log.Warn("service couldn't start on configured port")
|
||||
ms.log.Warn(logs.ServiceCouldntStartOnConfiguredPort)
|
||||
}
|
||||
} else {
|
||||
ms.log.Info("service hasn't started since it's disabled")
|
||||
ms.log.Info(logs.ServiceHasntStartedSinceItsDisabled)
|
||||
}
|
||||
}
|
||||
|
||||
// ShutDown stops the service.
|
||||
func (ms *Service) ShutDown(ctx context.Context) {
|
||||
ms.log.Info("shutting down service", zap.String("endpoint", ms.Addr))
|
||||
ms.log.Info(logs.ShuttingDownService, zap.String("endpoint", ms.Addr))
|
||||
err := ms.Shutdown(ctx)
|
||||
if err != nil {
|
||||
ms.log.Panic("can't shut down service")
|
||||
ms.log.Panic(logs.CantShutDownService)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -125,6 +125,8 @@ S3_GW_CORS_DEFAULT_MAX_AGE=600
|
|||
# to consider PUT to FrostFS successful.
|
||||
# `0` or empty list means that object will be processed according to the container's placement policy
|
||||
S3_GW_FROSTFS_SET_COPIES_NUMBER=0
|
||||
# This flag enables client side object preparing.
|
||||
S3_GW_FROSTFS_CLIENT_CUT=false
|
||||
|
||||
# List of allowed AccessKeyID prefixes
|
||||
# If not set, S3 GW will accept all AccessKeyIDs
|
||||
|
|
|
@ -150,6 +150,8 @@ frostfs:
|
|||
# Numbers of the object copies (for each replica) to consider PUT to FrostFS successful.
|
||||
# `[0]` or empty list means that object will be processed according to the container's placement policy
|
||||
set_copies_number: [0]
|
||||
# This flag enables client side object preparing.
|
||||
client_cut: false
|
||||
|
||||
# List of allowed AccessKeyID prefixes
|
||||
# If the parameter is omitted, S3 GW will accept all AccessKeyIDs
|
||||
|
|
|
@ -500,17 +500,20 @@ tracing:
|
|||
# `frostfs` section
|
||||
|
||||
Contains parameters of requests to FrostFS.
|
||||
This value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
|
||||
|
||||
The `set_copies_number` value can be overridden with `X-Amz-Meta-Frostfs-Copies-Number` (value is comma separated numbers: `1,2,3`)
|
||||
header for `PutObject`, `CopyObject`, `CreateMultipartUpload`.
|
||||
|
||||
```yaml
|
||||
frostfs:
|
||||
set_copies_number: [0]
|
||||
client_cut: false
|
||||
```
|
||||
|
||||
| Parameter | Type | Default value | Description |
|
||||
|---------------------|------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `set_copies_number` | `[]uint32` | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
|
||||
| Parameter | Type | SIGHUP reload | Default value | Description |
|
||||
|---------------------|------------|---------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `set_copies_number` | `[]uint32` | yes | `[0]` | Numbers of the object copies (for each replica) to consider PUT to FrostFS successful. <br/>Default value `[0]` or empty list means that object will be processed according to the container's placement policy |
|
||||
| `client_cut` | `bool` | yes | `false` | This flag enables client side object preparing. |
|
||||
|
||||
# `resolve_bucket` section
|
||||
|
||||
|
|
2
go.mod
2
go.mod
|
@ -1,6 +1,6 @@
|
|||
module git.frostfs.info/TrueCloudLab/frostfs-s3-gw
|
||||
|
||||
go 1.19
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44
|
||||
|
|
2
go.sum
2
go.sum
|
@ -44,8 +44,6 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
|
|||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3 h1:GBRTOTRrtIvxi2TgxG7z/J7uRXiyb1SxR4247FaYCgU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821073319-342524159ac3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw=
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
|
||||
|
|
|
@ -243,6 +243,7 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
|
|||
prmPut.SetHeader(*obj)
|
||||
prmPut.SetPayload(prm.Payload)
|
||||
prmPut.SetCopiesNumberVector(prm.CopiesNumber)
|
||||
prmPut.SetClientCut(prm.ClientCut)
|
||||
|
||||
if prm.BearerToken != nil {
|
||||
prmPut.UseBearer(*prm.BearerToken)
|
||||
|
@ -262,6 +263,9 @@ type payloadReader struct {
|
|||
|
||||
func (x payloadReader) Read(p []byte) (int, error) {
|
||||
n, err := x.ReadCloser.Read(p)
|
||||
if err != nil && errors.Is(err, io.EOF) {
|
||||
return n, err
|
||||
}
|
||||
return n, handleObjectError("read payload", err)
|
||||
}
|
||||
|
||||
|
|
115
internal/logs/logs.go
Normal file
115
internal/logs/logs.go
Normal file
|
@ -0,0 +1,115 @@
|
|||
package logs
|
||||
|
||||
const (
|
||||
RequestUnmatched = "request unmatched" // Error in ../../api/router.go
|
||||
CheckContainer = "check container" // Info in ../../authmate/authmate.go
|
||||
CreateContainer = "create container" // Info in ../../authmate/authmate.go
|
||||
StoreBearerTokenIntoFrostFS = "store bearer token into FrostFS" // Info in ../../authmate/authmate.go
|
||||
UpdateAccessCredObjectIntoFrostFS = "update access cred object into FrostFS" // Info in ../../authmate/authmate.go
|
||||
MetricsAreDisabled = "metrics are disabled" // Warn in ../../metrics/app.go
|
||||
FoundMoreThanOneUnversionedNode = "found more than one unversioned node" // Debug in ../../pkg/service/tree/tree.go
|
||||
ServiceIsRunning = "service is running" // Info in ../../cmd/s3-gw/service.go
|
||||
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../cmd/s3-gw/service.go
|
||||
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../cmd/s3-gw/service.go
|
||||
ShuttingDownService = "shutting down service" // Info in ../../cmd/s3-gw/service.go
|
||||
ContainerResolverWillBeDisabled = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../cmd/s3-gw/app.go
|
||||
FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../cmd/s3-gw/app.go
|
||||
TracingConfigUpdated = "tracing config updated" // Info in ../../cmd/s3-gw/app.go
|
||||
FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../cmd/s3-gw/app.go
|
||||
UsingCredentials = "using credentials" // Info in ../../cmd/s3-gw/app.go
|
||||
ApplicationStarted = "application started" // Info in ../../cmd/s3-gw/app.go
|
||||
ApplicationFinished = "application finished" // Info in ../../cmd/s3-gw/app.go
|
||||
FetchDomainsPrepareToUseAPI = "fetch domains, prepare to use API" // Info in ../../cmd/s3-gw/app.go
|
||||
StartingServer = "starting server" // Info in ../../cmd/s3-gw/app.go
|
||||
StoppingServer = "stopping server" // Info in ../../cmd/s3-gw/app.go
|
||||
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../cmd/s3-gw/app.go
|
||||
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../cmd/s3-gw/app.go
|
||||
FailedToReloadConfig = "failed to reload config" // Warn in ../../cmd/s3-gw/app.go
|
||||
FailedToReloadResolvers = "failed to reload resolvers" // Warn in ../../cmd/s3-gw/app.go
|
||||
FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../cmd/s3-gw/app.go
|
||||
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../cmd/s3-gw/app.go
|
||||
LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../cmd/s3-gw/app.go
|
||||
FailedToAddServer = "failed to add server" // Warn in ../../cmd/s3-gw/app.go
|
||||
AddServer = "add server" // Info in ../../cmd/s3-gw/app.go
|
||||
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver 'nns' won't be used since 'rpc_endpoint' isn't provided" // Warn in ../../cmd/s3-gw/app.go
|
||||
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/s3-gw/app_settings.go
|
||||
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/s3-gw/app_settings.go
|
||||
FailedToParseDefaultLocationConstraint = "failed to parse 'default' location constraint, default one will be used" // Warn in cmd/s3-gw/app_settings.go
|
||||
FailedToReadRegionMapFilePolicies = "failed to read region map file, policies will be empty" // Warn in cmd/s3-gw/app_settings.go
|
||||
DefaultLocationConstraintCantBeOverriden = "'default' location constraint can't be overriden by custom policy, use 'placement_policy.default'" // Warn in cmd/s3-gw/app_settings.go
|
||||
FailedToParseLocationConstraint = "failed to parse location constraint, it cannot be used" // Warn in cmd/s3-gw/app_settings.go
|
||||
FailedToParseDefaultCopiesNumbers = "failed to parse 'default' copies numbers, default one will be used" // Warn in cmd/s3-gw/app_settings.go
|
||||
FailedToParseCopiesNumbers = "failed to parse copies numbers, skip" // Warn in cmd/s3-gw/app_settings.go
|
||||
FailedToParseDefaultDefaultLocationConstraint = "failed to parse default 'default' location constraint" // Fatal in cmd/s3-gw/app_settings.go
|
||||
ConstraintAdded = "constraint added" // Info in ../../cmd/s3-gw/app_settings.go
|
||||
SkipEmptyAddress = "skip, empty address" // Warn in ../../cmd/s3-gw/app_settings.go
|
||||
AddedStoragePeer = "added storage peer" // Info in ../../cmd/s3-gw/app_settings.go
|
||||
PrepareConnectionPool = "prepare connection pool" // Debug in ../../cmd/s3-authmate/modules/utils.go
|
||||
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../../api/cache/*
|
||||
InvalidCacheKeyType = "invalid cache key type" // Warn in ../../api/cache/objectslist.go
|
||||
ObjectIsCopied = "object is copied" // Info in ../../api/handler/copy.go
|
||||
CouldntSendNotification = "couldn't send notification: %w" // Error in ../../api/handler/*
|
||||
FailedToSendTestEventBecauseNotificationsIsDisabled = "failed to send test event because notifications is disabled" // Warn in ../../api/handler/notifications.go
|
||||
RequestFailed = "request failed" // Error in ../../api/handler/util.go
|
||||
GetBucketInfo = "get bucket info" // Warn in ../../api/handler/cors.go
|
||||
GetBucketCors = "get bucket cors" // Warn in ../../api/handler/cors.go
|
||||
SomeACLNotFullyMapped = "some acl not fully mapped" // Warn in ../../api/handler/acl.go
|
||||
CouldntDeleteObjects = "couldn't delete objects" // Error in ../../api/handler/delete.go
|
||||
NotificatorIsDisabledS3WontProduceNotificationEvents = "notificator is disabled, s3 won't produce notification events" // Warn in ../../api/handler/api.go
|
||||
CouldntGetBucketVersioning = "couldn't get bucket versioning" // Warn in ../../api/handler/put.go
|
||||
BucketIsCreated = "bucket is created" // Info in ../../api/handler/put.go
|
||||
CouldntDeleteNotificationConfigurationObject = "couldn't delete notification configuration object" // Error in ../../api/layer/notifications.go
|
||||
CouldNotParseContainerObjectLockEnabledAttribute = "could not parse container object lock enabled attribute" // Error in ../../api/layer/container.go
|
||||
CouldNotListUserContainers = "could not list user containers" // Error in ../../api/layer/container.go
|
||||
CouldNotFetchContainerInfo = "could not fetch container info" // Error in ../../api/layer/container.go
|
||||
MismatchedObjEncryptionInfo = "mismatched obj encryptionInfo" // Warn in ../../api/layer/multipart_upload.go
|
||||
UploadPart = "upload part" // Debug in ../../api/layer/multipart_upload.go
|
||||
CouldntDeleteOldPartObject = "couldn't delete old part object" // Error in ../../api/layer/multipart_upload.go
|
||||
CouldNotPutCompletedObject = "could not put a completed object (multipart upload)" // Error in ../../api/layer/multipart_upload.go
|
||||
CouldNotDeleteUploadPart = "could not delete upload part" // Warn in ../../api/layer/multipart_upload.go
|
||||
CouldntDeletePart = "couldn't delete part" // Warn in ../../api/layer/multipart_upload.go
|
||||
PartDetails = "part details" // Debug in ../../api/layer/multipart_upload.go
|
||||
GetObject = "get object" // Debug in ../../api/layer/layer.go
|
||||
ObjectAlreadyRemoved = "object already removed" // Debug in ../../api/layer/layer.go
|
||||
ObjectNotFound = "object not found" // Debug in ../../api/layer/layer.go
|
||||
ResolveBucket = "resolve bucket" // Info in ../../api/layer/layer.go
|
||||
CouldntDeleteCorsObject = "couldn't delete cors object" // Error in ../../api/layer/cors.go
|
||||
PutObject = "put object" // Debug in ../../api/layer/object.go
|
||||
FailedToDiscardPutPayloadProbablyGoroutineLeaks = "failed to discard put payload, probably goroutine leaks" // Warn in ../../api/layer/object.go
|
||||
FailedToSubmitTaskToPool = "failed to submit task to pool" // Warn in ../../api/layer/object.go
|
||||
CouldNotFetchObjectMeta = "could not fetch object meta" // Warn in ../../api/layer/object.go
|
||||
GetTreeNode = "get tree node" // Debug in ../../api/layer/tagging.go
|
||||
CouldntPutBucketInfoIntoCache = "couldn't put bucket info into cache" // Warn in ../../api/layer/cache.go
|
||||
CouldntAddObjectToCache = "couldn't add object to cache" // Warn in ../../api/layer/cache.go
|
||||
CouldntCacheAccessControlOperation = "couldn't cache access control operation" // Warn in ../../api/layer/cache.go
|
||||
CouldntPutObjAddressToNameCache = "couldn't put obj address to name cache" // Warn in ../../api/layer/cache.go
|
||||
CouldntCacheListOfObjects = "couldn't cache list of objects" // Warn in ../../api/layer/cache.go
|
||||
CouldntCacheTags = "couldn't cache tags" // Error in ../../api/layer/cache.go
|
||||
CouldntCacheLockInfo = "couldn't cache lock info" // Error in ../../api/layer/cache.go
|
||||
CouldntCacheBucketSettings = "couldn't cache bucket settings" // Warn in ../../api/layer/cache.go
|
||||
CouldntCacheCors = "couldn't cache cors" // Warn in ../../api/layer/cache.go
|
||||
CouldntCacheNotificationConfiguration = "couldn't cache notification configuration" // Warn in ../../api/layer/cache.go
|
||||
RequestEnd = "request end" // Info in ../../api/middleware/response.go
|
||||
CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed = "couldn't receive access box for gate key, random key will be used" // Debug in ../../api/middleware/auth.go
|
||||
FailedToPassAuthentication = "failed to pass authentication" // Error in ../../api/middleware/auth.go
|
||||
FailedToResolveCID = "failed to resolve CID" // Debug in ../../api/middleware/metrics.go
|
||||
RequestStart = "request start" // Info in ../../api/middleware/reqinfo.go
|
||||
CouldNotHandleMessage = "could not handle message" // Error in ../../api/notifications/controller.go
|
||||
CouldNotACKMessage = "could not ACK message" // Error in ../../api/notifications/controller.go
|
||||
CouldntMarshalAnEvent = "couldn't marshal an event" // Error in ../../api/notifications/controller.go
|
||||
CouldntSendAnEventToTopic = "couldn't send an event to topic" // Error in ../../api/notifications/controller.go
|
||||
InvalidDefaultMaxAge = "invalid defaultMaxAge" // Fatal in ../../cmd/s3-gw/app_settings.go
|
||||
CantShutDownService = "can't shut down service" // Panic in ../../cmd/s3-gw/service.go
|
||||
CouldntGenerateRandomKey = "couldn't generate random key" // Fatal in ../../cmd/s3-gw/app.go
|
||||
FailedToEnableNotifications = "failed to enable notifications" // Fatal in ../../cmd/s3-gw/app.go
|
||||
CouldntInitializeLayer = "couldn't initialize layer" // Fatal in ../../cmd/s3-gw/app.go
|
||||
FailedToCreateResolver = "failed to create resolver" // Fatal in ../../cmd/s3-gw/app.go
|
||||
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../cmd/s3-gw/app.go
|
||||
FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||
FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||
FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||
FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../cmd/s3-gw/app.go
|
||||
ListenAndServe = "listen and serve" // Fatal in ../../cmd/s3-gw/app.go
|
||||
NoHealthyServers = "no healthy servers" // Fatal in ../../cmd/s3-gw/app.go
|
||||
CouldNotInitializeAPIHandler = "could not initialize API handler" // Fatal in ../../cmd/s3-gw/app.go
|
||||
)
|
|
@ -4,6 +4,7 @@ import (
|
|||
"net/http"
|
||||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
@ -17,7 +18,7 @@ type AppMetrics struct {
|
|||
|
||||
func NewAppMetrics(logger *zap.Logger, poolStatistics StatisticScraper, enabled bool) *AppMetrics {
|
||||
if !enabled {
|
||||
logger.Warn("metrics are disabled")
|
||||
logger.Warn(logs.MetricsAreDisabled)
|
||||
}
|
||||
return &AppMetrics{
|
||||
logger: logger,
|
||||
|
@ -28,7 +29,7 @@ func NewAppMetrics(logger *zap.Logger, poolStatistics StatisticScraper, enabled
|
|||
|
||||
func (m *AppMetrics) SetEnabled(enabled bool) {
|
||||
if !enabled {
|
||||
m.logger.Warn("metrics are disabled")
|
||||
m.logger.Warn(logs.MetricsAreDisabled)
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"go.uber.org/zap"
|
||||
|
@ -220,6 +221,30 @@ func newNodeVersionFromTreeNode(filePath string, treeNode *treeNode) *data.NodeV
|
|||
return version
|
||||
}
|
||||
|
||||
func newMultipartInfoFromTreeNode(filePath string, treeNode *treeNode) (*data.MultipartInfo, error) {
|
||||
uploadID, _ := treeNode.Get(uploadIDKV)
|
||||
if uploadID == "" {
|
||||
return nil, fmt.Errorf("it's not a multipart node")
|
||||
}
|
||||
|
||||
multipartInfo := &data.MultipartInfo{
|
||||
ID: treeNode.ID,
|
||||
Key: filePath,
|
||||
UploadID: uploadID,
|
||||
Meta: treeNode.Meta,
|
||||
}
|
||||
|
||||
ownerID, _ := treeNode.Get(ownerKV)
|
||||
_ = multipartInfo.Owner.DecodeString(ownerID)
|
||||
|
||||
created, _ := treeNode.Get(createdKV)
|
||||
if utcMilli, err := strconv.ParseInt(created, 10, 64); err == nil {
|
||||
multipartInfo.Created = time.UnixMilli(utcMilli)
|
||||
}
|
||||
|
||||
return multipartInfo, nil
|
||||
}
|
||||
|
||||
func newMultipartInfo(node NodeResponse) (*data.MultipartInfo, error) {
|
||||
multipartInfo := &data.MultipartInfo{
|
||||
ID: node.GetNodeID(),
|
||||
|
@ -829,7 +854,7 @@ func (c *Tree) getUnversioned(ctx context.Context, bktInfo *data.BucketInfo, tre
|
|||
}
|
||||
|
||||
if len(nodes) > 1 {
|
||||
c.reqLogger(ctx).Debug("found more than one unversioned node",
|
||||
c.reqLogger(ctx).Debug(logs.FoundMoreThanOneUnversionedNode,
|
||||
zap.String("treeID", treeID), zap.String("filepath", filepath))
|
||||
}
|
||||
|
||||
|
@ -857,14 +882,14 @@ func (c *Tree) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketIn
|
|||
}
|
||||
|
||||
func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.MultipartInfo, error) {
|
||||
subTreeNodes, _, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false)
|
||||
subTreeNodes, headPrefix, err := c.getSubTreeByPrefix(ctx, bktInfo, systemTree, prefix, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result []*data.MultipartInfo
|
||||
for _, node := range subTreeNodes {
|
||||
multipartUploads, err := c.getSubTreeMultipartUploads(ctx, bktInfo, node.GetNodeID())
|
||||
multipartUploads, err := c.getSubTreeMultipartUploads(ctx, bktInfo, node.GetNodeID(), headPrefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -874,19 +899,55 @@ func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.Bu
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) ([]*data.MultipartInfo, error) {
|
||||
func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, parentFilePath string) ([]*data.MultipartInfo, error) {
|
||||
subTree, err := c.service.GetSubTree(ctx, bktInfo, systemTree, nodeID, maxGetSubTreeDepth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make([]*data.MultipartInfo, 0, len(subTree))
|
||||
for _, node := range subTree {
|
||||
multipartInfo, err := newMultipartInfo(node)
|
||||
if err != nil { // missed uploadID (it's a part node)
|
||||
var parentPrefix string
|
||||
if parentFilePath != "" { // The root of subTree can also have a parent
|
||||
parentPrefix = strings.TrimSuffix(parentFilePath, separator) + separator // To avoid 'foo//bar'
|
||||
}
|
||||
|
||||
var filepath string
|
||||
namesMap := make(map[uint64]string, len(subTree))
|
||||
multiparts := make(map[string][]*data.MultipartInfo, len(subTree))
|
||||
|
||||
for i, node := range subTree {
|
||||
treeNode, fileName, err := parseTreeNode(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
result = append(result, multipartInfo)
|
||||
|
||||
if i != 0 {
|
||||
if filepath, err = formFilePath(node, fileName, namesMap); err != nil {
|
||||
return nil, fmt.Errorf("invalid node order: %w", err)
|
||||
}
|
||||
} else {
|
||||
filepath = parentPrefix + fileName
|
||||
namesMap[treeNode.ID] = filepath
|
||||
}
|
||||
|
||||
multipartInfo, err := newMultipartInfoFromTreeNode(filepath, treeNode)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
key := formLatestNodeKey(node.GetParentID(), fileName)
|
||||
multipartInfos, ok := multiparts[key]
|
||||
if !ok {
|
||||
multipartInfos = []*data.MultipartInfo{multipartInfo}
|
||||
} else {
|
||||
multipartInfos = append(multipartInfos, multipartInfo)
|
||||
}
|
||||
|
||||
multiparts[key] = multipartInfos
|
||||
}
|
||||
|
||||
result := make([]*data.MultipartInfo, 0, len(multiparts))
|
||||
for _, multipartInfo := range multiparts {
|
||||
result = append(result, multipartInfo...)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
|
|
Loading…
Reference in a new issue